leponceau.org

Programming And Stuff, You Know The Thing…

rsync examples

Posted at — Feb 24, 2006

Systemfiles Backup vor Distro Update

KEIN Backup von /home.

mkdir /backup && \
rsync -axvPHS --numeric-ids --delete-excluded '--exclude=/var/cache/fscache' '--exclude=/home/*' \
    '--exclude=/backup/' --exclude='/tmp/*' --exclude='/mnt/*' --exclude=/swapfile / /backup/.

.rsync-filter / -F

$ cat a/.rsync-filter 
+ /mnt/b/
- /mnt/*

$ find a
a
a/other
a/mnt
a/mnt/a
a/mnt/a/excl
a/mnt/.excluded_too
a/mnt/b
a/mnt/b/c
a/mnt/b/c/a.txt
a/.rsync-filter

$ rm -rf b; rsync -Fva a/ b
sending incremental file list
created directory b
./
.rsync-filter
other
mnt/
mnt/b/
mnt/b/c/
mnt/b/c/a.txt

sent 309 bytes  received 84 bytes  786.00 bytes/sec
total size is 24  speedup is 0.06

one level deeper (you need to exclude/re-include at every level!):

$ cat a/.rsync-filter 
+ /mnt/b/c/
- /mnt/b/*
+ /mnt/b/
- /mnt/*

$ find a
a
a/other
a/mnt
a/mnt/a
a/mnt/a/excl
a/mnt/.excluded_too
a/mnt/b
a/mnt/b/excluded_dir
a/mnt/b/c
a/mnt/b/c/a.txt
a/mnt/b/excluded_file
a/.rsync-filter

$ rm -rf b; rsync -Fva a/ b
sending incremental file list
created directory b
./
.rsync-filter
other
mnt/
mnt/b/
mnt/b/c/
mnt/b/c/a.txt

sent 333 bytes  received 84 bytes  834.00 bytes/sec
total size is 48  speedup is 0.12

converting history of snapshots into rsync batch updates

$ ls -l
insgesamt 296
drwxr-xr-x 3 root root   4096  1. Mär 23:12 201003030000
-rw------- 1 root root 155747  3. Mär 02:46 201003030000.log.bz2
drwxr-xr-x 3 root root   4096  1. Mär 23:12 201003040000
-rw------- 1 root root   4211  4. Mär 01:27 201003040000.log.bz2
drwxr-xr-x 3 root root   4096  1. Mär 23:12 201003050000
-rw------- 1 root root   4516  5. Mär 00:53 201003050000.log.bz2
drwxr-xr-x 3 root root   4096  1. Mär 23:12 201003060000
-rw------- 1 root root   4202  6. Mär 00:48 201003060000.log.bz2
drwxr-xr-x 3 root root   4096  6. Mär 19:58 201003070000
-rw------- 1 root root   3678  7. Mär 22:04 201003070000.log.bz2
drwxr-xr-x 3 root root   4096  6. Mär 19:58 201003080000
...

set -x
last=""
find * -maxdepth 0 -type d| sort -n | while read l; do
  if ((last)); then
    if ! rsync -a --delete --numeric-ids -AS --only-write-batch="$last.batch" "$last/" "$l/"; then
      rm -f "$last.batch"
    fi
  fi
  last="$l"
done

Easy Transfers

put the following script into your homedir and create in/out dirs on your local and on the remote machine.

#!/bin/bash

USER="mw"
MACHINE="myserver.de"
DELAY=60

if [[ "$1" == "up" ]]; then
  while ! rsync -avPz --remove-source-files out/ $USER@$MACHINE:in/. ; do
    sleep $DELAY
  done
elif [[ "$1" == "down" ]]; then
  while ! rsync -avPz --remove-source-files $USER@$MACHINE:out/ in/. ; do
    sleep $DELAY
  done
else
  xterm -e $0 up &
  pid1=$?
  xterm -e $0 down &
  pid2=$?
  echo "press ENTER to stop the transfers"
  read
  kill $pid1
  kill $pid2
fi

Space-Efficient Snapshots

#!/bin/bash

set -ex

if [[ -z $1 ]]; then
  echo "Uses rsync's --link-dest Option to create backup snapshots."
  echo
  echo "All options passed to this script are passed through to rsync."
  echo "IMPORTANT: The source location must be the last option!"
  echo "In addition to these, the --link-dest and the target options"
  echo "are added -- so don't add them yourself."
  echo
  echo "Example: ./backup-rsync-linkdest.sh -avzPAHS --numeric-ids root@myserver.com:/home"
  echo "This will create a backup in ./linkdest_backup_root@myserver.com:_home/2007100812 where"
  echo "the appended number represents the current date and hour. The --link-dest option is"
  echo "set such that it points to the last such backup location if one exists."
  exit 0
fi

ionice -c3 -p$$ || :

lastarg=${!#}
new=$(date +%Y%m%d%H%M%S)
backupdir=./linkdest_backup_${lastarg//\//_}

# save some inodes/space:
weeklytar="$backupdir.$(date +%Y%W).tar.bz2"
if test -d "$backupdir" && ! test -e "$weeklytar"; then
    tar cjf "$weeklytar.tmp" "$backupdir"
    sync
    tar djf "$weeklytar.tmp" "$backupdir"
    sync
    mv "$weeklytar.tmp" "$weeklytar"
    sync
    rm -rf "$backupdir"
fi

mkdir -p "${backupdir}"
last=$(cd "${backupdir}" && ls 2>/dev/null | grep ^20 | sort | tail -n 1)
[[ "$last" != "$new" ]]
rm -rf "${backupdir}/tmp"
mkdir "${backupdir}/tmp"
[[ ! -e "${backupdir}/$new" ]]
linkdest=""
if [[ -n $last ]]; then
  linkdest="--link-dest ../$last/"
fi
rsync $linkdest --delete "$@" "${backupdir}/tmp"
mv -f "$backupdir/tmp" "$backupdir/$new"
echo "backup $backupdir/$new created OK"

exit 0

Misc