From 2f602062be06050a1a1b8010dd59b0072833cf27 Mon Sep 17 00:00:00 2001 From: lookshe Date: Sat, 18 Mar 2017 23:15:36 +0100 Subject: changed to use borg executable instead of tar, gzip and rsync --- backup.sh | 153 ++++++++++++++++++++------------------------------------------ 1 file changed, 49 insertions(+), 104 deletions(-) diff --git a/backup.sh b/backup.sh index 733b889..5f50674 100755 --- a/backup.sh +++ b/backup.sh @@ -3,7 +3,6 @@ # get path to all needed tools # /bin -cp=$(which cp) date=$(which date) echo=$(which echo) grep=$(which grep) @@ -12,18 +11,13 @@ mv=$(which mv) pwd=$(which pwd) rm=$(which rm) sed=$(which sed) -tar=$(which tar) -umount=$(which umount) # /usr/bin cut=$(which cut) dirname=$(which dirname) expr=$(which expr) head=$(which head) nice=$(which nice) -rsync=$(which rsync) -seq=$(which seq) ssh=$(which ssh) -sshfs=$(which sshfs) tail=$(which tail) # check if all needed tools are installed (only the ones not installed under /bin/) @@ -33,10 +27,7 @@ test "$dirname" = "" && echo "'dirname' not installed or not found by 'which'" & test "$expr" = "" && echo "'expr' not installed or not found by 'which'" && alltoolsinstalled="no" test "$head" = "" && echo "'head' not installed or not found by 'which'" && alltoolsinstalled="no" test "$nice" = "" && echo "'nice' not installed or not found by 'which'" && alltoolsinstalled="no" -test "$rsync" = "" && echo "'rsync' not installed or not found by 'which'" && alltoolsinstalled="no" -test "$seq" = "" && echo "'sed' not installed or not found by 'which'" && alltoolsinstalled="no" test "$ssh" = "" && echo "'ssh' not installed or not found by 'which'" && alltoolsinstalled="no" -test "$sshfs" = "" && echo "'sshfs' not installed or not found by 'which'" && alltoolsinstalled="no" test "$tail" = "" && echo "'tail' not installed or not found by 'which'" && alltoolsinstalled="no" if [ "$alltoolsinstalled" = "no" ] @@ -50,12 +41,6 @@ config_file="$script_dir/main.cfg" # timestamps yyyy-mm-dd and unixtime for calculating backup_stamp=$($date +%F) backup_time=$($date -d$backup_stamp +%s) -# check if it is the first of the month to do monthly backups -backup_month="no" -if [ "$($echo $backup_stamp | $cut -d- -f3)" = "01" ] -then - backup_month="yes" -fi # source a specified section from a specified config function source_section { @@ -74,26 +59,57 @@ function source_section { $rm -f "$tmp_file" } -# copy file with rsync to server and save last timestamp to logfile -function rsync_server { - filefrom="$1" - dirto="$2" +# check if borg repo exists and create if not +function check_borg_repo { + repo="$1" + serverdir="$2" + usesamerepo="$3" + if [ "$usesamerepo" = "yes" ] + then + repo_path="$userserver:$backupdir/$serverdir" + else + repo_path="$userserver:$backupdir/$serverdir/$repo" + fi + #check if repo exists + $borg_local_path list --remote-path "$borg_remote_path" --no-files-cache "$repo_path" > /dev/null 2>&1 + if [ $? -ne 0 ] + then + # create repo if not exists + $borg_local_path init --remote-path "$borg_remote_path" --encryption "$borg_encryption" "$repo_path" + fi +} + +# backup single dir with borg +function backup_dir { + repo="$1" + serverdir="$2" logfileentry="$3" - #nice -n 19 rsync -rle "ssh -i $keyfile" "$filefrom" "$userserver:$dirto" - # limit the bandwith - $nice -n 19 $rsync --bwlimit=100000 -rle "$ssh -i $keyfile" "$filefrom" "$userserver:$dirto" - ret=$? - if [ $ret -ne 0 ] + usesamerepo="$4" + check_borg_repo "$repo" "$serverdir" "$usesamerepo" + # to get backup.ignore work with relative paths we need to change the directory + pushd "$repo" > /dev/null + if [ "$usesamerepo" = "yes" ] then - $echo "problem in rsync: $filefrom" + repo_path="$userserver:$backupdir/$serverdir::$repo-{now:$default_timestamp}" + else + repo_path="$userserver:$backupdir/$serverdir/$repo::{now:$default_timestamp}" fi - $echo "$filefrom" | $grep ".month." > /dev/null 2>&1 - ret2=$? - # on monthly backups and connection errors there should be no update of logfile - if [ $ret2 -ne 0 -a $ret -eq 0 ] + ret=0 + if [ -f "backup.ignore" ] + then + $borg_local_path create --remote-path "$borg_remote_path" --one-file-system --exclude-from "backup.ignore" --compression "$borg_compression" "$repo_path" . + ret=$? + else + $borg_local_path create --remote-path "$borg_remote_path" --one-file-system --compression "$borg_compression" "$repo_path" . + ret=$? + fi + popd > /dev/null + if [ $ret -eq 0 ] then $sed -e "s/^$logfileentry .*$/${logfileentry} $backup_time/" "$logfile" > "$logfile.tmp" $mv -f "$logfile.tmp" "$logfile" + else + $echo "problem in borg create $repo" fi } @@ -107,6 +123,8 @@ do config_act="$confdir/$config_single" if [ -e "$config_act" ] then + # reset default settings + usesamerepo="no" # general section from specific config source_section "$config_act" "general" # change folder for backup @@ -127,94 +145,21 @@ do then $echo "${config_single}_$dir_single 0" >> $logfile fi - # add /. to directory for also backing up softlinks - dir_single_soft_link_path="$dir_single/." # get the stamp for the last backup last_stamp=$($grep "^${config_single}_$dir_single " $logfile | $cut -d" " -f2) # get the stamp for the next backup now_stamp=$($expr $last_stamp + $rotate_stamp) do_backup="no" - do_monthly="no" - do_normal="no" # check wether it is time to do a backup by time and also month if [ $now_stamp -le $backup_time ] then do_backup="yes" - if [ "$rotate" != "inc" ] - then - do_normal="yes" - fi - fi - if [ "$backup_month" = "yes" -a "$monthly" = "yes" ] - then - do_backup="yes" - do_monthly="yes" fi # now we are sure to do a backup if [ "$do_backup" = "yes" ] then - # we want an incremental backup since last one - if [ "$rotate" = "inc" ] - then - # tar is not able to handle unix-timestamps, so we calculate the required one - tar_stamp=$($date -d "1970-01-01 $last_stamp sec" +%F) - # only changed files since last modification in backup - if [ -f "$dir_single_soft_link_path/backup.ignore" ] - then - $nice -n 19 $tar czf "$dir_single.$backup_stamp.tar.gz" --warning=none --exclude-from="$dir_single_soft_link_path/backup.ignore" -N "$tar_stamp" "$dir_single_soft_link_path" - else - $nice -n 19 $tar czf "$dir_single.$backup_stamp.tar.gz" --warning=none -N "$tar_stamp" "$dir_single_soft_link_path" - fi - rsync_server "$dir_single.$backup_stamp.tar.gz" "$backupdir/$backupdirsingle/" "${config_single}_$dir_single" - # delete backups from local - $rm -f "$dir_single.$backup_stamp.tar.gz" - fi # if [ "$rotate" = "inc" ] - # we want a full backup - if [ "$do_monthly" = "yes" -o "$do_normal" = "yes" ] - then - # backup everything - if [ -f "$dir_single_soft_link_path/backup.ignore" ] - then - $nice -n 19 $tar czf "$dir_single.tar.gz" --warning=none --exclude-from="$dir_single_soft_link_path/backup.ignore" "$dir_single_soft_link_path" - else - $nice -n 19 $tar czf "$dir_single.tar.gz" --warning=none "$dir_single_soft_link_path" - fi - # check wether we should rotate - if [ "$rotate" != "inc" -a "$do_normal" = "yes" ] - then - - # mount for renaming - $sshfs $userserver:$backupdir $backupdir_local -o IdentityFile=$keyfile -o IdentitiesOnly=yes - - # rotate old backups - for count in `seq $rotate -1 2` - do - count_last=$(expr $count - 1) - $rm -f "$backupdir_local/$backupdirsingle/$dir_single.tar.gz.$count" - $mv -f "$backupdir_local/$backupdirsingle/$dir_single.tar.gz.$count_last" "$backupdir_local/$backupdirsingle/$dir_single.tar.gz.$count" > /dev/null 2>&1 - done - $rm -f "$backupdir_local/$backupdirsingle/$dir_single.tar.gz.1" - $mv -f "$backupdir_local/$backupdirsingle/$dir_single.tar.gz" "$backupdir_local/$backupdirsingle/$dir_single.tar.gz.1" > /dev/null 2>&1 - - # unmount - $umount $backupdir_local - - fi # if [ "$rotate" != "inc" -a "$do_normal" = "yes" ] - # copy new backups - if [ "$do_normal" = "yes" ] - then - rsync_server "$dir_single.tar.gz" "$backupdir/$backupdirsingle/" "${config_single}_$dir_single" - fi - if [ "$do_monthly" = "yes" ] - then - #TODO: check why move sometimes causes problems - $nice -n 19 $cp "$dir_single.tar.gz" "$dir_single.$backup_stamp.month.tar.gz" - rsync_server "$dir_single.$backup_stamp.month.tar.gz" "$backupdir/$backupdirsingle/" "${config_single}_$dir_single" - fi - # delete backups from local - $rm -f "$dir_single.$backup_stamp.month.tar.gz" - $rm -f "$dir_single.tar.gz" - fi # if [ "$do_monthly" = "yes" -o "$do_normal" = "yes" ] + # backup everything + backup_dir "$dir_single" "$backupdirsingle" "${config_single}_$dir_single" "$usesamerepo" fi # if [ "$do_backup" = "yes" ] fi # if [ -e "$dir_single" ] done # for dir_single in $dirssingle -- cgit v1.2.3