From f3a265b07e708ce45cd7ed2d4dd4a86f0ebf39cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fatih=20=C3=9Cst=C3=BCnda=C4=9F?= Date: Fri, 3 Jan 2014 14:40:53 +0200 Subject: [PATCH 1/6] SSH support added. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Backups can be created at remote server via ssh. Destination folder should be given like “root@192.168.0.1:/bakcups” NOTE: Assumed public/private key config is ok. --- .gitignore | 1 + rsync_tmbackup.sh | 102 ++++++++++++++++++++++++++++++++++++++-------- 2 files changed, 85 insertions(+), 18 deletions(-) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..723ef36 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +.idea \ No newline at end of file diff --git a/rsync_tmbackup.sh b/rsync_tmbackup.sh index a467755..7758f90 100755 --- a/rsync_tmbackup.sh +++ b/rsync_tmbackup.sh @@ -9,6 +9,14 @@ APPNAME=$(basename $0 | sed "s/\.sh$//") fn_log_info() { echo "$APPNAME: $1"; } fn_log_warn() { echo "$APPNAME: [WARNING] $1" 1>&2; } fn_log_error() { echo "$APPNAME: [ERROR] $1" 1>&2; } +fn_log_info_cmd() { + if [ -n "$SSH_CMD" ] + then + echo "$APPNAME: $SSH_CMD '$1'"; + else + echo "$APPNAME: $1"; + fi +} # ----------------------------------------------------------------------------- # Make sure everything really stops when CTRL+C is pressed @@ -34,7 +42,7 @@ fn_parse_date() { } fn_find_backups() { - find "$DEST_FOLDER" -type d -name "????-??-??-??????" -prune | sort -r + fn_run_cmd "find "$DEST_FOLDER" -type d -name "????-??-??-??????" -prune | sort -r" } fn_expire_backup() { @@ -46,17 +54,72 @@ fn_expire_backup() { fi fn_log_info "Expiring $1" - rm -rf -- "$1" + fn_rm "$1" +} + +fn_parse_ssh() { + if [[ "$DEST_FOLDER" =~ ^[A-Za-z0-9\._%\+\-]+@[A-Za-z0-9.\-]+\:.+$ ]] + then + SSH_USER=$(echo "$DEST_FOLDER" | sed -E 's/^([A-Za-z0-9\._%\+\-]+)@([A-Za-z0-9.\-]+)\:(.+)$/\1/') + SSH_HOST=$(echo "$DEST_FOLDER" | sed -E 's/^([A-Za-z0-9\._%\+\-]+)@([A-Za-z0-9.\-]+)\:(.+)$/\2/') + SSH_DEST_FOLDER=$(echo "$DEST_FOLDER" | sed -E 's/^([A-Za-z0-9\._%\+\-]+)@([A-Za-z0-9.\-]+)\:(.+)$/\3/') + SSH_CMD="ssh ${SSH_USER}@${SSH_HOST}" + SSH_FOLDER_PREFIX="${SSH_USER}@${SSH_HOST}:" + fi +} + +fn_run_cmd() { + if [ -n "$SSH_CMD" ] + then + eval "$SSH_CMD '$1'" + else + eval $1 + fi +} + +fn_find() { + fn_run_cmd "find $1" 2>/dev/null +} + +fn_get_absolute_path() { + fn_run_cmd "cd $1;pwd" +} + +fn_mkdir() { + fn_run_cmd "mkdir -p -- $1" +} + +fn_rm() { + fn_run_cmd "rm -rf -- $1" +} + +fn_touch() { + fn_run_cmd "touch -- $1" +} + +fn_ln() { + fn_run_cmd "ln -vs -- $1 $2" } # ----------------------------------------------------------------------------- # Source and destination information # ----------------------------------------------------------------------------- +SSH_USER="" +SSH_HOST="" +SSH_DEST_FOLDER="" +SSH_CMD="" +SSH_FOLDER_PREFIX="" SRC_FOLDER="${1%/}" DEST_FOLDER="${2%/}" EXCLUSION_FILE="$3" +fn_parse_ssh + +if [ -n "$SSH_DEST_FOLDER" ]; then + DEST_FOLDER="$SSH_DEST_FOLDER" +fi + for ARG in "$SRC_FOLDER" "$DEST_FOLDER" "$EXCLUSION_FILE"; do if [[ "$ARG" == *"'"* ]]; then fn_log_error 'Arguments may not have any single quote characters.' @@ -71,13 +134,13 @@ done # TODO: check that the destination supports hard links fn_backup_marker_path() { echo "$1/backup.marker"; } -fn_find_backup_marker() { find "$(fn_backup_marker_path "$1")" 2>/dev/null; } +fn_find_backup_marker() { fn_find "$(fn_backup_marker_path "$1")" 2>/dev/null; } if [ -z "$(fn_find_backup_marker "$DEST_FOLDER")" ]; then fn_log_info "Safety check failed - the destination does not appear to be a backup folder or drive (marker file not found)." fn_log_info "If it is indeed a backup folder, you may add the marker file by running the following command:" fn_log_info "" - fn_log_info "touch \"$(fn_backup_marker_path "$DEST_FOLDER")\"" + fn_log_info_cmd "touch \"$(fn_backup_marker_path "$DEST_FOLDER")\"" fn_log_info "" exit 1 fi @@ -111,12 +174,12 @@ fi # Handle case where a previous backup failed or was interrupted. # ----------------------------------------------------------------------------- -if [ -f "$INPROGRESS_FILE" ]; then +if [ -n "$(fn_find "$INPROGRESS_FILE")" ]; then if [ -n "$PREVIOUS_DEST" ]; then # - Last backup is moved to current backup folder so that it can be resumed. # - 2nd to last backup becomes last backup. - fn_log_info "$INPROGRESS_FILE already exists - the previous backup failed or was interrupted. Backup will resume from there." - mv -- "$PREVIOUS_DEST" "$DEST" + fn_log_info "$SSH_FOLDER_PREFIX$INPROGRESS_FILE already exists - the previous backup failed or was interrupted. Backup will resume from there." + fn_run_cmd "mv -- $PREVIOUS_DEST $DEST" if [ "$(fn_find_backups | wc -l)" -gt 1 ]; then PREVIOUS_DEST="$(fn_find_backups | sed -n '2p')" else @@ -138,8 +201,8 @@ while : ; do else # If the path is relative, it needs to be relative to the destination. To keep # it simple, just use an absolute path. See http://serverfault.com/a/210058/118679 - PREVIOUS_DEST="$(cd "$PREVIOUS_DEST"; pwd)" - fn_log_info "Previous backup found - doing incremental backup from $PREVIOUS_DEST" + PREVIOUS_DEST="$(fn_get_absolute_path "$PREVIOUS_DEST")" + fn_log_info "Previous backup found - doing incremental backup from $SSH_FOLDER_PREFIX$PREVIOUS_DEST" LINK_DEST_OPTION="--link-dest='$PREVIOUS_DEST'" fi @@ -147,9 +210,9 @@ while : ; do # Create destination folder if it doesn't already exists # ----------------------------------------------------------------------------- - if [ ! -d "$DEST" ]; then - fn_log_info "Creating destination $DEST" - mkdir -p -- "$DEST" + if [ -z "$(fn_find "$DEST -type d" 2>/dev/null)" ]; then + fn_log_info "Creating destination $SSH_FOLDER_PREFIX$DEST" + fn_mkdir "$DEST" fi # ----------------------------------------------------------------------------- @@ -189,9 +252,12 @@ while : ; do fn_log_info "Starting backup..." fn_log_info "From: $SRC_FOLDER" - fn_log_info "To: $DEST" + fn_log_info "To: $SSH_FOLDER_PREFIX$DEST" CMD="rsync" + if [ -n "$SSH_CMD" ]; then + CMD="$CMD -e 'ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null'" + fi CMD="$CMD --compress" CMD="$CMD --numeric-ids" CMD="$CMD --links" @@ -206,13 +272,13 @@ while : ; do CMD="$CMD --exclude-from '$EXCLUSION_FILE'" fi CMD="$CMD $LINK_DEST_OPTION" - CMD="$CMD -- '$SRC_FOLDER/' '$DEST/'" + CMD="$CMD -- '$SRC_FOLDER/' '$SSH_FOLDER_PREFIX$DEST/'" CMD="$CMD | grep -E '^deleting|[^/]$'" fn_log_info "Running command:" fn_log_info "$CMD" - touch -- "$INPROGRESS_FILE" + fn_touch "$INPROGRESS_FILE" eval $CMD # ----------------------------------------------------------------------------- @@ -251,10 +317,10 @@ while : ; do # Add symlink to last successful backup # ----------------------------------------------------------------------------- - rm -rf -- "$DEST_FOLDER/latest" - ln -vs -- "$(basename -- "$DEST")" "$DEST_FOLDER/latest" + fn_rm "$DEST_FOLDER/latest" + fn_ln "$(basename -- "$DEST")" "$DEST_FOLDER/latest" - rm -f -- "$INPROGRESS_FILE" + fn_rm "$INPROGRESS_FILE" rm -f -- "$LOG_FILE" fn_log_info "Backup completed without errors." From 6c4607d8531f1b9d5d431f101f5bdf0438d9a5bd Mon Sep 17 00:00:00 2001 From: Elliot Jordan Date: Tue, 20 Jan 2015 13:36:48 -0800 Subject: [PATCH 2/6] Removed unnecessary $ from arithmetics. --- rsync_tmbackup.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rsync_tmbackup.sh b/rsync_tmbackup.sh index 7758f90..89e7f27 100755 --- a/rsync_tmbackup.sh +++ b/rsync_tmbackup.sh @@ -152,8 +152,8 @@ fi # Date logic NOW=$(date +"%Y-%m-%d-%H%M%S") EPOCH=$(date "+%s") -KEEP_ALL_DATE=$(($EPOCH - 86400)) # 1 day ago -KEEP_DAILIES_DATE=$(($EPOCH - 2678400)) # 31 days ago +KEEP_ALL_DATE=$((EPOCH - 86400)) # 1 day ago +KEEP_DAILIES_DATE=$((EPOCH - 2678400)) # 31 days ago export IFS=$'\n' # Better for handling spaces in filenames. PROFILE_FOLDER="$HOME/.$APPNAME" From 3202ca5da057a74d03183ff55c6ee7710fdd1534 Mon Sep 17 00:00:00 2001 From: Elliot Jordan Date: Tue, 20 Jan 2015 13:41:52 -0800 Subject: [PATCH 3/6] Consistent formatting for conditionals. --- rsync_tmbackup.sh | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/rsync_tmbackup.sh b/rsync_tmbackup.sh index 89e7f27..cb1ef0f 100755 --- a/rsync_tmbackup.sh +++ b/rsync_tmbackup.sh @@ -10,8 +10,7 @@ fn_log_info() { echo "$APPNAME: $1"; } fn_log_warn() { echo "$APPNAME: [WARNING] $1" 1>&2; } fn_log_error() { echo "$APPNAME: [ERROR] $1" 1>&2; } fn_log_info_cmd() { - if [ -n "$SSH_CMD" ] - then + if [ -n "$SSH_CMD" ]; then echo "$APPNAME: $SSH_CMD '$1'"; else echo "$APPNAME: $1"; @@ -58,8 +57,7 @@ fn_expire_backup() { } fn_parse_ssh() { - if [[ "$DEST_FOLDER" =~ ^[A-Za-z0-9\._%\+\-]+@[A-Za-z0-9.\-]+\:.+$ ]] - then + if [[ "$DEST_FOLDER" =~ ^[A-Za-z0-9\._%\+\-]+@[A-Za-z0-9.\-]+\:.+$ ]]; then SSH_USER=$(echo "$DEST_FOLDER" | sed -E 's/^([A-Za-z0-9\._%\+\-]+)@([A-Za-z0-9.\-]+)\:(.+)$/\1/') SSH_HOST=$(echo "$DEST_FOLDER" | sed -E 's/^([A-Za-z0-9\._%\+\-]+)@([A-Za-z0-9.\-]+)\:(.+)$/\2/') SSH_DEST_FOLDER=$(echo "$DEST_FOLDER" | sed -E 's/^([A-Za-z0-9\._%\+\-]+)@([A-Za-z0-9.\-]+)\:(.+)$/\3/') @@ -69,8 +67,7 @@ fn_parse_ssh() { } fn_run_cmd() { - if [ -n "$SSH_CMD" ] - then + if [ -n "$SSH_CMD" ]; then eval "$SSH_CMD '$1'" else eval $1 @@ -121,7 +118,7 @@ if [ -n "$SSH_DEST_FOLDER" ]; then fi for ARG in "$SRC_FOLDER" "$DEST_FOLDER" "$EXCLUSION_FILE"; do -if [[ "$ARG" == *"'"* ]]; then + if [[ "$ARG" == *"'"* ]]; then fn_log_error 'Arguments may not have any single quote characters.' exit 1 fi From 742412fb3e29039356c5c90ed8f7e26799ce8980 Mon Sep 17 00:00:00 2001 From: Elliot Jordan Date: Tue, 20 Jan 2015 13:42:53 -0800 Subject: [PATCH 4/6] Cleaned up whitespace (no changes to code). --- rsync_tmbackup.sh | 306 +++++++++++++++++++++++----------------------- 1 file changed, 153 insertions(+), 153 deletions(-) mode change 100755 => 100644 rsync_tmbackup.sh diff --git a/rsync_tmbackup.sh b/rsync_tmbackup.sh old mode 100755 new mode 100644 index cb1ef0f..6605ae4 --- a/rsync_tmbackup.sh +++ b/rsync_tmbackup.sh @@ -11,10 +11,10 @@ fn_log_warn() { echo "$APPNAME: [WARNING] $1" 1>&2; } fn_log_error() { echo "$APPNAME: [ERROR] $1" 1>&2; } fn_log_info_cmd() { if [ -n "$SSH_CMD" ]; then - echo "$APPNAME: $SSH_CMD '$1'"; - else - echo "$APPNAME: $1"; - fi + echo "$APPNAME: $SSH_CMD '$1'"; + else + echo "$APPNAME: $1"; + fi } # ----------------------------------------------------------------------------- @@ -22,8 +22,8 @@ fn_log_info_cmd() { # ----------------------------------------------------------------------------- fn_terminate_script() { - fn_log_info "SIGINT caught." - exit 1 + fn_log_info "SIGINT caught." + exit 1 } trap 'fn_terminate_script' SIGINT @@ -33,27 +33,27 @@ trap 'fn_terminate_script' SIGINT # ----------------------------------------------------------------------------- fn_parse_date() { - # Converts YYYY-MM-DD-HHMMSS to YYYY-MM-DD HH:MM:SS and then to Unix Epoch. - case "$OSTYPE" in - linux*) date -d "${1:0:10} ${1:11:2}:${1:13:2}:${1:15:2}" +%s ;; - darwin*) date -j -f "%Y-%m-%d-%H%M%S" "$1" "+%s" ;; - esac + # Converts YYYY-MM-DD-HHMMSS to YYYY-MM-DD HH:MM:SS and then to Unix Epoch. + case "$OSTYPE" in + linux*) date -d "${1:0:10} ${1:11:2}:${1:13:2}:${1:15:2}" +%s ;; + darwin*) date -j -f "%Y-%m-%d-%H%M%S" "$1" "+%s" ;; + esac } fn_find_backups() { - fn_run_cmd "find "$DEST_FOLDER" -type d -name "????-??-??-??????" -prune | sort -r" + fn_run_cmd "find "$DEST_FOLDER" -type d -name "????-??-??-??????" -prune | sort -r" } fn_expire_backup() { - # Double-check that we're on a backup destination to be completely - # sure we're deleting the right folder - if [ -z "$(fn_find_backup_marker "$(dirname -- "$1")")" ]; then - fn_log_error "$1 is not on a backup destination - aborting." - exit 1 - fi + # Double-check that we're on a backup destination to be completely + # sure we're deleting the right folder + if [ -z "$(fn_find_backup_marker "$(dirname -- "$1")")" ]; then + fn_log_error "$1 is not on a backup destination - aborting." + exit 1 + fi - fn_log_info "Expiring $1" - fn_rm "$1" + fn_log_info "Expiring $1" + fn_rm "$1" } fn_parse_ssh() { @@ -118,10 +118,10 @@ if [ -n "$SSH_DEST_FOLDER" ]; then fi for ARG in "$SRC_FOLDER" "$DEST_FOLDER" "$EXCLUSION_FILE"; do - if [[ "$ARG" == *"'"* ]]; then - fn_log_error 'Arguments may not have any single quote characters.' - exit 1 - fi + if [[ "$ARG" == *"'"* ]]; then + fn_log_error 'Arguments may not have any single quote characters.' + exit 1 + fi done # ----------------------------------------------------------------------------- @@ -134,12 +134,12 @@ fn_backup_marker_path() { echo "$1/backup.marker"; } fn_find_backup_marker() { fn_find "$(fn_backup_marker_path "$1")" 2>/dev/null; } if [ -z "$(fn_find_backup_marker "$DEST_FOLDER")" ]; then - fn_log_info "Safety check failed - the destination does not appear to be a backup folder or drive (marker file not found)." - fn_log_info "If it is indeed a backup folder, you may add the marker file by running the following command:" - fn_log_info "" - fn_log_info_cmd "touch \"$(fn_backup_marker_path "$DEST_FOLDER")\"" - fn_log_info "" - exit 1 + fn_log_info "Safety check failed - the destination does not appear to be a backup folder or drive (marker file not found)." + fn_log_info "If it is indeed a backup folder, you may add the marker file by running the following command:" + fn_log_info "" + fn_log_info_cmd "touch \"$(fn_backup_marker_path "$DEST_FOLDER")\"" + fn_log_info "" + exit 1 fi # ----------------------------------------------------------------------------- @@ -163,8 +163,8 @@ INPROGRESS_FILE="$DEST_FOLDER/backup.inprogress" # ----------------------------------------------------------------------------- if [ ! -d "$PROFILE_FOLDER" ]; then - fn_log_info "Creating profile folder in '$PROFILE_FOLDER'..." - mkdir -- "$PROFILE_FOLDER" + fn_log_info "Creating profile folder in '$PROFILE_FOLDER'..." + mkdir -- "$PROFILE_FOLDER" fi # ----------------------------------------------------------------------------- @@ -172,155 +172,155 @@ fi # ----------------------------------------------------------------------------- if [ -n "$(fn_find "$INPROGRESS_FILE")" ]; then - if [ -n "$PREVIOUS_DEST" ]; then - # - Last backup is moved to current backup folder so that it can be resumed. - # - 2nd to last backup becomes last backup. - fn_log_info "$SSH_FOLDER_PREFIX$INPROGRESS_FILE already exists - the previous backup failed or was interrupted. Backup will resume from there." - fn_run_cmd "mv -- $PREVIOUS_DEST $DEST" - if [ "$(fn_find_backups | wc -l)" -gt 1 ]; then - PREVIOUS_DEST="$(fn_find_backups | sed -n '2p')" - else - PREVIOUS_DEST="" - fi - fi + if [ -n "$PREVIOUS_DEST" ]; then + # - Last backup is moved to current backup folder so that it can be resumed. + # - 2nd to last backup becomes last backup. + fn_log_info "$SSH_FOLDER_PREFIX$INPROGRESS_FILE already exists - the previous backup failed or was interrupted. Backup will resume from there." + fn_run_cmd "mv -- $PREVIOUS_DEST $DEST" + if [ "$(fn_find_backups | wc -l)" -gt 1 ]; then + PREVIOUS_DEST="$(fn_find_backups | sed -n '2p')" + else + PREVIOUS_DEST="" + fi + fi fi # Run in a loop to handle the "No space left on device" logic. while : ; do - # ----------------------------------------------------------------------------- - # Check if we are doing an incremental backup (if previous backup exists). - # ----------------------------------------------------------------------------- + # ----------------------------------------------------------------------------- + # Check if we are doing an incremental backup (if previous backup exists). + # ----------------------------------------------------------------------------- - LINK_DEST_OPTION="" - if [ -z "$PREVIOUS_DEST" ]; then - fn_log_info "No previous backup - creating new one." - else - # If the path is relative, it needs to be relative to the destination. To keep - # it simple, just use an absolute path. See http://serverfault.com/a/210058/118679 - PREVIOUS_DEST="$(fn_get_absolute_path "$PREVIOUS_DEST")" - fn_log_info "Previous backup found - doing incremental backup from $SSH_FOLDER_PREFIX$PREVIOUS_DEST" - LINK_DEST_OPTION="--link-dest='$PREVIOUS_DEST'" - fi + LINK_DEST_OPTION="" + if [ -z "$PREVIOUS_DEST" ]; then + fn_log_info "No previous backup - creating new one." + else + # If the path is relative, it needs to be relative to the destination. To keep + # it simple, just use an absolute path. See http://serverfault.com/a/210058/118679 + PREVIOUS_DEST="$(fn_get_absolute_path "$PREVIOUS_DEST")" + fn_log_info "Previous backup found - doing incremental backup from $SSH_FOLDER_PREFIX$PREVIOUS_DEST" + LINK_DEST_OPTION="--link-dest='$PREVIOUS_DEST'" + fi - # ----------------------------------------------------------------------------- - # Create destination folder if it doesn't already exists - # ----------------------------------------------------------------------------- + # ----------------------------------------------------------------------------- + # Create destination folder if it doesn't already exists + # ----------------------------------------------------------------------------- - if [ -z "$(fn_find "$DEST -type d" 2>/dev/null)" ]; then - fn_log_info "Creating destination $SSH_FOLDER_PREFIX$DEST" - fn_mkdir "$DEST" - fi + if [ -z "$(fn_find "$DEST -type d" 2>/dev/null)" ]; then + fn_log_info "Creating destination $SSH_FOLDER_PREFIX$DEST" + fn_mkdir "$DEST" + fi - # ----------------------------------------------------------------------------- - # Purge certain old backups before beginning new backup. - # ----------------------------------------------------------------------------- + # ----------------------------------------------------------------------------- + # Purge certain old backups before beginning new backup. + # ----------------------------------------------------------------------------- - # Default value for $PREV ensures that the most recent backup is never deleted. - PREV="0000-00-00-000000" - for FILENAME in $(fn_find_backups | sort -r); do - BACKUP_DATE=$(basename "$FILENAME") - TIMESTAMP=$(fn_parse_date $BACKUP_DATE) + # Default value for $PREV ensures that the most recent backup is never deleted. + PREV="0000-00-00-000000" + for FILENAME in $(fn_find_backups | sort -r); do + BACKUP_DATE=$(basename "$FILENAME") + TIMESTAMP=$(fn_parse_date $BACKUP_DATE) - # Skip if failed to parse date... - if [ -z "$TIMESTAMP" ]; then - fn_log_warn "Could not parse date: $FILENAME" - continue - fi + # Skip if failed to parse date... + if [ -z "$TIMESTAMP" ]; then + fn_log_warn "Could not parse date: $FILENAME" + continue + fi - if [ $TIMESTAMP -ge $KEEP_ALL_DATE ]; then - true - elif [ $TIMESTAMP -ge $KEEP_DAILIES_DATE ]; then - # Delete all but the most recent of each day. - [ "${BACKUP_DATE:0:10}" == "${PREV:0:10}" ] && fn_expire_backup "$FILENAME" - else - # Delete all but the most recent of each month. - [ "${BACKUP_DATE:0:7}" == "${PREV:0:7}" ] && fn_expire_backup "$FILENAME" - fi + if [ $TIMESTAMP -ge $KEEP_ALL_DATE ]; then + true + elif [ $TIMESTAMP -ge $KEEP_DAILIES_DATE ]; then + # Delete all but the most recent of each day. + [ "${BACKUP_DATE:0:10}" == "${PREV:0:10}" ] && fn_expire_backup "$FILENAME" + else + # Delete all but the most recent of each month. + [ "${BACKUP_DATE:0:7}" == "${PREV:0:7}" ] && fn_expire_backup "$FILENAME" + fi - PREV=$BACKUP_DATE - done + PREV=$BACKUP_DATE + done - # ----------------------------------------------------------------------------- - # Start backup - # ----------------------------------------------------------------------------- + # ----------------------------------------------------------------------------- + # Start backup + # ----------------------------------------------------------------------------- - LOG_FILE="$PROFILE_FOLDER/$(date +"%Y-%m-%d-%H%M%S").log" + LOG_FILE="$PROFILE_FOLDER/$(date +"%Y-%m-%d-%H%M%S").log" - fn_log_info "Starting backup..." - fn_log_info "From: $SRC_FOLDER" - fn_log_info "To: $SSH_FOLDER_PREFIX$DEST" + fn_log_info "Starting backup..." + fn_log_info "From: $SRC_FOLDER" + fn_log_info "To: $SSH_FOLDER_PREFIX$DEST" - CMD="rsync" - if [ -n "$SSH_CMD" ]; then - CMD="$CMD -e 'ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null'" - fi - CMD="$CMD --compress" - CMD="$CMD --numeric-ids" - CMD="$CMD --links" - CMD="$CMD --hard-links" - CMD="$CMD --one-file-system" - CMD="$CMD --archive" - CMD="$CMD --itemize-changes" - CMD="$CMD --verbose" - CMD="$CMD --log-file '$LOG_FILE'" - if [ -n "$EXCLUSION_FILE" ]; then - # We've already checked that $EXCLUSION_FILE doesn't contain a single quote - CMD="$CMD --exclude-from '$EXCLUSION_FILE'" - fi - CMD="$CMD $LINK_DEST_OPTION" - CMD="$CMD -- '$SRC_FOLDER/' '$SSH_FOLDER_PREFIX$DEST/'" - CMD="$CMD | grep -E '^deleting|[^/]$'" + CMD="rsync" + if [ -n "$SSH_CMD" ]; then + CMD="$CMD -e 'ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null'" + fi + CMD="$CMD --compress" + CMD="$CMD --numeric-ids" + CMD="$CMD --links" + CMD="$CMD --hard-links" + CMD="$CMD --one-file-system" + CMD="$CMD --archive" + CMD="$CMD --itemize-changes" + CMD="$CMD --verbose" + CMD="$CMD --log-file '$LOG_FILE'" + if [ -n "$EXCLUSION_FILE" ]; then + # We've already checked that $EXCLUSION_FILE doesn't contain a single quote + CMD="$CMD --exclude-from '$EXCLUSION_FILE'" + fi + CMD="$CMD $LINK_DEST_OPTION" + CMD="$CMD -- '$SRC_FOLDER/' '$SSH_FOLDER_PREFIX$DEST/'" + CMD="$CMD | grep -E '^deleting|[^/]$'" - fn_log_info "Running command:" - fn_log_info "$CMD" + fn_log_info "Running command:" + fn_log_info "$CMD" - fn_touch "$INPROGRESS_FILE" - eval $CMD + fn_touch "$INPROGRESS_FILE" + eval $CMD - # ----------------------------------------------------------------------------- - # Check if we ran out of space - # ----------------------------------------------------------------------------- + # ----------------------------------------------------------------------------- + # Check if we ran out of space + # ----------------------------------------------------------------------------- - # TODO: find better way to check for out of space condition without parsing log. - NO_SPACE_LEFT="$(grep "No space left on device (28)\|Result too large (34)" "$LOG_FILE")" + # TODO: find better way to check for out of space condition without parsing log. + NO_SPACE_LEFT="$(grep "No space left on device (28)\|Result too large (34)" "$LOG_FILE")" - if [ -n "$NO_SPACE_LEFT" ]; then - fn_log_warn "No space left on device - removing oldest backup and resuming." + if [ -n "$NO_SPACE_LEFT" ]; then + fn_log_warn "No space left on device - removing oldest backup and resuming." - if [[ "$(fn_find_backups | wc -l)" -lt "2" ]]; then - fn_log_error "No space left on device, and no old backup to delete." - exit 1 - fi + if [[ "$(fn_find_backups | wc -l)" -lt "2" ]]; then + fn_log_error "No space left on device, and no old backup to delete." + exit 1 + fi - fn_expire_backup "$(fn_find_backups | tail -n 1)" + fn_expire_backup "$(fn_find_backups | tail -n 1)" - # Resume backup - continue - fi + # Resume backup + continue + fi - # ----------------------------------------------------------------------------- - # Check whether rsync reported any errors - # ----------------------------------------------------------------------------- - if [ -n "$(grep "rsync:" "$LOG_FILE")" ]; then - fn_log_warn "Rsync reported a warning, please check '$LOG_FILE' for more details." - fi - if [ -n "$(grep "rsync error:" "$LOG_FILE")" ]; then - fn_log_error "Rsync reported an error, please check '$LOG_FILE' for more details." - exit 1 - fi + # ----------------------------------------------------------------------------- + # Check whether rsync reported any errors + # ----------------------------------------------------------------------------- + if [ -n "$(grep "rsync:" "$LOG_FILE")" ]; then + fn_log_warn "Rsync reported a warning, please check '$LOG_FILE' for more details." + fi + if [ -n "$(grep "rsync error:" "$LOG_FILE")" ]; then + fn_log_error "Rsync reported an error, please check '$LOG_FILE' for more details." + exit 1 + fi - # ----------------------------------------------------------------------------- - # Add symlink to last successful backup - # ----------------------------------------------------------------------------- + # ----------------------------------------------------------------------------- + # Add symlink to last successful backup + # ----------------------------------------------------------------------------- - fn_rm "$DEST_FOLDER/latest" - fn_ln "$(basename -- "$DEST")" "$DEST_FOLDER/latest" + fn_rm "$DEST_FOLDER/latest" + fn_ln "$(basename -- "$DEST")" "$DEST_FOLDER/latest" - fn_rm "$INPROGRESS_FILE" - rm -f -- "$LOG_FILE" - - fn_log_info "Backup completed without errors." + fn_rm "$INPROGRESS_FILE" + rm -f -- "$LOG_FILE" - exit 0 + fn_log_info "Backup completed without errors." + + exit 0 done From 5b2296658948a81b2bbccab5ee4fb5e3d1824320 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fatih=20=C3=9Cst=C3=BCnda=C4=9F?= Date: Sat, 26 Sep 2015 01:11:05 +0300 Subject: [PATCH 5/6] Merge branch 'master' of /Users/fatihustundag/Projects/rsync-time-backup with conflicts. --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 723ef36..e8d64cc 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ -.idea \ No newline at end of file +.idea +test.sh \ No newline at end of file From e07f7e71231d6e2fe99788c2f012e2631de4c57e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fatih=20=C3=9Cst=C3=BCnda=C4=9F?= Date: Sat, 26 Sep 2015 01:15:35 +0300 Subject: [PATCH 6/6] Tested at Cygwin 2.2.1 at Windows Server 2012 R2 64bit --- rsync_tmbackup.sh | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/rsync_tmbackup.sh b/rsync_tmbackup.sh index 6605ae4..6ca2b0f 100644 --- a/rsync_tmbackup.sh +++ b/rsync_tmbackup.sh @@ -36,6 +36,7 @@ fn_parse_date() { # Converts YYYY-MM-DD-HHMMSS to YYYY-MM-DD HH:MM:SS and then to Unix Epoch. case "$OSTYPE" in linux*) date -d "${1:0:10} ${1:11:2}:${1:13:2}:${1:15:2}" +%s ;; + cygwin*) date -d "${1:0:10} ${1:11:2}:${1:13:2}:${1:15:2}" +%s ;; darwin*) date -j -f "%Y-%m-%d-%H%M%S" "$1" "+%s" ;; esac } @@ -57,7 +58,8 @@ fn_expire_backup() { } fn_parse_ssh() { - if [[ "$DEST_FOLDER" =~ ^[A-Za-z0-9\._%\+\-]+@[A-Za-z0-9.\-]+\:.+$ ]]; then + if [[ "$DEST_FOLDER" =~ ^[A-Za-z0-9\._%\+\-]+@[A-Za-z0-9.\-]+\:.+$ ]] + then SSH_USER=$(echo "$DEST_FOLDER" | sed -E 's/^([A-Za-z0-9\._%\+\-]+)@([A-Za-z0-9.\-]+)\:(.+)$/\1/') SSH_HOST=$(echo "$DEST_FOLDER" | sed -E 's/^([A-Za-z0-9\._%\+\-]+)@([A-Za-z0-9.\-]+)\:(.+)$/\2/') SSH_DEST_FOLDER=$(echo "$DEST_FOLDER" | sed -E 's/^([A-Za-z0-9\._%\+\-]+)@([A-Za-z0-9.\-]+)\:(.+)$/\3/') @@ -67,7 +69,8 @@ fn_parse_ssh() { } fn_run_cmd() { - if [ -n "$SSH_CMD" ]; then + if [ -n "$SSH_CMD" ] + then eval "$SSH_CMD '$1'" else eval $1 @@ -118,7 +121,7 @@ if [ -n "$SSH_DEST_FOLDER" ]; then fi for ARG in "$SRC_FOLDER" "$DEST_FOLDER" "$EXCLUSION_FILE"; do - if [[ "$ARG" == *"'"* ]]; then +if [[ "$ARG" == *"'"* ]]; then fn_log_error 'Arguments may not have any single quote characters.' exit 1 fi @@ -137,7 +140,7 @@ if [ -z "$(fn_find_backup_marker "$DEST_FOLDER")" ]; then fn_log_info "Safety check failed - the destination does not appear to be a backup folder or drive (marker file not found)." fn_log_info "If it is indeed a backup folder, you may add the marker file by running the following command:" fn_log_info "" - fn_log_info_cmd "touch \"$(fn_backup_marker_path "$DEST_FOLDER")\"" + fn_log_info_cmd "mkdir -p -- \"$DEST_FOLDER\" ; touch \"$(fn_backup_marker_path "$DEST_FOLDER")\"" fn_log_info "" exit 1 fi @@ -157,6 +160,7 @@ PROFILE_FOLDER="$HOME/.$APPNAME" DEST="$DEST_FOLDER/$NOW" PREVIOUS_DEST="$(fn_find_backups | head -n 1)" INPROGRESS_FILE="$DEST_FOLDER/backup.inprogress" +MYPID="$$" # ----------------------------------------------------------------------------- # Create profile folder if it doesn't exist @@ -172,6 +176,12 @@ fi # ----------------------------------------------------------------------------- if [ -n "$(fn_find "$INPROGRESS_FILE")" ]; then + RUNNINGPID="$(fn_run_cmd "cat $INPROGRESS_FILE")" + if [ "$RUNNINGPID"="$(pgrep "$APPNAME")" ]; then + fn_log_error "Previous backup task is still active - aborting." + exit 1 + fi + if [ -n "$PREVIOUS_DEST" ]; then # - Last backup is moved to current backup folder so that it can be resumed. # - 2nd to last backup becomes last backup. @@ -182,6 +192,8 @@ if [ -n "$(fn_find "$INPROGRESS_FILE")" ]; then else PREVIOUS_DEST="" fi + # update PID to current process to avoid multiple concurrent resumes + fn_run_cmd "echo $MYPID > $INPROGRESS_FILE" fi fi @@ -275,7 +287,8 @@ while : ; do fn_log_info "Running command:" fn_log_info "$CMD" - fn_touch "$INPROGRESS_FILE" + fn_run_cmd "echo $MYPID > $INPROGRESS_FILE" + eval $CMD # -----------------------------------------------------------------------------