Mirrors/rsync-time-backup
2
0
Fork 0
mirror of https://github.com/laurent22/rsync-time-backup synced 2024-11-10 14:14:16 +00:00
Code Issues Projects Releases Packages Wiki Activity

Merge pull request #29 from fustundag/master

Browse source 
SSH support added.
This commit is contained in:
Laurent Cozic 2015-11-08 12:00:37 +01:00
commit b7d2cae0b3
2 changed files with 227 additions and 157 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.gitignore vendored Normal file
View file

@ -0,0 +1,2 @@
.idea
test.sh

382
rsync_tmbackup.sh Executable file → Normal file
View file

@ -9,14 +9,21 @@ APPNAME=$(basename $0 | sed "s/\.sh$//")
fn_log_info() { echo "$APPNAME: $1"; } fn_log_info() { echo "$APPNAME: $1"; }
fn_log_warn() { echo "$APPNAME: [WARNING] $1" 1>&2; } fn_log_warn() { echo "$APPNAME: [WARNING] $1" 1>&2; }
fn_log_error() { echo "$APPNAME: [ERROR] $1" 1>&2; } fn_log_error() { echo "$APPNAME: [ERROR] $1" 1>&2; }
fn_log_info_cmd() {
if [ -n "$SSH_CMD" ]; then
echo "$APPNAME: $SSH_CMD '$1'";
else
echo "$APPNAME: $1";
fi
}
# ----------------------------------------------------------------------------- # -----------------------------------------------------------------------------
# Make sure everything really stops when CTRL+C is pressed # Make sure everything really stops when CTRL+C is pressed
# ----------------------------------------------------------------------------- # -----------------------------------------------------------------------------
fn_terminate_script() { fn_terminate_script() {
fn_log_info "SIGINT caught." fn_log_info "SIGINT caught."
exit 1 exit 1
} }
trap 'fn_terminate_script' SIGINT trap 'fn_terminate_script' SIGINT
@ -26,43 +33,98 @@ trap 'fn_terminate_script' SIGINT
# ----------------------------------------------------------------------------- # -----------------------------------------------------------------------------
fn_parse_date() { fn_parse_date() {
# Converts YYYY-MM-DD-HHMMSS to YYYY-MM-DD HH:MM:SS and then to Unix Epoch. # Converts YYYY-MM-DD-HHMMSS to YYYY-MM-DD HH:MM:SS and then to Unix Epoch.
case "$OSTYPE" in case "$OSTYPE" in
linux*) date -d "${1:0:10} ${1:11:2}:${1:13:2}:${1:15:2}" +%s ;; linux*) date -d "${1:0:10} ${1:11:2}:${1:13:2}:${1:15:2}" +%s ;;
cygwin*) date -d "${1:0:10} ${1:11:2}:${1:13:2}:${1:15:2}" +%s ;; cygwin*) date -d "${1:0:10} ${1:11:2}:${1:13:2}:${1:15:2}" +%s ;;
darwin*) date -j -f "%Y-%m-%d-%H%M%S" "$1" "+%s" ;; darwin*) date -j -f "%Y-%m-%d-%H%M%S" "$1" "+%s" ;;
esac esac
} }
fn_find_backups() { fn_find_backups() {
find "$DEST_FOLDER" -type d -name "????-??-??-??????" -prune | sort -r fn_run_cmd "find "$DEST_FOLDER" -type d -name "????-??-??-??????" -prune | sort -r"
} }
fn_expire_backup() { fn_expire_backup() {
# Double-check that we're on a backup destination to be completely # Double-check that we're on a backup destination to be completely
# sure we're deleting the right folder # sure we're deleting the right folder
if [ -z "$(fn_find_backup_marker "$(dirname -- "$1")")" ]; then if [ -z "$(fn_find_backup_marker "$(dirname -- "$1")")" ]; then
fn_log_error "$1 is not on a backup destination - aborting." fn_log_error "$1 is not on a backup destination - aborting."
exit 1 exit 1
fi fi
fn_log_info "Expiring $1" fn_log_info "Expiring $1"
rm -rf -- "$1" fn_rm "$1"
}
fn_parse_ssh() {
if [[ "$DEST_FOLDER" =~ ^[A-Za-z0-9\._%\+\-]+@[A-Za-z0-9.\-]+\:.+$ ]]
then
SSH_USER=$(echo "$DEST_FOLDER" | sed -E 's/^([A-Za-z0-9\._%\+\-]+)@([A-Za-z0-9.\-]+)\:(.+)$/\1/')
SSH_HOST=$(echo "$DEST_FOLDER" | sed -E 's/^([A-Za-z0-9\._%\+\-]+)@([A-Za-z0-9.\-]+)\:(.+)$/\2/')
SSH_DEST_FOLDER=$(echo "$DEST_FOLDER" | sed -E 's/^([A-Za-z0-9\._%\+\-]+)@([A-Za-z0-9.\-]+)\:(.+)$/\3/')
SSH_CMD="ssh ${SSH_USER}@${SSH_HOST}"
SSH_FOLDER_PREFIX="${SSH_USER}@${SSH_HOST}:"
fi
}
fn_run_cmd() {
if [ -n "$SSH_CMD" ]
then
eval "$SSH_CMD '$1'"
else
eval $1
fi
}
fn_find() {
fn_run_cmd "find $1" 2>/dev/null
}
fn_get_absolute_path() {
fn_run_cmd "cd $1;pwd"
}
fn_mkdir() {
fn_run_cmd "mkdir -p -- $1"
}
fn_rm() {
fn_run_cmd "rm -rf -- $1"
}
fn_touch() {
fn_run_cmd "touch -- $1"
}
fn_ln() {
fn_run_cmd "ln -vs -- $1 $2"
} }
# ----------------------------------------------------------------------------- # -----------------------------------------------------------------------------
# Source and destination information # Source and destination information
# ----------------------------------------------------------------------------- # -----------------------------------------------------------------------------
SSH_USER=""
SSH_HOST=""
SSH_DEST_FOLDER=""
SSH_CMD=""
SSH_FOLDER_PREFIX=""
SRC_FOLDER="${1%/}" SRC_FOLDER="${1%/}"
DEST_FOLDER="${2%/}" DEST_FOLDER="${2%/}"
EXCLUSION_FILE="$3" EXCLUSION_FILE="$3"
fn_parse_ssh
if [ -n "$SSH_DEST_FOLDER" ]; then
DEST_FOLDER="$SSH_DEST_FOLDER"
fi
for ARG in "$SRC_FOLDER" "$DEST_FOLDER" "$EXCLUSION_FILE"; do for ARG in "$SRC_FOLDER" "$DEST_FOLDER" "$EXCLUSION_FILE"; do
if [[ "$ARG" == *"'"* ]]; then if [[ "$ARG" == *"'"* ]]; then
fn_log_error 'Arguments may not have any single quote characters.' fn_log_error 'Arguments may not have any single quote characters.'
exit 1 exit 1
fi fi
done done
# ----------------------------------------------------------------------------- # -----------------------------------------------------------------------------
@ -72,15 +134,15 @@ done
# TODO: check that the destination supports hard links # TODO: check that the destination supports hard links
fn_backup_marker_path() { echo "$1/backup.marker"; } fn_backup_marker_path() { echo "$1/backup.marker"; }
fn_find_backup_marker() { find "$(fn_backup_marker_path "$1")" 2>/dev/null; } fn_find_backup_marker() { fn_find "$(fn_backup_marker_path "$1")" 2>/dev/null; }
if [ -z "$(fn_find_backup_marker "$DEST_FOLDER")" ]; then if [ -z "$(fn_find_backup_marker "$DEST_FOLDER")" ]; then
fn_log_info "Safety check failed - the destination does not appear to be a backup folder or drive (marker file not found)." fn_log_info "Safety check failed - the destination does not appear to be a backup folder or drive (marker file not found)."
fn_log_info "If it is indeed a backup folder, you may add the marker file by running the following command:" fn_log_info "If it is indeed a backup folder, you may add the marker file by running the following command:"
fn_log_info "" fn_log_info ""
fn_log_info "mkdir -p -- \"$DEST_FOLDER\" ; touch \"$(fn_backup_marker_path "$DEST_FOLDER")\"" fn_log_info_cmd "mkdir -p -- \"$DEST_FOLDER\" ; touch \"$(fn_backup_marker_path "$DEST_FOLDER")\""
fn_log_info "" fn_log_info ""
exit 1 exit 1
fi fi
# ----------------------------------------------------------------------------- # -----------------------------------------------------------------------------
@ -90,182 +152,188 @@ fi
# Date logic # Date logic
NOW=$(date +"%Y-%m-%d-%H%M%S") NOW=$(date +"%Y-%m-%d-%H%M%S")
EPOCH=$(date "+%s") EPOCH=$(date "+%s")
KEEP_ALL_DATE=$(($EPOCH - 86400)) # 1 day ago KEEP_ALL_DATE=$((EPOCH - 86400)) # 1 day ago
KEEP_DAILIES_DATE=$(($EPOCH - 2678400)) # 31 days ago KEEP_DAILIES_DATE=$((EPOCH - 2678400)) # 31 days ago
export IFS=$'\n' # Better for handling spaces in filenames. export IFS=$'\n' # Better for handling spaces in filenames.
PROFILE_FOLDER="$HOME/.$APPNAME" PROFILE_FOLDER="$HOME/.$APPNAME"
DEST="$DEST_FOLDER/$NOW" DEST="$DEST_FOLDER/$NOW"
PREVIOUS_DEST="$(fn_find_backups | head -n 1)" PREVIOUS_DEST="$(fn_find_backups | head -n 1)"
INPROGRESS_FILE="$DEST_FOLDER/backup.inprogress" INPROGRESS_FILE="$DEST_FOLDER/backup.inprogress"
MYPID="$$"
# ----------------------------------------------------------------------------- # -----------------------------------------------------------------------------
# Create profile folder if it doesn't exist # Create profile folder if it doesn't exist
# ----------------------------------------------------------------------------- # -----------------------------------------------------------------------------
if [ ! -d "$PROFILE_FOLDER" ]; then if [ ! -d "$PROFILE_FOLDER" ]; then
fn_log_info "Creating profile folder in '$PROFILE_FOLDER'..." fn_log_info "Creating profile folder in '$PROFILE_FOLDER'..."
mkdir -- "$PROFILE_FOLDER" mkdir -- "$PROFILE_FOLDER"
fi fi
# ----------------------------------------------------------------------------- # -----------------------------------------------------------------------------
# Handle case where a previous backup failed or was interrupted. # Handle case where a previous backup failed or was interrupted.
# ----------------------------------------------------------------------------- # -----------------------------------------------------------------------------
if [ -f "$INPROGRESS_FILE" ]; then if [ -n "$(fn_find "$INPROGRESS_FILE")" ]; then
if pgrep -F "$INPROGRESS_FILE" "$APPNAME"> /dev/null 2>&1 ; then RUNNINGPID="$(fn_run_cmd "cat $INPROGRESS_FILE")"
fn_log_error "Previous backup task is still active - aborting." if [ "$RUNNINGPID"="$(pgrep "$APPNAME")" ]; then
exit 1 fn_log_error "Previous backup task is still active - aborting."
fi exit 1
if [ -n "$PREVIOUS_DEST" ]; then fi
# - Last backup is moved to current backup folder so that it can be resumed.
# - 2nd to last backup becomes last backup. if [ -n "$PREVIOUS_DEST" ]; then
fn_log_info "$INPROGRESS_FILE already exists - the previous backup failed or was interrupted. Backup will resume from there." # - Last backup is moved to current backup folder so that it can be resumed.
mv -- "$PREVIOUS_DEST" "$DEST" # - 2nd to last backup becomes last backup.
if [ "$(fn_find_backups | wc -l)" -gt 1 ]; then fn_log_info "$SSH_FOLDER_PREFIX$INPROGRESS_FILE already exists - the previous backup failed or was interrupted. Backup will resume from there."
PREVIOUS_DEST="$(fn_find_backups | sed -n '2p')" fn_run_cmd "mv -- $PREVIOUS_DEST $DEST"
else if [ "$(fn_find_backups | wc -l)" -gt 1 ]; then
PREVIOUS_DEST="" PREVIOUS_DEST="$(fn_find_backups | sed -n '2p')"
fi else
# update PID to current process to avoid multiple concurrent resumes PREVIOUS_DEST=""
echo "$$" > "$INPROGRESS_FILE" fi
fi # update PID to current process to avoid multiple concurrent resumes
fn_run_cmd "echo $MYPID > $INPROGRESS_FILE"
fi
fi fi
# Run in a loop to handle the "No space left on device" logic. # Run in a loop to handle the "No space left on device" logic.
while : ; do while : ; do
# ----------------------------------------------------------------------------- # -----------------------------------------------------------------------------
# Check if we are doing an incremental backup (if previous backup exists). # Check if we are doing an incremental backup (if previous backup exists).
# ----------------------------------------------------------------------------- # -----------------------------------------------------------------------------
LINK_DEST_OPTION="" LINK_DEST_OPTION=""
if [ -z "$PREVIOUS_DEST" ]; then if [ -z "$PREVIOUS_DEST" ]; then
fn_log_info "No previous backup - creating new one." fn_log_info "No previous backup - creating new one."
else else
# If the path is relative, it needs to be relative to the destination. To keep # If the path is relative, it needs to be relative to the destination. To keep
# it simple, just use an absolute path. See http://serverfault.com/a/210058/118679 # it simple, just use an absolute path. See http://serverfault.com/a/210058/118679
PREVIOUS_DEST="$(cd "$PREVIOUS_DEST"; pwd)" PREVIOUS_DEST="$(fn_get_absolute_path "$PREVIOUS_DEST")"
fn_log_info "Previous backup found - doing incremental backup from $PREVIOUS_DEST" fn_log_info "Previous backup found - doing incremental backup from $SSH_FOLDER_PREFIX$PREVIOUS_DEST"
LINK_DEST_OPTION="--link-dest='$PREVIOUS_DEST'" LINK_DEST_OPTION="--link-dest='$PREVIOUS_DEST'"
fi fi
# ----------------------------------------------------------------------------- # -----------------------------------------------------------------------------
# Create destination folder if it doesn't already exists # Create destination folder if it doesn't already exists
# ----------------------------------------------------------------------------- # -----------------------------------------------------------------------------
if [ ! -d "$DEST" ]; then if [ -z "$(fn_find "$DEST -type d" 2>/dev/null)" ]; then
fn_log_info "Creating destination $DEST" fn_log_info "Creating destination $SSH_FOLDER_PREFIX$DEST"
mkdir -p -- "$DEST" fn_mkdir "$DEST"
fi fi
# ----------------------------------------------------------------------------- # -----------------------------------------------------------------------------
# Purge certain old backups before beginning new backup. # Purge certain old backups before beginning new backup.
# ----------------------------------------------------------------------------- # -----------------------------------------------------------------------------
# Default value for $PREV ensures that the most recent backup is never deleted. # Default value for $PREV ensures that the most recent backup is never deleted.
PREV="0000-00-00-000000" PREV="0000-00-00-000000"
for FILENAME in $(fn_find_backups | sort -r); do for FILENAME in $(fn_find_backups | sort -r); do
BACKUP_DATE=$(basename "$FILENAME") BACKUP_DATE=$(basename "$FILENAME")
TIMESTAMP=$(fn_parse_date $BACKUP_DATE) TIMESTAMP=$(fn_parse_date $BACKUP_DATE)
# Skip if failed to parse date... # Skip if failed to parse date...
if [ -z "$TIMESTAMP" ]; then if [ -z "$TIMESTAMP" ]; then
fn_log_warn "Could not parse date: $FILENAME" fn_log_warn "Could not parse date: $FILENAME"
continue continue
fi fi
if [ $TIMESTAMP -ge $KEEP_ALL_DATE ]; then if [ $TIMESTAMP -ge $KEEP_ALL_DATE ]; then
true true
elif [ $TIMESTAMP -ge $KEEP_DAILIES_DATE ]; then elif [ $TIMESTAMP -ge $KEEP_DAILIES_DATE ]; then
# Delete all but the most recent of each day. # Delete all but the most recent of each day.
[ "${BACKUP_DATE:0:10}" == "${PREV:0:10}" ] && fn_expire_backup "$FILENAME" [ "${BACKUP_DATE:0:10}" == "${PREV:0:10}" ] && fn_expire_backup "$FILENAME"
else else
# Delete all but the most recent of each month. # Delete all but the most recent of each month.
[ "${BACKUP_DATE:0:7}" == "${PREV:0:7}" ] && fn_expire_backup "$FILENAME" [ "${BACKUP_DATE:0:7}" == "${PREV:0:7}" ] && fn_expire_backup "$FILENAME"
fi fi
PREV=$BACKUP_DATE PREV=$BACKUP_DATE
done done
# ----------------------------------------------------------------------------- # -----------------------------------------------------------------------------
# Start backup # Start backup
# ----------------------------------------------------------------------------- # -----------------------------------------------------------------------------
LOG_FILE="$PROFILE_FOLDER/$(date +"%Y-%m-%d-%H%M%S").log" LOG_FILE="$PROFILE_FOLDER/$(date +"%Y-%m-%d-%H%M%S").log"
fn_log_info "Starting backup..." fn_log_info "Starting backup..."
fn_log_info "From: $SRC_FOLDER" fn_log_info "From: $SRC_FOLDER"
fn_log_info "To: $DEST" fn_log_info "To: $SSH_FOLDER_PREFIX$DEST"
CMD="rsync" CMD="rsync"
CMD="$CMD --compress" if [ -n "$SSH_CMD" ]; then
CMD="$CMD --numeric-ids" CMD="$CMD -e 'ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null'"
CMD="$CMD --links" fi
CMD="$CMD --hard-links" CMD="$CMD --compress"
CMD="$CMD --one-file-system" CMD="$CMD --numeric-ids"
CMD="$CMD --archive" CMD="$CMD --links"
CMD="$CMD --itemize-changes" CMD="$CMD --hard-links"
CMD="$CMD --verbose" CMD="$CMD --one-file-system"
CMD="$CMD --log-file '$LOG_FILE'" CMD="$CMD --archive"
if [ -n "$EXCLUSION_FILE" ]; then CMD="$CMD --itemize-changes"
# We've already checked that $EXCLUSION_FILE doesn't contain a single quote CMD="$CMD --verbose"
CMD="$CMD --exclude-from '$EXCLUSION_FILE'" CMD="$CMD --log-file '$LOG_FILE'"
fi if [ -n "$EXCLUSION_FILE" ]; then
CMD="$CMD $LINK_DEST_OPTION" # We've already checked that $EXCLUSION_FILE doesn't contain a single quote
CMD="$CMD -- '$SRC_FOLDER/' '$DEST/'" CMD="$CMD --exclude-from '$EXCLUSION_FILE'"
CMD="$CMD | grep -E '^deleting|[^/]$'" fi
CMD="$CMD $LINK_DEST_OPTION"
CMD="$CMD -- '$SRC_FOLDER/' '$SSH_FOLDER_PREFIX$DEST/'"
CMD="$CMD | grep -E '^deleting|[^/]$'"
fn_log_info "Running command:" fn_log_info "Running command:"
fn_log_info "$CMD" fn_log_info "$CMD"
echo "$$" > "$INPROGRESS_FILE" fn_run_cmd "echo $MYPID > $INPROGRESS_FILE"
eval $CMD
# ----------------------------------------------------------------------------- eval $CMD
# Check if we ran out of space
# -----------------------------------------------------------------------------
# TODO: find better way to check for out of space condition without parsing log. # -----------------------------------------------------------------------------
NO_SPACE_LEFT="$(grep "No space left on device (28)\|Result too large (34)" "$LOG_FILE")" # Check if we ran out of space
# -----------------------------------------------------------------------------
if [ -n "$NO_SPACE_LEFT" ]; then # TODO: find better way to check for out of space condition without parsing log.
fn_log_warn "No space left on device - removing oldest backup and resuming." NO_SPACE_LEFT="$(grep "No space left on device (28)\|Result too large (34)" "$LOG_FILE")"
if [[ "$(fn_find_backups | wc -l)" -lt "2" ]]; then if [ -n "$NO_SPACE_LEFT" ]; then
fn_log_error "No space left on device, and no old backup to delete." fn_log_warn "No space left on device - removing oldest backup and resuming."
exit 1
fi
fn_expire_backup "$(fn_find_backups | tail -n 1)" if [[ "$(fn_find_backups | wc -l)" -lt "2" ]]; then
fn_log_error "No space left on device, and no old backup to delete."
exit 1
fi
# Resume backup fn_expire_backup "$(fn_find_backups | tail -n 1)"
continue
fi
# ----------------------------------------------------------------------------- # Resume backup
# Check whether rsync reported any errors continue
# ----------------------------------------------------------------------------- fi
if [ -n "$(grep "rsync:" "$LOG_FILE")" ]; then
fn_log_warn "Rsync reported a warning, please check '$LOG_FILE' for more details."
fi
if [ -n "$(grep "rsync error:" "$LOG_FILE")" ]; then
fn_log_error "Rsync reported an error, please check '$LOG_FILE' for more details."
exit 1
fi
# ----------------------------------------------------------------------------- # -----------------------------------------------------------------------------
# Add symlink to last successful backup # Check whether rsync reported any errors
# ----------------------------------------------------------------------------- # -----------------------------------------------------------------------------
if [ -n "$(grep "rsync:" "$LOG_FILE")" ]; then
fn_log_warn "Rsync reported a warning, please check '$LOG_FILE' for more details."
fi
if [ -n "$(grep "rsync error:" "$LOG_FILE")" ]; then
fn_log_error "Rsync reported an error, please check '$LOG_FILE' for more details."
exit 1
fi
rm -rf -- "$DEST_FOLDER/latest" # -----------------------------------------------------------------------------
ln -vs -- "$(basename -- "$DEST")" "$DEST_FOLDER/latest" # Add symlink to last successful backup
# -----------------------------------------------------------------------------
rm -f -- "$INPROGRESS_FILE" fn_rm "$DEST_FOLDER/latest"
rm -f -- "$LOG_FILE" fn_ln "$(basename -- "$DEST")" "$DEST_FOLDER/latest"
fn_log_info "Backup completed without errors."
exit 0 fn_rm "$INPROGRESS_FILE"
rm -f -- "$LOG_FILE"
fn_log_info "Backup completed without errors."
exit 0
done done