rsync-time-backup/rsync_tmbackup.sh

434 lines
14 KiB
Bash
Raw Normal View History

#!/usr/bin/env bash
2013-10-27 04:00:28 +00:00
2013-11-26 20:59:00 +00:00
APPNAME=$(basename $0 | sed "s/\.sh$//")
# -----------------------------------------------------------------------------
# Log functions
# -----------------------------------------------------------------------------
2013-11-26 20:59:00 +00:00
fn_log_info() { echo "$APPNAME: $1"; }
fn_log_warn() { echo "$APPNAME: [WARNING] $1" 1>&2; }
fn_log_error() { echo "$APPNAME: [ERROR] $1" 1>&2; }
fn_log_info_cmd() {
if [ -n "$SSH_CMD" ]; then
echo "$APPNAME: $SSH_CMD '$1'";
else
echo "$APPNAME: $1";
fi
}
2013-10-27 04:00:28 +00:00
# -----------------------------------------------------------------------------
# Make sure everything really stops when CTRL+C is pressed
# -----------------------------------------------------------------------------
fn_terminate_script() {
fn_log_info "SIGINT caught."
exit 1
2013-10-27 04:00:28 +00:00
}
trap 'fn_terminate_script' SIGINT
2013-10-27 04:00:28 +00:00
# -----------------------------------------------------------------------------
2013-11-13 18:21:47 +00:00
# Small utility functions for reducing code duplication
2013-10-27 04:00:28 +00:00
# -----------------------------------------------------------------------------
fn_display_usage() {
echo "Usage: $(basename $0) [OPTION]... <SOURCE> <[USER@HOST:]DESTINATION> [exclude-pattern-file]"
echo ""
echo "Options"
echo " -p, --port SSH port."
echo " -h, --help Display this help message."
echo " --rsync-get-flags Display the default rsync flags that are used for backup."
echo " --rsync-set-flags Set the rsync flags that are going to be used for backup."
2017-05-02 20:14:36 +00:00
echo " --log-dir Set the log file directory. If this flag is set, generated files will"
echo " not be managed by the script - in particular they will not be"
echo " automatically deleted."
echo " Default: $LOG_DIR"
echo ""
echo "For more detailed help, please see the README file:"
echo ""
echo "https://github.com/laurent22/rsync-time-backup/blob/master/README.md"
}
2013-10-27 04:00:28 +00:00
fn_parse_date() {
# Converts YYYY-MM-DD-HHMMSS to YYYY-MM-DD HH:MM:SS and then to Unix Epoch.
case "$OSTYPE" in
linux*) date -d "${1:0:10} ${1:11:2}:${1:13:2}:${1:15:2}" +%s ;;
cygwin*) date -d "${1:0:10} ${1:11:2}:${1:13:2}:${1:15:2}" +%s ;;
darwin*) date -j -f "%Y-%m-%d-%H%M%S" "$1" "+%s" ;;
2016-10-05 20:51:08 +00:00
FreeBSD*) date -j -f "%Y-%m-%d-%H%M%S" "$1" "+%s" ;;
esac
}
2013-11-13 18:21:47 +00:00
fn_find_backups() {
fn_run_cmd "find "$DEST_FOLDER" -type d -name "????-??-??-??????" -prune | sort -r"
2013-11-13 18:21:47 +00:00
}
fn_expire_backup() {
# Double-check that we're on a backup destination to be completely
# sure we're deleting the right folder
if [ -z "$(fn_find_backup_marker "$(dirname -- "$1")")" ]; then
fn_log_error "$1 is not on a backup destination - aborting."
exit 1
fi
fn_log_info "Expiring $1"
fn_rm "$1"
}
2013-11-15 02:50:34 +00:00
fn_parse_ssh() {
if [[ "$DEST_FOLDER" =~ ^[A-Za-z0-9\._%\+\-]+@[A-Za-z0-9.\-]+\:.+$ ]]
then
SSH_USER=$(echo "$DEST_FOLDER" | sed -E 's/^([A-Za-z0-9\._%\+\-]+)@([A-Za-z0-9.\-]+)\:(.+)$/\1/')
SSH_HOST=$(echo "$DEST_FOLDER" | sed -E 's/^([A-Za-z0-9\._%\+\-]+)@([A-Za-z0-9.\-]+)\:(.+)$/\2/')
SSH_DEST_FOLDER=$(echo "$DEST_FOLDER" | sed -E 's/^([A-Za-z0-9\._%\+\-]+)@([A-Za-z0-9.\-]+)\:(.+)$/\3/')
2016-08-15 11:01:37 +00:00
SSH_CMD="ssh -p $SSH_PORT ${SSH_USER}@${SSH_HOST}"
SSH_FOLDER_PREFIX="${SSH_USER}@${SSH_HOST}:"
fi
}
fn_run_cmd() {
if [ -n "$SSH_CMD" ]
then
eval "$SSH_CMD '$1'"
else
eval $1
fi
}
fn_find() {
fn_run_cmd "find '$1'" 2>/dev/null
}
fn_get_absolute_path() {
fn_run_cmd "cd '$1';pwd"
}
fn_mkdir() {
fn_run_cmd "mkdir -p -- '$1'"
}
fn_rm() {
fn_run_cmd "mkdir /tmp/rsync-time-backup-emptydir"
if [ -d '$1' ] ; then
# when deleting a directory use rsyny for performance reasons
fn_run_cmd "rsync -a --delete /tmp/rsync-time-backup-emptydir/ '$1'"
elif [ -f '$1' ]; then
# when deleting a file use regular rm
fn_run_cmd "rm -f '$1'"
fi
fn_run_cmd "rm -rf /tmp/rsync-time-backup-emptydir '$1'"
}
fn_touch() {
fn_run_cmd "touch -- '$1'"
}
fn_ln() {
fn_run_cmd "ln -s -- '$1' '$2'"
2013-11-13 18:21:47 +00:00
}
# -----------------------------------------------------------------------------
# Source and destination information
# -----------------------------------------------------------------------------
SSH_USER=""
SSH_HOST=""
SSH_DEST_FOLDER=""
SSH_CMD=""
SSH_FOLDER_PREFIX=""
2016-08-15 11:01:37 +00:00
SSH_PORT="22"
2013-11-13 18:21:47 +00:00
2016-08-15 10:29:15 +00:00
SRC_FOLDER=""
DEST_FOLDER=""
EXCLUSION_FILE=""
2017-05-02 20:14:36 +00:00
LOG_DIR="$HOME/.$APPNAME"
AUTO_DELETE_LOG="1"
2016-08-15 10:29:15 +00:00
RSYNC_FLAGS="-D --compress --numeric-ids --links --hard-links --one-file-system --itemize-changes --times --recursive --perms --owner --group"
2016-08-15 10:29:15 +00:00
while :; do
case $1 in
-h|-\?|--help)
fn_display_usage
exit
;;
2016-08-15 11:01:37 +00:00
-p|--port)
shift
SSH_PORT=$1
;;
--rsync-get-flags)
shift
echo $RSYNC_FLAGS
exit
;;
--rsync-set-flags)
shift
RSYNC_FLAGS="$1"
;;
2017-05-02 20:14:36 +00:00
--log-dir)
shift
LOG_DIR="$1"
AUTO_DELETE_LOG="0"
;;
2016-08-15 10:29:15 +00:00
--)
shift
SRC_FOLDER="$1"
DEST_FOLDER="$2"
2016-08-15 10:29:15 +00:00
EXCLUSION_FILE="$3"
break
;;
-?*)
fn_log_error "Unknown option: \"$1\""
fn_log_info ""
fn_display_usage
exit 1
;;
*)
SRC_FOLDER="$1"
DEST_FOLDER="$2"
2016-08-15 10:29:15 +00:00
EXCLUSION_FILE="$3"
break
esac
shift
done
# Display usage information if required arguments are not passed
if [[ -z "$SRC_FOLDER" || -z "$DEST_FOLDER" ]]; then
fn_display_usage
exit 1
fi
2013-10-27 04:00:28 +00:00
# Strips off last slash. Note that it means the root folder "/"
# will be represented as an empty string "", which is fine
# with the current script (since a "/" is added when needed)
# but still something to keep in mind.
# Don't think it would with DEST_FOLDER set to "/" though,
# but there's probably not a use case for this anyway.
SRC_FOLDER="${SRC_FOLDER%/}"
DEST_FOLDER="${DEST_FOLDER%/}"
fn_parse_ssh
if [ -n "$SSH_DEST_FOLDER" ]; then
DEST_FOLDER="$SSH_DEST_FOLDER"
fi
for ARG in "$SRC_FOLDER" "$DEST_FOLDER" "$EXCLUSION_FILE"; do
2016-08-15 10:29:15 +00:00
if [[ "$ARG" == *"'"* ]]; then
fn_log_error 'Source and destination directories may not contain single quote characters.'
exit 1
fi
done
2013-10-27 04:00:28 +00:00
# -----------------------------------------------------------------------------
# Check that the destination drive is a backup drive
# -----------------------------------------------------------------------------
# TODO: check that the destination supports hard links
fn_backup_marker_path() { echo "$1/backup.marker"; }
fn_find_backup_marker() { fn_find "$(fn_backup_marker_path "$1")" 2>/dev/null; }
if [ -z "$(fn_find_backup_marker "$DEST_FOLDER")" ]; then
fn_log_info "Safety check failed - the destination does not appear to be a backup folder or drive (marker file not found)."
fn_log_info "If it is indeed a backup folder, you may add the marker file by running the following command:"
fn_log_info ""
fn_log_info_cmd "mkdir -p -- \"$DEST_FOLDER\" ; touch \"$(fn_backup_marker_path "$DEST_FOLDER")\""
fn_log_info ""
exit 1
2013-10-27 04:00:28 +00:00
fi
# -----------------------------------------------------------------------------
# Setup additional variables
# -----------------------------------------------------------------------------
# Date logic
2013-10-27 04:00:28 +00:00
NOW=$(date +"%Y-%m-%d-%H%M%S")
EPOCH=$(date "+%s")
KEEP_ALL_DATE=$((EPOCH - 86400)) # 1 day ago
KEEP_DAILIES_DATE=$((EPOCH - 2678400)) # 31 days ago
export IFS=$'\n' # Better for handling spaces in filenames.
2013-11-26 21:08:52 +00:00
DEST="$DEST_FOLDER/$NOW"
PREVIOUS_DEST="$(fn_find_backups | head -n 1)"
INPROGRESS_FILE="$DEST_FOLDER/backup.inprogress"
MYPID="$$"
2013-11-15 18:24:52 +00:00
# -----------------------------------------------------------------------------
2017-05-02 20:14:36 +00:00
# Create log folder if it doesn't exist
# -----------------------------------------------------------------------------
2017-05-02 20:14:36 +00:00
if [ ! -d "$LOG_DIR" ]; then
fn_log_info "Creating log folder in '$LOG_DIR'..."
mkdir -- "$LOG_DIR"
fi
2013-10-27 04:00:28 +00:00
# -----------------------------------------------------------------------------
# Handle case where a previous backup failed or was interrupted.
# -----------------------------------------------------------------------------
if [ -n "$(fn_find "$INPROGRESS_FILE")" ]; then
if [ "$OSTYPE" == "cygwin" ]; then
# TODO: Cygwin reports the name of currently running Bash scripts as just "/usr/bin/bash"
# TODO: so the pgrep solution below won't work. Need to use "procps -wwFAH", grep
# TODO: the script name, and extract the process ID from it.
fn_log_warn "Cygwin only: Previous backup task has either been interrupted or it might still be active, but there is currently no check for this. Assuming that the task was simply interrupted."
else
RUNNINGPID="$(fn_run_cmd "cat $INPROGRESS_FILE")"
if [ "$RUNNINGPID" = "$(pgrep "$APPNAME")" ]; then
fn_log_error "Previous backup task is still active - aborting."
exit 1
fi
2013-10-27 04:00:28 +00:00
fi
if [ -n "$PREVIOUS_DEST" ]; then
# - Last backup is moved to current backup folder so that it can be resumed.
# - 2nd to last backup becomes last backup.
fn_log_info "$SSH_FOLDER_PREFIX$INPROGRESS_FILE already exists - the previous backup failed or was interrupted. Backup will resume from there."
fn_run_cmd "mv -- $PREVIOUS_DEST $DEST"
if [ "$(fn_find_backups | wc -l)" -gt 1 ]; then
PREVIOUS_DEST="$(fn_find_backups | sed -n '2p')"
else
PREVIOUS_DEST=""
fi
# update PID to current process to avoid multiple concurrent resumes
fn_run_cmd "echo $MYPID > $INPROGRESS_FILE"
fi
2013-10-27 04:00:28 +00:00
fi
2013-11-12 03:14:02 +00:00
# Run in a loop to handle the "No space left on device" logic.
2013-11-26 21:02:29 +00:00
while : ; do
2013-10-27 04:00:28 +00:00
# -----------------------------------------------------------------------------
# Check if we are doing an incremental backup (if previous backup exists).
# -----------------------------------------------------------------------------
LINK_DEST_OPTION=""
if [ -z "$PREVIOUS_DEST" ]; then
fn_log_info "No previous backup - creating new one."
else
# If the path is relative, it needs to be relative to the destination. To keep
# it simple, just use an absolute path. See http://serverfault.com/a/210058/118679
PREVIOUS_DEST="$(fn_get_absolute_path "$PREVIOUS_DEST")"
fn_log_info "Previous backup found - doing incremental backup from $SSH_FOLDER_PREFIX$PREVIOUS_DEST"
LINK_DEST_OPTION="--link-dest='$PREVIOUS_DEST'"
fi
# -----------------------------------------------------------------------------
# Create destination folder if it doesn't already exists
# -----------------------------------------------------------------------------
if [ -z "$(fn_find "$DEST -type d" 2>/dev/null)" ]; then
fn_log_info "Creating destination $SSH_FOLDER_PREFIX$DEST"
fn_mkdir "$DEST"
fi
# -----------------------------------------------------------------------------
# Purge certain old backups before beginning new backup.
# -----------------------------------------------------------------------------
# Default value for $PREV ensures that the most recent backup is never deleted.
PREV="0000-00-00-000000"
for FILENAME in $(fn_find_backups | sort -r); do
BACKUP_DATE=$(basename "$FILENAME")
TIMESTAMP=$(fn_parse_date $BACKUP_DATE)
# Skip if failed to parse date...
if [ -z "$TIMESTAMP" ]; then
fn_log_warn "Could not parse date: $FILENAME"
continue
fi
if [ $TIMESTAMP -ge $KEEP_ALL_DATE ]; then
true
elif [ $TIMESTAMP -ge $KEEP_DAILIES_DATE ]; then
# Delete all but the most recent of each day.
[ "${BACKUP_DATE:0:10}" == "${PREV:0:10}" ] && fn_expire_backup "$FILENAME"
else
# Delete all but the most recent of each month.
[ "${BACKUP_DATE:0:7}" == "${PREV:0:7}" ] && fn_expire_backup "$FILENAME"
fi
PREV=$BACKUP_DATE
done
# -----------------------------------------------------------------------------
# Start backup
# -----------------------------------------------------------------------------
2017-05-02 20:14:36 +00:00
LOG_FILE="$LOG_DIR/$(date +"%Y-%m-%d-%H%M%S").log"
fn_log_info "Starting backup..."
fn_log_info "From: $SRC_FOLDER/"
fn_log_info "To: $SSH_FOLDER_PREFIX$DEST/"
CMD="rsync"
if [ -n "$SSH_CMD" ]; then
2016-08-15 11:01:37 +00:00
CMD="$CMD -e 'ssh -p $SSH_PORT -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null'"
fi
CMD="$CMD $RSYNC_FLAGS"
CMD="$CMD --log-file '$LOG_FILE'"
if [ -n "$EXCLUSION_FILE" ]; then
# We've already checked that $EXCLUSION_FILE doesn't contain a single quote
CMD="$CMD --exclude-from '$EXCLUSION_FILE'"
fi
CMD="$CMD $LINK_DEST_OPTION"
CMD="$CMD -- '$SRC_FOLDER/' '$SSH_FOLDER_PREFIX$DEST/'"
CMD="$CMD | grep -E '^deleting|[^/]$'"
fn_log_info "Running command:"
fn_log_info "$CMD"
fn_run_cmd "echo $MYPID > $INPROGRESS_FILE"
eval $CMD
# -----------------------------------------------------------------------------
# Check if we ran out of space
# -----------------------------------------------------------------------------
# TODO: find better way to check for out of space condition without parsing log.
NO_SPACE_LEFT="$(grep "No space left on device (28)\|Result too large (34)" "$LOG_FILE")"
if [ -n "$NO_SPACE_LEFT" ]; then
fn_log_warn "No space left on device - removing oldest backup and resuming."
if [[ "$(fn_find_backups | wc -l)" -lt "2" ]]; then
fn_log_error "No space left on device, and no old backup to delete."
exit 1
fi
fn_expire_backup "$(fn_find_backups | tail -n 1)"
# Resume backup
continue
fi
# -----------------------------------------------------------------------------
# Check whether rsync reported any errors
# -----------------------------------------------------------------------------
if [ -n "$(grep "rsync:" "$LOG_FILE")" ]; then
fn_log_warn "Rsync reported a warning, please check '$LOG_FILE' for more details."
fi
if [ -n "$(grep "rsync error:" "$LOG_FILE")" ]; then
fn_log_error "Rsync reported an error, please check '$LOG_FILE' for more details."
exit 1
fi
# -----------------------------------------------------------------------------
# Add symlink to last successful backup
# -----------------------------------------------------------------------------
fn_rm "$DEST_FOLDER/latest"
fn_ln "$(basename -- "$DEST")" "$DEST_FOLDER/latest"
fn_rm "$INPROGRESS_FILE"
2017-05-02 20:14:36 +00:00
if [[ $AUTO_DELETE_LOG == "1" ]]; then
rm -f -- "$LOG_FILE"
fi
fn_log_info "Backup completed without errors."
exit 0
done