2013-10-28 08:26:12 +00:00
#!/usr/bin/env bash
2013-10-27 04:00:28 +00:00
2013-11-26 20:59:00 +00:00
APPNAME = $( basename $0 | sed " s/\.sh $// " )
2013-11-05 08:08:23 +00:00
# -----------------------------------------------------------------------------
# Log functions
# -----------------------------------------------------------------------------
2013-11-26 20:59:00 +00:00
fn_log_info( ) { echo " $APPNAME : $1 " ; }
2013-12-16 20:45:43 +00:00
fn_log_warn( ) { echo " $APPNAME : [WARNING] $1 " 1>& 2; }
fn_log_error( ) { echo " $APPNAME : [ERROR] $1 " 1>& 2; }
2014-01-03 12:40:53 +00:00
fn_log_info_cmd( ) {
2016-08-12 16:26:44 +00:00
if [ -n " $SSH_CMD " ] ; then
echo " $APPNAME : $SSH_CMD ' $1 ' " ;
else
echo " $APPNAME : $1 " ;
fi
2014-01-03 12:40:53 +00:00
}
2013-11-05 08:08:23 +00:00
2013-10-27 04:00:28 +00:00
# -----------------------------------------------------------------------------
# Make sure everything really stops when CTRL+C is pressed
# -----------------------------------------------------------------------------
2013-11-05 08:08:23 +00:00
fn_terminate_script( ) {
2016-08-12 16:26:44 +00:00
fn_log_info "SIGINT caught."
exit 1
2013-10-27 04:00:28 +00:00
}
2013-11-05 08:08:23 +00:00
trap 'fn_terminate_script' SIGINT
2013-10-27 04:00:28 +00:00
# -----------------------------------------------------------------------------
2013-11-13 18:21:47 +00:00
# Small utility functions for reducing code duplication
2013-10-27 04:00:28 +00:00
# -----------------------------------------------------------------------------
2014-01-28 18:48:38 +00:00
fn_display_usage( ) {
2016-09-10 10:32:11 +00:00
echo " Usage: $( basename $0 ) [OPTION]... <SOURCE> <[USER@HOST:]DESTINATION> [exclude-pattern-file] "
echo ""
echo "Options"
echo " -p, --port SSH port."
echo " -h, --help Display this help message."
echo " --rsync-get-flags Display the default rsync flags that are used for backup."
echo " --rsync-set-flags Set the rsync flags that are going to be used for backup."
2017-05-02 20:14:36 +00:00
echo " --log-dir Set the log file directory. If this flag is set, generated files will"
echo " not be managed by the script - in particular they will not be"
echo " automatically deleted."
echo " Default: $LOG_DIR "
echo ""
echo "For more detailed help, please see the README file:"
echo ""
echo "https://github.com/laurent22/rsync-time-backup/blob/master/README.md"
2014-01-28 18:48:38 +00:00
}
2013-10-27 04:00:28 +00:00
2013-11-13 08:01:17 +00:00
fn_parse_date( ) {
2016-08-12 16:26:44 +00:00
# Converts YYYY-MM-DD-HHMMSS to YYYY-MM-DD HH:MM:SS and then to Unix Epoch.
case " $OSTYPE " in
linux*) date -d " ${ 1 : 0 : 10 } ${ 1 : 11 : 2 } : ${ 1 : 13 : 2 } : ${ 1 : 15 : 2 } " +%s ; ;
cygwin*) date -d " ${ 1 : 0 : 10 } ${ 1 : 11 : 2 } : ${ 1 : 13 : 2 } : ${ 1 : 15 : 2 } " +%s ; ;
darwin*) date -j -f "%Y-%m-%d-%H%M%S" " $1 " "+%s" ; ;
2016-10-05 20:51:08 +00:00
FreeBSD*) date -j -f "%Y-%m-%d-%H%M%S" " $1 " "+%s" ; ;
2016-08-12 16:26:44 +00:00
esac
2013-11-13 08:01:17 +00:00
}
2013-11-13 18:21:47 +00:00
fn_find_backups( ) {
2016-08-12 16:26:44 +00:00
fn_run_cmd "find " $DEST_FOLDER " -type d -name " ????-??-??-??????" -prune | sort -r"
2013-11-13 18:21:47 +00:00
}
fn_expire_backup( ) {
2016-08-12 16:26:44 +00:00
# Double-check that we're on a backup destination to be completely
# sure we're deleting the right folder
if [ -z " $( fn_find_backup_marker " $( dirname -- " $1 " ) " ) " ] ; then
fn_log_error " $1 is not on a backup destination - aborting. "
exit 1
fi
fn_log_info " Expiring $1 "
fn_rm " $1 "
2014-01-03 12:40:53 +00:00
}
2013-11-15 02:50:34 +00:00
2014-01-03 12:40:53 +00:00
fn_parse_ssh( ) {
2016-08-12 16:26:44 +00:00
if [ [ " $DEST_FOLDER " = ~ ^[ A-Za-z0-9\. _%\+ \- ] +@[ A-Za-z0-9.\- ] +\: .+$ ] ]
then
SSH_USER = $( echo " $DEST_FOLDER " | sed -E 's/^([A-Za-z0-9\._%\+\-]+)@([A-Za-z0-9.\-]+)\:(.+)$/\1/' )
SSH_HOST = $( echo " $DEST_FOLDER " | sed -E 's/^([A-Za-z0-9\._%\+\-]+)@([A-Za-z0-9.\-]+)\:(.+)$/\2/' )
SSH_DEST_FOLDER = $( echo " $DEST_FOLDER " | sed -E 's/^([A-Za-z0-9\._%\+\-]+)@([A-Za-z0-9.\-]+)\:(.+)$/\3/' )
2016-08-15 11:01:37 +00:00
SSH_CMD = " ssh -p $SSH_PORT ${ SSH_USER } @ ${ SSH_HOST } "
2016-08-12 16:26:44 +00:00
SSH_FOLDER_PREFIX = " ${ SSH_USER } @ ${ SSH_HOST } : "
fi
2014-01-03 12:40:53 +00:00
}
fn_run_cmd( ) {
2016-08-12 16:26:44 +00:00
if [ -n " $SSH_CMD " ]
then
eval " $SSH_CMD ' $1 ' "
else
eval $1
fi
2014-01-03 12:40:53 +00:00
}
fn_find( ) {
2016-10-01 12:44:56 +00:00
fn_run_cmd " find ' $1 ' " 2>/dev/null
2014-01-03 12:40:53 +00:00
}
fn_get_absolute_path( ) {
2016-10-01 12:44:56 +00:00
fn_run_cmd " cd ' $1 ';pwd "
2014-01-03 12:40:53 +00:00
}
fn_mkdir( ) {
2016-10-01 12:44:56 +00:00
fn_run_cmd " mkdir -p -- ' $1 ' "
2014-01-03 12:40:53 +00:00
}
fn_rm( ) {
2017-05-02 21:58:01 +00:00
if [ [ -d $1 ] ] ; then
# when deleting a directory use rsync for performance reasons
fn_run_cmd "mkdir -p /tmp/rsync-time-backup-emptydir"
2017-04-19 08:01:30 +00:00
fn_run_cmd " rsync -a --delete /tmp/rsync-time-backup-emptydir/ ' $1 ' "
2017-05-02 21:58:01 +00:00
fn_run_cmd " rm -rf /tmp/rsync-time-backup-emptydir ' $1 ' "
else
2017-04-19 08:01:30 +00:00
# when deleting a file use regular rm
2017-05-02 21:58:01 +00:00
fn_run_cmd " rm -f ' $1 ' "
2017-04-19 08:01:30 +00:00
fi
2014-01-03 12:40:53 +00:00
}
fn_touch( ) {
2016-10-01 12:44:56 +00:00
fn_run_cmd " touch -- ' $1 ' "
2014-01-03 12:40:53 +00:00
}
fn_ln( ) {
2016-10-01 12:44:56 +00:00
fn_run_cmd " ln -s -- ' $1 ' ' $2 ' "
2013-11-13 18:21:47 +00:00
}
# -----------------------------------------------------------------------------
# Source and destination information
# -----------------------------------------------------------------------------
2014-01-03 12:40:53 +00:00
SSH_USER = ""
SSH_HOST = ""
SSH_DEST_FOLDER = ""
SSH_CMD = ""
SSH_FOLDER_PREFIX = ""
2016-08-15 11:01:37 +00:00
SSH_PORT = "22"
2013-11-13 18:21:47 +00:00
2016-08-15 10:29:15 +00:00
SRC_FOLDER = ""
DEST_FOLDER = ""
EXCLUSION_FILE = ""
2017-05-02 20:14:36 +00:00
LOG_DIR = " $HOME /. $APPNAME "
AUTO_DELETE_LOG = "1"
2016-08-15 10:29:15 +00:00
2017-05-02 21:58:01 +00:00
RSYNC_FLAGS = "-D --compress --numeric-ids --links --hard-links --one-file-system --itemize-changes --times --recursive --perms --owner --group --stats --human-readable"
2016-09-10 10:32:11 +00:00
2016-08-15 10:29:15 +00:00
while :; do
case $1 in
-h| -\? | --help)
fn_display_usage
exit
; ;
2016-08-15 11:01:37 +00:00
-p| --port)
shift
SSH_PORT = $1
; ;
2016-09-10 10:32:11 +00:00
--rsync-get-flags)
shift
echo $RSYNC_FLAGS
exit
; ;
--rsync-set-flags)
shift
RSYNC_FLAGS = " $1 "
; ;
2017-05-02 20:14:36 +00:00
--log-dir)
shift
LOG_DIR = " $1 "
AUTO_DELETE_LOG = "0"
; ;
2016-08-15 10:29:15 +00:00
--)
shift
2016-11-29 12:28:10 +00:00
SRC_FOLDER = " $1 "
DEST_FOLDER = " $2 "
2016-08-15 10:29:15 +00:00
EXCLUSION_FILE = " $3 "
break
; ;
-?*)
fn_log_error " Unknown option: \" $1 \" "
fn_log_info ""
fn_display_usage
exit 1
; ;
*)
2016-11-29 12:28:10 +00:00
SRC_FOLDER = " $1 "
DEST_FOLDER = " $2 "
2016-08-15 10:29:15 +00:00
EXCLUSION_FILE = " $3 "
break
esac
shift
done
# Display usage information if required arguments are not passed
if [ [ -z " $SRC_FOLDER " || -z " $DEST_FOLDER " ] ] ; then
2014-01-28 18:48:38 +00:00
fn_display_usage
exit 1
fi
2013-10-27 04:00:28 +00:00
2016-11-29 12:28:10 +00:00
# Strips off last slash. Note that it means the root folder "/"
# will be represented as an empty string "", which is fine
# with the current script (since a "/" is added when needed)
# but still something to keep in mind.
# Don't think it would with DEST_FOLDER set to "/" though,
# but there's probably not a use case for this anyway.
SRC_FOLDER = " ${ SRC_FOLDER %/ } "
DEST_FOLDER = " ${ DEST_FOLDER %/ } "
2014-01-03 12:40:53 +00:00
fn_parse_ssh
if [ -n " $SSH_DEST_FOLDER " ] ; then
2016-08-12 16:26:44 +00:00
DEST_FOLDER = " $SSH_DEST_FOLDER "
2014-01-03 12:40:53 +00:00
fi
2013-11-16 14:50:10 +00:00
for ARG in " $SRC_FOLDER " " $DEST_FOLDER " " $EXCLUSION_FILE " ; do
2016-08-15 10:29:15 +00:00
if [ [ " $ARG " = = *"'" * ] ] ; then
fn_log_error 'Source and destination directories may not contain single quote characters.'
2016-08-12 16:26:44 +00:00
exit 1
fi
2013-10-30 13:45:48 +00:00
done
2013-10-27 04:00:28 +00:00
# -----------------------------------------------------------------------------
# Check that the destination drive is a backup drive
# -----------------------------------------------------------------------------
2013-11-05 08:08:23 +00:00
# TODO: check that the destination supports hard links
2013-11-26 20:55:17 +00:00
fn_backup_marker_path( ) { echo " $1 /backup.marker " ; }
2014-01-03 12:40:53 +00:00
fn_find_backup_marker( ) { fn_find " $( fn_backup_marker_path " $1 " ) " 2>/dev/null; }
2013-11-05 08:08:23 +00:00
2013-11-26 20:55:17 +00:00
if [ -z " $( fn_find_backup_marker " $DEST_FOLDER " ) " ] ; then
2016-08-12 16:26:44 +00:00
fn_log_info "Safety check failed - the destination does not appear to be a backup folder or drive (marker file not found)."
fn_log_info "If it is indeed a backup folder, you may add the marker file by running the following command:"
fn_log_info ""
fn_log_info_cmd " mkdir -p -- \" $DEST_FOLDER \" ; touch \" $( fn_backup_marker_path " $DEST_FOLDER " ) \" "
fn_log_info ""
exit 1
2013-10-27 04:00:28 +00:00
fi
# -----------------------------------------------------------------------------
# Setup additional variables
# -----------------------------------------------------------------------------
2013-11-15 19:08:28 +00:00
# Date logic
2013-10-27 04:00:28 +00:00
NOW = $( date +"%Y-%m-%d-%H%M%S" )
2013-11-15 19:08:28 +00:00
EPOCH = $( date "+%s" )
2015-01-20 21:36:48 +00:00
KEEP_ALL_DATE = $(( EPOCH - 86400 )) # 1 day ago
KEEP_DAILIES_DATE = $(( EPOCH - 2678400 )) # 31 days ago
2013-11-15 19:08:28 +00:00
export IFS = $'\n' # Better for handling spaces in filenames.
2013-11-26 21:08:52 +00:00
DEST = " $DEST_FOLDER / $NOW "
PREVIOUS_DEST = " $( fn_find_backups | head -n 1) "
INPROGRESS_FILE = " $DEST_FOLDER /backup.inprogress "
2015-09-25 22:15:35 +00:00
MYPID = " $$ "
2013-11-15 18:24:52 +00:00
2013-11-05 08:08:23 +00:00
# -----------------------------------------------------------------------------
2017-05-02 20:14:36 +00:00
# Create log folder if it doesn't exist
2013-11-05 08:08:23 +00:00
# -----------------------------------------------------------------------------
2017-05-02 20:14:36 +00:00
if [ ! -d " $LOG_DIR " ] ; then
fn_log_info " Creating log folder in ' $LOG_DIR '... "
mkdir -- " $LOG_DIR "
2013-11-05 08:08:23 +00:00
fi
2013-10-27 04:00:28 +00:00
# -----------------------------------------------------------------------------
# Handle case where a previous backup failed or was interrupted.
# -----------------------------------------------------------------------------
2014-01-03 12:40:53 +00:00
if [ -n " $( fn_find " $INPROGRESS_FILE " ) " ] ; then
2015-11-08 11:09:28 +00:00
if [ " $OSTYPE " = = "cygwin" ] ; then
# TODO: Cygwin reports the name of currently running Bash scripts as just "/usr/bin/bash"
# TODO: so the pgrep solution below won't work. Need to use "procps -wwFAH", grep
# TODO: the script name, and extract the process ID from it.
fn_log_warn "Cygwin only: Previous backup task has either been interrupted or it might still be active, but there is currently no check for this. Assuming that the task was simply interrupted."
else
2016-08-12 16:26:44 +00:00
RUNNINGPID = " $( fn_run_cmd " cat $INPROGRESS_FILE " ) "
if [ " $RUNNINGPID " = " $( pgrep " $APPNAME " ) " ] ; then
fn_log_error "Previous backup task is still active - aborting."
exit 1
fi
2013-10-27 04:00:28 +00:00
fi
2015-09-25 22:15:35 +00:00
2016-08-12 16:26:44 +00:00
if [ -n " $PREVIOUS_DEST " ] ; then
# - Last backup is moved to current backup folder so that it can be resumed.
# - 2nd to last backup becomes last backup.
fn_log_info " $SSH_FOLDER_PREFIX $INPROGRESS_FILE already exists - the previous backup failed or was interrupted. Backup will resume from there. "
fn_run_cmd " mv -- $PREVIOUS_DEST $DEST "
if [ " $( fn_find_backups | wc -l) " -gt 1 ] ; then
PREVIOUS_DEST = " $( fn_find_backups | sed -n '2p' ) "
else
PREVIOUS_DEST = ""
fi
# update PID to current process to avoid multiple concurrent resumes
fn_run_cmd " echo $MYPID > $INPROGRESS_FILE "
fi
2013-10-27 04:00:28 +00:00
fi
2013-11-12 03:14:02 +00:00
# Run in a loop to handle the "No space left on device" logic.
2013-11-26 21:02:29 +00:00
while : ; do
2013-10-27 04:00:28 +00:00
2016-08-12 16:26:44 +00:00
# -----------------------------------------------------------------------------
# Check if we are doing an incremental backup (if previous backup exists).
# -----------------------------------------------------------------------------
LINK_DEST_OPTION = ""
if [ -z " $PREVIOUS_DEST " ] ; then
fn_log_info "No previous backup - creating new one."
else
# If the path is relative, it needs to be relative to the destination. To keep
# it simple, just use an absolute path. See http://serverfault.com/a/210058/118679
PREVIOUS_DEST = " $( fn_get_absolute_path " $PREVIOUS_DEST " ) "
fn_log_info " Previous backup found - doing incremental backup from $SSH_FOLDER_PREFIX $PREVIOUS_DEST "
LINK_DEST_OPTION = " --link-dest=' $PREVIOUS_DEST ' "
fi
# -----------------------------------------------------------------------------
# Create destination folder if it doesn't already exists
# -----------------------------------------------------------------------------
if [ -z " $( fn_find " $DEST -type d " 2>/dev/null) " ] ; then
fn_log_info " Creating destination $SSH_FOLDER_PREFIX $DEST "
fn_mkdir " $DEST "
fi
# -----------------------------------------------------------------------------
# Purge certain old backups before beginning new backup.
# -----------------------------------------------------------------------------
# Default value for $PREV ensures that the most recent backup is never deleted.
PREV = "0000-00-00-000000"
for FILENAME in $( fn_find_backups | sort -r) ; do
BACKUP_DATE = $( basename " $FILENAME " )
TIMESTAMP = $( fn_parse_date $BACKUP_DATE )
# Skip if failed to parse date...
if [ -z " $TIMESTAMP " ] ; then
fn_log_warn " Could not parse date: $FILENAME "
continue
fi
if [ $TIMESTAMP -ge $KEEP_ALL_DATE ] ; then
true
elif [ $TIMESTAMP -ge $KEEP_DAILIES_DATE ] ; then
# Delete all but the most recent of each day.
[ " ${ BACKUP_DATE : 0 : 10 } " = = " ${ PREV : 0 : 10 } " ] && fn_expire_backup " $FILENAME "
else
# Delete all but the most recent of each month.
[ " ${ BACKUP_DATE : 0 : 7 } " = = " ${ PREV : 0 : 7 } " ] && fn_expire_backup " $FILENAME "
fi
PREV = $BACKUP_DATE
done
# -----------------------------------------------------------------------------
# Start backup
# -----------------------------------------------------------------------------
2017-05-02 20:14:36 +00:00
LOG_FILE = " $LOG_DIR / $( date +"%Y-%m-%d-%H%M%S" ) .log "
2016-08-12 16:26:44 +00:00
fn_log_info "Starting backup..."
2016-11-29 12:28:10 +00:00
fn_log_info " From: $SRC_FOLDER / "
fn_log_info " To: $SSH_FOLDER_PREFIX $DEST / "
2016-08-12 16:26:44 +00:00
CMD = "rsync"
if [ -n " $SSH_CMD " ] ; then
2016-08-15 11:01:37 +00:00
CMD = " $CMD -e 'ssh -p $SSH_PORT -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null' "
2016-08-12 16:26:44 +00:00
fi
2016-09-10 10:32:11 +00:00
CMD = " $CMD $RSYNC_FLAGS "
2016-08-12 16:26:44 +00:00
CMD = " $CMD --log-file ' $LOG_FILE ' "
if [ -n " $EXCLUSION_FILE " ] ; then
# We've already checked that $EXCLUSION_FILE doesn't contain a single quote
CMD = " $CMD --exclude-from ' $EXCLUSION_FILE ' "
fi
CMD = " $CMD $LINK_DEST_OPTION "
CMD = " $CMD -- ' $SRC_FOLDER /' ' $SSH_FOLDER_PREFIX $DEST /' "
fn_log_info "Running command:"
fn_log_info " $CMD "
fn_run_cmd " echo $MYPID > $INPROGRESS_FILE "
eval $CMD
# -----------------------------------------------------------------------------
# Check if we ran out of space
# -----------------------------------------------------------------------------
NO_SPACE_LEFT = " $( grep "No space left on device (28)\|Result too large (34)" " $LOG_FILE " ) "
if [ -n " $NO_SPACE_LEFT " ] ; then
fn_log_warn "No space left on device - removing oldest backup and resuming."
if [ [ " $( fn_find_backups | wc -l) " -lt "2" ] ] ; then
fn_log_error "No space left on device, and no old backup to delete."
exit 1
fi
fn_expire_backup " $( fn_find_backups | tail -n 1) "
# Resume backup
continue
fi
# -----------------------------------------------------------------------------
# Check whether rsync reported any errors
# -----------------------------------------------------------------------------
2017-05-02 21:58:01 +00:00
2016-08-12 16:26:44 +00:00
if [ -n " $( grep "rsync error:" " $LOG_FILE " ) " ] ; then
2017-05-02 21:14:22 +00:00
fn_log_error " Rsync reported an error. Run this command for more details: grep -E 'rsync:|rsync error:' ' $LOG_FILE ' "
2016-08-12 16:26:44 +00:00
exit 1
fi
2017-05-02 21:14:22 +00:00
if [ -n " $( grep "rsync:" " $LOG_FILE " ) " ] ; then
fn_log_warn " Rsync reported a warning. Run this command for more details: grep -E 'rsync:|rsync error:' ' $LOG_FILE ' "
fi
2016-08-12 16:26:44 +00:00
# -----------------------------------------------------------------------------
# Add symlink to last successful backup
# -----------------------------------------------------------------------------
fn_rm " $DEST_FOLDER /latest "
fn_ln " $( basename -- " $DEST " ) " " $DEST_FOLDER /latest "
fn_rm " $INPROGRESS_FILE "
2017-05-02 20:14:36 +00:00
if [ [ $AUTO_DELETE_LOG = = "1" ] ] ; then
rm -f -- " $LOG_FILE "
fi
2016-08-12 16:26:44 +00:00
fn_log_info "Backup completed without errors."
exit 0
2013-11-05 08:08:23 +00:00
done