diff --git a/README b/README new file mode 100644 index 0000000..5786338 --- /dev/null +++ b/README @@ -0,0 +1,57 @@ +Copyright 2008, Robert Pufky (github.com/r-pufky) + +ReadyNAS Backup Scripts +----------------------- +These were a collection of scripts I wrote to overcome the restricted backup +functionality provided by the backup web-face on the ReadyNAS. Any questions or +comments, please let me know! More information, and updated versions can be +found at: + + http://www.crazymonkies.com/projects.php?type=readynas + +1) If you have never run cronjob tasks before, or not comfortable with using the + commandline, STOP NOW. These scripts are fore more advanced users who want + more control over their backups. + +2) Read the scripts and the cronjob file and verify they are setup the way you + want them. Please note the generation of public key authentication for the + NAS, and also note that as of this release, the ReadyNAS was affected by the + openssl bug. If you don't want to use a script, you can just remove it from + the cronjobs file, and delete the script. + +3) Prep the ReadyNAS for backup scripts: + + - Install SSH Addons if they are not installed already. They are located + here: + + http://www.readynas.com/?page_id=93 + + You want to select your current ReadyNAS firmware version, then download + and install the following packages: + + ToggleSSH + EnableRootSSH + + - Create a backup share, only allowing READ access to normal users. I named + my backup share "Backup" + +4) Copy the script to the root directory on your NAS; the command would be + something like the following: + + scp -r ReadyNAS-Backup-Scripts/ root@YOURNAS:/root/ + +5) SSH to your box to verify the setup. I would recommend logging into your NAS + and verifying your scripts are working as intended by first running the + scripts with the --check and --log options: + + ./[script]_backup --check --log + + After that, run the initial backup with that script. Please note that the + first actual backup of data will take the longest. You should probably run + the first backup with --log to make sure everything goes well: + + ./[script]_backup --log + +6) Install your cronjob, and go. + + crontab /root/cronjobs diff --git a/backup_scripts/critical_backup b/backup_scripts/critical_backup new file mode 100755 index 0000000..c368e62 --- /dev/null +++ b/backup_scripts/critical_backup @@ -0,0 +1,322 @@ +#!/usr/bin/perl -w +# +# Sync's critical data to user read-only directory on NAS. +# Copyright 2008, Robert Pufky (github.com/r-pufky) +# http://www.crazymonkies.com/projects.php?type=readynas +# +# Installation / Usage: +# --------------------- +# run './critical_backup --help' for more information +# +# Recommended cronjob: +# 30 23 * * * /root/backup_scripts/critical_backup +# +# Notes: +# This is meant to be run on very small critical pieces of data, which will be +# backed up very fast. If you want to backup larger sets of data, use the +# regular_backup script, which can be run in a separate cronjob. Remote data +# is backed up VIA rsync over SSH. +# +# The rsync over SSH setup that is used in this script requires public-key +# authentication, which a basic setup is described below. Google for more +# information. +# +# This job should run relatively quickly after the initial sync, as critical +# data shouldn't change that much. Should be run relatively frequently. +# +# If NO directories are being backed up locally or remotely, just leave the +# directory list blank. It should look like: +# - local NAS directories: $NAS{'LOCAL'} = []; +# - remote directories: $NAS{'REMOTE'} = []; +# +# Initially, this should be run manually with logging enabled to make sure you +# are grabbing everything that you want. The initial sync will take the +# longest, and it's best to monitor the first sync to adjust your backups +# accordingly. +# +# ./critical_backup --log +# +# Public-key Setup: +# - Setup local (NAS) .ssh directory +# ssh root@YOURNAS +# mkdir -p ${HOME}/.ssh +# chmod 0700 ${HOME}/.ssh +# - Generate a passwordless DSA public/private keyset ** +# ssh-keygen -t dsa -f ${HOME}/.ssh/id_dsa -P '' +# - Upload public key to your server VIA scp +# scp -P PORT ${HOME}/.ssh/id_dsa.pub USER@SERVER:/REMOTEHOME/ +# - Setup remote system directory +# ssh USER@HOST -p PORT +# mkdir -p ${HOME}/.ssh +# cat ${HOME}/id_dsa.pub >> ${HOME}/.ssh/authorized_keys +# cat ${HOME}/id_dsa.pub >> ${HOME}/.ssh/authorized_keys2 +# chmod 0600 ${HOME}/.ssh/authorized_keys* +# +# ** At the time of this script, there was an openssl bug that affected the +# ReadyNAS, which created cryptographically weak keys. The work-around is +# to generate the keys on a Mac or linux box that isn't affected by the keys +# using the folloing command, then copy the resulting (id_dsa,id_dsa.pub) +# keys to your ReadNAS: +# +# ssh-keygen -t dsa -f ~/id_dsa -C 'root@YOURNAS' -P '' +# scp -P PORT ~/id_dsa* root@YOURNAS:/root/.ssh/ +# +use strict; +my %NAS; +# ----------------------------------------------------------------------------- +# Critical local NAS directories to backup +$NAS{'LOCAL'} = [ + '/me_home/Pastor/', + '/me_home/Documents']; +# Critical remote system directories to backup +$NAS{'REMOTE'} = [ + '/Users/me/Library/Keychains/', + '/Users/me/Library/Calendars', + '/Users/me/Library/iTunes', + '/Users/me/Library/Application Support/AddressBook', + '/Users/me/Library/Application Support/Little Snitch']; +# Local NAS destination backup directory, should be from root of drive +$NAS{'DESTINATION'} = '/Backup/Critical/'; +# remote system to connect to +$NAS{'HOST'} = '192.168.0.10'; +# remote user to authenticate with +$NAS{'USER'} = 'me'; +# remote port for SSH server +$NAS{'PORT'} = '22'; +# E-mail message settings +$NAS{'TO'} = 'readynas@example.com'; +$NAS{'FROM'} = 'readynas@example.com'; +# rsync binary location (run 'which rsync' from the command line to find it) +$NAS{'RSYNC'} = '/usr/bin/rsync'; +# ----------------------------------------------------------------------------- + + + +my $start_time = time(); +my $check = 0; +my $escape_spaces = 1; +my $log = 0; +foreach(@ARGV) { + if( $_ eq '--check' ) { $check = 1; } + if( $_ eq '--log' ) { $log = 1; } + if( $_ eq '--no-space-escape' ) { $escape_spaces = 0; } + if( $_ eq '--help' or $_ eq '-h' ) { usage(); exit(0); } +} +%NAS = clean_options(\%NAS, $check, $escape_spaces, $log); +if( !-d $NAS{'DESTINATION'} && !-w $NAS{'DESTINATION'} ) { + my $error = 'Local NAS directory '.$NAS{'DESTINATION'}.' not found, not a '. + 'directory or not writable by this program!'; + if( $check ) { print "\n".$error."\n";} + send_mail($NAS{'TO'},$NAS{'FROM'},"Critical backup FAILED.",$error); +} else { + send_mail($NAS{'TO'},$NAS{'FROM'},"Starting backup of critical data...", + "All options verified,\n\nStarting critical backup now..."); + %NAS = backup(\%NAS, $check); + my $body = "Successful backups:\n-------------------\n" . + join("\n",@{$NAS{'successes'}}); + if( scalar(@{$NAS{'failures'}}) == 0 ) { + $body .= "\n\nThere were no failed backup jobs."; + } else { + $body .= "\n\nFailed backups:\n---------------\n" . + join("\n",@{$NAS{'failures'}}); + } + my ($secs,$mins,$hours) = gmtime(time() - $start_time); + $body .= "\n\nJob completed in ".$hours." hours, ". + $mins." minutes, ".$secs." seconds."; + if( scalar(@{$NAS{'failures'}}) == 0 ) { + send_mail($NAS{'TO'},$NAS{'FROM'},"Critical backup success!!",$body); + } else { + send_mail($NAS{'TO'},$NAS{'FROM'},"Critical backup FAILED.",$body); + } +} + +# Function: clean_options +# Purpose: cleans initial options and sets additional key options: +# RSYNC_OPTIONS, RSYNC_REMOTE_OPTIONS, failures, successes +# Requires: NAS - a hash reference containing the configuration keys +# check - boolean, True to run rsync in dry-run mode +# escape_spaces - boolean, True to escape destination string +# log - boolean, True to ryn rsync in verbose mode +# Returns: A NAS hash with new and cleaned keys +sub clean_options { + # grab the NAS reference, and create a nice hash pointer to the NAS data + my($NAS_reference, $check, $escape_spaces, $log) = @_; + my %NAS = %{$NAS_reference}; + + $NAS{'RSYNC_OPTIONS'} = + '--size-only --copy-unsafe-links --archive --delete'; + if( $check ) { $NAS{'RSYNC_OPTIONS'} .= ' --dry-run'; } + if( $log ) { $NAS{'RSYNC_OPTIONS'} .= ' --verbose'; } + + # ensure trailing / on destination directory + $NAS{'DESTINATION'} =~ s/(.*)([^\/]$)/$1$2\//; + + # remove trailing / on local directories, ensure no duplicate 'source' dirs + foreach(@{$NAS{'LOCAL'}}) { $_ =~ s/\/$//; } + check_duplicates(\@{$NAS{'LOCAL'}}); + + # remote trailing /, escape spaces if specified, ensure no duplicate 'remote' + # directories + foreach(@{$NAS{'REMOTE'}}) { + if( $escape_spaces ) { $_ =~ s/ /\\ /g; } + $_ =~ s/\/$//; + } + check_duplicates(\@{$NAS{'REMOTE'}}); + + $NAS{'RSYNC_REMOTE_OPTIONS'} = + '--rsh="ssh -p '.$NAS{'PORT'}.'" '.$NAS{'USER'}.'@'.$NAS{'HOST'}.':'; + $NAS{'failures'} = []; + $NAS{'successes'} = []; + + if( $check ) { + print "\nCleaned options:\n----------------\n"; + print "Local backup directories:\n"; + print join("\n",@{$NAS{'LOCAL'}}); + print "\n\nRemote backup directories:\n"; + print join("\n",@{$NAS{'REMOTE'}}); + print "\n\nEscape spaces in remote directories?: "; + if( $escape_spaces == 1 ) { + print "YES\n"; + } else { + print "NO\n"; + } + print "NAS destination directory: " . $NAS{'DESTINATION'} . "\n"; + print "Remote host: " . $NAS{'HOST'} . "\n"; + print "Remote user: " . $NAS{'USER'} . "\n"; + print "Remote port: " . $NAS{'PORT'} . "\n"; + print "To address: " . $NAS{'TO'} . "\n"; + print "From address: " . $NAS{'FROM'} . "\n"; + print "Rsync binary location: " . $NAS{'RSYNC'} . "\n"; + print "Rsync options to use: " . $NAS{'RSYNC_OPTIONS'} . "\n"; + print "Rsync remote options to use: " . $NAS{'RSYNC_REMOTE_OPTIONS'} . + "\n\n"; + print "Simulating backup with dry rsync run:\n"; + } + return %NAS; +} + +# Function: send_mail +# Purpose: sends a mail VIA sendmail with given content +# Requires: to - string To e-mail address +# from - string From e-mail address +# subject - string subject +# body - string e-mail body +sub send_mail { + my ($to, $from, $subject, $body) = @_; + my $sendmail = "/usr/sbin/sendmail -t"; + open(SENDMAIL, "|$sendmail") or return 0; + print SENDMAIL "To: $to\n"; + print SENDMAIL "From: $from\n"; + print SENDMAIL "Subject: $subject\n"; + print SENDMAIL "Content-type: text/plain\n\n"; + print SENDMAIL "$body\n"; + close(SENDMAIL); +} + +# Function: backup +# Purpose: rsyncs the given NAS local and remote source directories to +# destination directories +# Requires: NAS - a hash reference containing the configuration keys, must be +# run through clean_options first +# Returns: True on success, False on non-writable or non-directory destination +# Returns: A NAS hash with processed successes and failures. +sub backup { + # grab the NAS reference, and create a nice hash pointer to the NAS data + my($NAS_reference, $check) = @_; + my $NAS = %{$NAS_reference}; + + my $rsync_command = ''; + foreach(@{$NAS{'LOCAL'}}) { + $rsync_command = $NAS{'RSYNC'}." ". + $NAS{'RSYNC_OPTIONS'}." '".$_. + "' '".$NAS{'DESTINATION'}."'"; + if( $check ) { print $rsync_command . "\n"; } + if( system($rsync_command) != 0 ) { + push(@{$NAS{'failures'}},$_); + } else { + push(@{$NAS{'successes'}},$_); + } + } + foreach(@{$NAS{'REMOTE'}}) { + $rsync_command = $NAS{'RSYNC'}." ". + $NAS{'RSYNC_OPTIONS'}." ". + $NAS{'RSYNC_REMOTE_OPTIONS'}."'".$_. + "' '".$NAS{'DESTINATION'}."'"; + if( $check ) { print $rsync_command . "\n"; } + if( system($rsync_command) != 0 ) { + push(@{$NAS{'failures'}},$_); + } else { + push(@{$NAS{'successes'}},$_); + } + } + return %NAS; +} + +# Function: check_duplicates +# Purpose: checks for duplicate source directories in a given array +# Requires: array - a array reference containing the directories to check +# Returns: True on success, exits with e-mail and error(1) if failed +sub check_duplicates { + my($directories_reference) = @_; + my @check_directories = @{$directories_reference}; + my %duplicate_counter; + my $last_dir_name; + + foreach(@check_directories) { + # grab the last directory in the source string, and lowercase it to compare + $last_dir_name = (split(/\//, $_))[-1]; + $last_dir_name =~ tr/[A-Z]/[a-z]/; + $duplicate_counter{$last_dir_name}++; + if( $duplicate_counter{$last_dir_name} > 1 ) { + my $body = "Your source directories have the same LAST directory name. ". + "This WILL LEAD TO DATA LOSS, as only the last duplicate directory ". + "is copied to the destination (not the full path). This will ". + "overwrite the first directory backed up on the remote server.\n\n". + "ABORTING TO PRESERVE DATA.\n--------------------------\n"; + $body .= join("\n",@check_directories); + send_mail($NAS{'TO'},$NAS{'FROM'}, + "Critical backup ABORTED - duplicate sources detected!",$body); + exit(1); + } + } + return 1; +} + +# Function: usage +# Purpose: prints usage information, and exits with no error +# Requires: none +sub usage { + print " + critical_backup + + This will sync critical backup data (from the NAS and/or a remote system) to a + directory on NAS (preferably read-only by normal users). By default, this + program runs sliently. Edit the script to add/change directories that are + backed up. + + OPTIONS: + + --check Turns on checking mode, showing processed options before + execution, and DISABLES actual rsync transfers. This is + useful to test that everything is setup correctly before + running the script. + + --log Logs rync status, and transfer details to the terminal + window. If run in a cronjob, you should redirect this + output to a logfile somewhere. Useful for debugging, and + verifying copied files. + + --no-space-escape By default, remote directories are quoted and escaped. + However, some systems only handle quoting or escaping, but + not both. Enable this flag if your remote directories + have spaces in them, and FAIL for no apparent reason. + This is most easily determined by running this program + with the --check option, and looking for (code 23) errors + during the rsync transfers. If you see those, setting + this option should fix those errors and allow transfers on + remote servers. + + --help / -h This help message. + "; + exit(0); +} diff --git a/backup_scripts/offsite_backup b/backup_scripts/offsite_backup new file mode 100755 index 0000000..3449930 --- /dev/null +++ b/backup_scripts/offsite_backup @@ -0,0 +1,280 @@ +#!/usr/bin/perl -w +# +# Sync's NAS backup data to remote site VIA rsync over SSH. +# Copyright 2008, Robert Pufky (github.com/r-pufky) +# http://www.crazymonkies.com/projects.php?type=readynas +# +# Installation / Usage: +# --------------------- +# run './offsite_backup --help' for more information +# +# Recommended cronjob: +# 30 0 * * 3 /root/backup_scripts/offsite_backup +# +# Notes: +# This is meant to backup your entire local NAS backup directory to a remote +# site. This potentially can be very large sets of data being sync'ed, and +# will be much slower than critical or regular backups. Remote data is backed +# up VIA rsync over SSH. +# +# The rsync over SSH setup that is used in this script requires public-key +# authentication, which a basic setup is described below. Google for more +# information. +# +# This job should run relatively slowly. Should be run relatively +# infrequently. +# +# Initially, this should be run manually with logging enabled to make sure you +# are grabbing everything that you want. The initial sync will take the +# longest, and it's best to monitor the first sync to adjust your backups +# accordingly. +# +# ./offsite_backup --log +# +# Public-key Setup: +# - Setup local (NAS) .ssh directory +# ssh root@YOURNAS +# mkdir -p ${HOME}/.ssh +# chmod 0700 ${HOME}/.ssh +# - Generate a passwordless DSA public/private keyset ** +# ssh-keygen -t dsa -f ${HOME}/.ssh/id_dsa -P '' +# - Upload public key to your server VIA scp +# scp -P PORT ${HOME}/.ssh/id_dsa.pub USER@SERVER:/REMOTEHOME/ +# - Setup remote system directory +# ssh USER@HOST -p PORT +# mkdir -p ${HOME}/.ssh +# cat ${HOME}/id_dsa.pub >> ${HOME}/.ssh/authorized_keys +# cat ${HOME}/id_dsa.pub >> ${HOME}/.ssh/authorized_keys2 +# chmod 0600 ${HOME}/.ssh/authorized_keys* +# +# ** At the time of this script, there was an openssl bug that affected the +# ReadyNAS, which created cryptographically weak keys. The work-around is +# to generate the keys on a Mac or linux box that isn't affected by the keys +# using the folloing command, then copy the resulting (id_dsa,id_dsa.pub) +# keys to your ReadNAS: +# +# ssh-keygen -t dsa -f ~/id_dsa -C 'root@YOURNAS' -P '' +# scp -P PORT ~/id_dsa* root@YOURNAS:/root/.ssh/ +# +use strict; +my %NAS; +# ----------------------------------------------------------------------------- +# Local NAS backup directories +$NAS{'LOCAL'} = [ + '/Backup']; +# Offsite remote system destination directory (use full path for safety) +$NAS{'DESTINATION'} = '/home/me/data/nas-backup/'; +# remote system to connect to +$NAS{'HOST'} = 'remoteserver.example.com'; +# remote user to authenticate with +$NAS{'USER'} = 'you'; +# remote port for SSH server +$NAS{'PORT'} = '22'; +# E-mail message settings +$NAS{'TO'} = 'readynas@example.com'; +$NAS{'FROM'} = 'readynas@example.com'; +# rsync binary location (run 'which rsync' from the command line to find it) +$NAS{'RSYNC'} = '/usr/bin/rsync'; +# ----------------------------------------------------------------------------- + + + +my $start_time = time(); +my $check = 0; +my $escape_spaces = 1; +my $log = 0; +foreach(@ARGV) { + if( $_ eq '--check' ) { $check = 1; } + if( $_ eq '--log' ) { $log = 1; } + if( $_ eq '--no-space-escape' ) { $escape_spaces = 0; } + if( $_ eq '--help' or $_ eq '-h' ) { usage(); exit(0); } +} +%NAS = clean_options(\%NAS, $check, $escape_spaces, $log); +send_mail($NAS{'TO'},$NAS{'FROM'},"Starting backup to offsite...", + "All options verified,\n\nStarting offsite backup now..."); +%NAS = backup(\%NAS, $check); +my $body = "Successful backups:\n-------------------\n" . + join("\n",@{$NAS{'successes'}}); +if( scalar(@{$NAS{'failures'}}) == 0 ) { + $body .= "\n\nThere were no failed backup jobs."; +} else { + $body .= "\n\nFailed backups:\n---------------\n" . + join("\n",@{$NAS{'failures'}}); +} +my ($secs,$mins,$hours) = gmtime(time() - $start_time); +$body .= "\n\nJob completed in ".$hours." hours, ". + $mins." minutes, ".$secs." seconds."; +if( scalar(@{$NAS{'failures'}}) == 0 ) { + send_mail($NAS{'TO'},$NAS{'FROM'},"offsite backup success!!",$body); +} else { + send_mail($NAS{'TO'},$NAS{'FROM'},"offsite backup FAILED.",$body); +} + +# Function: clean_options +# Purpose: cleans initial options and sets additional key options: +# RSYNC_OPTIONS, RSYNC_REMOTE_OPTIONS, failures, successes +# Requires: NAS - a hash reference containing the configuration keys +# check - boolean, True to run rsync in dry-run mode +# escape_spaces - boolean, True to escape destination string +# log - boolean, True to ryn rsync in verbose mode +# Returns: A NAS hash with new and cleaned keys +sub clean_options { + # grab the NAS reference, and create a nice hash pointer to the NAS data + my($NAS_reference, $check, $escape_space, $log) = @_; + my %NAS = %{$NAS_reference}; + + $NAS{'RSYNC_OPTIONS'} = + '--size-only --copy-unsafe-links --archive --delete'; + if( $check ) { $NAS{'RSYNC_OPTIONS'} .= ' --dry-run'; } + if( $log ) { $NAS{'RSYNC_OPTIONS'} .= ' --verbose'; } + + # escape spaces if specified, and ensure trailing / for remote server + if( $escape_spaces ) { $NAS{'DESTINATION'} =~ s/ /\\ /g; } + $NAS{'DESTINATION'} =~ s/(.*)([^\/]$)/$1$2\//; + + # remove trailing / on local directories, ensure no duplicate 'source' dirs + foreach(@{$NAS{'LOCAL'}}) { $_ =~ s/\/$//; } + check_duplicates(\@{$NAS{'LOCAL'}}); + + $NAS{'RSYNC_REMOTE_OPTIONS'} = + '--rsh="ssh -p '.$NAS{'PORT'}.'" '.$NAS{'USER'}.'@'.$NAS{'HOST'}.':'; + $NAS{'failures'} = []; + $NAS{'successes'} = []; + + if( $check ) { + print "\nCleaned options:\n----------------\n"; + print "Local NAS backup directory:\n"; + print join("\n",@{$NAS{'LOCAL'}}); + print "\n\nOffsite remote system destination directory : " . + $NAS{'DESTINATION'} . "\n"; + print "\n\nEscape spaces in remote directories?: "; + if( $escape_spaces == 1 ) { + print "YES\n"; + } else { + print "NO\n"; + } + print "Remote host: " . $NAS{'HOST'} . "\n"; + print "Remote user: " . $NAS{'USER'} . "\n"; + print "Remote port: " . $NAS{'PORT'} . "\n"; + print "To address: " . $NAS{'TO'} . "\n"; + print "From address: " . $NAS{'FROM'} . "\n"; + print "Rsync binary location: " . $NAS{'RSYNC'} . "\n"; + print "Rsync options to use: " . $NAS{'RSYNC_OPTIONS'} . "\n"; + print "Rsync remote options to use: " . $NAS{'RSYNC_REMOTE_OPTIONS'} . + "\n\n"; + print "Simulating backup with dry rsync run:\n"; + } + return %NAS; +} + +# Function: send_mail +# Purpose: sends a mail VIA sendmail with given content +# Requires: to - string To e-mail address +# from - string From e-mail address +# subject - string subject +# body - string e-mail body +sub send_mail { + my ($to, $from, $subject, $body) = @_; + my $sendmail = "/usr/sbin/sendmail -t"; + open(SENDMAIL, "|$sendmail") or return 0; + print SENDMAIL "To: $to\n"; + print SENDMAIL "From: $from\n"; + print SENDMAIL "Subject: $subject\n"; + print SENDMAIL "Content-type: text/plain\n\n"; + print SENDMAIL "$body\n"; + close(SENDMAIL); +} + +# Function: backup +# Purpose: rsyncs the given NAS local and remote source directories to +# destination directories +# Requires: NAS - a hash reference containing the configuration keys, must be +# run through clean_options first +# Returns: True on success, False on non-writable or non-directory destination +# Returns: A NAS hash with processed successes and failures. +sub backup { + # grab the NAS reference, and create a nice hash pointer to the NAS data + my($NAS_reference, $check) = @_; + my $NAS = %{$NAS_reference}; + + my $rsync_command = ''; + foreach(@{$NAS{'LOCAL'}}) { + $rsync_command = $NAS{'RSYNC'}." ". + $NAS{'RSYNC_OPTIONS'}." ".$_." ". + $NAS{'RSYNC_REMOTE_OPTIONS'}."'".$NAS{'DESTINATION'}."'"; + if( $check ) { print $rsync_command . "\n"; } + if( system($rsync_command) != 0 ) { + push(@{$NAS{'failures'}},$_); + } else { + push(@{$NAS{'successes'}},$_); + } + } + return %NAS; +} + +# Function: check_duplicates +# Purpose: checks for duplicate source directories in a given array +# Requires: array - a array reference containing the directories to check +# Returns: True on success, exits with e-mail and error(1) if failed +sub check_duplicates { + my($directories_reference) = @_; + my @check_directories = @{$directories_reference}; + my %duplicate_counter; + my $last_dir_name; + + foreach(@check_directories) { + # grab the last directory in the source string, and lowercase it to compare + $last_dir_name = (split(/\//, $_))[-1]; + $last_dir_name =~ tr/[A-Z]/[a-z]/; + $duplicate_counter{$last_dir_name}++; + if( $duplicate_counter{$last_dir_name} > 1 ) { + my $body = "Your source directories have the same LAST directory name. ". + "This WILL LEAD TO DATA LOSS, as only the last duplicate directory ". + "is copied to the destination (not the full path). This will ". + "overwrite the first directory backed up on the remote server.\n\n". + "ABORTING TO PRESERVE DATA.\n--------------------------\n"; + $body .= join("\n",@check_directories); + send_mail($NAS{'TO'},$NAS{'FROM'}, + "Critical backup ABORTED - duplicate sources detected!",$body); + exit(1); + } + } + return 1; +} + +# Function: usage +# Purpose: prints usage information, and exits with no error +# Requires: none +sub usage { + print " + offsite_backup + + This will sync NAS data to an offsite server. By default, this program runs + sliently. Edit the script to add/change directories that are backed up. + + OPTIONS: + + --check Turns on checking mode, showing processed options before + execution, and DISABLES actual rsync transfers. This is + useful to test that everything is setup correctly before + running the script. + + --log Logs rync status, and transfer details to the terminal + window. If run in a cronjob, you should redirect this + output to a logfile somewhere. Useful for debugging, and + verifying copied files. + + --no-space-escape By default, remote directories are quoted and escaped. + However, some systems only handle quoting or escaping, but + not both. Enable this flag if your remote directories + have spaces in them, and FAIL for no apparent reason. + This is most easily determined by running this program + with the --check option, and looking for (code 23) errors + during the rsync transfers. If you see those, setting + this option should fix those errors and allow transfers on + remote servers. + + --help / -h This help message. + "; + exit(0); +} diff --git a/backup_scripts/regular_backup b/backup_scripts/regular_backup new file mode 100755 index 0000000..2689bfe --- /dev/null +++ b/backup_scripts/regular_backup @@ -0,0 +1,321 @@ +#!/usr/bin/perl -w +# +# Sync's regular data to user read-only directory on NAS. +# Copyright 2008, Robert Pufky (github.com/r-pufky) +# http://www.crazymonkies.com/projects.php?type=readynas +# +# Installation / Usage: +# --------------------- +# run './regular_backup --help' for more information +# +# Recommended cronjob: +# 30 1 * * 0 /root/backup_scripts/regular_backup +# +# Notes: +# This is meant to be run on large sets of regular data that need to be backed +# up, but don't require the backups to be completed in a nightly manner (i.e. +# music, pictures, etc). If you want to backup critical sets of data, use the +# critical_backup script, which can be run in a separate cronjob. Remote data +# is backed up VIA rsync over SSH. +# +# The rsync over SSH setup that is used in this script requires public-key +# authentication, which a basic setup is described below. Google for more +# information. +# +# This job will run slower than critical_backup, and should be timed in your +# cronjob to start after your critical backup. Should be run relatively +# infrequently, as large data sets can take a multiple hours. +# +# If NO directories are being backed up locally or remotely, just leave the +# directory list blank. It should look like: +# - local NAS directories: $NAS{'LOCAL'} = []; +# - remote directories: $NAS{'REMOTE'} = []; +# +# Initially, this should be run manually with logging enabled to make sure you +# are grabbing everything that you want. The initial sync will take the +# longest, and it's best to monitor the first sync to adjust your backups +# accordingly. +# +# ./regular_backup --log +# +# Public-key Setup: +# - Setup local (NAS) .ssh directory +# ssh root@YOURNAS +# mkdir -p ${HOME}/.ssh +# chmod 0700 ${HOME}/.ssh +# - Generate a passwordless DSA public/private keyset ** +# ssh-keygen -t dsa -f ${HOME}/.ssh/id_dsa -P '' +# - Upload public key to your server VIA scp +# scp -P PORT ${HOME}/.ssh/id_dsa.pub USER@SERVER:/REMOTEHOME/ +# - Setup remote system directory +# ssh USER@HOST -p PORT +# mkdir -p ${HOME}/.ssh +# cat ${HOME}/id_dsa.pub >> ${HOME}/.ssh/authorized_keys +# cat ${HOME}/id_dsa.pub >> ${HOME}/.ssh/authorized_keys2 +# chmod 0600 ${HOME}/.ssh/authorized_keys* +# +# ** At the time of this script, there was an openssl bug that affected the +# ReadyNAS, which created cryptographically weak keys. The work-around is +# to generate the keys on a Mac or linux box that isn't affected by the keys +# using the folloing command, then copy the resulting (id_dsa,id_dsa.pub) +# keys to your ReadNAS: +# +# ssh-keygen -t dsa -f ~/id_dsa -C 'root@YOURNAS' -P '' +# scp -P PORT ~/id_dsa* root@YOURNAS:/root/.ssh/ +# +use strict; +my %NAS; +# ----------------------------------------------------------------------------- +# Regular local NAS directories to backup +$NAS{'LOCAL'} = [ + '/Pictures', + '/Music', + '/me_home/Projects']; +# Regular remote system directories to backup +$NAS{'REMOTE'} = [ + '/Users/me/svn-checkouts']; +# Local NAS destination backup directory, should be from root of drive +$NAS{'DESTINATION'} = '/Backup/Regular/'; +# remote system to connect to +$NAS{'HOST'} = '192.168.0.10'; +# remote user to authenticate with +$NAS{'USER'} = 'me'; +# remote port for SSH server +$NAS{'PORT'} = '22'; +# E-mail message settings +$NAS{'TO'} = 'readynas@example.com'; +$NAS{'FROM'} = 'readynas@example.com'; +# rsync binary location (run 'which rsync' from the command line to find it) +$NAS{'RSYNC'} = '/usr/bin/rsync'; +# ----------------------------------------------------------------------------- + + + +my $start_time = time(); +my $check = 0; +my $escape_spaces = 1; +my $log = 0; +foreach(@ARGV) { + if( $_ eq '--check' ) { $check = 1; } + if( $_ eq '--log' ) { $log = 1; } + if( $_ eq '--no-space-escape' ) { $escape_spaces = 0; } + if( $_ eq '--help' or $_ eq '-h' ) { usage(); exit(0); } +} +%NAS = clean_options(\%NAS, $check, $escape_spaces, $log); +if( !-d $NAS{'DESTINATION'} && !-w $NAS{'DESTINATION'} ) { + my $error = 'Local NAS directory '.$NAS{'DESTINATION'}.' not found, not a '. + 'directory or not writable by this program!'; + if( $check ) { print "\n".$error."\n";} + send_mail($NAS{'TO'},$NAS{'FROM'},"Regular backup FAILED.",$error); +} else { + send_mail($NAS{'TO'},$NAS{'FROM'},"Starting backup of regular data...", + "All options verified,\n\nStarting regular backup now..."); + %NAS = backup(\%NAS, $check); + my $body = "Successful backups:\n-------------------\n" . + join("\n",@{$NAS{'successes'}}); + if( scalar(@{$NAS{'failures'}}) == 0 ) { + $body .= "\n\nThere were no failed backup jobs."; + } else { + $body .= "\n\nFailed backups:\n---------------\n" . + join("\n",@{$NAS{'failures'}}); + } + my ($secs,$mins,$hours) = gmtime(time() - $start_time); + $body .= "\n\nJob completed in ".$hours." hours, ". + $mins." minutes, ".$secs." seconds."; + if( scalar(@{$NAS{'failures'}}) == 0 ) { + send_mail($NAS{'TO'},$NAS{'FROM'},"Regular backup success!!",$body); + } else { + send_mail($NAS{'TO'},$NAS{'FROM'},"Regular backup FAILED.",$body); + } +} + +# Function: clean_options +# Purpose: cleans initial options and sets additional key options: +# RSYNC_OPTIONS, RSYNC_REMOTE_OPTIONS, failures, successes +# Requires: NAS - a hash reference containing the configuration keys +# check - boolean, True to run rsync in dry-run mode +# escape_spaces - boolean, True to escape destination string +# log - boolean, True to ryn rsync in verbose mode +# Returns: A NAS hash with new and cleaned keys +sub clean_options { + # grab the NAS reference, and create a nice hash pointer to the NAS data + my($NAS_reference, $check, $escape_spaces, $log) = @_; + my %NAS = %{$NAS_reference}; + + $NAS{'RSYNC_OPTIONS'} = + '--size-only --copy-unsafe-links --archive --delete'; + if( $check ) { $NAS{'RSYNC_OPTIONS'} .= ' --dry-run'; } + if( $log ) { $NAS{'RSYNC_OPTIONS'} .= ' --verbose'; } + + # ensure trailing / on destination directory + $NAS{'DESTINATION'} =~ s/(.*)([^\/]$)/$1$2\//; + + # remove trailing / on local directories, ensure no duplicate 'source' dirs + foreach(@{$NAS{'LOCAL'}}) { $_ =~ s/\/$//; } + check_duplicates(\@{$NAS{'LOCAL'}}); + + # remote trailing /, escape spaces if specified, ensure no duplicate 'remote' + # directories + foreach(@{$NAS{'REMOTE'}}) { + if( $escape_spaces ) { $_ =~ s/ /\\ /g; } + $_ =~ s/\/$//; + } + check_duplicates(\@{$NAS{'REMOTE'}}); + + $NAS{'RSYNC_REMOTE_OPTIONS'} = + '--rsh="ssh -p '.$NAS{'PORT'}.'" '.$NAS{'USER'}.'@'.$NAS{'HOST'}.':'; + $NAS{'failures'} = []; + $NAS{'successes'} = []; + + if( $check ) { + print "\nCleaned options:\n----------------\n"; + print "Local backup directories:\n"; + print join("\n",@{$NAS{'LOCAL'}}); + print "\n\nRemote backup directories:\n"; + print join("\n",@{$NAS{'REMOTE'}}); + print "\n\nEscape spaces in remote directories?: "; + if( $escape_spaces == 1 ) { + print "YES\n"; + } else { + print "NO\n"; + } + print "NAS destination directory: " . $NAS{'DESTINATION'} . "\n"; + print "Remote host: " . $NAS{'HOST'} . "\n"; + print "Remote user: " . $NAS{'USER'} . "\n"; + print "Remote port: " . $NAS{'PORT'} . "\n"; + print "To address: " . $NAS{'TO'} . "\n"; + print "From address: " . $NAS{'FROM'} . "\n"; + print "Rsync binary location: " . $NAS{'RSYNC'} . "\n"; + print "Rsync options to use: " . $NAS{'RSYNC_OPTIONS'} . "\n"; + print "Rsync remote options to use: " . $NAS{'RSYNC_REMOTE_OPTIONS'} . + "\n\n"; + print "Simulating backup with dry rsync run:\n"; + } + return %NAS; +} + +# Function: send_mail +# Purpose: sends a mail VIA sendmail with given content +# Requires: to - string To e-mail address +# from - string From e-mail address +# subject - string subject +# body - string e-mail body +sub send_mail { + my ($to, $from, $subject, $body) = @_; + my $sendmail = "/usr/sbin/sendmail -t"; + open(SENDMAIL, "|$sendmail") or return 0; + print SENDMAIL "To: $to\n"; + print SENDMAIL "From: $from\n"; + print SENDMAIL "Subject: $subject\n"; + print SENDMAIL "Content-type: text/plain\n\n"; + print SENDMAIL "$body\n"; + close(SENDMAIL); +} + +# Function: backup +# Purpose: rsyncs the given NAS local and remote source directories to +# destination directories +# Requires: NAS - a hash reference containing the configuration keys, must be +# run through clean_options first +# Returns: True on success, False on non-writable or non-directory destination +# Returns: A NAS hash with processed successes and failures. +sub backup { + # grab the NAS reference, and create a nice hash pointer to the NAS data + my($NAS_reference, $check) = @_; + my $NAS = %{$NAS_reference}; + + my $rsync_command = ''; + foreach(@{$NAS{'LOCAL'}}) { + $rsync_command = $NAS{'RSYNC'}." ". + $NAS{'RSYNC_OPTIONS'}." '".$_. + "' '".$NAS{'DESTINATION'}."'"; + if( $check ) { print $rsync_command . "\n"; } + if( system($rsync_command) != 0 ) { + push(@{$NAS{'failures'}},$_); + } else { + push(@{$NAS{'successes'}},$_); + } + } + foreach(@{$NAS{'REMOTE'}}) { + $rsync_command = $NAS{'RSYNC'}." ". + $NAS{'RSYNC_OPTIONS'}." ". + $NAS{'RSYNC_REMOTE_OPTIONS'}."'".$_. + "' '".$NAS{'DESTINATION'}."'"; + if( $check ) { print $rsync_command . "\n"; } + if( system($rsync_command) != 0 ) { + push(@{$NAS{'failures'}},$_); + } else { + push(@{$NAS{'successes'}},$_); + } + } + return %NAS; +} + +# Function: check_duplicates +# Purpose: checks for duplicate source directories in a given array +# Requires: array - a array reference containing the directories to check +# Returns: True on success, exits with e-mail and error(1) if failed +sub check_duplicates { + my($directories_reference) = @_; + my @check_directories = @{$directories_reference}; + my %duplicate_counter; + my $last_dir_name; + + foreach(@check_directories) { + # grab the last directory in the source string, and lowercase it to compare + $last_dir_name = (split(/\//, $_))[-1]; + $last_dir_name =~ tr/[A-Z]/[a-z]/; + $duplicate_counter{$last_dir_name}++; + if( $duplicate_counter{$last_dir_name} > 1 ) { + my $body = "Your source directories have the same LAST directory name. ". + "This WILL LEAD TO DATA LOSS, as only the last duplicate directory ". + "is copied to the destination (not the full path). This will ". + "overwrite the first directory backed up on the remote server.\n\n". + "ABORTING TO PRESERVE DATA.\n--------------------------\n"; + $body .= join("\n",@check_directories); + send_mail($NAS{'TO'},$NAS{'FROM'}, + "Critical backup ABORTED - duplicate sources detected!",$body); + exit(1); + } + } + return 1; +} + +# Function: usage +# Purpose: prints usage information, and exits with no error +# Requires: none +sub usage { + print " + regular_backup + + This will sync regular backup data (from the NAS and/or a remote system) to a + directory on NAS (preferably read-only by normal users). By default, this + program runs sliently. Edit the script to add/change directories that are + backed up. + + OPTIONS: + + --check Turns on checking mode, showing processed options before + execution, and DISABLES actual rsync transfers. This is + useful to test that everything is setup correctly before + running the script. + + --log Logs rync status, and transfer details to the terminal + window. If run in a cronjob, you should redirect this + output to a logfile somewhere. Useful for debugging, and + verifying copied files. + + --no-space-escape By default, remote directories are quoted and escaped. + However, some systems only handle quoting or escaping, but + not both. Enable this flag if your remote directories + have spaces in them, and FAIL for no apparent reason. + This is most easily determined by running this program + with the --check option, and looking for (code 23) errors + during the rsync transfers. If you see those, setting + this option should fix those errors and allow transfers on + remote servers. + + --help / -h This help message. + "; + exit(0); +} diff --git a/cronjobs b/cronjobs new file mode 100644 index 0000000..e378706 --- /dev/null +++ b/cronjobs @@ -0,0 +1,5 @@ +# cronjob file for readynas-root +# import into cronjob with: 'crontab /root/cronjobs' +30 23 * * * /root/backup_scripts/critical_backup >/dev/null 2>&1 +30 1 * * 0 /root/backup_scripts/regular_backup >/dev/null 2>&1 +30 0 * * 3 /root/backup_scripts/offsite_backup >/dev/null 2>&1