From 131d91090614382780d16daec1815b53ebda97f2 Mon Sep 17 00:00:00 2001 From: David Steele Date: Thu, 8 Jan 2015 15:43:43 -0500 Subject: [PATCH] New structure for --delta and --force params. --- bin/pg_backrest.pl | 12 ++++---- lib/BackRest/Config.pm | 20 ++++++------- lib/BackRest/Manifest.pm | 23 ++++++++++++++- lib/BackRest/Restore.pm | 44 +++++++++++++++++++++++------ lib/BackRest/ThreadGroup.pm | 2 +- test/lib/BackRestTest/BackupTest.pm | 2 +- 6 files changed, 76 insertions(+), 27 deletions(-) diff --git a/bin/pg_backrest.pl b/bin/pg_backrest.pl index f6abb3754..f2e3cd2a8 100755 --- a/bin/pg_backrest.pl +++ b/bin/pg_backrest.pl @@ -56,10 +56,9 @@ pg_backrest.pl [options] [operation] Restore Options: --set backup set to restore (defaults to latest set). - --remap remaps the base or a tablespace to another path. - --thread # of threads to use for restore (defaults to 1). - --force force restore when destination paths are not empty. - Use with extreme caution as this will delete data in those paths! + --delta perform a delta restore using checksums when available. + --force force a restore and overwrite all existing files. + with --delta forces size/timestamp delta even if checksums are present. =cut #################################################################################################################################### @@ -436,9 +435,10 @@ if (operation_get() eq OP_RESTORE) ( config_key_load(CONFIG_SECTION_STANZA, CONFIG_KEY_PATH), param_get(PARAM_SET), - param_get(PARAM_REMAP), + undef, #param_get(PARAM_REMAP), $oFile, - param_get(PARAM_THREAD), + undef, #param_get(PARAM_THREAD), + param_get(PARAM_DELTA), param_get(PARAM_FORCE) )->restore; diff --git a/lib/BackRest/Config.pm b/lib/BackRest/Config.pm index a0b7651dc..06c6680c2 100644 --- a/lib/BackRest/Config.pm +++ b/lib/BackRest/Config.pm @@ -26,7 +26,7 @@ our @EXPORT = qw(config_load config_key_load operation_get operation_set param_g BACKUP_TYPE_FULL BACKUP_TYPE_DIFF BACKUP_TYPE_INCR - PARAM_CONFIG PARAM_STANZA PARAM_TYPE PARAM_REMAP PARAM_SET PARAM_NO_START_STOP PARAM_THREAD PARAM_FORCE + PARAM_CONFIG PARAM_STANZA PARAM_TYPE PARAM_DELTA PARAM_SET PARAM_NO_START_STOP PARAM_FORCE PARAM_VERSION PARAM_HELP PARAM_TEST PARAM_TEST_DELAY PARAM_TEST_NO_FORK CONFIG_SECTION_COMMAND CONFIG_SECTION_COMMAND_OPTION CONFIG_SECTION_LOG CONFIG_SECTION_BACKUP @@ -87,9 +87,8 @@ use constant PARAM_STANZA => 'stanza', PARAM_TYPE => 'type', PARAM_NO_START_STOP => 'no-start-stop', - PARAM_REMAP => 'remap', + PARAM_DELTA => 'delta', PARAM_SET => 'set', - PARAM_THREAD => 'thread', PARAM_FORCE => 'force', PARAM_VERSION => 'version', PARAM_HELP => 'help', @@ -109,6 +108,7 @@ use constant CONFIG_SECTION_LOG => 'log', CONFIG_SECTION_BACKUP => 'backup', CONFIG_SECTION_RESTORE => 'restore', + CONFIG_SECTION_RESTORE_REMAP => 'restore:remap', CONFIG_SECTION_ARCHIVE => 'archive', CONFIG_SECTION_RETENTION => 'retention', CONFIG_SECTION_STANZA => 'stanza', @@ -167,8 +167,8 @@ sub config_load param_set(PARAM_TEST_DELAY, 5); # Seconds to delay after a test point (default is not enough for manual tests) # Get command line parameters - GetOptions (\%oParam, PARAM_CONFIG . '=s', PARAM_STANZA . '=s', PARAM_TYPE . '=s', PARAM_REMAP . '=s%', PARAM_SET . '=s', - PARAM_THREAD . '=s', PARAM_NO_START_STOP, PARAM_FORCE, PARAM_VERSION, PARAM_HELP, + GetOptions (\%oParam, PARAM_CONFIG . '=s', PARAM_STANZA . '=s', PARAM_TYPE . '=s', PARAM_DELTA, PARAM_SET . '=s', + PARAM_NO_START_STOP, PARAM_FORCE, PARAM_VERSION, PARAM_HELP, PARAM_TEST, PARAM_TEST_DELAY . '=s', PARAM_TEST_NO_FORK) or pod2usage(2); @@ -227,11 +227,11 @@ sub config_load } } - # Validate thread parameter - if (defined(param_get(PARAM_THREAD)) && !(param_get(PARAM_THREAD) >= 1)) - { - confess &log(ERROR, 'thread parameter should be >= 1'); - } + # # Validate thread parameter + # if (defined(param_get(PARAM_THREAD)) && !(param_get(PARAM_THREAD) >= 1)) + # { + # confess &log(ERROR, 'thread parameter should be >= 1'); + # } # Get configuration parameter and load it if (!defined(param_get(PARAM_CONFIG))) diff --git a/lib/BackRest/Manifest.pm b/lib/BackRest/Manifest.pm index d5bdad225..616a1a38a 100644 --- a/lib/BackRest/Manifest.pm +++ b/lib/BackRest/Manifest.pm @@ -9,6 +9,7 @@ use warnings; use Carp; use File::Basename qw(dirname); +use Time::Local qw(timelocal); use lib dirname($0); use BackRest::Utility; @@ -18,7 +19,7 @@ use Exporter qw(import); our @EXPORT = qw(MANIFEST_SECTION_BACKUP MANIFEST_SECTION_BACKUP_OPTION MANIFEST_SECTION_BACKUP_PATH MANIFEST_SECTION_BACKUP_TABLESPACE - MANIFEST_KEY_LABEL + MANIFEST_KEY_LABEL MANIFEST_KEY_TIMESTAMP_COPY_START MANIFEST_SUBKEY_CHECKSUM MANIFEST_SUBKEY_DESTINATION MANIFEST_SUBKEY_FUTURE MANIFEST_SUBKEY_GROUP MANIFEST_SUBKEY_LINK MANIFEST_SUBKEY_MODE MANIFEST_SUBKEY_MODIFICATION_TIME MANIFEST_SUBKEY_PATH @@ -35,6 +36,7 @@ use constant MANIFEST_SECTION_BACKUP_TABLESPACE => 'backup:tablespace', MANIFEST_KEY_LABEL => 'label', + MANIFEST_KEY_TIMESTAMP_COPY_START => 'timestamp-copy-start', MANIFEST_SUBKEY_CHECKSUM => 'checksum', MANIFEST_SUBKEY_DESTINATION => 'link_destination', @@ -178,6 +180,25 @@ sub valid (defined($strSubKey) ? ", subkey '$strSubKey'" : '') . ' is not valid'); } +#################################################################################################################################### +# epoch +# +# Retrieves a value in the format YYYY-MM-DD HH24:MI:SS and converts to epoch time. +#################################################################################################################################### +sub epoch +{ + my $self = shift; + my $strSection = shift; + my $strKey = shift; + my $strSubKey = shift; + + my $strValue = $self->get($strSection, $strKey, $strSubKey); + + my ($iYear, $iMonth, $iDay, $iHour, $iMinute, $iSecond) = split(/[\s\-\:]+/, $strValue); + + return timelocal($iSecond, $iMinute, $iHour, $iDay , $iMonth - 1, $iYear); +} + #################################################################################################################################### # KEYS # diff --git a/lib/BackRest/Restore.pm b/lib/BackRest/Restore.pm index 024dff941..ee2676a48 100644 --- a/lib/BackRest/Restore.pm +++ b/lib/BackRest/Restore.pm @@ -10,7 +10,8 @@ use strict; use warnings; use Carp; -use File::Basename; +use File::Basename qw(dirname); +use File::stat qw(lstat); use lib dirname($0); use BackRest::Utility; @@ -30,7 +31,8 @@ sub new my $oRemapRef = shift; # Tablespace remaps my $oFile = shift; # Default file object my $iThreadTotal = shift; # Total threads to run for restore - my $bForce = shift; # Force the restore even if files are present + my $bDelta = shift; # perform delta restore + my $bForce = shift; # force a restore # Create the class hash my $self = {}; @@ -40,6 +42,7 @@ sub new $self->{strDbClusterPath} = $strDbClusterPath; $self->{oFile} = $oFile; $self->{iThreadTotal} = defined($iThreadTotal) ? $iThreadTotal : 1; + $self->{bDelta} = $bDelta; $self->{bForce} = $bForce; $self->{oRemapRef} = $oRemapRef; @@ -269,7 +272,7 @@ sub clean } # If force was not specified then error if any file is found - if (!$self->{bForce}) + if (!$self->{bForce} && !$self->{bDelta}) { confess &log(ERROR, "db path '${strPath}' contains files"); } @@ -473,7 +476,7 @@ sub restore_thread my $self = shift; # Class hash my $iThreadIdx = shift; # Defines the index of this thread my $oyRestoreQueueRef = shift; # Restore queues - my $oManifest = shift; # Backup manifest + my $oManifest = shift; # Backup manifest my $iDirection = $iThreadIdx % 2 == 0 ? 1 : -1; # Size of files currently copied by this thread my $oFileThread = $self->{oFile}->clone($iThreadIdx); # Thread local file object @@ -482,6 +485,9 @@ sub restore_thread my $iQueueStartIdx = int((@{$oyRestoreQueueRef} / $self->{iThreadTotal}) * $iThreadIdx); my $iQueueIdx = $iQueueStartIdx; + # Time when the backup copying began - used for size/timestamp deltas + my $lCopyTimeBegin = $oManifest->epoch(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TIMESTAMP_COPY_START); + # Set source compression my $bSourceCompression = ${$oManifest}{'backup:option'}{compress} eq 'y' ? true : false; @@ -504,7 +510,7 @@ sub restore_thread my $strName = (split(/\|/, $strMessage))[1]; # Name of file to be restored # If the file is a reference to a previous backup and hardlinks are off, then fetch it from that backup - my $strReference = ${$oManifest}{'backup:option'}{hardlink} eq 'y' ? undef : + my $strReference = defined(${$oManifest}{'backup:option'}{hardlink}) && ${$oManifest}{'backup:option'}{hardlink} eq 'y' ? undef : ${$oManifest}{$strSection}{$strName}{reference}; # Generate destination file name @@ -515,10 +521,32 @@ sub restore_thread if ($oFileThread->exists(PATH_DB_ABSOLUTE, $strDestinationFile)) { - if (defined($strChecksum) && $oFileThread->hash(PATH_DB_ABSOLUTE, $strDestinationFile) eq $strChecksum) + # Perform delta if requested + if ($self->{bDelta}) { - &log(DEBUG, "${strDestinationFile} exists and matches backup checksum ${strChecksum}"); - next; + # Do checksum delta if --force was not requested and checksums exist + if (!$self->{bForce} && defined($strChecksum) && + $oFileThread->hash(PATH_DB_ABSOLUTE, $strDestinationFile) eq $strChecksum) + { + &log(DEBUG, "${strDestinationFile} exists and matches backup checksum ${strChecksum}"); + next; + } + # Else use size/timestamp delta + else + { + my $oStat = lstat($strDestinationFile); + + # Make sure that timestamp/size are equal and that timestamp is before the copy start time of the backup + if (defined($oStat) && + $oStat->size == $oManifest->get($strSection, $strName, MANIFEST_SUBKEY_SIZE) && + $oStat->mtime == $oManifest->get($strSection, $strName, MANIFEST_SUBKEY_MODIFICATION_TIME) && + $oStat->mtime < $lCopyTimeBegin) + { + &log(DEBUG, "${strDestinationFile} exists and matches size " . $oStat->size . + " and modification time " . $oStat->mtime); + next; + } + } } $oFileThread->remove(PATH_DB_ABSOLUTE, $strDestinationFile); diff --git a/lib/BackRest/ThreadGroup.pm b/lib/BackRest/ThreadGroup.pm index 48431cce3..e816f5694 100644 --- a/lib/BackRest/ThreadGroup.pm +++ b/lib/BackRest/ThreadGroup.pm @@ -64,7 +64,7 @@ sub complete my $bConfessOnError = shift; # Set defaults - $bConfessOnError = defined($bConfessOnError) ? true : $bConfessOnError; + $bConfessOnError = defined($bConfessOnError) ? $bConfessOnError : true; # Wait for all threads to complete and handle errors my $iThreadComplete = 0; diff --git a/test/lib/BackRestTest/BackupTest.pm b/test/lib/BackRestTest/BackupTest.pm index d0677cf63..68d76dd39 100755 --- a/test/lib/BackRestTest/BackupTest.pm +++ b/test/lib/BackRestTest/BackupTest.pm @@ -719,7 +719,7 @@ sub BackRestTestBackup_CompareRestore # Create the backup command BackRestTestCommon_Execute(BackRestTestCommon_CommandMainGet() . ' --config=' . BackRestTestCommon_DbPathGet() . - '/pg_backrest.conf ' . (defined($bForce)? '--force ' : '') . "--stanza=${strStanza} restore"); + '/pg_backrest.conf ' . (defined($bForce)? '--delta ' : '') . "--stanza=${strStanza} restore"); } ####################################################################################################################################