From be337c951aee0121dc3d1a251bdb5218d72769a0 Mon Sep 17 00:00:00 2001 From: Cynthia Shang Date: Wed, 4 Jan 2017 10:12:29 -0500 Subject: [PATCH] Improvements to stanza-create: * Force only required when a change is needed for backup.info or archive.info. * Identical files are not overwritten. --- lib/pgBackRest/Stanza.pm | 167 ++++++++++-------- test/expect/backup-stanza-create-001.log | 141 ++++++++++++++- test/expect/backup-stanza-create-002.log | 141 ++++++++++++++- test/expect/backup-synthetic-001.log | 2 +- test/expect/backup-synthetic-005.log | 2 +- .../Backup/BackupStanzaCreateTest.pm | 29 ++- .../Common/Host/HostBackupTest.pm | 11 +- 7 files changed, 412 insertions(+), 81 deletions(-) diff --git a/lib/pgBackRest/Stanza.pm b/lib/pgBackRest/Stanza.pm index 89998c9b5..b03866a90 100644 --- a/lib/pgBackRest/Stanza.pm +++ b/lib/pgBackRest/Stanza.pm @@ -29,7 +29,7 @@ use pgBackRest::Protocol::Protocol; #################################################################################################################################### # Global variables #################################################################################################################################### -my $strStanzaCreateErrorMsg = "not empty, stanza-create has already been attempted\n" . +my $strStanzaCreateErrorMsg = "not empty\n" . "HINT: Use --force to force the stanza data to be created."; #################################################################################################################################### @@ -130,28 +130,35 @@ sub stanzaCreate my @stryFileListArchive = fileList($strParentPathArchive, undef, 'forward', true); my @stryFileListBackup = fileList($strParentPathBackup, undef, 'forward', true); - # If force was not used then error if any dir has data + # If force not used and at least one directory is not empty, then check to see if the info files exist if (!optionGet(OPTION_FORCE) && (@stryFileListArchive || @stryFileListBackup)) { - confess &log(ERROR, - (@stryFileListBackup ? 'backup directory ' : '') . - ((@stryFileListBackup && @stryFileListArchive) ? 'and ' : '') . - (@stryFileListArchive ? 'archive directory ' : '') . - $strStanzaCreateErrorMsg, ERROR_PATH_NOT_EMPTY); + my $strBackupInfoFile = &FILE_BACKUP_INFO; + my $strArchiveInfoFile = &ARCHIVE_INFO_FILE; + + # If either info file is not in the file list, then something exists in the directories so need to use force option + if (@stryFileListBackup && !grep(/^$strBackupInfoFile/i, @stryFileListBackup) + || @stryFileListArchive && !grep(/^$strArchiveInfoFile/i, @stryFileListArchive)) + { + confess &log(ERROR, + (@stryFileListBackup ? 'backup directory ' : '') . + ((@stryFileListBackup && @stryFileListArchive) ? 'and/or ' : '') . + (@stryFileListArchive ? 'archive directory ' : '') . + $strStanzaCreateErrorMsg, ERROR_PATH_NOT_EMPTY); + } } - # If we get here, then the directories are either empty or force was used - # Create the archive.info file + # Create the archive.info file and local variables my ($iResult, $strResultMessage) = $self->infoFileCreate((new pgBackRest::ArchiveInfo($strParentPathArchive, false)), $oFile, - PATH_BACKUP_ARCHIVE, (!@stryFileListArchive ? true : false)); + PATH_BACKUP_ARCHIVE, $strParentPathArchive, \@stryFileListArchive); if ($iResult == 0) { # Create the backup.info file ($iResult, $strResultMessage) = $self->infoFileCreate((new pgBackRest::BackupInfo($strParentPathBackup, false, false)), $oFile, - PATH_BACKUP_CLUSTER, (!@stryFileListBackup ? true : false)); + PATH_BACKUP_CLUSTER, $strParentPathBackup, \@stryFileListBackup); } if ($iResult != 0) @@ -204,7 +211,7 @@ sub parentPathGet return logDebugReturn ( $strOperation, - {name => 'strParentPath', value => $strParentPath, trace => true}, + {name => 'strParentPath', value => $strParentPath}, ); } @@ -224,55 +231,91 @@ sub infoFileCreate $oInfo, $oFile, $strPathType, - $bDirEmpty, + $strParentPath, + $stryFileList, ) = logDebugParam ( __PACKAGE__ . '->infoFileCreate', \@_, {name => 'oInfo', trace => true}, {name => 'oFile', trace => true}, - {name => 'strPathType', trace => true}, - {name => 'bDirEmpty', trace => true}, + {name => 'strPathType'}, + {name => 'strParentPath'}, + {name => 'stryFileList'}, ); my $iResult = 0; my $strResultMessage = undef; my $strWarningMsgArchive = undef; + my $bSave = true; - # If force option used, reconstruct the files regardless of whether they exist or not - if (optionGet(OPTION_FORCE)) + # Turn off console logging to control when to display the error + logLevelSet(undef, OFF); + + eval { - # Turn off console logging to control when to display the error - logLevelSet(undef, OFF); + # ??? File init will need to be addressed with stanza-upgrade since there could then be more than one DB and db-id + # so the DB section, esp for backup.info, cannot be initialized before we attempt to reconstruct the file from the + # directories since the history id would be wrong. Also need to handle if the reconstruction fails - if any file in + # the backup directory or archive directory are missing or mal-formed, then currently an error will be thrown, which + # may not be desireable. - eval + # If the info file does not exist, initialize it internally but do not save until complete reconstruction + if (!$oInfo->{bExists}) { - # ??? File init will need to be addressed with stanza-upgrade since there could then be more than one DB and db-id - # so the DB section, esp for backup.info, cannot be initialized before we attempt to reconstruct the file from the - # directories since the history id would be wrong. Also need to handle if the reconstruction fails - if any file in - # the backup directory or archive directory are missing or mal-formed, then currently an error will be thrown, which - # may not be desireable. + ($strPathType eq PATH_BACKUP_CLUSTER) + ? $oInfo->create($self->{oDb}{strDbVersion}, $self->{oDb}{ullDbSysId}, $self->{oDb}{iControlVersion}, + $self->{oDb}{iCatalogVersion}, false) + : $oInfo->create($self->{oDb}{strDbVersion}, $self->{oDb}{ullDbSysId}, false); + } - # If the file backup.info file does not exist, initialize it internally but do not save until complete reconstruction - if (!$oInfo->{bExists}) - { - ($strPathType eq PATH_BACKUP_CLUSTER) - ? $oInfo->create($self->{oDb}{strDbVersion}, $self->{oDb}{ullDbSysId}, $self->{oDb}{iControlVersion}, - $self->{oDb}{iCatalogVersion}, false) - : $oInfo->create($self->{oDb}{strDbVersion}, $self->{oDb}{ullDbSysId}, false); - } + # Reconstruct the file from the data in the directory if there is any + if ($strPathType eq PATH_BACKUP_CLUSTER) + { + $oInfo->reconstruct(false, false); + } + # If this is the archive.info reconstruction then catch any warnings + else + { + $strWarningMsgArchive = $oInfo->reconstruct($oFile, $self->{oDb}{strDbVersion}, $self->{oDb}{ullDbSysId}); + } - # Reconstruct the file from the data in the directory - if ($strPathType eq PATH_BACKUP_CLUSTER) + # If the file exists on disk, then check if the reconstructed data is the same as what is on disk + if ($oInfo->{bExists}) + { + my $oInfoOnDisk = + ($strPathType eq PATH_BACKUP_CLUSTER ? new pgBackRest::BackupInfo($strParentPath) + : new pgBackRest::ArchiveInfo($strParentPath)); + + # If force was not used and the hashes are different then error + if ($oInfoOnDisk->hash() ne $oInfo->hash()) { - $oInfo->reconstruct(false, false); + if (!optionGet(OPTION_FORCE)) + { + $iResult = ERROR_FILE_INVALID; + $strResultMessage = + ($strPathType eq PATH_BACKUP_CLUSTER ? 'backup file ' : 'archive file ') . + ' invalid; to correct, use --force'; + } } - # If this is the archive.info reconstruction then catch any warnings + # If the hashes are the same, then don't save the file since it already exists and is valid else { - $strWarningMsgArchive = $oInfo->reconstruct($oFile, $self->{oDb}{strDbVersion}, $self->{oDb}{ullDbSysId}); + $bSave = false; } + } + # If force was not used and the info file does not exist and the directory is not empty, then error + # This should also be performed by the calling routine before this function is called, so this is just a safety check + if ($iResult == 0 && !optionGet(OPTION_FORCE) && !$oInfo->{bExists} && @$stryFileList) + { + $iResult = ERROR_PATH_NOT_EMPTY; + $strResultMessage = + ($strPathType eq PATH_BACKUP_CLUSTER ? 'backup directory ' : 'archive directory ') . $strStanzaCreateErrorMsg; + } + + if ($iResult == 0) + { # ??? With stanza-upgrade we will want ability to force the DB section to match but for now, if it doesn't match, # then something is wrong. ($strPathType eq PATH_BACKUP_CLUSTER) @@ -281,7 +324,10 @@ sub infoFileCreate : $oInfo->check($self->{oDb}{strDbVersion}, $self->{oDb}{ullDbSysId}, false); # Save the reconstructed file - $oInfo->save(); + if ($bSave) + { + $oInfo->save(); + } # Sync path if requested if (optionGet(OPTION_REPO_SYNC)) @@ -290,38 +336,19 @@ sub infoFileCreate PATH_BACKUP_ABSOLUTE, defined($oInfo->{strArchiveClusterPath}) ? $oInfo->{strArchiveClusterPath} : $oInfo->{strBackupClusterPath}); } - - return true; } - or do - { - # Confess unhandled errors - confess $EVAL_ERROR if (!isException($EVAL_ERROR)); - # If this is a backrest error then capture the last code and message - $iResult = $EVAL_ERROR->code(); - $strResultMessage = $EVAL_ERROR->message(); - }; - - # Reset the console logging - logLevelSet(undef, optionGet(OPTION_LOG_LEVEL_CONSOLE)); - } - else - { # If the cluster repo path is empty then create it - if ($bDirEmpty) - { - # Create and save the info file - $oInfo->create( - $self->{oDb}{strDbVersion}, $self->{oDb}{ullDbSysId}, $self->{oDb}{iControlVersion}, $self->{oDb}{iCatalogVersion}); - } - # Else somehow we got here when we should not have so error - else - { - $iResult = ERROR_PATH_NOT_EMPTY; - $strResultMessage = - ($strPathType eq PATH_BACKUP_CLUSTER ? 'backup directory ' : 'archive directory ') . $strStanzaCreateErrorMsg; - } + return true; } + or do + { + # Confess unhandled errors + confess $EVAL_ERROR if (!isException($EVAL_ERROR)); + + # If this is a backrest error then capture the last code and message + $iResult = $EVAL_ERROR->code(); + $strResultMessage = $EVAL_ERROR->message(); + }; # Reset the console logging logLevelSet(undef, optionGet(OPTION_LOG_LEVEL_CONSOLE)); @@ -336,8 +363,8 @@ sub infoFileCreate return logDebugReturn ( $strOperation, - {name => 'iResult', value => $iResult, trace => true}, - {name => 'strResultMessage', value => $strResultMessage, trace => true} + {name => 'iResult', value => $iResult}, + {name => 'strResultMessage', value => $strResultMessage}, ); } diff --git a/test/expect/backup-stanza-create-001.log b/test/expect/backup-stanza-create-001.log index 0276795ab..7c8c92c4c 100644 --- a/test/expect/backup-stanza-create-001.log +++ b/test/expect/backup-stanza-create-001.log @@ -46,6 +46,44 @@ db-version="9.4" [db:history] 1={"db-id":6353949018581704918,"db-version":"9.4"} +stanza-create db - successful rerun of stanza-create (db-master host) +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db --log-level-console=detail --no-online stanza-create +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: stanza-create command begin [BACKREST-VERSION]: --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --lock-path=[TEST_PATH]/db-master/repo/lock --log-level-console=detail --log-level-file=trace --log-path=[TEST_PATH]/db-master/repo/log --no-online --repo-path=[TEST_PATH]/db-master/repo --stanza=db +P00 INFO: stanza-create command end: completed successfully + ++ supplemental file: [TEST_PATH]/db-master/repo/backup/db/backup.info +--------------------------------------------------------------------- +[backrest] +backrest-checksum="[CHECKSUM]" +backrest-format=5 +backrest-version="[VERSION-1]" + +[db] +db-catalog-version=201409291 +db-control-version=942 +db-id=1 +db-system-id=6353949018581704918 +db-version="9.4" + +[db:history] +1={"db-catalog-version":201409291,"db-control-version":942,"db-system-id":6353949018581704918,"db-version":"9.4"} + ++ supplemental file: [TEST_PATH]/db-master/repo/archive/db/archive.info +----------------------------------------------------------------------- +[backrest] +backrest-checksum="[CHECKSUM]" +backrest-format=5 +backrest-version="[VERSION-1]" + +[db] +db-id=1 +db-system-id=6353949018581704918 +db-version="9.4" + +[db:history] +1={"db-id":6353949018581704918,"db-version":"9.4"} + > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --no-fork --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001 ------------------------------------------------------------------------------------------------------------------------------------ P00 INFO: archive-push command begin [BACKREST-VERSION]: --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --no-fork --lock-path=[TEST_PATH]/db-master/repo/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/repo/log --repo-path=[TEST_PATH]/db-master/repo --stanza=db @@ -78,7 +116,7 @@ stanza-create db - fail on archive info file missing from non-empty dir (db-mast > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db --log-level-console=detail --no-online stanza-create ------------------------------------------------------------------------------------------------------------------------------------ P00 INFO: stanza-create command begin [BACKREST-VERSION]: --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --lock-path=[TEST_PATH]/db-master/repo/lock --log-level-console=detail --log-level-file=trace --log-path=[TEST_PATH]/db-master/repo/log --no-online --repo-path=[TEST_PATH]/db-master/repo --stanza=db -P00 ERROR: [115]: backup directory and archive directory not empty, stanza-create has already been attempted +P00 ERROR: [115]: backup directory and/or archive directory not empty HINT: Use --force to force the stanza data to be created. P00 INFO: stanza-create command end: aborted with exception [115] @@ -162,7 +200,106 @@ db-version="9.4" [db:history] 1={"db-id":6353949018581704918,"db-version":"9.4"} -stanza-create db - repeat force (db-master host) +stanza-create db - repeat create (db-master host) +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db --log-level-console=detail --no-online stanza-create +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: stanza-create command begin [BACKREST-VERSION]: --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --lock-path=[TEST_PATH]/db-master/repo/lock --log-level-console=detail --log-level-file=trace --log-path=[TEST_PATH]/db-master/repo/log --no-online --repo-path=[TEST_PATH]/db-master/repo --stanza=db +P00 INFO: stanza-create command end: completed successfully + ++ supplemental file: [TEST_PATH]/db-master/repo/backup/db/backup.info +--------------------------------------------------------------------- +[backrest] +backrest-checksum="[CHECKSUM]" +backrest-format=5 +backrest-version="[VERSION-1]" + +[db] +db-catalog-version=201409291 +db-control-version=942 +db-id=1 +db-system-id=6353949018581704918 +db-version="9.4" + +[db:history] +1={"db-catalog-version":201409291,"db-control-version":942,"db-system-id":6353949018581704918,"db-version":"9.4"} + ++ supplemental file: [TEST_PATH]/db-master/repo/archive/db/archive.info +----------------------------------------------------------------------- +[backrest] +backrest-checksum="[CHECKSUM]" +backrest-format=5 +backrest-version="[VERSION-1]" + +[db] +db-id=1 +db-system-id=6353949018581704918 +db-version="9.4" + +[db:history] +1={"db-id":6353949018581704918,"db-version":"9.4"} + +stanza-create db - force not needed when backup dir empty, archive.info exists but backup.info is missing (db-master host) +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db --log-level-console=detail --no-online stanza-create +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: stanza-create command begin [BACKREST-VERSION]: --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --lock-path=[TEST_PATH]/db-master/repo/lock --log-level-console=detail --log-level-file=trace --log-path=[TEST_PATH]/db-master/repo/log --no-online --repo-path=[TEST_PATH]/db-master/repo --stanza=db +P00 INFO: stanza-create command end: completed successfully + ++ supplemental file: [TEST_PATH]/db-master/repo/backup/db/backup.info +--------------------------------------------------------------------- +[backrest] +backrest-checksum="[CHECKSUM]" +backrest-format=5 +backrest-version="[VERSION-1]" + +[db] +db-catalog-version=201409291 +db-control-version=942 +db-id=1 +db-system-id=6353949018581704918 +db-version="9.4" + +[db:history] +1={"db-catalog-version":201409291,"db-control-version":942,"db-system-id":6353949018581704918,"db-version":"9.4"} + ++ supplemental file: [TEST_PATH]/db-master/repo/archive/db/archive.info +----------------------------------------------------------------------- +[backrest] +backrest-checksum="[CHECKSUM]" +backrest-format=5 +backrest-version="[VERSION-1]" + +[db] +db-id=1 +db-system-id=6353949018581704918 +db-version="9.4" + +[db:history] +1={"db-id":6353949018581704918,"db-version":"9.4"} + +stanza-create db - hash check fails requiring force (db-master host) +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db --log-level-console=detail --no-online stanza-create +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: stanza-create command begin [BACKREST-VERSION]: --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --lock-path=[TEST_PATH]/db-master/repo/lock --log-level-console=detail --log-level-file=trace --log-path=[TEST_PATH]/db-master/repo/log --no-online --repo-path=[TEST_PATH]/db-master/repo --stanza=db +P00 WARN: unable to create stanza 'db' +P00 ERROR: [103]: archive file invalid; to correct, use --force +P00 INFO: stanza-create command end: aborted with exception [103] + ++ supplemental file: [TEST_PATH]/db-master/repo/archive/db/archive.info +----------------------------------------------------------------------- +[backrest] +backrest-checksum="[CHECKSUM]" +backrest-format=5 +backrest-version="[VERSION-1]" + +[db] +db-id=1 +db-system-id=6353949018581704918 +db-version="8.0" + +[db:history] +1={"db-id":6353949018581704918,"db-version":"9.4"} + +stanza-create db - use force to overwrite the invalid file (db-master host) > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db --log-level-console=detail --no-online --force stanza-create ------------------------------------------------------------------------------------------------------------------------------------ P00 INFO: stanza-create command begin [BACKREST-VERSION]: --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --force --lock-path=[TEST_PATH]/db-master/repo/lock --log-level-console=detail --log-level-file=trace --log-path=[TEST_PATH]/db-master/repo/log --no-online --repo-path=[TEST_PATH]/db-master/repo --stanza=db diff --git a/test/expect/backup-stanza-create-002.log b/test/expect/backup-stanza-create-002.log index 998bad419..76e1ec37f 100644 --- a/test/expect/backup-stanza-create-002.log +++ b/test/expect/backup-stanza-create-002.log @@ -46,6 +46,44 @@ db-version="9.4" [db:history] 1={"db-id":6353949018581704918,"db-version":"9.4"} +stanza-create db - successful rerun of stanza-create (backup host) +> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db --log-level-console=detail --no-online stanza-create +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: stanza-create command begin [BACKREST-VERSION]: --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-path=[TEST_PATH]/db-master/db/base --db-user=[USER-1] --lock-path=[TEST_PATH]/backup/repo/lock --log-level-console=detail --log-level-file=trace --log-path=[TEST_PATH]/backup/repo/log --no-online --repo-path=[TEST_PATH]/backup/repo --stanza=db +P00 INFO: stanza-create command end: completed successfully + ++ supplemental file: [TEST_PATH]/backup/repo/backup/db/backup.info +------------------------------------------------------------------ +[backrest] +backrest-checksum="[CHECKSUM]" +backrest-format=5 +backrest-version="[VERSION-1]" + +[db] +db-catalog-version=201409291 +db-control-version=942 +db-id=1 +db-system-id=6353949018581704918 +db-version="9.4" + +[db:history] +1={"db-catalog-version":201409291,"db-control-version":942,"db-system-id":6353949018581704918,"db-version":"9.4"} + ++ supplemental file: [TEST_PATH]/backup/repo/archive/db/archive.info +-------------------------------------------------------------------- +[backrest] +backrest-checksum="[CHECKSUM]" +backrest-format=5 +backrest-version="[VERSION-1]" + +[db] +db-id=1 +db-system-id=6353949018581704918 +db-version="9.4" + +[db:history] +1={"db-id":6353949018581704918,"db-version":"9.4"} + > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --no-fork --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001 ------------------------------------------------------------------------------------------------------------------------------------ P00 INFO: archive-push command begin [BACKREST-VERSION]: --backup-cmd=[BACKREST-BIN] --backup-config=[TEST_PATH]/backup/pgbackrest.conf --backup-host=backup --backup-user=[USER-2] --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base --no-fork --lock-path=[TEST_PATH]/db-master/spool/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/spool/log --repo-path=[TEST_PATH]/backup/repo --stanza=db @@ -74,7 +112,7 @@ stanza-create db - fail on archive info file missing from non-empty dir (backup > [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db --log-level-console=detail --no-online stanza-create ------------------------------------------------------------------------------------------------------------------------------------ P00 INFO: stanza-create command begin [BACKREST-VERSION]: --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-path=[TEST_PATH]/db-master/db/base --db-user=[USER-1] --lock-path=[TEST_PATH]/backup/repo/lock --log-level-console=detail --log-level-file=trace --log-path=[TEST_PATH]/backup/repo/log --no-online --repo-path=[TEST_PATH]/backup/repo --stanza=db -P00 ERROR: [115]: backup directory and archive directory not empty, stanza-create has already been attempted +P00 ERROR: [115]: backup directory and/or archive directory not empty HINT: Use --force to force the stanza data to be created. P00 INFO: stanza-create command end: aborted with exception [115] @@ -158,7 +196,106 @@ db-version="9.4" [db:history] 1={"db-id":6353949018581704918,"db-version":"9.4"} -stanza-create db - repeat force (backup host) +stanza-create db - repeat create (backup host) +> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db --log-level-console=detail --no-online stanza-create +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: stanza-create command begin [BACKREST-VERSION]: --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-path=[TEST_PATH]/db-master/db/base --db-user=[USER-1] --lock-path=[TEST_PATH]/backup/repo/lock --log-level-console=detail --log-level-file=trace --log-path=[TEST_PATH]/backup/repo/log --no-online --repo-path=[TEST_PATH]/backup/repo --stanza=db +P00 INFO: stanza-create command end: completed successfully + ++ supplemental file: [TEST_PATH]/backup/repo/backup/db/backup.info +------------------------------------------------------------------ +[backrest] +backrest-checksum="[CHECKSUM]" +backrest-format=5 +backrest-version="[VERSION-1]" + +[db] +db-catalog-version=201409291 +db-control-version=942 +db-id=1 +db-system-id=6353949018581704918 +db-version="9.4" + +[db:history] +1={"db-catalog-version":201409291,"db-control-version":942,"db-system-id":6353949018581704918,"db-version":"9.4"} + ++ supplemental file: [TEST_PATH]/backup/repo/archive/db/archive.info +-------------------------------------------------------------------- +[backrest] +backrest-checksum="[CHECKSUM]" +backrest-format=5 +backrest-version="[VERSION-1]" + +[db] +db-id=1 +db-system-id=6353949018581704918 +db-version="9.4" + +[db:history] +1={"db-id":6353949018581704918,"db-version":"9.4"} + +stanza-create db - force not needed when backup dir empty, archive.info exists but backup.info is missing (backup host) +> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db --log-level-console=detail --no-online stanza-create +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: stanza-create command begin [BACKREST-VERSION]: --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-path=[TEST_PATH]/db-master/db/base --db-user=[USER-1] --lock-path=[TEST_PATH]/backup/repo/lock --log-level-console=detail --log-level-file=trace --log-path=[TEST_PATH]/backup/repo/log --no-online --repo-path=[TEST_PATH]/backup/repo --stanza=db +P00 INFO: stanza-create command end: completed successfully + ++ supplemental file: [TEST_PATH]/backup/repo/backup/db/backup.info +------------------------------------------------------------------ +[backrest] +backrest-checksum="[CHECKSUM]" +backrest-format=5 +backrest-version="[VERSION-1]" + +[db] +db-catalog-version=201409291 +db-control-version=942 +db-id=1 +db-system-id=6353949018581704918 +db-version="9.4" + +[db:history] +1={"db-catalog-version":201409291,"db-control-version":942,"db-system-id":6353949018581704918,"db-version":"9.4"} + ++ supplemental file: [TEST_PATH]/backup/repo/archive/db/archive.info +-------------------------------------------------------------------- +[backrest] +backrest-checksum="[CHECKSUM]" +backrest-format=5 +backrest-version="[VERSION-1]" + +[db] +db-id=1 +db-system-id=6353949018581704918 +db-version="9.4" + +[db:history] +1={"db-id":6353949018581704918,"db-version":"9.4"} + +stanza-create db - hash check fails requiring force (backup host) +> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db --log-level-console=detail --no-online stanza-create +------------------------------------------------------------------------------------------------------------------------------------ +P00 INFO: stanza-create command begin [BACKREST-VERSION]: --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-path=[TEST_PATH]/db-master/db/base --db-user=[USER-1] --lock-path=[TEST_PATH]/backup/repo/lock --log-level-console=detail --log-level-file=trace --log-path=[TEST_PATH]/backup/repo/log --no-online --repo-path=[TEST_PATH]/backup/repo --stanza=db +P00 WARN: unable to create stanza 'db' +P00 ERROR: [103]: archive file invalid; to correct, use --force +P00 INFO: stanza-create command end: aborted with exception [103] + ++ supplemental file: [TEST_PATH]/backup/repo/archive/db/archive.info +-------------------------------------------------------------------- +[backrest] +backrest-checksum="[CHECKSUM]" +backrest-format=5 +backrest-version="[VERSION-1]" + +[db] +db-id=1 +db-system-id=6353949018581704918 +db-version="8.0" + +[db:history] +1={"db-id":6353949018581704918,"db-version":"9.4"} + +stanza-create db - use force to overwrite the invalid file (backup host) > [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db --log-level-console=detail --no-online --force stanza-create ------------------------------------------------------------------------------------------------------------------------------------ P00 INFO: stanza-create command begin [BACKREST-VERSION]: --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-path=[TEST_PATH]/db-master/db/base --db-user=[USER-1] --force --lock-path=[TEST_PATH]/backup/repo/lock --log-level-console=detail --log-level-file=trace --log-path=[TEST_PATH]/backup/repo/log --no-online --repo-path=[TEST_PATH]/backup/repo --stanza=db diff --git a/test/expect/backup-synthetic-001.log b/test/expect/backup-synthetic-001.log index 6ef0d1a5c..2538408b3 100644 --- a/test/expect/backup-synthetic-001.log +++ b/test/expect/backup-synthetic-001.log @@ -2729,7 +2729,7 @@ stanza-create db - fail on backup directory not empty and missing backup.info (d > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db --log-level-console=detail --no-online stanza-create ------------------------------------------------------------------------------------------------------------------------------------ P00 INFO: stanza-create command begin [BACKREST-VERSION]: --config=[TEST_PATH]/db-master/pgbackrest.conf --db-path=[TEST_PATH]/db-master/db/base-2 --lock-path=[TEST_PATH]/db-master/repo/lock --log-level-console=detail --log-level-file=trace --log-path=[TEST_PATH]/db-master/repo/log --no-online --repo-path=[TEST_PATH]/db-master/repo --stanza=db -P00 ERROR: [115]: backup directory and archive directory not empty, stanza-create has already been attempted +P00 ERROR: [115]: backup directory and/or archive directory not empty HINT: Use --force to force the stanza data to be created. P00 INFO: stanza-create command end: aborted with exception [115] diff --git a/test/expect/backup-synthetic-005.log b/test/expect/backup-synthetic-005.log index dbb2002d6..eb1bdb78b 100644 --- a/test/expect/backup-synthetic-005.log +++ b/test/expect/backup-synthetic-005.log @@ -2691,7 +2691,7 @@ stanza-create db - fail on backup directory not empty and missing backup.info (b > [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db --log-level-console=detail --no-online stanza-create ------------------------------------------------------------------------------------------------------------------------------------ P00 INFO: stanza-create command begin [BACKREST-VERSION]: --config=[TEST_PATH]/backup/pgbackrest.conf --db-cmd=[BACKREST-BIN] --db-config=[TEST_PATH]/db-master/pgbackrest.conf --db-host=db-master --db-path=[TEST_PATH]/db-master/db/base-2 --db-user=[USER-1] --lock-path=[TEST_PATH]/backup/repo/lock --log-level-console=detail --log-level-file=trace --log-path=[TEST_PATH]/backup/repo/log --no-online --repo-path=[TEST_PATH]/backup/repo --stanza=db -P00 ERROR: [115]: backup directory and archive directory not empty, stanza-create has already been attempted +P00 ERROR: [115]: backup directory and/or archive directory not empty HINT: Use --force to force the stanza data to be created. P00 INFO: stanza-create command end: aborted with exception [115] diff --git a/test/lib/pgBackRestTest/Backup/BackupStanzaCreateTest.pm b/test/lib/pgBackRestTest/Backup/BackupStanzaCreateTest.pm index a79c567ce..e7e016f49 100644 --- a/test/lib/pgBackRestTest/Backup/BackupStanzaCreateTest.pm +++ b/test/lib/pgBackRestTest/Backup/BackupStanzaCreateTest.pm @@ -60,6 +60,9 @@ sub run $oHostBackup->stanzaCreate('successfully create the stanza', {strOptionalParam => '--no-' . OPTION_ONLINE}); + # Rerun stanza-create and confirm success without the need to use force on empty directories + $oHostBackup->stanzaCreate('successful rerun of stanza-create', {strOptionalParam => '--no-' . OPTION_ONLINE}); + # Create the xlog path my $strXlogPath = $oHostDbMaster->dbBasePath() . '/pg_xlog'; filePathCreate($strXlogPath, undef, false, true); @@ -90,8 +93,30 @@ sub run $oHostBackup->stanzaCreate('force create archive.info from gz file', {strOptionalParam => '--no-' . OPTION_ONLINE . ' --' . OPTION_FORCE}); - # Run force again to ensure the format is still valid - $oHostBackup->stanzaCreate('repeat force', {strOptionalParam => '--no-' . OPTION_ONLINE . ' --' . OPTION_FORCE}); + # Rerun without the force to ensure the format is still valid - this will hash check the info files and indicate the + # stanza already exists + $oHostBackup->stanzaCreate('repeat create', {strOptionalParam => '--no-' . OPTION_ONLINE}); + + # Remove the backup info file and confirm success with backup dir empty + # Backup Full tests will confirm failure when backup dir not empty + $oHostBackup->executeSimple('rm ' . $oFile->pathGet(PATH_BACKUP_CLUSTER, FILE_BACKUP_INFO)); + $oHostBackup->stanzaCreate('force not needed when backup dir empty, archive.info exists but backup.info is missing', + {strOptionalParam => '--no-' . OPTION_ONLINE}); + + # Remove the backup.info file then munge and save the archive info file + $oHostBackup->executeSimple('rm ' . $oFile->pathGet(PATH_BACKUP_CLUSTER, FILE_BACKUP_INFO)); + $oHostBackup->infoMunge( + $oFile->pathGet(PATH_BACKUP_ARCHIVE, ARCHIVE_INFO_FILE), + {&INFO_BACKUP_SECTION_DB => {&INFO_BACKUP_KEY_DB_VERSION => '8.0'}}); + + $oHostBackup->stanzaCreate('hash check fails requiring force', + {iExpectedExitStatus => ERROR_FILE_INVALID, strOptionalParam => '--no-' . OPTION_ONLINE}); + + $oHostBackup->stanzaCreate('use force to overwrite the invalid file', + {strOptionalParam => '--no-' . OPTION_ONLINE . ' --' . OPTION_FORCE}); + + # Cleanup the global hash but don't save the file (permission issues may prevent it anyway) + $oHostBackup->infoRestore($oFile->pathGet(PATH_BACKUP_ARCHIVE, ARCHIVE_INFO_FILE), false); # Change the database version by copying a new pg_control file executeTest('sudo rm ' . $oHostDbMaster->dbBasePath() . '/' . DB_FILE_PGCONTROL); diff --git a/test/lib/pgBackRestTest/Common/Host/HostBackupTest.pm b/test/lib/pgBackRestTest/Common/Host/HostBackupTest.pm index 88b457981..cb4c20deb 100644 --- a/test/lib/pgBackRestTest/Common/Host/HostBackupTest.pm +++ b/test/lib/pgBackRestTest/Common/Host/HostBackupTest.pm @@ -1109,18 +1109,23 @@ sub infoRestore my ( $strOperation, - $strFileName + $strFileName, + $bSave, ) = logDebugParam ( __PACKAGE__ . '->infoRestore', \@_, - {name => 'strFileName'} + {name => 'strFileName'}, + {name => 'bSave', default => true, required => false}, ); # If the original file content exists in the global hash, then save it to the file if (defined($self->{hInfoFile}{$strFileName})) { - iniSave($strFileName, $self->{hInfoFile}{$strFileName}); + if ($bSave) + { + iniSave($strFileName, $self->{hInfoFile}{$strFileName}); + } } else {