diff --git a/doc/xml/release.xml b/doc/xml/release.xml index cf90aa3bd..86ee75f03 100644 --- a/doc/xml/release.xml +++ b/doc/xml/release.xml @@ -29,6 +29,14 @@ + + + + + +

Improve check command to verify that the backup manifest can be built.

+ +

The C library is now required. This eliminates conditional loading and eases development of new library features.

 diff --git a/doc/xml/user-guide.xml b/doc/xml/user-guide.xml index badfcc760..dbc67b34e 100644 --- a/doc/xml/user-guide.xml +++ b/doc/xml/user-guide.xml @@ -2496,7 +2496,7 @@ -

Test configuration using the check command. The warning on the backup host regarding the standby being down is expected and can be ignored.

+

Test configuration using the check command.

Check configuration @@ -2510,14 +2510,6 @@ - - Check configuration - - - {[project-exe]} {[dash]}-stanza={[postgres-cluster-demo]} check - - -

Remove the old cluster.

@@ -2532,16 +2524,6 @@ -

Run a full backup on the new cluster and then restore the standby from the backup. The backup type will automatically be changed to full if incr or diff is requested.

- - - Run a full backup - - - {[project-exe]} {[dash]}-stanza={[postgres-cluster-demo]} {[dash]}-type=full backup - - -

Install the new binaries on the standby and create the cluster.

@@ -2566,6 +2548,26 @@ +

Run the check on the backup host. The warning regarding the standby being down is expected since the standby cluster is down. Running this command demostrates that the backup server is aware of the standby and is confirgured properly for the primary server.

+ + + Check configuration + + + {[project-exe]} {[dash]}-stanza={[postgres-cluster-demo]} check + + + +

Run a full backup on the new cluster and then restore the standby from the backup. The backup type will automatically be changed to full if incr or diff is requested.

+ + + Run a full backup + + + {[project-exe]} {[dash]}-stanza={[postgres-cluster-demo]} {[dash]}-type=full backup + + + Restore the {[postgres-cluster-demo]} standby cluster @@ -2581,7 +2583,7 @@ - Start <postgres/> + Start <postgres/> and check the <backrest/> configuration {[db-cluster-start-upgrade]} @@ -2590,6 +2592,10 @@ {[db-cluster-wait]} + + + {[project-exe]} {[dash]}-stanza={[postgres-cluster-demo]} check +

Backup from standby can be enabled now that the standby is restored.

diff --git a/lib/pgBackRest/Check/Check.pm b/lib/pgBackRest/Check/Check.pm index 42d7d1575..1d47d6ee6 100644 --- a/lib/pgBackRest/Check/Check.pm +++ b/lib/pgBackRest/Check/Check.pm @@ -55,17 +55,9 @@ sub process # Assign function parameters, defaults, and log debug info my $strOperation = logDebugParam(__PACKAGE__ . '->process'); - # Initialize the database object. This will also check the configured replicas and throw an error if at least one is not - # able to be connected to and warnings for any that cannot be properly connected to. - my ($oDb) = dbObjectGet(); - - # Validate the database configuration - $oDb->configValidate(); - - # Get the timeout and error message to display - if it is 0 we are testing + # Initialize the local variables my $iArchiveTimeout = cfgOption(CFGOPT_ARCHIVE_TIMEOUT); - # Initialize the result variables my $iResult = 0; my $strResultMessage = undef; @@ -73,63 +65,117 @@ sub process my $strArchiveFile = undef; my $strWalSegment = undef; + # Get the master database object to test to see if the manifest can be built + my ($oDb) = dbMasterGet(); + + # Get the databse version to pass to the manifest constructor and the system-id in the event of a failure + my ($strDbVersion, $iControlVersion, $iCatalogVersion, $ullDbSysId) = $oDb->info(); + # Turn off console logging to control when to display the error logLevelSet(undef, OFF); - # Check backup.info - if the archive check fails below (e.g --no-archive-check set) then at least know backup.info succeeded - eval + # Loop through all defined databases and attempt to build a manifest + for (my $iRemoteIdx = 1; $iRemoteIdx <= cfgOptionIndexTotal(CFGOPT_DB_HOST); $iRemoteIdx++) { - # Check that the backup info file is written and is valid for the current database of the stanza - $self->backupInfoCheck(); - return true; - } - # If there is an unhandled error then confess - or do - { - # Capture error information - $iResult = exceptionCode($EVAL_ERROR); - $strResultMessage = exceptionMessage($EVAL_ERROR); - }; - - # Check archive.info - if ($iResult == 0) - { - eval + # Make sure a db is defined for this index + if (cfgOptionTest(cfgOptionIdFromIndex(CFGOPT_DB_PATH, $iRemoteIdx)) || + cfgOptionTest(cfgOptionIdFromIndex(CFGOPT_DB_HOST, $iRemoteIdx))) { - # Check that the archive info file is written and is valid for the current database of the stanza - ($strArchiveId) = new pgBackRest::Archive::Get::Get()->getCheck(); - return true; + eval + { + # Passing file location dev/null so that the save will fail if it is ever attempted. Pass a miscellaneus value for + # encryption key since the file will not be saved. + my $oBackupManifest = new pgBackRest::Manifest("/dev/null/manifest.chk", + {bLoad => false, strDbVersion => $strDbVersion, + strCipherPass => 'x', + strCipherPassSub => 'x'}); + + $oBackupManifest->build(storageDb({iRemoteIdx => $iRemoteIdx}), + cfgOption(cfgOptionIdFromIndex(CFGOPT_DB_PATH, $iRemoteIdx)), undef, cfgOption(CFGOPT_ONLINE)); + + return true; + } + or do + { + # Capture error information + $strResultMessage = "Database: ${strDbVersion} ${ullDbSysId} " . exceptionMessage($EVAL_ERROR) . + (($iResult != 0) ? "\n[$iResult] : $strResultMessage" : ""); + $iResult = exceptionCode($EVAL_ERROR); + }; } - or do - { - # Capture error information - $iResult = exceptionCode($EVAL_ERROR); - $strResultMessage = exceptionMessage($EVAL_ERROR); - }; - } - - # If able to get the archive id then force archiving and check the arrival of the archived WAL file with the time specified - if ($iResult == 0 && !$oDb->isStandby()) - { - $strWalSegment = $oDb->walSwitch(); - - eval - { - $strArchiveFile = walSegmentFind(storageRepo(), $strArchiveId, $strWalSegment, $iArchiveTimeout); - return true; - } - # If this is a backrest error then capture the code and message else confess - or do - { - # Capture error information - $iResult = exceptionCode($EVAL_ERROR); - $strResultMessage = exceptionMessage($EVAL_ERROR); - }; } # Reset the console logging logLevelSet(undef, cfgOption(CFGOPT_LOG_LEVEL_CONSOLE)); + # If the manifest builds are ok, then proceed with the other checks + if ($iResult == 0) + { + # Reinitialize the database object in order to check the configured replicas. This will throw an error if at least one is not + # able to be connected to and warnings for any that cannot be properly connected to. + ($oDb) = dbObjectGet(); + + # Validate the database configuration + $oDb->configValidate(); + + # Turn off console logging to control when to display the error + logLevelSet(undef, OFF); + + # Check backup.info - if the archive check fails below (e.g --no-archive-check set) then at least know backup.info succeeded + eval + { + # Check that the backup info file is written and is valid for the current database of the stanza + $self->backupInfoCheck(); + return true; + } + # If there is an unhandled error then confess + or do + { + # Capture error information + $iResult = exceptionCode($EVAL_ERROR); + $strResultMessage = exceptionMessage($EVAL_ERROR); + }; + + # Check archive.info + if ($iResult == 0) + { + eval + { + # Check that the archive info file is written and is valid for the current database of the stanza + ($strArchiveId) = new pgBackRest::Archive::Get::Get()->getCheck(); + return true; + } + or do + { + # Capture error information + $iResult = exceptionCode($EVAL_ERROR); + $strResultMessage = exceptionMessage($EVAL_ERROR); + }; + } + + # If able to get the archive id then force archiving and check the arrival of the archived WAL file with the time specified + if ($iResult == 0 && !$oDb->isStandby()) + { + $strWalSegment = $oDb->walSwitch(); + + eval + { + $strArchiveFile = walSegmentFind(storageRepo(), $strArchiveId, $strWalSegment, $iArchiveTimeout); + return true; + } + # If this is a backrest error then capture the code and message else confess + or do + { + # Capture error information + $iResult = exceptionCode($EVAL_ERROR); + $strResultMessage = exceptionMessage($EVAL_ERROR); + }; + } + + # Reset the console logging + logLevelSet(undef, cfgOption(CFGOPT_LOG_LEVEL_CONSOLE)); + } + # If the archiving was successful and backup.info check did not error in an unexpected way, then indicate success # Else, log the error. if ($iResult == 0) @@ -147,10 +193,11 @@ sub process } else { + # Throw the captured error &log(ERROR, $strResultMessage, $iResult); # If a WAL switch was attempted, then alert the user that the WAL that did not reach the archive - if (defined($strWalSegment)) + if (defined($strWalSegment) && !defined($strArchiveFile)) { &log(WARN, "WAL segment ${strWalSegment} did not reach the archive:" . (defined($strArchiveId) ? $strArchiveId : '') . "\n" . diff --git a/test/expect/real-all-001.log b/test/expect/real-all-001.log index c12f4def4..ac4f40b4b 100644 --- a/test/expect/real-all-001.log +++ b/test/expect/real-all-001.log @@ -49,6 +49,10 @@ check db - fail on backup info mismatch (db-master host) > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --archive-timeout=5 --stanza=db check ------------------------------------------------------------------------------------------------------------------------------------ +check db - confirm master manifest->build executed (db-master host) +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --archive-timeout=5 --stanza=db check +------------------------------------------------------------------------------------------------------------------------------------ + check db - verify success after backup (db-master host) > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --archive-timeout=5 --stanza=db check ------------------------------------------------------------------------------------------------------------------------------------ diff --git a/test/expect/real-all-002.log b/test/expect/real-all-002.log index 1c12acb87..8002b50d9 100644 --- a/test/expect/real-all-002.log +++ b/test/expect/real-all-002.log @@ -144,6 +144,10 @@ log-path=[TEST_PATH]/db-standby/log protocol-timeout=60 spool-path=[TEST_PATH]/db-standby/spool +check db - confirm standby manifest->build executed (db-standby host) +> [CONTAINER-EXEC] db-standby [BACKREST-BIN] --config=[TEST_PATH]/db-standby/pgbackrest.conf --log-level-console=detail --archive-timeout=5 --stanza=db check +------------------------------------------------------------------------------------------------------------------------------------ + check db - verify check command on standby (db-standby host) > [CONTAINER-EXEC] db-standby [BACKREST-BIN] --config=[TEST_PATH]/db-standby/pgbackrest.conf --log-level-console=detail --stanza=db check ------------------------------------------------------------------------------------------------------------------------------------ diff --git a/test/expect/real-all-003.log b/test/expect/real-all-003.log index fbccca3b7..bbc7a2aa2 100644 --- a/test/expect/real-all-003.log +++ b/test/expect/real-all-003.log @@ -148,8 +148,8 @@ spool-path=[TEST_PATH]/db-standby/spool archive-copy=y start-fast=y -check db - verify check command on standby (db-standby host) -> [CONTAINER-EXEC] db-standby [BACKREST-BIN] --config=[TEST_PATH]/db-standby/pgbackrest.conf --log-level-console=detail --stanza=db check +check db - confirm standby manifest->build executed (db-standby host) +> [CONTAINER-EXEC] db-standby [BACKREST-BIN] --config=[TEST_PATH]/db-standby/pgbackrest.conf --log-level-console=detail --archive-timeout=5 --stanza=db check ------------------------------------------------------------------------------------------------------------------------------------ incr backup - update during backup (db-standby host) diff --git a/test/expect/real-all-004.log b/test/expect/real-all-004.log index 31e7618b2..4b1fd5530 100644 --- a/test/expect/real-all-004.log +++ b/test/expect/real-all-004.log @@ -73,6 +73,10 @@ check db - fail on backup info mismatch (backup host) > [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --log-level-console=detail --archive-timeout=5 --stanza=db check ------------------------------------------------------------------------------------------------------------------------------------ +check db - confirm master manifest->build executed (db-master host) +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --archive-timeout=5 --stanza=db check +------------------------------------------------------------------------------------------------------------------------------------ + check db - verify success after backup (db-master host) > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --archive-timeout=5 --stanza=db check ------------------------------------------------------------------------------------------------------------------------------------ diff --git a/test/expect/real-all-005.log b/test/expect/real-all-005.log index a166a9eae..d488e3c3e 100644 --- a/test/expect/real-all-005.log +++ b/test/expect/real-all-005.log @@ -202,6 +202,10 @@ repo-path=[TEST_PATH]/backup/repo archive-copy=y start-fast=y +check db - confirm standby manifest->build executed (db-standby host) +> [CONTAINER-EXEC] db-standby [BACKREST-BIN] --config=[TEST_PATH]/db-standby/pgbackrest.conf --log-level-console=detail --archive-timeout=5 --stanza=db check +------------------------------------------------------------------------------------------------------------------------------------ + check db - verify check command on standby (db-standby host) > [CONTAINER-EXEC] db-standby [BACKREST-BIN] --config=[TEST_PATH]/db-standby/pgbackrest.conf --log-level-console=detail --stanza=db check ------------------------------------------------------------------------------------------------------------------------------------ diff --git a/test/expect/real-all-006.log b/test/expect/real-all-006.log index ada1315c1..1cbc520ee 100644 --- a/test/expect/real-all-006.log +++ b/test/expect/real-all-006.log @@ -73,6 +73,10 @@ check db - fail on backup info mismatch (backup host) > [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --log-level-console=detail --archive-timeout=5 --stanza=db check ------------------------------------------------------------------------------------------------------------------------------------ +check db - confirm master manifest->build executed (db-master host) +> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --archive-timeout=5 --stanza=db check +------------------------------------------------------------------------------------------------------------------------------------ + check db - verify success after backup (db-master host) > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --log-level-console=detail --archive-timeout=5 --stanza=db check ------------------------------------------------------------------------------------------------------------------------------------ diff --git a/test/lib/pgBackRestTest/Module/Real/RealAllTest.pm b/test/lib/pgBackRestTest/Module/Real/RealAllTest.pm index 39078799e..3ca9bf807 100644 --- a/test/lib/pgBackRestTest/Module/Real/RealAllTest.pm +++ b/test/lib/pgBackRestTest/Module/Real/RealAllTest.pm @@ -262,6 +262,16 @@ sub run # Restore the file to its original condition $oHostBackup->infoRestore(storageRepo()->pathGet(STORAGE_REPO_BACKUP . qw{/} . FILE_BACKUP_INFO)); + # Create a directory in pg_data location that is only readable by root to ensure manifest->build is called by check + my $strDir = $oHostDbMaster->dbBasePath() . '/rootreaddir'; + executeTest('sudo mkdir ' . $strDir); + executeTest("sudo chown root:root ${strDir}"); + executeTest("sudo chmod 400 ${strDir}"); + + $strComment = 'confirm master manifest->build executed'; + $oHostDbMaster->check($strComment, {iTimeout => 5, iExpectedExitStatus => ERROR_FILE_OPEN}); + executeTest("sudo rmdir ${strDir}"); + # Providing a sufficient archive-timeout, verify that the check command runs successfully now with valid # archive.info and backup.info files $strComment = 'verify success after backup'; @@ -483,8 +493,47 @@ sub run $strFullBackup = $strStandbyBackup; } - # Confirm the check command runs without error on a standby - $oHostDbStandby->check('verify check command on standby'); + # Create a directory in pg_data location that is only readable by root to ensure manifest->build is called by check + my $strDir = $oHostDbStandby->dbBasePath() . '/rootreaddir'; + executeTest('sudo mkdir ' . $strDir); + executeTest("sudo chown root:root ${strDir}"); + executeTest("sudo chmod 400 ${strDir}"); + + # Determine if there is an invalid db-host from the config file + my $rhConfig = iniParse(${storageTest()->get($oHostDbStandby->backrestConfig())}, {bRelaxed => true}); + my $bBogusHost = false; + + for (my $iRemoteIdx = 1; $iRemoteIdx <= cfgOptionIndexTotal(CFGOPT_DB_HOST); $iRemoteIdx++) + { + if (defined($rhConfig->{$self->stanza()}{$oHostDbStandby->optionIndexName(CFGOPT_DB_HOST, $iRemoteIdx)}) && + ($rhConfig->{$self->stanza()}{$oHostDbStandby->optionIndexName(CFGOPT_DB_HOST, $iRemoteIdx)} eq BOGUS)) + { + $bBogusHost = true; + last; + } + } + + my $strComment = 'confirm standby manifest->build executed'; + + # If there is an invalid host, the final error returned from check will be the inability to resolve the name which is + # a read error instead of an open error + if (!$bBogusHost) + { + $oHostDbStandby->check($strComment, {iTimeout => 5, iExpectedExitStatus => ERROR_FILE_OPEN}); + } + else + { + $oHostDbStandby->check($strComment, {iTimeout => 5, iExpectedExitStatus => ERROR_FILE_READ}); + } + + # Remove the directory in pg_data location that is only readable by root + executeTest("sudo rmdir ${strDir}"); + + # Confirm the check command runs without error on a standby (when a bogus host is not configured) + if (!$bBogusHost) + { + $oHostDbStandby->check('verify check command on standby'); + } # Shutdown the stanby before creating tablespaces (this will error since paths are different) $oHostDbStandby->clusterStop({bIgnoreLogError => true});