db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2025-07-07 00:35:37 +02:00
Code Issues Releases Activity

Improve info command fault tolerance.

Browse Source 
This improvement reduces the number of errors thrown; these errors will now be reported as a status for the stanza or repo as appropriate. Invalid option configurations are still thrown but all other errors are caught, formatted and reported. This was necessary for multiple repositories so that the command can complete gathering information from each repository and report the results rather than immediately aborting when an error occurs.

Two new error codes were introduced:
6 = requested backup not found
99 = other, which is used to indicate an error has occurred that requires more details to be provided

A new stanza name of "[invalid]" was created for instances where a stanza was not specified and no stanza can be found.

If there is only one repository configured the error will move up to the stanza level with the standard error formatting of 'error (message)' where the message will be "other" and the details of the error will be listed on the next line(s):

stanza: stanza1
    status: error (other)
            [CryptoError] unable to load info file '/var/lib/pgbackrest/repo/backup/stanza1/backup.info' or '/var/lib/pgbackrest/repo/backup/stanza1/backup.info.copy':
            CryptoError: cipher header invalid
            HINT: is or was the repo encrypted?
            FileMissingError: unable to open missing file '/var/lib/pgbackrest/repo/backup/stanza1/backup.info.copy' for read
            HINT: backup.info cannot be opened and is required to perform a backup.
            HINT: has a stanza-create been performed?
            HINT: use option --stanza if encryption settings are different for the stanza than the global
    cipher: aes-256-cbc

If a backup set is requested but is not found on any repo, a stanza-level status error of 'requested backup not found' is reported when there are no other errors:

pgbackrest info --stanza=demo --set=bogus
stanza: demo
    status: error (requested backup not found)
    cipher: mixed
        repo1: aes-256-cbc
        repo2: none

If there are multiple repositories configured and a single repo is in error but the other repos are ok or have a different error:

pgbackrest info --stanza=demo --set=20210322-171211F
stanza: demo
    status: mixed
        repo1: error
               [CryptoError] unable to load info file '/var/lib/pgbackrest/repo/backup/stanza1/backup.info' or '/var/lib/pgbackrest/repo/backup/stanza1/backup.info.copy':
               CryptoError: cipher header invalid
               HINT: is or was the repo encrypted?
               FileMissingError: unable to open missing file '/var/lib/pgbackrest/repo/backup/stanza1/backup.info.copy' for read
               HINT: backup.info cannot be opened and is required to perform a backup.
               HINT: has a stanza-create been performed?
               HINT: use option --stanza if encryption settings are different for the stanza than the global
        repo2: ok
    cipher: mixed
        repo1: aes-256-cbc
        repo2: none

    db (current)
        wal archive min/max (12): 000000010000000000000001/000000010000000000000003

        full backup: 20210322-171211F
            timestamp start/stop: 2021-03-22 17:12:11 / 2021-03-22 17:12:28
            wal start/stop: 000000010000000000000002 / 000000010000000000000002
            database size: 23.4MB, database backup size: 23.4MB
            repo2: backup set size: 2.8MB, backup size: 2.8MB
            database list: postgres (13359)

Json output will include the repository information and any error information. If no stanzas are found, then [invalid] will be set as the name:

[
   {
       "archive":[],
       "backup":[],
       "cipher":"none",
       "db":[],
       "name":"[invalid]",
       "repo":[
           {
               "cipher":"none",
               "key":1,
               "status":{
                   "code":99,
                   "message":"[PathOpenError] unable to list file info for path '/var/lib/pgbackrest/repo2/backup': [13] Permission denied"
               }
           }
       ],
       "status":{
           "code":99,
           "lock":{"backup":{"held":false}},
           "message":"other"
           }
   }
]
This commit is contained in:
Cynthia Shang
2021-03-25 12:29:36 -04:00
committed by GitHub
parent 7d7ac0e0eb
commit 2789d3b620
5 changed files with 617 additions and 226 deletions
Download Patch File Download Diff File Expand all files Collapse all files

290
test/src/module/command/infoTest.c
View File

@ -154,17 +154,57 @@ testRun(void)
harnessInfoChecksum(content)),
"put backup info to file");
TEST_ERROR_FMT(infoRender(), FileMissingError,
"unable to load info file '%s/archive.info' or '%s/archive.info.copy':\n"
"FileMissingError: " STORAGE_ERROR_READ_MISSING "\n"
"FileMissingError: " STORAGE_ERROR_READ_MISSING "\n"
"HINT: archive.info cannot be opened but is required to push/get WAL segments.\n"
"HINT: is archive_command configured correctly in postgresql.conf?\n"
"HINT: has a stanza-create been performed?\n"
"HINT: use --no-archive-check to disable archive checks during backup if you have an alternate archiving scheme.",
strZ(archiveStanza1Path), strZ(archiveStanza1Path),
strZ(strNewFmt("%s/archive.info", strZ(archiveStanza1Path))),
strZ(strNewFmt("%s/archive.info.copy", strZ(archiveStanza1Path))));
TEST_RESULT_STR(
infoRender(), strNewFmt(
"["
"{"
"\"archive\":[],"
"\"backup\":[],"
"\"cipher\":\"none\","
"\"db\":[],"
"\"name\":\"stanza1\","
"\"repo\":["
"{"
"\"cipher\":\"none\","
"\"key\":1,"
"\"status\":{"
"\"code\":99,"
"\"message\":\"[FileMissingError] unable to load info file '%s/archive.info' or"
" '%s/archive.info.copy':\\n"
"FileMissingError: unable to open missing file '%s/archive.info' for read\\n"
"FileMissingError: unable to open missing file '%s/archive.info.copy' for read\\n"
"HINT: archive.info cannot be opened but is required to push/get WAL segments.\\n"
"HINT: is archive_command configured correctly in postgresql.conf?\\n"
"HINT: has a stanza-create been performed?\\n"
"HINT: use --no-archive-check to disable archive checks during backup if you have an alternate"
" archiving scheme.\""
"}"
"}"
"],"
"\"status\":{"
"\"code\":99,"
"\"lock\":{\"backup\":{\"held\":false}},"
"\"message\":\"other\""
"}"
"}"
"]", strZ(archiveStanza1Path), strZ(archiveStanza1Path), strZ(archiveStanza1Path), strZ(archiveStanza1Path)),
"json - other error, single repo");
harnessCfgLoad(cfgCmdInfo, argListTextStanzaOpt);
TEST_RESULT_STR(
infoRender(), strNewFmt(
"stanza: stanza1\n"
" status: error (other)\n"
" [FileMissingError] unable to load info file '%s/stanza1/archive.info' or '%s/stanza1/archive.info.copy':\n"
" FileMissingError: unable to open missing file '%s/stanza1/archive.info' for read\n"
" FileMissingError: unable to open missing file '%s/stanza1/archive.info.copy' for read\n"
" HINT: archive.info cannot be opened but is required to push/get WAL segments.\n"
" HINT: is archive_command configured correctly in postgresql.conf?\n"
" HINT: has a stanza-create been performed?\n"
" HINT: use --no-archive-check to disable archive checks during backup if you have an alternate archiving"
" scheme.\n"
" cipher: none\n", strZ(archivePath), strZ(archivePath), strZ(archivePath), strZ(archivePath)),
"text - other error, single repo");
// backup.info/archive.info files exist, mismatched db ids, no backup:current section so no valid backups
// Only the current db information from the db:history will be processed.
@ -1566,6 +1606,22 @@ testRun(void)
}
HARNESS_FORK_END();
// Stanza exists but set requested backup does not
//--------------------------------------------------------------------------------------------------------------------------
argList2 = strLstDup(argListMultiRepo);
strLstAddZ(argList2, "--stanza=stanza1");
strLstAddZ(argList2, "--set=bogus");
harnessCfgLoad(cfgCmdInfo, argList2);
TEST_RESULT_STR_Z(
infoRender(),
"stanza: stanza1\n"
" status: error (requested backup not found)\n"
" cipher: mixed\n"
" repo1: none\n"
" repo2: aes-256-cbc\n",
"text, multi-repo, backup not found");
// Backup set requested, with 1 checksum error
//--------------------------------------------------------------------------------------------------------------------------
argList2 = strLstDup(argListMultiRepo);
@ -2293,6 +2349,7 @@ testRun(void)
//--------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("encryption error");
// Change repo1 to have the same cipher type as repo2 even though on disk it does not
content = strNew
(
"[global]\n"
@ -2302,19 +2359,123 @@ testRun(void)
TEST_RESULT_VOID(
storagePutP(storageNewWriteP(storageLocalWrite(), strNewFmt("%s/pgbackrest.conf", testPath())),
BUFSTR(content)), "put pgbackrest.conf file");
strLstAddZ(argListText, "--repo-cipher-type=aes-256-cbc");
strLstAdd(argListText, strNewFmt("--config=%s/pgbackrest.conf", testPath()));
harnessCfgLoad(cfgCmdInfo, argListText);
TEST_ERROR_FMT(
infoRender(), CryptoError,
"unable to load info file '%s/backup.info' or '%s/backup.info.copy':\n"
"CryptoError: cipher header invalid\n"
"HINT: is or was the repo encrypted?\n"
"FileMissingError: " STORAGE_ERROR_READ_MISSING "\n"
"HINT: backup.info cannot be opened and is required to perform a backup.\n"
"HINT: has a stanza-create been performed?\n"
"HINT: use option --stanza if encryption settings are different for the stanza than the global settings.",
strZ(backupStanza1Path), strZ(backupStanza1Path), strZ(strNewFmt("%s/backup.info.copy", strZ(backupStanza1Path))));
argList2 = strLstDup(argListMultiRepo);
strLstAddZ(argList2, "--repo-cipher-type=aes-256-cbc");
strLstAdd(argList2, strNewFmt("--config=%s/pgbackrest.conf", testPath()));
harnessCfgLoad(cfgCmdInfo, argList2);
TEST_RESULT_STR(
infoRender(), strNewFmt(
"stanza: stanza1\n"
" status: mixed\n"
" repo1: error (other)\n"
" [CryptoError] unable to load info file '%s/stanza1/backup.info' or '%s/stanza1/backup.info.copy':\n"
" CryptoError: cipher header invalid\n"
" HINT: is or was the repo encrypted?\n"
" FileMissingError: unable to open missing file '%s/stanza1/backup.info.copy' for read\n"
" HINT: backup.info cannot be opened and is required to perform a backup.\n"
" HINT: has a stanza-create been performed?\n"
" HINT: use option --stanza if encryption settings are different for the stanza than the global"
" settings.\n"
" repo2: error (no valid backups)\n"
" cipher: aes-256-cbc\n"
"\n"
" db (current)\n"
" wal archive min/max (9.5): 000000010000000000000003/000000010000000000000004\n"
"\n"
"stanza: stanza2\n"
" status: mixed\n"
" repo1: error (other)\n"
" [CryptoError] unable to load info file '%s/stanza2/backup.info' or '%s/stanza2/backup.info.copy':\n"
" CryptoError: cipher header invalid\n"
" HINT: is or was the repo encrypted?\n"
" FileMissingError: unable to open missing file '%s/stanza2/backup.info.copy' for read\n"
" HINT: backup.info cannot be opened and is required to perform a backup.\n"
" HINT: has a stanza-create been performed?\n"
" HINT: use option --stanza if encryption settings are different for the stanza than the global"
" settings.\n"
" repo2: error (missing stanza path)\n"
" cipher: aes-256-cbc\n"
"\n"
"stanza: stanza3\n"
" status: mixed\n"
" repo1: error (other)\n"
" [CryptoError] unable to load info file '%s/stanza3/backup.info' or '%s/stanza3/backup.info.copy':\n"
" CryptoError: cipher header invalid\n"
" HINT: is or was the repo encrypted?\n"
" FileMissingError: unable to open missing file '%s/stanza3/backup.info.copy' for read\n"
" HINT: backup.info cannot be opened and is required to perform a backup.\n"
" HINT: has a stanza-create been performed?\n"
" HINT: use option --stanza if encryption settings are different for the stanza than the global"
" settings.\n"
" repo2: ok\n"
" cipher: aes-256-cbc\n"
"\n"
" db (current)\n"
" wal archive min/max (9.4): 000000010000000000000001/000000010000000000000002\n"
"\n"
" full backup: 20201110-100000F\n"
" timestamp start/stop: 2020-11-10 10:00:00 / 2020-11-10 10:00:02\n"
" wal start/stop: 000000010000000000000001 / 000000010000000000000002\n"
" database size: 25.7MB, database backup size: 25.7MB\n"
" repo2: backup set size: 3MB, backup size: 3KB\n",
strZ(backupPath), strZ(backupPath), strZ(backupPath), strZ(backupPath), strZ(backupPath), strZ(backupPath),
strZ(backupPath), strZ(backupPath), strZ(backupPath)),
"text - multi-repo, multi-stanza cipher error");
// Backup label not found, one repo in error
//--------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("backup label exists on one repo, other repo in error");
hrnCfgArgRawZ(argList2, cfgOptStanza, "stanza3");
hrnCfgArgRawZ(argList2, cfgOptSet, "20201110-100000F");
harnessCfgLoad(cfgCmdInfo, argList2);
TEST_RESULT_STR(
infoRender(), strNewFmt(
"stanza: stanza3\n"
" status: mixed\n"
" repo1: error (other)\n"
" [CryptoError] unable to load info file '%s/stanza3/backup.info' or '%s/stanza3/backup.info.copy':\n"
" CryptoError: cipher header invalid\n"
" HINT: is or was the repo encrypted?\n"
" FileMissingError: unable to open missing file '%s/stanza3/backup.info.copy' for read\n"
" HINT: backup.info cannot be opened and is required to perform a backup.\n"
" HINT: has a stanza-create been performed?\n"
" HINT: use option --stanza if encryption settings are different for the stanza than the global"
" settings.\n"
" repo2: error (requested backup not found)\n"
" cipher: aes-256-cbc\n", strZ(backupPath), strZ(backupPath), strZ(backupPath)),
"backup label not found, one repo in error");
// Crypto error
//--------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("WAL read error");
TEST_RESULT_VOID(
storagePathCreateP(
storageLocalWrite(), strNewFmt("%s/9.4-1", strZ(archiveStanza1Path)), .mode = 0200),
"WAL directory with bad permissions");
argList2 = strLstDup(argListMultiRepo);
hrnCfgArgRawZ(argList2, cfgOptStanza, "stanza1");
harnessCfgLoad(cfgCmdInfo, argList2);
TEST_RESULT_STR(
infoRender(), strNewFmt(
"stanza: stanza1\n"
" status: mixed\n"
" repo1: error (other)\n"
" [PathOpenError] unable to list file info for path '%s/stanza1/9.4-1': [13] Permission denied\n"
" repo2: error (no valid backups)\n"
" cipher: mixed\n"
" repo1: none\n"
" repo2: aes-256-cbc\n"
"\n"
" db (current)\n"
" wal archive min/max (9.5): 000000010000000000000003/000000010000000000000004\n", strZ(archivePath)),
"WAL directory read error");
// Unset environment key
hrnCfgEnvKeyRemoveRaw(cfgOptRepoCipherPass, 2);
@ -2591,21 +2752,82 @@ testRun(void)
TEST_ERROR_FMT(
harnessCfgLoad(cfgCmdInfo, argList), OptionInvalidError, "option 'set' not valid without option 'stanza'");
// Option --repo not required when only 1 repo configured
strLstAddZ(argList, "--stanza=stanza1");
//--------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("repo-level error");
TEST_RESULT_VOID(
storagePathCreateP(
storageLocalWrite(), strNewFmt("%s/repo2", testPath()), .mode = 0200), "repo directory with bad permissions");
argList = strLstNew();
hrnCfgArgKeyRawFmt(argList, cfgOptRepoPath, 1, "%s/repo2", testPath());
harnessCfgLoad(cfgCmdInfo, argList);
TEST_ERROR_FMT(
cmdInfo(), FileMissingError, "manifest does not exist for backup 'bogus'\n"
"HINT: is the backup listed when running the info command with --stanza option only?");
TEST_RESULT_STR(
infoRender(), strNewFmt(
"stanza: [invalid]\n"
" status: error (other)\n"
" [PathOpenError] unable to list file info for path '%s/repo2/backup': [13] Permission denied\n"
" cipher: none\n", testPath()),
"text - invalid stanza");
// Option --repo when only 1 repo configured but will search the repo provided
strLstAddZ(argList, "--repo=1");
hrnCfgArgRawZ(argList, cfgOptOutput, "json");
harnessCfgLoad(cfgCmdInfo, argList);
TEST_ERROR_FMT(
cmdInfo(), FileMissingError, "manifest does not exist for backup 'bogus'\n"
"HINT: is the backup listed when running the info command with --stanza option only?");
TEST_RESULT_STR(
infoRender(), strNewFmt(
"["
"{"
"\"archive\":[],"
"\"backup\":[],"
"\"cipher\":\"none\","
"\"db\":[],"
"\"name\":\"[invalid]\","
"\"repo\":["
"{"
"\"cipher\":\"none\","
"\"key\":1,"
"\"status\":{"
"\"code\":99,"
"\"message\":\"[PathOpenError] unable to list file info for path '%s/repo2/backup': [13] Permission"
" denied\""
"}"
"}"
"],"
"\"status\":{"
"\"code\":99,"
"\"lock\":{\"backup\":{\"held\":false}},"
"\"message\":\"other\""
"}"
"}"
"]", testPath()),
"json - invalid stanza");
argList = strLstNew();
hrnCfgArgKeyRawFmt(argList, cfgOptRepoPath, 1, "%s/repo2", testPath());
hrnCfgArgRawZ(argList, cfgOptStanza, "stanza1");
harnessCfgLoad(cfgCmdInfo, argList);
TEST_RESULT_STR(
infoRender(), strNewFmt(
"stanza: stanza1\n"
" status: error (other)\n"
" [PathOpenError] unable to list file info for path '%s/repo2/backup': [13] Permission denied\n"
" cipher: none\n", testPath()),
"text - stanza requested");
hrnCfgArgKeyRawFmt(argList, cfgOptRepoPath, 2, "%s/repo", testPath());
harnessCfgLoad(cfgCmdInfo, argList);
TEST_RESULT_STR(
infoRender(), strNewFmt(
"stanza: stanza1\n"
" status: mixed\n"
" repo1: error (other)\n"
" [PathOpenError] unable to list file info for path '%s/repo2/backup': [13] Permission denied\n"
" repo2: error (missing stanza path)\n"
" cipher: none\n", testPath()),
"text - stanza repo structure exists");
}
FUNCTION_HARNESS_RETURN_VOID();