db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2025-07-13 01:00:23 +02:00
Code Issues Releases Activity

Partial multi-repository implementation.

Browse Source 
Multi-repository implementations for the archive-push, check, info, stanza-create, stanza-upgrade, and stanza-delete commands.

Multi-repo configuration is disabled so there should be no behavioral changes between these commands and their current single-repo implementations.

Multi-repo documentation and integration tests are still in the multi-repo development branch. All unit tests work as multi-repo since they are able to bypass the configuration restrictions.
This commit is contained in:
Cynthia Shang
2021-01-21 15:21:50 -05:00
committed by GitHub
parent 1333748550
commit f32eb9b94e
52 changed files with 6103 additions and 1903 deletions
Download Patch File Download Diff File Expand all files Collapse all files

244
test/src/module/command/archivePushTest.c
View File

@ -133,8 +133,9 @@ testRun(void)
"1={\"db-id\":5555555555555555555,\"db-version\":\"9.4\"}\n"));
TEST_ERROR(
archivePushCheck(true, cipherTypeNone, NULL), ArchiveMismatchError,
"PostgreSQL version 9.6, system-id 18072658121562454734 do not match stanza version 9.4, system-id 5555555555555555555"
archivePushCheck(true), ArchiveMismatchError,
"PostgreSQL version 9.6, system-id 18072658121562454734 do not match repo1 stanza version 9.4, system-id"
" 5555555555555555555"
"\nHINT: are you archiving to the correct stanza?");
// Fix the version
@ -148,8 +149,9 @@ testRun(void)
"1={\"db-id\":5555555555555555555,\"db-version\":\"9.6\"}\n"));
TEST_ERROR(
archivePushCheck(true, cipherTypeNone, NULL), ArchiveMismatchError,
"PostgreSQL version 9.6, system-id 18072658121562454734 do not match stanza version 9.6, system-id 5555555555555555555"
archivePushCheck(true), ArchiveMismatchError,
"PostgreSQL version 9.6, system-id 18072658121562454734 do not match repo1 stanza version 9.6, system-id"
" 5555555555555555555"
"\nHINT: are you archiving to the correct stanza?");
// Fix archive info
@ -163,12 +165,73 @@ testRun(void)
"1={\"db-id\":18072658121562454734,\"db-version\":\"9.6\"}\n"));
ArchivePushCheckResult result = {0};
TEST_ASSIGN(result, archivePushCheck(true, cipherTypeNone, NULL), "get archive check result");
TEST_ASSIGN(result, archivePushCheck(true), "get archive check result");
TEST_RESULT_UINT(result.pgVersion, PG_VERSION_96, "check pg version");
TEST_RESULT_UINT(result.pgSystemId, 0xFACEFACEFACEFACE, "check pg system id");
TEST_RESULT_STR_Z(result.archiveId, "9.6-1", "check archive id");
TEST_RESULT_STR_Z(result.archiveCipherPass, NULL, "check archive cipher pass (not set in this test)");
TEST_RESULT_STR_Z(result.repoData[0].archiveId, "9.6-1", "check archive id");
TEST_RESULT_UINT(result.repoData[0].cipherType, cipherTypeNone, "check cipher type");
TEST_RESULT_STR_Z(result.repoData[0].cipherPass, NULL, "check cipher pass (not set in this test)");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("mismatched repos when pg-path not present");
argList = strLstNew();
strLstAddZ(argList, "--stanza=test");
strLstAdd(argList, strNewFmt("--repo2-path=%s/repo2", testPath()));
strLstAdd(argList, strNewFmt("--repo4-path=%s/repo4", testPath()));
harnessCfgLoad(cfgCmdArchivePush, argList);
// repo2 has correct info
storagePutP(
storageNewWriteP(storageTest, strNew("repo2/archive/test/archive.info")),
harnessInfoChecksumZ(
"[db]\n"
"db-id=1\n"
"\n"
"[db:history]\n"
"1={\"db-id\":18072658121562454734,\"db-version\":\"9.6\"}\n"));
// repo4 has incorrect info
storagePutP(
storageNewWriteP(storageTest, strNew("repo4/archive/test/archive.info")),
harnessInfoChecksumZ(
"[db]\n"
"db-id=1\n"
"\n"
"[db:history]\n"
"1={\"db-id\":5555555555555555555,\"db-version\":\"9.4\"}\n"));
TEST_ERROR(
archivePushCheck(false), ArchiveMismatchError,
"repo2 stanza version 9.6, system-id 18072658121562454734 do not match repo4 stanza version 9.4, system-id"
" 5555555555555555555"
"\nHINT: are you archiving to the correct stanza?");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("matched repos when pg-path not present");
// repo4 has correct info
storagePutP(
storageNewWriteP(storageTest, strNew("repo4/archive/test/archive.info")),
harnessInfoChecksumZ(
"[db]\n"
"db-id=2\n"
"\n"
"[db:history]\n"
"1={\"db-id\":5555555555555555555,\"db-version\":\"9.4\"}\n"
"2={\"db-id\":18072658121562454734,\"db-version\":\"9.6\"}\n"));
TEST_ASSIGN(result, archivePushCheck(false), "get archive check result");
TEST_RESULT_UINT(result.pgVersion, PG_VERSION_96, "check pg version");
TEST_RESULT_UINT(result.pgSystemId, 0xFACEFACEFACEFACE, "check pg system id");
TEST_RESULT_STR_Z(result.repoData[0].archiveId, "9.6-1", "check repo2 archive id");
TEST_RESULT_UINT(result.repoData[0].cipherType, cipherTypeNone, "check repo2 cipher pass");
TEST_RESULT_STR_Z(result.repoData[0].cipherPass, NULL, "check repo2 cipher pass (not set in this test)");
TEST_RESULT_STR_Z(result.repoData[1].archiveId, "9.6-2", "check repo4 archive id");
TEST_RESULT_UINT(result.repoData[1].cipherType, cipherTypeNone, "check repo4 cipher type");
TEST_RESULT_STR_Z(result.repoData[1].cipherPass, NULL, "check repo4 cipher pass (not set in this test)");
}
// *****************************************************************************************************************************
@ -279,7 +342,7 @@ testRun(void)
TEST_RESULT_VOID(cmdArchivePush(), "push the WAL segment again");
harnessLogResult(
"P00 WARN: WAL file '000000010000000100000001' already exists in the archive with the same checksum\n"
"P00 WARN: WAL file '000000010000000100000001' already exists in the repo1 archive with the same checksum\n"
" HINT: this is valid in some recovery scenarios but may also indicate a problem.\n"
"P00 INFO: pushed WAL file '000000010000000100000001' to the archive");
@ -292,7 +355,9 @@ testRun(void)
storagePutP(storageNewWriteP(storagePgWrite(), strNew("pg_wal/000000010000000100000001")), walBuffer2);
TEST_ERROR(cmdArchivePush(), ArchiveDuplicateError, "WAL file '000000010000000100000001' already exists in the archive");
TEST_ERROR(
cmdArchivePush(), ArchiveDuplicateError,
"WAL file '000000010000000100000001' already exists in the repo1 archive with a different checksum");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("WAL with absolute path and no pg1-path");
@ -363,7 +428,7 @@ testRun(void)
TEST_RESULT_VOID(cmdArchivePush(), "push WAL file again");
harnessLogResult(
"P00 WARN: WAL file '000000010000000100000002' already exists in the archive with the same checksum\n"
"P00 WARN: WAL file '000000010000000100000002' already exists in the repo1 archive with the same checksum\n"
" HINT: this is valid in some recovery scenarios but may also indicate a problem.\n"
"P00 INFO: pushed WAL file '000000010000000100000002' to the archive");
@ -371,20 +436,20 @@ testRun(void)
// -------------------------------------------------------------------------------------------------------------------------
VariantList *paramList = varLstNew();
varLstAdd(paramList, varNewStr(strNewFmt("%s/pg/pg_wal/000000010000000100000002", testPath())));
varLstAdd(paramList, varNewStrZ("11-1"));
varLstAdd(paramList, varNewUInt64(PG_VERSION_11));
varLstAdd(paramList, varNewUInt64(0xFACEFACEFACEFACE));
varLstAdd(paramList, varNewStrZ("000000010000000100000002"));
varLstAdd(paramList, varNewUInt64(cipherTypeNone));
varLstAdd(paramList, NULL);
varLstAdd(paramList, varNewBool(false));
varLstAdd(paramList, varNewInt(6));
varLstAdd(paramList, varNewStrZ("11-1"));
varLstAdd(paramList, varNewUInt64(cipherTypeNone));
varLstAdd(paramList, NULL);
TEST_RESULT_BOOL(
archivePushProtocol(PROTOCOL_COMMAND_ARCHIVE_PUSH_STR, paramList, server), true, "protocol archive put");
TEST_RESULT_STR_Z(
strNewBuf(serverWrite),
"{\"out\":\"WAL file '000000010000000100000002' already exists in the archive with the same checksum"
"{\"out\":\"WAL file '000000010000000100000002' already exists in the repo1 archive with the same checksum"
"\\nHINT: this is valid in some recovery scenarios but may also indicate a problem.\"}\n",
"check result");
@ -394,11 +459,14 @@ testRun(void)
// -------------------------------------------------------------------------------------------------------------------------
TEST_RESULT_BOOL(archivePushProtocol(strNew(BOGUS_STR), paramList, server), false, "invalid function");
// Create a new encrypted repo to test encryption
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("multiple repos, one encrypted");
// Remove old repo
storagePathRemoveP(storageTest, strNew("repo"), .errorOnMissing = true, .recurse = true);
StorageWrite *infoWrite = storageNewWriteP(storageTest, strNew("repo/archive/test/archive.info"));
// repo2 is encrypted
StorageWrite *infoWrite = storageNewWriteP(storageTest, strNew("repo2/archive/test/archive.info"));
ioFilterGroupAdd(
ioWriteFilterGroup(storageWriteIo(infoWrite)), cipherBlockNew(cipherModeEncrypt, cipherTypeAes256Cbc,
@ -416,22 +484,70 @@ testRun(void)
"[db:history]\n"
"1={\"db-id\":18072658121562454734,\"db-version\":\"11\"}"));
// repo3 is not encrypted
storagePutP(
storageNewWriteP(storageTest, strNew("repo3/archive/test/archive.info")),
harnessInfoChecksumZ(
"[db]\n"
"db-id=1\n"
"\n"
"[db:history]\n"
"1={\"db-id\":18072658121562454734,\"db-version\":\"11\"}"));
// Push encrypted WAL segment
argListTemp = strLstDup(argList);
argListTemp = strLstNew();
hrnCfgArgRawZ(argListTemp, cfgOptStanza, "test");
hrnCfgArgKeyRawFmt(argListTemp, cfgOptPgPath, 1, "%s/pg", testPath());
hrnCfgArgKeyRawFmt(argListTemp, cfgOptRepoPath, 2, "%s/repo2", testPath());
hrnCfgArgKeyRawZ(argListTemp, cfgOptRepoCipherType, 2, CIPHER_TYPE_AES_256_CBC);
hrnCfgEnvKeyRawZ(cfgOptRepoCipherPass, 2, "badpassphrase");
hrnCfgArgKeyRawFmt(argListTemp, cfgOptRepoPath, 3, "%s/repo3", testPath());
hrnCfgArgRawNegate(argListTemp, cfgOptCompress);
strLstAddZ(argListTemp, "pg_wal/000000010000000100000002");
strLstAddZ(argListTemp, "--repo1-cipher-type=aes-256-cbc");
strLstAddZ(argListTemp, "--no-compress");
setenv("PGBACKREST_REPO1_CIPHER_PASS", "badpassphrase", true);
harnessCfgLoad(cfgCmdArchivePush, argListTemp);
unsetenv("PGBACKREST_REPO1_CIPHER_PASS");
hrnCfgEnvKeyRemoveRaw(cfgOptRepoCipherPass, 2);
TEST_RESULT_VOID(cmdArchivePush(), "push the WAL segment");
harnessLogResult("P00 INFO: pushed WAL file '000000010000000100000002' to the archive");
TEST_RESULT_BOOL(
storageExistsP(
storageTest, strNewFmt("repo/archive/test/11-1/0000000100000001/000000010000000100000002-%s", walBuffer2Sha1)),
true, "check repo for WAL file");
storageTest, strNewFmt("repo2/archive/test/11-1/0000000100000001/000000010000000100000002-%s", walBuffer2Sha1)),
true, "check repo2 for WAL file");
TEST_RESULT_BOOL(
storageExistsP(
storageTest, strNewFmt("repo3/archive/test/11-1/0000000100000001/000000010000000100000002-%s", walBuffer2Sha1)),
true, "check repo3 for WAL file");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("remove WAL from one repo and push again");
storageRemoveP(
storageTest, strNewFmt("repo2/archive/test/11-1/0000000100000001/000000010000000100000002-%s", walBuffer2Sha1),
.errorOnMissing = true);
TEST_RESULT_VOID(cmdArchivePush(), "push the WAL segment");
harnessLogResult(
"P00 WARN: WAL file '000000010000000100000002' already exists in the repo3 archive with the same checksum\n"
" HINT: this is valid in some recovery scenarios but may also indicate a problem.\n"
"P00 INFO: pushed WAL file '000000010000000100000002' to the archive");
TEST_RESULT_BOOL(
storageExistsP(
storageTest, strNewFmt("repo2/archive/test/11-1/0000000100000001/000000010000000100000002-%s", walBuffer2Sha1)),
true, "check repo2 for WAL file");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("WAL already exists in both repos");
TEST_RESULT_VOID(cmdArchivePush(), "push the WAL segment");
harnessLogResult(
"P00 WARN: WAL file '000000010000000100000002' already exists in the repo2 archive with the same checksum\n"
" HINT: this is valid in some recovery scenarios but may also indicate a problem.\n"
" WAL file '000000010000000100000002' already exists in the repo3 archive with the same checksum\n"
" HINT: this is valid in some recovery scenarios but may also indicate a problem.\n"
"P00 INFO: pushed WAL file '000000010000000100000002' to the archive");
}
// *****************************************************************************************************************************
@ -638,11 +754,29 @@ testRun(void)
"global.error\n", "check status files");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("push already pushed WAL and error on missing WAL");
TEST_TITLE("add repo, push already pushed WAL and new WAL");
// Add repo3
hrnCfgArgKeyRawFmt(argList, cfgOptRepoPath, 3, "%s/repo3", testPath());
harnessCfgLoadRole(cfgCmdArchivePush, cfgCmdRoleAsync, argList);
storagePutP(
storageNewWriteP(storageTest, strNew("repo3/archive/test/archive.info")),
harnessInfoChecksumZ(
"[db]\n"
"db-id=1\n"
"\n"
"[db:history]\n"
"1={\"db-id\":12297848147757817309,\"db-version\":\"9.4\"}\n"));
// Recreate ready file for WAL 1
storagePutP(storageNewWriteP(storagePgWrite(), strNew("pg_xlog/archive_status/000000010000000100000001.ready")), NULL);
TEST_RESULT_BOOL(
storageExistsP(
storageTest, strNewFmt("repo/archive/test/9.4-1/0000000100000001/000000010000000100000001-%s", walBuffer1Sha1)),
true, "check repo1 for WAL 1 file");
// Create a ready file for WAL 2 but don't create the segment yet -- this will test the file error
storagePutP(storageNewWriteP(storagePgWrite(), strNew("pg_xlog/archive_status/000000010000000100000002.ready")), NULL);
@ -651,7 +785,7 @@ testRun(void)
strZ(
strNewFmt(
"P00 INFO: push 2 WAL file(s) to archive: 000000010000000100000001...000000010000000100000002\n"
"P01 WARN: WAL file '000000010000000100000001' already exists in the archive with the same checksum\n"
"P01 WARN: WAL file '000000010000000100000001' already exists in the repo1 archive with the same checksum\n"
" HINT: this is valid in some recovery scenarios but may also indicate a problem.\n"
"P01 DETAIL: pushed WAL file '000000010000000100000001' to the archive\n"
"P01 WARN: could not push WAL file '000000010000000100000002' to the archive (will be retried): "
@ -661,7 +795,12 @@ testRun(void)
TEST_RESULT_BOOL(
storageExistsP(
storageTest, strNewFmt("repo/archive/test/9.4-1/0000000100000001/000000010000000100000001-%s", walBuffer1Sha1)),
true, "check repo for WAL 1 file");
true, "check repo1 for WAL 1 file");
TEST_RESULT_BOOL(
storageExistsP(
storageTest, strNewFmt("repo/archive/test/9.4-1/0000000100000001/000000010000000100000001-%s", walBuffer1Sha1)),
true, "check repo3 for WAL 1 file");
TEST_RESULT_STRLST_Z(
strLstSort(storageListP(storageSpool(), strNew(STORAGE_SPOOL_ARCHIVE_OUT)), sortOrderAsc),
@ -691,12 +830,63 @@ testRun(void)
TEST_RESULT_BOOL(
storageExistsP(
storageTest, strNewFmt("repo/archive/test/9.4-1/0000000100000001/000000010000000100000002-%s", walBuffer2Sha1)),
true, "check repo for WAL 2 file");
true, "check repo1 for WAL 2 file");
TEST_RESULT_BOOL(
storageExistsP(
storageTest, strNewFmt("repo3/archive/test/9.4-1/0000000100000001/000000010000000100000002-%s", walBuffer2Sha1)),
true, "check repo3 for WAL 2 file");
TEST_RESULT_STRLST_Z(
strLstSort(storageListP(storageSpool(), strNew(STORAGE_SPOOL_ARCHIVE_OUT)), sortOrderAsc),
"000000010000000100000001.ok\n000000010000000100000002.ok\n", "check status files");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("push wal 2 again to get warnings from both repos");
// Remove the OK file so the WAL gets pushed again
storageRemoveP(storageSpoolWrite(), STRDEF(STORAGE_SPOOL_ARCHIVE_OUT "/000000010000000100000002.ok"));
TEST_RESULT_VOID(cmdArchivePushAsync(), "push WAL segments");
harnessLogResult(
"P00 INFO: push 1 WAL file(s) to archive: 000000010000000100000002\n"
"P01 WARN: WAL file '000000010000000100000002' already exists in the repo1 archive with the same checksum\n"
" HINT: this is valid in some recovery scenarios but may also indicate a problem.\n"
" WAL file '000000010000000100000002' already exists in the repo3 archive with the same checksum\n"
" HINT: this is valid in some recovery scenarios but may also indicate a problem.\n"
"P01 DETAIL: pushed WAL file '000000010000000100000002' to the archive");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("create and push WAL 3 to both repos");
// Create WAL 3 segment
Buffer *walBuffer3 = bufNew((size_t)16 * 1024 * 1024);
bufUsedSet(walBuffer3, bufSize(walBuffer3));
memset(bufPtr(walBuffer3), 0x44, bufSize(walBuffer3));
pgWalTestToBuffer((PgWal){.version = PG_VERSION_94, .systemId = 0xAAAABBBBCCCCDDDD}, walBuffer3);
const char *walBuffer3Sha1 = strZ(bufHex(cryptoHashOne(HASH_TYPE_SHA1_STR, walBuffer3)));
storagePutP(storageNewWriteP(storagePgWrite(), strNew("pg_xlog/000000010000000100000003")), walBuffer3);
// Create ready file
storagePutP(storageNewWriteP(storagePgWrite(), strNew("pg_xlog/archive_status/000000010000000100000003.ready")), NULL);
TEST_RESULT_VOID(cmdArchivePushAsync(), "push WAL segment");
harnessLogResult(
"P00 INFO: push 1 WAL file(s) to archive: 000000010000000100000003\n"
"P01 DETAIL: pushed WAL file '000000010000000100000003' to the archive");
TEST_RESULT_BOOL(
storageExistsP(
storageTest, strNewFmt("repo/archive/test/9.4-1/0000000100000001/000000010000000100000003-%s", walBuffer3Sha1)),
true, "check repo1 for WAL 3 file");
TEST_RESULT_BOOL(
storageExistsP(
storageTest, strNewFmt("repo3/archive/test/9.4-1/0000000100000001/000000010000000100000003-%s", walBuffer3Sha1)),
true, "check repo3 for WAL 3 file");
// Remove the ready file to prevent WAL 3 from being considered for the next test
storageRemoveP(storagePgWrite(), strNew("pg_xlog/archive_status/000000010000000100000003.ready"), .errorOnMissing = true);
// Check that drop functionality works
// -------------------------------------------------------------------------------------------------------------------------
// Remove status files