diff --git a/test/src/module/command/archiveGetTest.c b/test/src/module/command/archiveGetTest.c index ef86e8b2f..838bbdf59 100644 --- a/test/src/module/command/archiveGetTest.c +++ b/test/src/module/command/archiveGetTest.c @@ -269,32 +269,35 @@ testRun(void) TEST_TITLE("multiple segments where some are missing or errored"); argList = strLstDup(argBaseList); - strLstAddZ(argList, "000000010000000100000001"); - strLstAddZ(argList, "000000010000000100000002"); - strLstAddZ(argList, "000000010000000100000003"); + strLstAddZ(argList, "0000000100000001000000FE"); + strLstAddZ(argList, "0000000100000001000000FF"); + strLstAddZ(argList, "000000010000000200000000"); harnessCfgLoadRole(cfgCmdArchiveGet, cfgCmdRoleAsync, argList); + HRN_STORAGE_PUT_EMPTY( + storageRepoWrite(), STORAGE_REPO_ARCHIVE "/10-1/0000000100000001000000FE-abcdabcdabcdabcdabcdabcdabcdabcdabcdabcd"); + // Create segment duplicates HRN_STORAGE_PUT_EMPTY( - storageRepoWrite(), STORAGE_REPO_ARCHIVE "/10-1/000000010000000100000003-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); + storageRepoWrite(), STORAGE_REPO_ARCHIVE "/10-1/000000010000000200000000-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); HRN_STORAGE_PUT_EMPTY( - storageRepoWrite(), STORAGE_REPO_ARCHIVE "/10-1/000000010000000100000003-bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"); + storageRepoWrite(), STORAGE_REPO_ARCHIVE "/10-1/000000010000000200000000-bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"); TEST_RESULT_VOID(cmdArchiveGetAsync(), "archive async"); harnessLogResult( - "P00 INFO: get 3 WAL file(s) from archive: 000000010000000100000001...000000010000000100000003\n" - "P01 DETAIL: found 000000010000000100000001 in the archive\n" - "P01 DETAIL: unable to find 000000010000000100000002 in the archive\n" - "P01 WARN: could not get 000000010000000100000003 from the archive (will be retried): " - "[45] raised from local-1 protocol: duplicates found in archive for WAL segment 000000010000000100000003: " - "000000010000000100000003-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa, " - "000000010000000100000003-bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\n" + "P00 INFO: get 3 WAL file(s) from archive: 0000000100000001000000FE...000000010000000200000000\n" + "P01 DETAIL: found 0000000100000001000000FE in the archive\n" + "P01 DETAIL: unable to find 0000000100000001000000FF in the archive\n" + "P01 WARN: could not get 000000010000000200000000 from the archive (will be retried): " + "[45] raised from local-1 protocol: duplicates found in archive for WAL segment 000000010000000200000000: " + "000000010000000200000000-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa, " + "000000010000000200000000-bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\n" " HINT: are multiple primaries archiving to this stanza?"); TEST_STORAGE_LIST( storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_IN, - "000000010000000100000001\n000000010000000100000002.ok\n000000010000000100000003.error\n", + "0000000100000001000000FE\n0000000100000001000000FF.ok\n000000010000000200000000.error\n", .remove = true); // ------------------------------------------------------------------------------------------------------------------------- @@ -308,9 +311,9 @@ testRun(void) hrnCfgArgRawBool(argList, cfgOptArchiveAsync, true); hrnCfgArgRawZ(argList, cfgOptStanza, "test2"); strLstAddZ(argList, CFGCMD_ARCHIVE_GET ":" CONFIG_COMMAND_ROLE_ASYNC); - strLstAddZ(argList, "000000010000000100000001"); - strLstAddZ(argList, "000000010000000100000002"); - strLstAddZ(argList, "000000010000000100000003"); + strLstAddZ(argList, "0000000100000001000000FE"); + strLstAddZ(argList, "0000000100000001000000FF"); + strLstAddZ(argList, "000000010000000200000000"); harnessCfgLoadRaw(strLstSize(argList), strLstPtr(argList)); TEST_ERROR( @@ -318,7 +321,7 @@ testRun(void) "local-1 process terminated unexpectedly [102]: unable to execute 'pgbackrest-bogus': [2] No such file or directory"); harnessLogResult( - "P00 INFO: get 3 WAL file(s) from archive: 000000010000000100000001...000000010000000100000003"); + "P00 INFO: get 3 WAL file(s) from archive: 0000000100000001000000FE...000000010000000200000000"); TEST_RESULT_STR_Z( strNewBuf(storageGetP(storageNewReadP(storageSpool(), strNew(STORAGE_SPOOL_ARCHIVE_IN "/global.error")))), @@ -482,11 +485,11 @@ testRun(void) TEST_ERROR(cmdArchiveGet(), ParamInvalidError, "extra parameters found"); // ------------------------------------------------------------------------------------------------------------------------- - TEST_TITLE("file is missing"); + TEST_TITLE("pg version does not match archive.info"); HRN_STORAGE_PUT( storagePgWrite(), PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL, - pgControlTestToBuffer((PgControl){.version = PG_VERSION_10, .systemId = 0xFACEFACEFACEFACE})); + pgControlTestToBuffer((PgControl){.version = PG_VERSION_11, .systemId = 0xFACEFACEFACEFACE})); HRN_INFO_PUT( storageRepoWrite(), INFO_ARCHIVE_PATH_FILE, @@ -496,14 +499,38 @@ testRun(void) "[db:history]\n" "1={\"db-id\":18072658121562454734,\"db-version\":\"10\"}"); - argList = strLstNew(); - hrnCfgArgRawZ(argList, cfgOptPgPath, TEST_PATH_PG); - hrnCfgArgRawZ(argList, cfgOptRepoPath, TEST_PATH_REPO); - hrnCfgArgRawZ(argList, cfgOptStanza, "test1"); + argBaseList = strLstNew(); + hrnCfgArgRawZ(argBaseList, cfgOptPgPath, TEST_PATH_PG); + hrnCfgArgRawZ(argBaseList, cfgOptRepoPath, TEST_PATH_REPO); + hrnCfgArgRawZ(argBaseList, cfgOptStanza, "test1"); + + argList = strLstDup(argBaseList); strLstAddZ(argList, "01ABCDEF01ABCDEF01ABCDEF"); strLstAddZ(argList, TEST_PATH_PG "/pg_wal/RECOVERYXLOG"); harnessCfgLoad(cfgCmdArchiveGet, argList); + TEST_ERROR( + cmdArchiveGet(), ArchiveMismatchError, + "unable to retrieve the archive id for database version '11' and system-id '18072658121562454734'"); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("pg version does not match archive.info"); + + HRN_STORAGE_PUT( + storagePgWrite(), PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL, + pgControlTestToBuffer((PgControl){.version = PG_VERSION_10, .systemId = 0x8888888888888888})); + + TEST_ERROR( + cmdArchiveGet(), ArchiveMismatchError, + "unable to retrieve the archive id for database version '10' and system-id '9838263505978427528'"); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("file is missing"); + + HRN_STORAGE_PUT( + storagePgWrite(), PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL, + pgControlTestToBuffer((PgControl){.version = PG_VERSION_10, .systemId = 0xFACEFACEFACEFACE})); + TEST_RESULT_INT(cmdArchiveGet(), 1, "get"); harnessLogResult("P00 INFO: unable to find 01ABCDEF01ABCDEF01ABCDEF in the archive"); @@ -529,9 +556,79 @@ testRun(void) storageInfoP(storageTest, STRDEF(TEST_PATH_PG "/pg_wal/RECOVERYXLOG")).size, 16 * 1024 * 1024, "check size"); TEST_STORAGE_LIST(storageTest, TEST_PATH_PG "/pg_wal", "RECOVERYXLOG\n", .remove = true); + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("get from prior db-id"); + + HRN_INFO_PUT( + storageRepoWrite(), INFO_ARCHIVE_PATH_FILE, + "[db]\n" + "db-id=1\n" + "\n" + "[db:history]\n" + "1={\"db-id\":18072658121562454734,\"db-version\":\"10\"}\n" + "2={\"db-id\":10000000000000000000,\"db-version\":\"11\"}\n" + "3={\"db-id\":18072658121562454734,\"db-version\":\"10\"}"); + + TEST_RESULT_INT(cmdArchiveGet(), 0, "get"); + + harnessLogResult("P00 INFO: found 01ABCDEF01ABCDEF01ABCDEF in the archive"); + + TEST_STORAGE_LIST(storageTest, TEST_PATH_PG "/pg_wal", "RECOVERYXLOG\n", .remove = true); TEST_STORAGE_REMOVE( storageRepoWrite(), STORAGE_REPO_ARCHIVE "/10-1/01ABCDEF01ABCDEF01ABCDEF-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("get partial"); + + buffer = bufNew(16 * 1024 * 1024); + memset(bufPtr(buffer), 0xFF, bufSize(buffer)); + bufUsedSet(buffer, bufSize(buffer)); + + HRN_STORAGE_PUT( + storageRepoWrite(), + STORAGE_REPO_ARCHIVE "/10-3/000000010000000100000001.partial-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + buffer); + + argList = strLstDup(argBaseList); + strLstAddZ(argList, "000000010000000100000001.partial"); + strLstAddZ(argList, TEST_PATH_PG "/pg_wal/RECOVERYXLOG"); + harnessCfgLoad(cfgCmdArchiveGet, argList); + + TEST_RESULT_INT(cmdArchiveGet(), 0, "get"); + + harnessLogResult("P00 INFO: found 000000010000000100000001.partial in the archive"); + + TEST_STORAGE_LIST(storageTest, TEST_PATH_PG "/pg_wal", "RECOVERYXLOG\n", .remove = true); + TEST_STORAGE_REMOVE( + storageRepoWrite(), + STORAGE_REPO_ARCHIVE "/10-3/000000010000000100000001.partial-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("get missing history"); + + argList = strLstDup(argBaseList); + strLstAddZ(argList, "00000001.history"); + strLstAddZ(argList, TEST_PATH_PG "/pg_wal/RECOVERYHISTORY"); + harnessCfgLoad(cfgCmdArchiveGet, argList); + + TEST_RESULT_INT(cmdArchiveGet(), 1, "get"); + + harnessLogResult("P00 INFO: unable to find 00000001.history in the archive"); + + TEST_STORAGE_LIST(storageTest, TEST_PATH_PG "/pg_wal", NULL); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("get history"); + + HRN_STORAGE_PUT(storageRepoWrite(), STORAGE_REPO_ARCHIVE "/10-1/00000001.history", BUFSTRDEF("HISTORY")); + + TEST_RESULT_INT(cmdArchiveGet(), 0, "get"); + + harnessLogResult("P00 INFO: found 00000001.history in the archive"); + + TEST_RESULT_UINT(storageInfoP(storageTest, STRDEF(TEST_PATH_PG "/pg_wal/RECOVERYHISTORY")).size, 7, "check size"); + TEST_STORAGE_LIST(storageTest, TEST_PATH_PG "/pg_wal", "RECOVERYHISTORY\n", .remove = true); + // ------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("get compressed and encrypted WAL segment"); @@ -552,8 +649,11 @@ testRun(void) buffer, .compressType = compressTypeGz, .cipherType = cipherTypeAes256Cbc, .cipherPass = TEST_CIPHER_PASS_ARCHIVE); // Add encryption options + argList = strLstDup(argBaseList); hrnCfgArgRawZ(argList, cfgOptRepoCipherType, CIPHER_TYPE_AES_256_CBC); hrnCfgEnvRawZ(cfgOptRepoCipherPass, TEST_CIPHER_PASS); + strLstAddZ(argList, "01ABCDEF01ABCDEF01ABCDEF"); + strLstAddZ(argList, TEST_PATH_PG "/pg_wal/RECOVERYXLOG"); harnessCfgLoad(cfgCmdArchiveGet, argList); hrnCfgEnvRemoveRaw(cfgOptRepoCipherPass); @@ -578,12 +678,7 @@ testRun(void) bufUsedSet(serverWrite, 0); - // Add spool path - argList = strLstNew(); - hrnCfgArgRawZ(argList, cfgOptPgPath, TEST_PATH_PG); - hrnCfgArgRawZ(argList, cfgOptRepoPath, TEST_PATH_REPO); - hrnCfgArgRawZ(argList, cfgOptStanza, "test1"); - hrnCfgArgRawZ(argList, cfgOptRepoCipherType, CIPHER_TYPE_AES_256_CBC); + // Add archive-async and spool path hrnCfgArgRawZ(argList, cfgOptSpoolPath, TEST_PATH_SPOOL); hrnCfgArgRawBool(argList, cfgOptArchiveAsync, true); hrnCfgEnvRawZ(cfgOptRepoCipherPass, TEST_CIPHER_PASS);