diff --git a/test/define.yaml b/test/define.yaml index a4949fef8..d213da9c3 100644 --- a/test/define.yaml +++ b/test/define.yaml @@ -565,7 +565,7 @@ unit: # ---------------------------------------------------------------------------------------------------------------------------- - name: archive-get - total: 5 + total: 4 binReq: true coverage: diff --git a/test/src/module/command/archiveGetTest.c b/test/src/module/command/archiveGetTest.c index 4940c9877..ef86e8b2f 100644 --- a/test/src/module/command/archiveGetTest.c +++ b/test/src/module/command/archiveGetTest.c @@ -11,6 +11,7 @@ Test Archive Get Command #include "storage/posix/storage.h" #include "common/harnessInfo.h" +#include "common/harnessStorage.h" /*********************************************************************************************************************************** Test Run @@ -22,16 +23,6 @@ testRun(void) Storage *storageTest = storagePosixNewP(strNew(testPath()), .write = true); - // Start a protocol server to test the protocol directly - Buffer *serverWrite = bufNew(8192); - IoWrite *serverWriteIo = ioBufferWriteNew(serverWrite); - ioWriteOpen(serverWriteIo); - - ProtocolServer *server = protocolServerNew( - strNew("test"), strNew("test"), ioBufferReadNew(bufNew(0)), serverWriteIo); - - bufUsedSet(serverWrite, 0); - // ***************************************************************************************************************************** if (testBegin("archiveGetCheck()")) { @@ -39,12 +30,12 @@ testRun(void) StringList *argList = strLstNew(); strLstAddZ(argList, "--stanza=test1"); strLstAdd(argList, strNewFmt("--repo1-path=%s/repo", testPath())); - strLstAdd(argList, strNewFmt("--pg1-path=%s/db", testPath())); + strLstAdd(argList, strNewFmt("--pg1-path=%s/pg", testPath())); harnessCfgLoad(cfgCmdArchiveGet, argList); // Create pg_control file storagePutP( - storageNewWriteP(storageTest, strNew("db/" PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL)), + storageNewWriteP(storageTest, strNew("pg/" PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL)), pgControlTestToBuffer((PgControl){.version = PG_VERSION_10, .systemId = 0xFACEFACEFACEFACE})); // Control and archive info mismatch @@ -117,132 +108,6 @@ testRun(void) "history file found"); } - // ***************************************************************************************************************************** - if (testBegin("archiveGetFile()")) - { - // Load Parameters - StringList *argList = strLstNew(); - strLstAddZ(argList, "--stanza=test1"); - strLstAdd(argList, strNewFmt("--repo1-path=%s/repo", testPath())); - strLstAdd(argList, strNewFmt("--pg1-path=%s/db", testPath())); - harnessCfgLoad(cfgCmdArchiveGet, argList); - - // Create pg_control file - storagePutP( - storageNewWriteP(storageTest, strNew("db/" PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL)), - pgControlTestToBuffer((PgControl){.version = PG_VERSION_10, .systemId = 0xFACEFACEFACEFACE})); - - // Create archive.info - storagePutP( - storageNewWriteP(storageTest, strNew("repo/archive/test1/archive.info")), - harnessInfoChecksumZ( - "[db]\n" - "db-id=1\n" - "\n" - "[db:history]\n" - "1={\"db-id\":18072658121562454734,\"db-version\":\"10\"}")); - - // Nothing to copy - // ------------------------------------------------------------------------------------------------------------------------- - String *archiveFile = strNew("01ABCDEF01ABCDEF01ABCDEF"); - String *walDestination = strNewFmt("%s/db/pg_wal/RECOVERYXLOG", testPath()); - storagePathCreateP(storageTest, strPath(walDestination)); - - TEST_RESULT_INT( - archiveGetFile(storageTest, archiveFile, walDestination, false, cipherTypeNone, NULL), 1, "WAL segment missing"); - - // Create a WAL segment to copy - // ------------------------------------------------------------------------------------------------------------------------- - Buffer *buffer = bufNew(16 * 1024 * 1024); - memset(bufPtr(buffer), 0, bufSize(buffer)); - bufUsedSet(buffer, bufSize(buffer)); - - storagePutP( - storageNewWriteP( - storageTest, - strNew( - "repo/archive/test1/10-1/01ABCDEF01ABCDEF/01ABCDEF01ABCDEF01ABCDEF-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")), - buffer); - - TEST_RESULT_INT( - archiveGetFile(storageTest, archiveFile, walDestination, false, cipherTypeNone, NULL), 0, "WAL segment copied"); - TEST_RESULT_BOOL(storageExistsP(storageTest, walDestination), true, " check exists"); - TEST_RESULT_UINT(storageInfoP(storageTest, walDestination).size, 16 * 1024 * 1024, " check size"); - - storageRemoveP( - storageTest, - strNew("repo/archive/test1/10-1/01ABCDEF01ABCDEF/01ABCDEF01ABCDEF01ABCDEF-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), - .errorOnMissing = true); - storageRemoveP(storageTest, walDestination, .errorOnMissing = true); - - // Create a compressed WAL segment to copy - // ------------------------------------------------------------------------------------------------------------------------- - StorageWrite *infoWrite = storageNewWriteP(storageTest, strNew("repo/archive/test1/archive.info")); - - ioFilterGroupAdd( - ioWriteFilterGroup(storageWriteIo(infoWrite)), cipherBlockNew(cipherModeEncrypt, cipherTypeAes256Cbc, - BUFSTRDEF("12345678"), NULL)); - - storagePutP( - infoWrite, - harnessInfoChecksumZ( - "[cipher]\n" - "cipher-pass=\"worstpassphraseever\"\n" - "\n" - "[db]\n" - "db-id=1\n" - "\n" - "[db:history]\n" - "1={\"db-id\":18072658121562454734,\"db-version\":\"10\"}")); - - StorageWrite *destination = storageNewWriteP( - storageTest, - strNew( - "repo/archive/test1/10-1/01ABCDEF01ABCDEF/01ABCDEF01ABCDEF01ABCDEF-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.gz")); - - IoFilterGroup *filterGroup = ioWriteFilterGroup(storageWriteIo(destination)); - ioFilterGroupAdd(filterGroup, compressFilter(compressTypeGz, 3)); - ioFilterGroupAdd( - filterGroup, cipherBlockNew(cipherModeEncrypt, cipherTypeAes256Cbc, BUFSTRDEF("worstpassphraseever"), NULL)); - storagePutP(destination, buffer); - - TEST_RESULT_INT( - archiveGetFile( - storageTest, archiveFile, walDestination, false, cipherTypeAes256Cbc, strNew("12345678")), 0, "WAL segment copied"); - TEST_RESULT_BOOL(storageExistsP(storageTest, walDestination), true, " check exists"); - TEST_RESULT_UINT(storageInfoP(storageTest, walDestination).size, 16 * 1024 * 1024, " check size"); - - // Check protocol function directly - // ------------------------------------------------------------------------------------------------------------------------- - argList = strLstNew(); - strLstAddZ(argList, "--stanza=test1"); - strLstAdd(argList, strNewFmt("--repo1-path=%s/repo", testPath())); - strLstAdd(argList, strNewFmt("--pg1-path=%s/db", testPath())); - strLstAdd(argList, strNewFmt("--spool-path=%s/spool", testPath())); - strLstAddZ(argList, "--" CFGOPT_ARCHIVE_ASYNC); - strLstAddZ(argList, "--repo1-cipher-type=aes-256-cbc"); - setenv("PGBACKREST_REPO1_CIPHER_PASS", "12345678", true); - harnessCfgLoadRole(cfgCmdArchiveGet, cfgCmdRoleAsync, argList); - unsetenv("PGBACKREST_REPO1_CIPHER_PASS"); - - storagePathCreateP(storageTest, strNew("spool/archive/test1/in")); - - VariantList *paramList = varLstNew(); - varLstAdd(paramList, varNewStr(archiveFile)); - - TEST_RESULT_BOOL( - archiveGetProtocol(PROTOCOL_COMMAND_ARCHIVE_GET_STR, paramList, server), true, "protocol archive get"); - TEST_RESULT_STR_Z(strNewBuf(serverWrite), "{\"out\":0}\n", "check result"); - TEST_RESULT_BOOL( - storageExistsP(storageTest, strNewFmt("spool/archive/test1/in/%s", strZ(archiveFile))), true, " check exists"); - - bufUsedSet(serverWrite, 0); - - // Check invalid protocol function - // ------------------------------------------------------------------------------------------------------------------------- - TEST_RESULT_BOOL(archiveGetProtocol(strNew(BOGUS_STR), paramList, server), false, "invalid function"); - } - // ***************************************************************************************************************************** if (testBegin("queueNeed()")) { @@ -303,124 +168,120 @@ testRun(void) TEST_STORAGE_LIST(storageSpool(), STORAGE_SPOOL_ARCHIVE_IN, "000000010000000A00000FFE\n000000010000000A00000FFF\n"); } - // ***************************************************************************************************************************** if (testBegin("cmdArchiveGetAsync()")) { harnessLogLevelSet(logLevelDetail); - StringList *argCleanList = strLstNew(); - strLstAdd(argCleanList, strNewFmt("--pg1-path=%s/pg", testPath())); - strLstAdd(argCleanList, strNewFmt("--repo1-path=%s/repo", testPath())); - strLstAdd(argCleanList, strNewFmt("--spool-path=%s/spool", testPath())); - strLstAddZ(argCleanList, "--" CFGOPT_ARCHIVE_ASYNC); - strLstAddZ(argCleanList, "--stanza=test2"); - harnessCfgLoadRole(cfgCmdArchiveGet, cfgCmdRoleAsync, argCleanList); - - TEST_ERROR(cmdArchiveGetAsync(), ParamInvalidError, "at least one wal segment is required"); - - TEST_STORAGE_LIST(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_IN, "global.error\n", .remove = true); + // Arguments that must be included + StringList *argBaseList = strLstNew(); + hrnCfgArgRawZ(argBaseList, cfgOptPgPath, TEST_PATH_PG); + hrnCfgArgRawZ(argBaseList, cfgOptRepoPath, TEST_PATH_REPO); + hrnCfgArgRawZ(argBaseList, cfgOptSpoolPath, TEST_PATH_SPOOL); + hrnCfgArgRawBool(argBaseList, cfgOptArchiveAsync, true); + hrnCfgArgRawZ(argBaseList, cfgOptStanza, "test2"); // ------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("command must be run on the pg host"); - StringList *argList = strLstNew(); - hrnCfgArgRawZ(argList, cfgOptPgHost, "host"); - hrnCfgArgRawZ(argList, cfgOptPgPath, "/pg"); - hrnCfgArgRawZ(argList, cfgOptRepoPath, "/repo"); - hrnCfgArgRawZ(argList, cfgOptSpoolPath, TEST_PATH "/spool"); - strLstAddZ(argList, "--" CFGOPT_ARCHIVE_ASYNC); - strLstAddZ(argList, "--" CFGOPT_STANZA "=test2"); - strLstAddZ(argList, "000000010000000100000001"); + StringList *argList = strLstDup(argBaseList); + hrnCfgArgRawZ(argList, cfgOptPgHost, BOGUS_STR); harnessCfgLoadRole(cfgCmdArchiveGet, cfgCmdRoleAsync, argList); TEST_ERROR(cmdArchiveGetAsync(), HostInvalidError, "archive-get command must be run on the PostgreSQL host"); TEST_STORAGE_LIST(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_IN, "global.error\n", .remove = true); - // Create pg_control file and archive.info // ------------------------------------------------------------------------------------------------------------------------- - storagePutP( - storageNewWriteP(storageTest, strNew("pg/" PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL)), + TEST_TITLE("error on no segments"); + + argList = strLstDup(argBaseList); + harnessCfgLoadRole(cfgCmdArchiveGet, cfgCmdRoleAsync, argList); + + TEST_ERROR(cmdArchiveGetAsync(), ParamInvalidError, "at least one wal segment is required"); + + TEST_STORAGE_LIST(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_IN, "global.error\n", .remove = true); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("no segments to find"); + + HRN_STORAGE_PUT( + storagePgWrite(), PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL, pgControlTestToBuffer((PgControl){.version = PG_VERSION_10, .systemId = 0xFACEFACEFACEFACE})); - storagePutP( - storageNewWriteP(storageTest, strNew("repo/archive/test2/archive.info")), - harnessInfoChecksumZ( - "[db]\n" - "db-id=1\n" - "\n" - "[db:history]\n" - "1={\"db-id\":18072658121562454734,\"db-version\":\"10\"}\n")); + HRN_INFO_PUT( + storageRepoWrite(), INFO_ARCHIVE_PATH_FILE, + "[db]\n" + "db-id=1\n" + "\n" + "[db:history]\n" + "1={\"db-id\":18072658121562454734,\"db-version\":\"10\"}\n"); - // Get a single segment - // ------------------------------------------------------------------------------------------------------------------------- - argList = strLstDup(argCleanList); strLstAddZ(argList, "000000010000000100000001"); harnessCfgLoadRole(cfgCmdArchiveGet, cfgCmdRoleAsync, argList); - storagePathCreateP(storageSpoolWrite(), strNew(STORAGE_SPOOL_ARCHIVE_IN)); + TEST_RESULT_VOID(cmdArchiveGetAsync(), "get async"); - TEST_RESULT_VOID( - storagePutP( - storageNewWriteP( - storageTest, - strNew( - "repo/archive/test2/10-1/0000000100000001/" - "000000010000000100000001-abcdabcdabcdabcdabcdabcdabcdabcdabcdabcd")), - NULL), - "normal WAL segment"); + harnessLogResult( + "P00 INFO: get 1 WAL file(s) from archive: 000000010000000100000001\n" + "P01 DETAIL: unable to find 000000010000000100000001 in the archive"); - // Create tmp file to make it look like a prior async get failed partway through to ensure that retries work - TEST_RESULT_VOID( - storagePutP( - storageNewWriteP(storageSpoolWrite(), strNew(STORAGE_SPOOL_ARCHIVE_IN "/000000010000000100000001.pgbackrest.tmp")), - NULL), - "normal WAL segment"); + TEST_STORAGE_LIST(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_IN, "000000010000000100000001.ok\n", .remove = true); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("error on invalid compressed segment"); + + HRN_STORAGE_PUT_EMPTY( + storageRepoWrite(), STORAGE_REPO_ARCHIVE "/10-1/000000010000000100000001-abcdabcdabcdabcdabcdabcdabcdabcdabcdabcd.gz"); + + TEST_RESULT_VOID(cmdArchiveGetAsync(), "get async"); + + harnessLogResult( + "P00 INFO: get 1 WAL file(s) from archive: 000000010000000100000001\n" + "P01 WARN: could not get 000000010000000100000001 from the archive (will be retried):" + " [29] raised from local-1 protocol: unexpected eof in compressed data"); + + TEST_STORAGE_LIST( + storageSpool(), STORAGE_SPOOL_ARCHIVE_IN, "000000010000000100000001.error\n000000010000000100000001.pgbackrest.tmp\n"); + TEST_STORAGE_REMOVE(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_IN "/000000010000000100000001.error"); + + TEST_STORAGE_REMOVE( + storageRepoWrite(), STORAGE_REPO_ARCHIVE "/10-1/000000010000000100000001-abcdabcdabcdabcdabcdabcdabcdabcdabcdabcd.gz"); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("single segment"); + + HRN_STORAGE_PUT_EMPTY( + storageRepoWrite(), STORAGE_REPO_ARCHIVE "/10-1/000000010000000100000001-abcdabcdabcdabcdabcdabcdabcdabcdabcdabcd"); + + // There should be a temp file left over. Make sure it still exists to test that temp files are removed on retry. + TEST_STORAGE_EXISTS(storageSpool(), STORAGE_SPOOL_ARCHIVE_IN "/000000010000000100000001.pgbackrest.tmp"); TEST_RESULT_VOID(cmdArchiveGetAsync(), "archive async"); + harnessLogResult( "P00 INFO: get 1 WAL file(s) from archive: 000000010000000100000001\n" "P01 DETAIL: found 000000010000000100000001 in the archive"); - TEST_RESULT_BOOL( - storageExistsP(storageSpool(), strNew(STORAGE_SPOOL_ARCHIVE_IN "/000000010000000100000001")), true, - "check 000000010000000100000001 in spool"); - TEST_RESULT_BOOL( - storageExistsP(storageSpool(), strNew(STORAGE_SPOOL_ARCHIVE_IN "/000000010000000100000001.pgbackrest.tmp")), false, - "check 000000010000000100000001 tmp not in spool"); + TEST_STORAGE_LIST(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_IN, "000000010000000100000001\n", .remove = true); - // Get multiple segments where some are missing or errored // ------------------------------------------------------------------------------------------------------------------------- - argList = strLstDup(argCleanList); + TEST_TITLE("multiple segments where some are missing or errored"); + + argList = strLstDup(argBaseList); strLstAddZ(argList, "000000010000000100000001"); strLstAddZ(argList, "000000010000000100000002"); strLstAddZ(argList, "000000010000000100000003"); harnessCfgLoadRole(cfgCmdArchiveGet, cfgCmdRoleAsync, argList); - storagePathCreateP(storageSpoolWrite(), strNew(STORAGE_SPOOL_ARCHIVE_IN)); - - TEST_RESULT_VOID( - storagePutP( - storageNewWriteP( - storageTest, - strNew( - "repo/archive/test2/10-1/0000000100000001/" - "000000010000000100000003-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")), - NULL), - "normal WAL segment"); - - TEST_RESULT_VOID( - storagePutP( - storageNewWriteP( - storageTest, - strNew( - "repo/archive/test2/10-1/0000000100000001/" - "000000010000000100000003-bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb")), - NULL), - "duplicate WAL segment"); + // Create segment duplicates + HRN_STORAGE_PUT_EMPTY( + storageRepoWrite(), STORAGE_REPO_ARCHIVE "/10-1/000000010000000100000003-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); + HRN_STORAGE_PUT_EMPTY( + storageRepoWrite(), STORAGE_REPO_ARCHIVE "/10-1/000000010000000100000003-bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"); TEST_RESULT_VOID(cmdArchiveGetAsync(), "archive async"); + harnessLogResult( "P00 INFO: get 3 WAL file(s) from archive: 000000010000000100000001...000000010000000100000003\n" "P01 DETAIL: found 000000010000000100000001 in the archive\n" @@ -431,35 +292,21 @@ testRun(void) "000000010000000100000003-bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\n" " HINT: are multiple primaries archiving to this stanza?"); - TEST_RESULT_BOOL( - storageExistsP(storageSpool(), strNew(STORAGE_SPOOL_ARCHIVE_IN "/000000010000000100000001")), true, - "check 000000010000000100000001 in spool"); - TEST_RESULT_BOOL( - storageExistsP(storageSpool(), strNew(STORAGE_SPOOL_ARCHIVE_IN "/000000010000000100000002")), false, - "check 000000010000000100000002 not in spool"); - TEST_RESULT_BOOL( - storageExistsP(storageSpool(), strNew(STORAGE_SPOOL_ARCHIVE_IN "/000000010000000100000002.ok")), true, - "check 000000010000000100000002.ok in spool"); - TEST_RESULT_BOOL( - storageExistsP(storageSpool(), strNew(STORAGE_SPOOL_ARCHIVE_IN "/000000010000000100000003")), false, - "check 000000010000000100000003 not in spool"); - TEST_RESULT_BOOL( - storageExistsP(storageSpool(), strNew(STORAGE_SPOOL_ARCHIVE_IN "/000000010000000100000003.error")), true, - "check 000000010000000100000003.error in spool"); - - protocolFree(); + TEST_STORAGE_LIST( + storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_IN, + "000000010000000100000001\n000000010000000100000002.ok\n000000010000000100000003.error\n", + .remove = true); // ------------------------------------------------------------------------------------------------------------------------- - storageRemoveP( - storageSpoolWrite(), strNew(STORAGE_SPOOL_ARCHIVE_IN "/000000010000000100000003.error"), .errorOnMissing = true); + TEST_TITLE("global error on invalid executable"); argList = strLstNew(); strLstAddZ(argList, "pgbackrest-bogus"); - strLstAdd(argList, strNewFmt("--pg1-path=%s/pg", testPath())); - strLstAdd(argList, strNewFmt("--repo1-path=%s/repo", testPath())); - strLstAdd(argList, strNewFmt("--spool-path=%s/spool", testPath())); - strLstAddZ(argList, "--" CFGOPT_ARCHIVE_ASYNC); - strLstAddZ(argList, "--stanza=test2"); + hrnCfgArgRawZ(argList, cfgOptPgPath, TEST_PATH_PG); + hrnCfgArgRawZ(argList, cfgOptRepoPath, TEST_PATH_REPO); + hrnCfgArgRawZ(argList, cfgOptSpoolPath, TEST_PATH_SPOOL); + hrnCfgArgRawBool(argList, cfgOptArchiveAsync, true); + hrnCfgArgRawZ(argList, cfgOptStanza, "test2"); strLstAddZ(argList, CFGCMD_ARCHIVE_GET ":" CONFIG_COMMAND_ROLE_ASYNC); strLstAddZ(argList, "000000010000000100000001"); strLstAddZ(argList, "000000010000000100000002"); @@ -473,71 +320,60 @@ testRun(void) harnessLogResult( "P00 INFO: get 3 WAL file(s) from archive: 000000010000000100000001...000000010000000100000003"); - TEST_RESULT_BOOL( - storageExistsP(storageSpool(), strNew(STORAGE_SPOOL_ARCHIVE_IN "/000000010000000100000001.error")), false, - "check 000000010000000100000001.error not in spool"); - TEST_RESULT_BOOL( - storageExistsP(storageSpool(), strNew(STORAGE_SPOOL_ARCHIVE_IN "/000000010000000100000002.error")), false, - "check 000000010000000100000002.error not in spool"); - TEST_RESULT_BOOL( - storageExistsP(storageSpool(), strNew(STORAGE_SPOOL_ARCHIVE_IN "/000000010000000100000003.error")), false, - "check 000000010000000100000003.error not in spool"); TEST_RESULT_STR_Z( strNewBuf(storageGetP(storageNewReadP(storageSpool(), strNew(STORAGE_SPOOL_ARCHIVE_IN "/global.error")))), "102\nlocal-1 process terminated unexpectedly [102]: unable to execute 'pgbackrest-bogus': " "[2] No such file or directory", "check global error"); + + TEST_STORAGE_LIST(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_IN, "global.error\n", .remove = true); } // ***************************************************************************************************************************** if (testBegin("cmdArchiveGet()")) { + harnessLogLevelSet(logLevelDetail); + + // Arguments that must be included. Use raw config here because we need to keep the + StringList *argBaseList = strLstNew(); + strLstAddZ(argBaseList, "pgbackrest-bogus"); + hrnCfgArgRawZ(argBaseList, cfgOptPgPath, TEST_PATH_PG); + hrnCfgArgRawZ(argBaseList, cfgOptRepoPath, TEST_PATH_REPO); + hrnCfgArgRawZ(argBaseList, cfgOptStanza, "test1"); + hrnCfgArgRawZ(argBaseList, cfgOptArchiveTimeout, "1"); + strLstAddZ(argBaseList, CFGCMD_ARCHIVE_GET); + + // ------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("command must be run on the pg host"); - StringList *argList = strLstNew(); - hrnCfgArgRawZ(argList, cfgOptPgHost, "host"); - hrnCfgArgRawZ(argList, cfgOptPgPath, "/pg"); - hrnCfgArgRawZ(argList, cfgOptRepoPath, "/repo"); - strLstAddZ(argList, "--" CFGOPT_STANZA "=test2"); - strLstAddZ(argList, "000000010000000100000001"); - strLstAddZ(argList, "pg_wal/000000010000000100000001"); - harnessCfgLoadRole(cfgCmdArchiveGet, cfgCmdRoleDefault, argList); + StringList *argList = strLstDup(argBaseList); + hrnCfgArgRawZ(argList, cfgOptPgHost, BOGUS_STR); + harnessCfgLoadRaw(strLstSize(argList), strLstPtr(argList)); TEST_ERROR(cmdArchiveGet(), HostInvalidError, "archive-get command must be run on the PostgreSQL host"); // ------------------------------------------------------------------------------------------------------------------------- - argList = strLstNew(); - strLstAddZ(argList, "pgbackrest-bogus"); // Break this until async tests are setup correctly - strLstAddZ(argList, "--archive-timeout=1"); - strLstAdd(argList, strNewFmt("--lock-path=%s/lock", testPath())); - strLstAdd(argList, strNewFmt("--log-path=%s", testPath())); - strLstAdd(argList, strNewFmt("--log-level-file=debug")); - strLstAdd(argList, strNewFmt("--repo1-path=%s/repo", testPath())); - hrnCfgArgRawFmt(argList, cfgOptPgPath, "%s/db", testPath()); - strLstAddZ(argList, "--stanza=test1"); - strLstAddZ(argList, "archive-get"); + argList = strLstDup(argBaseList); harnessCfgLoadRaw(strLstSize(argList), strLstPtr(argList)); TEST_ERROR(cmdArchiveGet(), ParamRequiredError, "WAL segment to get required"); // ------------------------------------------------------------------------------------------------------------------------- - StringList *argListTemp = strLstDup(argList); - String *walSegment = strNew("000000010000000100000001"); - strLstAdd(argListTemp, walSegment); - harnessCfgLoadRaw(strLstSize(argListTemp), strLstPtr(argListTemp)); + argList = strLstDup(argBaseList); + strLstAddZ(argList, "000000010000000100000001"); + harnessCfgLoadRaw(strLstSize(argList), strLstPtr(argList)); TEST_ERROR(cmdArchiveGet(), ParamRequiredError, "path to copy WAL segment required"); // ------------------------------------------------------------------------------------------------------------------------- - storagePutP( - storageNewWriteP(storageTest, strNew("db/" PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL)), + HRN_STORAGE_PUT( + storagePgWrite(), PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL, pgControlTestToBuffer((PgControl){.version = PG_VERSION_10, .systemId = 0xFACEFACEFACEFACE})); - storagePathCreateP(storageTest, strNewFmt("%s/db/pg_wal", testPath())); + storagePathCreateP(storageTest, strNewFmt("%s/pg/pg_wal", testPath())); - String *walFile = strNewFmt("%s/db/pg_wal/RECOVERYXLOG", testPath()); - strLstAdd(argListTemp, walFile); - harnessCfgLoadRaw(strLstSize(argListTemp), strLstPtr(argListTemp)); + strLstAddZ(argList, TEST_PATH_PG "/pg_wal/RECOVERYXLOG"); + harnessCfgLoadRaw(strLstSize(argList), strLstPtr(argList)); TEST_ERROR_FMT( cmdArchiveGet(), FileMissingError, @@ -554,11 +390,12 @@ testRun(void) strZ(strNewFmt("%s/archive/test1/archive.info.copy", strZ(cfgOptionStr(cfgOptRepoPath))))); // ------------------------------------------------------------------------------------------------------------------------- - argListTemp = strLstDup(argList); - strLstAddZ(argListTemp, "00000001.history"); - strLstAdd(argListTemp, walFile); - strLstAddZ(argListTemp, "--archive-async"); - harnessCfgLoadRaw(strLstSize(argListTemp), strLstPtr(argListTemp)); + // !!! IS THIS TEST NEEDED + argList = strLstDup(argBaseList); + strLstAddZ(argList, "00000001.history"); + strLstAddZ(argList, TEST_PATH_PG "/pg_wal/RECOVERYHISTORY"); + strLstAddZ(argList, "--archive-async"); + harnessCfgLoadRaw(strLstSize(argList), strLstPtr(argList)); TEST_ERROR_FMT( cmdArchiveGet(), FileMissingError, @@ -576,9 +413,10 @@ testRun(void) // Make sure the process times out when there is nothing to get // ------------------------------------------------------------------------------------------------------------------------- - strLstAdd(argList, strNewFmt("--spool-path=%s/spool", testPath())); - strLstAddZ(argList, "--archive-async"); - strLstAdd(argList, walSegment); + argList = strLstDup(argBaseList); + hrnCfgArgRawZ(argList, cfgOptSpoolPath, TEST_PATH_SPOOL); + hrnCfgArgRawBool(argList, cfgOptArchiveAsync, true); + strLstAddZ(argList, "000000010000000100000001"); strLstAddZ(argList, "pg_wal/RECOVERYXLOG"); harnessCfgLoadRaw(strLstSize(argList), strLstPtr(argList)); @@ -590,51 +428,40 @@ testRun(void) // Check for missing WAL // ------------------------------------------------------------------------------------------------------------------------- - storagePutP(storageNewWriteP(storageSpoolWrite(), strNewFmt(STORAGE_SPOOL_ARCHIVE_IN "/%s.ok", strZ(walSegment))), NULL); + HRN_STORAGE_PUT_EMPTY(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_IN "/000000010000000100000001.ok"); TEST_RESULT_INT(cmdArchiveGet(), 1, "successful get of missing WAL"); harnessLogResult("P00 INFO: unable to find 000000010000000100000001 in the archive"); TEST_RESULT_BOOL( - storageExistsP(storageSpool(), strNewFmt(STORAGE_SPOOL_ARCHIVE_IN "/%s.ok", strZ(walSegment))), false, + storageExistsP(storageSpool(), STRDEF(STORAGE_SPOOL_ARCHIVE_IN "/000000010000000100000001.ok")), false, "check OK file was removed"); // Write out a WAL segment for success // ------------------------------------------------------------------------------------------------------------------------- - storagePutP( - storageNewWriteP(storageSpoolWrite(), strNewFmt(STORAGE_SPOOL_ARCHIVE_IN "/%s", strZ(walSegment))), - BUFSTRDEF("SHOULD-BE-A-REAL-WAL-FILE")); + HRN_STORAGE_PUT_Z(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_IN "/000000010000000100000001", "SHOULD-BE-A-REAL-WAL-FILE"); TEST_RESULT_INT(cmdArchiveGet(), 0, "successful get"); TEST_RESULT_VOID(harnessLogResult("P00 INFO: found 000000010000000100000001 in the archive"), "check log"); - TEST_RESULT_BOOL( - storageExistsP(storageSpoolWrite(), strNewFmt(STORAGE_SPOOL_ARCHIVE_IN "/%s", strZ(walSegment))), false, - "check WAL segment was removed from source"); - TEST_RESULT_BOOL(storageExistsP(storageTest, walFile), true, "check WAL segment was moved to destination"); - storageRemoveP(storageTest, walFile, .errorOnMissing = true); + TEST_STORAGE_LIST_EMPTY(storageSpool(), STORAGE_SPOOL_ARCHIVE_IN); + TEST_STORAGE_LIST(storageTest, TEST_PATH_PG "/pg_wal", "RECOVERYXLOG\n", .remove = true); // Write more WAL segments (in this case queue should be full) // ------------------------------------------------------------------------------------------------------------------------- strLstAddZ(argList, "--archive-get-queue-max=48"); harnessCfgLoadRaw(strLstSize(argList), strLstPtr(argList)); - String *walSegment2 = strNew("000000010000000100000002"); - - storagePutP( - storageNewWriteP(storageSpoolWrite(), strNewFmt(STORAGE_SPOOL_ARCHIVE_IN "/%s", strZ(walSegment))), - BUFSTRDEF("SHOULD-BE-A-REAL-WAL-FILE")); - storagePutP( - storageNewWriteP(storageSpoolWrite(), strNewFmt(STORAGE_SPOOL_ARCHIVE_IN "/%s", strZ(walSegment2))), - BUFSTRDEF("SHOULD-BE-A-REAL-WAL-FILE")); + HRN_STORAGE_PUT_Z(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_IN "/000000010000000100000001", "SHOULD-BE-A-REAL-WAL-FILE"); + HRN_STORAGE_PUT_Z(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_IN "/000000010000000100000002", "SHOULD-BE-A-REAL-WAL-FILE"); TEST_RESULT_INT(cmdArchiveGet(), 0, "successful get"); TEST_RESULT_VOID(harnessLogResult("P00 INFO: found 000000010000000100000001 in the archive"), "check log"); - TEST_RESULT_BOOL(storageExistsP(storageTest, walFile), true, "check WAL segment was moved"); + TEST_STORAGE_LIST(storageTest, TEST_PATH_PG "/pg_wal", "RECOVERYXLOG\n", .remove = true); // Make sure the process times out when it can't get a lock // ------------------------------------------------------------------------------------------------------------------------- @@ -653,6 +480,132 @@ testRun(void) harnessCfgLoadRaw(strLstSize(argList), strLstPtr(argList)); TEST_ERROR(cmdArchiveGet(), ParamInvalidError, "extra parameters found"); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("file is missing"); + + HRN_STORAGE_PUT( + storagePgWrite(), PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL, + pgControlTestToBuffer((PgControl){.version = PG_VERSION_10, .systemId = 0xFACEFACEFACEFACE})); + + HRN_INFO_PUT( + storageRepoWrite(), INFO_ARCHIVE_PATH_FILE, + "[db]\n" + "db-id=1\n" + "\n" + "[db:history]\n" + "1={\"db-id\":18072658121562454734,\"db-version\":\"10\"}"); + + argList = strLstNew(); + hrnCfgArgRawZ(argList, cfgOptPgPath, TEST_PATH_PG); + hrnCfgArgRawZ(argList, cfgOptRepoPath, TEST_PATH_REPO); + hrnCfgArgRawZ(argList, cfgOptStanza, "test1"); + strLstAddZ(argList, "01ABCDEF01ABCDEF01ABCDEF"); + strLstAddZ(argList, TEST_PATH_PG "/pg_wal/RECOVERYXLOG"); + harnessCfgLoad(cfgCmdArchiveGet, argList); + + TEST_RESULT_INT(cmdArchiveGet(), 1, "get"); + + harnessLogResult("P00 INFO: unable to find 01ABCDEF01ABCDEF01ABCDEF in the archive"); + + TEST_STORAGE_LIST_EMPTY(storageTest, TEST_PATH_PG "/pg_wal"); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("get WAL segment"); + + Buffer *buffer = bufNew(16 * 1024 * 1024); + memset(bufPtr(buffer), 0, bufSize(buffer)); + bufUsedSet(buffer, bufSize(buffer)); + + HRN_STORAGE_PUT( + storageRepoWrite(), STORAGE_REPO_ARCHIVE "/10-1/01ABCDEF01ABCDEF01ABCDEF-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + buffer); + + TEST_RESULT_INT(cmdArchiveGet(), 0, "get"); + + harnessLogResult("P00 INFO: found 01ABCDEF01ABCDEF01ABCDEF in the archive"); + + TEST_RESULT_UINT( + storageInfoP(storageTest, STRDEF(TEST_PATH_PG "/pg_wal/RECOVERYXLOG")).size, 16 * 1024 * 1024, "check size"); + TEST_STORAGE_LIST(storageTest, TEST_PATH_PG "/pg_wal", "RECOVERYXLOG\n", .remove = true); + + TEST_STORAGE_REMOVE( + storageRepoWrite(), STORAGE_REPO_ARCHIVE "/10-1/01ABCDEF01ABCDEF01ABCDEF-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("get compressed and encrypted WAL segment"); + + HRN_INFO_PUT( + storageRepoWrite(), INFO_ARCHIVE_PATH_FILE, + "[cipher]\n" + "cipher-pass=\"" TEST_CIPHER_PASS_ARCHIVE "\"\n" + "\n" + "[db]\n" + "db-id=1\n" + "\n" + "[db:history]\n" + "1={\"db-id\":18072658121562454734,\"db-version\":\"10\"}", + .cipherType = cipherTypeAes256Cbc); + + HRN_STORAGE_PUT( + storageRepoWrite(), STORAGE_REPO_ARCHIVE "/10-1/01ABCDEF01ABCDEF01ABCDEF-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + buffer, .compressType = compressTypeGz, .cipherType = cipherTypeAes256Cbc, .cipherPass = TEST_CIPHER_PASS_ARCHIVE); + + // Add encryption options + hrnCfgArgRawZ(argList, cfgOptRepoCipherType, CIPHER_TYPE_AES_256_CBC); + hrnCfgEnvRawZ(cfgOptRepoCipherPass, TEST_CIPHER_PASS); + harnessCfgLoad(cfgCmdArchiveGet, argList); + hrnCfgEnvRemoveRaw(cfgOptRepoCipherPass); + + TEST_RESULT_INT(cmdArchiveGet(), 0, "get"); + + harnessLogResult("P00 INFO: found 01ABCDEF01ABCDEF01ABCDEF in the archive"); + + TEST_STORAGE_LIST(storageTest, TEST_PATH_PG "/pg_wal", "RECOVERYXLOG\n"); + TEST_RESULT_UINT( + storageInfoP(storageTest, STRDEF(TEST_PATH_PG "/pg_wal/RECOVERYXLOG")).size, 16 * 1024 * 1024, "check size"); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("call protocol function directly"); + + // Start a protocol server + Buffer *serverWrite = bufNew(8192); + IoWrite *serverWriteIo = ioBufferWriteNew(serverWrite); + ioWriteOpen(serverWriteIo); + + ProtocolServer *server = protocolServerNew( + strNew("test"), strNew("test"), ioBufferReadNew(bufNew(0)), serverWriteIo); + + bufUsedSet(serverWrite, 0); + + // Add spool path + argList = strLstNew(); + hrnCfgArgRawZ(argList, cfgOptPgPath, TEST_PATH_PG); + hrnCfgArgRawZ(argList, cfgOptRepoPath, TEST_PATH_REPO); + hrnCfgArgRawZ(argList, cfgOptStanza, "test1"); + hrnCfgArgRawZ(argList, cfgOptRepoCipherType, CIPHER_TYPE_AES_256_CBC); + hrnCfgArgRawZ(argList, cfgOptSpoolPath, TEST_PATH_SPOOL); + hrnCfgArgRawBool(argList, cfgOptArchiveAsync, true); + hrnCfgEnvRawZ(cfgOptRepoCipherPass, TEST_CIPHER_PASS); + harnessCfgLoadRole(cfgCmdArchiveGet, cfgCmdRoleLocal, argList); + hrnCfgEnvRemoveRaw(cfgOptRepoCipherPass); + + // Setup protocol command + VariantList *paramList = varLstNew(); + varLstAdd(paramList, varNewStrZ("01ABCDEF01ABCDEF01ABCDEF")); + + TEST_RESULT_BOOL( + archiveGetProtocol(PROTOCOL_COMMAND_ARCHIVE_GET_STR, paramList, server), true, "protocol archive get"); + + TEST_RESULT_STR_Z(strNewBuf(serverWrite), "{\"out\":0}\n", "check result"); + TEST_STORAGE_LIST(storageSpool(), STORAGE_SPOOL_ARCHIVE_IN, "000000010000000100000002\n01ABCDEF01ABCDEF01ABCDEF\n"); + + bufUsedSet(serverWrite, 0); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("invalid protocol command"); + + TEST_RESULT_BOOL(archiveGetProtocol(strNew(BOGUS_STR), paramList, server), false, "invalid function"); } FUNCTION_HARNESS_RESULT_VOID();