db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2025-07-13 01:00:23 +02:00
Code Issues Releases Activity

Update command/archive tests to use standard patterns.

Browse Source 
Includes archiveCommon, archiveGet and archivePush.

Also fixed a test that was looking in repo instead of repo3 in the original archivePush to use the repo3 path as stated by the comment (line 879 in original tests and line 855 in new tests).
This commit is contained in:
Cynthia Shang
2021-07-02 12:22:11 -04:00
committed by GitHub
parent 23bdc3deb6
commit 62e8d97af0
4 changed files with 501 additions and 480 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
doc/xml/release.xml
View File

@ -96,6 +96,9 @@
<commit subject="Update info/manifest test to use standard patterns."> <commit subject="Update info/manifest test to use standard patterns.">
<github-pull-request id="1444"/> <github-pull-request id="1444"/>
</commit> </commit>
<commit subject="Update command/archive tests to use standard patterns.">
<github-pull-request id="1449"/>
</commit>
<release-item-contributor-list> <release-item-contributor-list>
<release-item-contributor id="cynthia.shang"/> <release-item-contributor id="cynthia.shang"/>

189
test/src/module/command/archiveCommonTest.c
View File

@ -25,67 +25,61 @@ testRun(void)
if (testBegin("archiveAsyncErrorClear() and archiveAsyncStatus()")) if (testBegin("archiveAsyncErrorClear() and archiveAsyncStatus()"))
{ {
StringList *argList = strLstNew(); StringList *argList = strLstNew();
strLstAddZ(argList, "--spool-path=" TEST_PATH); hrnCfgArgRawZ(argList, cfgOptStanza, "db");
strLstAddZ(argList, "--archive-async"); hrnCfgArgRawZ(argList, cfgOptSpoolPath, TEST_PATH);
strLstAddZ(argList, "--archive-timeout=1"); hrnCfgArgRawBool(argList, cfgOptArchiveAsync, true);
strLstAddZ(argList, "--stanza=db"); hrnCfgArgRawZ(argList, cfgOptArchiveTimeout, "1");
HRN_CFG_LOAD(cfgCmdArchivePush, argList); HRN_CFG_LOAD(cfgCmdArchivePush, argList);
// ------------------------------------------------------------------------------------------------------------------------- // -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("archiveAsyncStatus() - directory and status file not present");
const String *segment = STRDEF("000000010000000100000001"); const String *segment = STRDEF("000000010000000100000001");
TEST_RESULT_BOOL(archiveAsyncStatus(archiveModePush, segment, false, true), false, "directory and status file not present"); TEST_RESULT_BOOL(archiveAsyncStatus(archiveModePush, segment, false, true), false, "directory and status file not present");
TEST_RESULT_BOOL(archiveAsyncStatus(archiveModeGet, segment, false, true), false, "directory and status file not present"); TEST_RESULT_BOOL(archiveAsyncStatus(archiveModeGet, segment, false, true), false, "directory and status file not present");
// ------------------------------------------------------------------------------------------------------------------------- // -------------------------------------------------------------------------------------------------------------------------
mkdir(TEST_PATH "/archive", 0750); TEST_TITLE("archiveAsyncStatus() - directories present and status file not present");
mkdir(TEST_PATH "/archive/db", 0750);
mkdir(TEST_PATH "/archive/db/out", 0750); HRN_STORAGE_PATH_CREATE(storageTest, "archive/db/out", .mode = 0750);
TEST_RESULT_BOOL(archiveAsyncStatus(archiveModePush, segment, false, true), false, "status file not present"); TEST_RESULT_BOOL(archiveAsyncStatus(archiveModePush, segment, false, true), false, "status file not present");
// ------------------------------------------------------------------------------------------------------------------------- // -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("clear archive file errors"); TEST_TITLE("clear archive file errors");
const String *errorSegment = strNewFmt(STORAGE_SPOOL_ARCHIVE_OUT "/%s.error", strZ(segment)); HRN_STORAGE_PUT_EMPTY(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_OUT "/000000010000000100000001.error");
const String *errorGlobal = STRDEF(STORAGE_SPOOL_ARCHIVE_OUT "/global.error"); HRN_STORAGE_PUT_EMPTY(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_OUT "/global.error");
storagePutP(storageNewWriteP(storageSpoolWrite(), errorSegment), NULL); TEST_STORAGE_LIST(storageSpool(), STORAGE_SPOOL_ARCHIVE_OUT, "000000010000000100000001.error\nglobal.error\n");
storagePutP(storageNewWriteP(storageSpoolWrite(), errorGlobal), NULL);
TEST_RESULT_VOID(archiveAsyncErrorClear(archiveModePush, segment), "clear error"); TEST_RESULT_VOID(archiveAsyncErrorClear(archiveModePush, segment), "clear error");
TEST_RESULT_BOOL(storageExistsP(storageSpool(), errorSegment), false, " check segment error"); TEST_STORAGE_LIST_EMPTY(storageSpool(), STORAGE_SPOOL_ARCHIVE_OUT, .comment = "segment and global error cleared");
TEST_RESULT_BOOL(storageExistsP(storageSpool(), errorGlobal), false, " check global error");
// ------------------------------------------------------------------------------------------------------------------------- // -------------------------------------------------------------------------------------------------------------------------
storagePutP( TEST_TITLE("check ok file");
storageNewWriteP(storageSpoolWrite(), strNewFmt(STORAGE_SPOOL_ARCHIVE_OUT "/%s.ok", strZ(segment))),
BUFSTRDEF(BOGUS_STR)); HRN_STORAGE_PUT_Z(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_OUT "/000000010000000100000001.ok", BOGUS_STR);
TEST_ERROR( TEST_ERROR(
archiveAsyncStatus(archiveModePush, segment, false, true), FormatError, archiveAsyncStatus(archiveModePush, segment, false, true), FormatError,
"000000010000000100000001.ok content must have at least two lines"); "000000010000000100000001.ok content must have at least two lines");
storagePutP( HRN_STORAGE_PUT_Z(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_OUT "/000000010000000100000001.ok", BOGUS_STR "\n");
storageNewWriteP(storageSpoolWrite(), strNewFmt(STORAGE_SPOOL_ARCHIVE_OUT "/%s.ok", strZ(segment))),
BUFSTRDEF(BOGUS_STR "\n"));
TEST_ERROR( TEST_ERROR(
archiveAsyncStatus(archiveModePush, segment, false, true), FormatError, archiveAsyncStatus(archiveModePush, segment, false, true), FormatError,
"000000010000000100000001.ok message must be > 0"); "000000010000000100000001.ok message must be > 0");
storagePutP( HRN_STORAGE_PUT_Z(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_OUT "/000000010000000100000001.ok", BOGUS_STR "\nmessage");
storageNewWriteP(storageSpoolWrite(), strNewFmt(STORAGE_SPOOL_ARCHIVE_OUT "/%s.ok", strZ(segment))),
BUFSTRDEF(BOGUS_STR "\nmessage"));
TEST_ERROR( TEST_ERROR(
archiveAsyncStatus(archiveModePush, segment, false, true), archiveAsyncStatus(archiveModePush, segment, false, true),
FormatError, "unable to convert base 10 string 'BOGUS' to int"); FormatError, "unable to convert base 10 string 'BOGUS' to int");
storagePutP(storageNewWriteP(storageSpoolWrite(), strNewFmt(STORAGE_SPOOL_ARCHIVE_OUT "/%s.ok", strZ(segment))), NULL); HRN_STORAGE_PUT_EMPTY(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_OUT "/000000010000000100000001.ok");
TEST_RESULT_BOOL(archiveAsyncStatus(archiveModePush, segment, false, true), true, "ok file"); TEST_RESULT_BOOL(archiveAsyncStatus(archiveModePush, segment, false, true), true, "ok file");
storagePutP( HRN_STORAGE_PUT_Z(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_OUT "/000000010000000100000001.ok", "0\nwarning");
storageNewWriteP(storageSpoolWrite(), strNewFmt(STORAGE_SPOOL_ARCHIVE_OUT "/%s.ok", strZ(segment))),
BUFSTRDEF("0\nwarning"));
TEST_RESULT_BOOL(archiveAsyncStatus(archiveModePush, segment, false, true), true, "ok file with warning"); TEST_RESULT_BOOL(archiveAsyncStatus(archiveModePush, segment, false, true), true, "ok file with warning");
TEST_RESULT_LOG("P00 WARN: warning"); TEST_RESULT_LOG("P00 WARN: warning");
@ -97,36 +91,31 @@ testRun(void)
TEST_RESULT_LOG(""); TEST_RESULT_LOG("");
// ------------------------------------------------------------------------------------------------------------------------- // -------------------------------------------------------------------------------------------------------------------------
storagePutP( TEST_TITLE("error status renamed to ok");
storageNewWriteP(storageSpoolWrite(), strNewFmt(STORAGE_SPOOL_ARCHIVE_OUT "/%s.ok", strZ(segment))),
BUFSTRDEF("25\nerror")); HRN_STORAGE_PUT_Z(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_OUT "/000000010000000100000001.ok", "25\nerror");
TEST_RESULT_BOOL(archiveAsyncStatus(archiveModePush, segment, false, true), true, "error status renamed to ok"); TEST_RESULT_BOOL(archiveAsyncStatus(archiveModePush, segment, false, true), true, "error status renamed to ok");
TEST_RESULT_LOG( TEST_RESULT_LOG(
"P00 WARN: WAL segment '000000010000000100000001' was not pushed due to error [25] and was manually skipped: error"); "P00 WARN: WAL segment '000000010000000100000001' was not pushed due to error [25] and was manually skipped: error");
TEST_RESULT_VOID( HRN_STORAGE_REMOVE(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_OUT "/000000010000000100000001.ok", .errorOnMissing = true);
storageRemoveP(
storageSpoolWrite(), strNewFmt(STORAGE_SPOOL_ARCHIVE_OUT "/%s.ok", strZ(segment)), .errorOnMissing = true),
"remove ok");
// ------------------------------------------------------------------------------------------------------------------------- // -------------------------------------------------------------------------------------------------------------------------
storagePutP( TEST_TITLE("segment error file - AssertError");
storageNewWriteP(storageSpoolWrite(), strNewFmt(STORAGE_SPOOL_ARCHIVE_OUT "/%s.error", strZ(segment))), bufNew(0));
HRN_STORAGE_PUT_EMPTY(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_OUT "/000000010000000100000001.error");
TEST_ERROR( TEST_ERROR(
archiveAsyncStatus(archiveModePush, segment, true, true), AssertError, archiveAsyncStatus(archiveModePush, segment, true, true), AssertError,
"status file '000000010000000100000001.error' has no content"); "status file '000000010000000100000001.error' has no content");
storagePutP( HRN_STORAGE_PUT_Z(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_OUT "/000000010000000100000001.error", "25\nmessage");
storageNewWriteP(storageSpoolWrite(), strNewFmt(STORAGE_SPOOL_ARCHIVE_OUT "/%s.error", strZ(segment))),
BUFSTRDEF("25\nmessage"));
TEST_ERROR(archiveAsyncStatus(archiveModePush, segment, true, true), AssertError, "message"); TEST_ERROR(archiveAsyncStatus(archiveModePush, segment, true, true), AssertError, "message");
TEST_RESULT_BOOL(archiveAsyncStatus(archiveModePush, segment, false, true), false, "suppress error"); TEST_RESULT_BOOL(archiveAsyncStatus(archiveModePush, segment, false, true), false, "suppress error");
// ------------------------------------------------------------------------------------------------------------------------- // -------------------------------------------------------------------------------------------------------------------------
storagePutP( TEST_TITLE("global error file - ExecuteError");
storageNewWriteP(storageSpoolWrite(), STRDEF(STORAGE_SPOOL_ARCHIVE_OUT "/global.error")),
BUFSTRDEF("102\nexecute error"));
HRN_STORAGE_PUT_Z(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_OUT "/global.error", "102\nexecute error");
TEST_ERROR(archiveAsyncStatus(archiveModePush, STRDEF("anyfile"), true, true), ExecuteError, "execute error"); TEST_ERROR(archiveAsyncStatus(archiveModePush, STRDEF("anyfile"), true, true), ExecuteError, "execute error");
} }
@ -134,49 +123,40 @@ testRun(void)
if (testBegin("archiveAsyncStatusErrorWrite() and archiveAsyncStatusOkWrite()")) if (testBegin("archiveAsyncStatusErrorWrite() and archiveAsyncStatusOkWrite()"))
{ {
StringList *argList = strLstNew(); StringList *argList = strLstNew();
strLstAddZ(argList, "--spool-path=" TEST_PATH); hrnCfgArgRawZ(argList, cfgOptStanza, "db");
strLstAddZ(argList, "--stanza=db"); hrnCfgArgRawZ(argList, cfgOptSpoolPath, TEST_PATH);
hrnCfgArgRawZ(argList, cfgOptPgPath, "/path/to/pg"); hrnCfgArgRawZ(argList, cfgOptPgPath, "/path/to/pg");
strLstAddZ(argList, "--" CFGOPT_ARCHIVE_ASYNC); hrnCfgArgRawBool(argList, cfgOptArchiveAsync, true);
HRN_CFG_LOAD(cfgCmdArchiveGet, argList, .role = cfgCmdRoleAsync); HRN_CFG_LOAD(cfgCmdArchiveGet, argList, .role = cfgCmdRoleAsync);
const String *walSegment = STRDEF("000000010000000100000001"); const String *walSegment = STRDEF("000000010000000100000001");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("archiveAsyncStatusErrorWrite()");
TEST_RESULT_VOID( TEST_RESULT_VOID(
archiveAsyncStatusErrorWrite(archiveModeGet, walSegment, 25, STRDEF("error message")), "write error"); archiveAsyncStatusErrorWrite(archiveModeGet, walSegment, 25, STRDEF("error message")), "write segment error");
TEST_RESULT_STR_Z( TEST_STORAGE_GET(
strNewBuf(storageGetP(storageNewReadP(storageTest, STRDEF("archive/db/in/000000010000000100000001.error")))), storageTest, "archive/db/in/000000010000000100000001.error", "25\nerror message", .remove = true,
"25\nerror message", "check error"); .comment = "check segment error and remove");
TEST_RESULT_VOID(
storageRemoveP(storageTest, STRDEF("archive/db/in/000000010000000100000001.error"), .errorOnMissing = true),
"remove error");
TEST_RESULT_VOID( TEST_RESULT_VOID(
archiveAsyncStatusErrorWrite(archiveModeGet, NULL, 25, STRDEF("global error message")), "write global error"); archiveAsyncStatusErrorWrite(archiveModeGet, NULL, 25, STRDEF("global error message")), "write global error");
TEST_RESULT_STR_Z( TEST_STORAGE_GET(
strNewBuf(storageGetP(storageNewReadP(storageTest, STRDEF("archive/db/in/global.error")))), storageTest, "archive/db/in/global.error", "25\nglobal error message", .remove = true,
"25\nglobal error message", "check global error"); .comment = "check global error and remove");
TEST_RESULT_VOID(
storageRemoveP(storageTest, STRDEF("archive/db/in/global.error"), .errorOnMissing = true),
"remove global error");
TEST_RESULT_VOID( // -------------------------------------------------------------------------------------------------------------------------
archiveAsyncStatusOkWrite(archiveModeGet, walSegment, NULL), "write ok file"); TEST_TITLE("archiveAsyncStatusOkWrite()");
TEST_RESULT_STR_Z(
strNewBuf(storageGetP(storageNewReadP(storageTest, STRDEF("archive/db/in/000000010000000100000001.ok")))),
"", "check ok");
TEST_RESULT_VOID(
storageRemoveP(storageTest, STRDEF("archive/db/in/000000010000000100000001.ok"), .errorOnMissing = true),
"remove ok");
TEST_RESULT_VOID( TEST_RESULT_VOID(archiveAsyncStatusOkWrite(archiveModeGet, walSegment, NULL), "write ok file");
archiveAsyncStatusOkWrite(archiveModeGet, walSegment, STRDEF("WARNING")), "write ok file with warning"); TEST_STORAGE_GET(
TEST_RESULT_STR_Z( storageTest, "archive/db/in/000000010000000100000001.ok", "", .remove = true, .comment = "check ok and remove");
strNewBuf(storageGetP(storageNewReadP(storageTest, STRDEF("archive/db/in/000000010000000100000001.ok")))),
"0\nWARNING", "check ok warning"); TEST_RESULT_VOID(archiveAsyncStatusOkWrite(archiveModeGet, walSegment, STRDEF("WARNING")), "write ok file with warning");
TEST_RESULT_VOID( TEST_STORAGE_GET(
storageRemoveP(storageTest, STRDEF("archive/db/in/000000010000000100000001.ok"), .errorOnMissing = true), storageTest, "archive/db/in/000000010000000100000001.ok", "0\nWARNING", .remove = true,
"remove ok"); .comment = "check ok warning and remove");
} }
// ***************************************************************************************************************************** // *****************************************************************************************************************************
@ -197,8 +177,16 @@ testRun(void)
// ***************************************************************************************************************************** // *****************************************************************************************************************************
if (testBegin("walPath()")) if (testBegin("walPath()"))
{ {
StringList *argList = strLstNew();
hrnCfgArgRawZ(argList, cfgOptStanza, "db");
hrnCfgArgRawZ(argList, cfgOptPgPath, "/path/to/pg");
HRN_CFG_LOAD(cfgCmdArchiveGet, argList);
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("absolute and relative paths");
const String *pgPath = storagePathP(storageTest, STRDEF("pg")); const String *pgPath = storagePathP(storageTest, STRDEF("pg"));
storagePathCreateP(storageTest, pgPath); HRN_STORAGE_PATH_CREATE(storageTest, strZ(pgPath));
TEST_RESULT_STR_Z(walPath(STRDEF("/absolute/path"), pgPath, STRDEF("test")), "/absolute/path", "absolute path"); TEST_RESULT_STR_Z(walPath(STRDEF("/absolute/path"), pgPath, STRDEF("test")), "/absolute/path", "absolute path");
@ -206,6 +194,9 @@ testRun(void)
TEST_RESULT_STR( TEST_RESULT_STR(
walPath(STRDEF("relative/path"), pgPath, STRDEF("test")), strNewFmt("%s/relative/path", strZ(pgPath)), "relative path"); walPath(STRDEF("relative/path"), pgPath, STRDEF("test")), strNewFmt("%s/relative/path", strZ(pgPath)), "relative path");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("symlink path");
const String *pgPathLink = storagePathP(storageTest, STRDEF("pg-link")); const String *pgPathLink = storagePathP(storageTest, STRDEF("pg-link"));
THROW_ON_SYS_ERROR_FMT( THROW_ON_SYS_ERROR_FMT(
symlink(strZ(pgPath), strZ(pgPathLink)) == -1, FileOpenError, "unable to create symlink '%s' to '%s'", strZ(pgPath), symlink(strZ(pgPath), strZ(pgPathLink)) == -1, FileOpenError, "unable to create symlink '%s' to '%s'", strZ(pgPath),
@ -216,6 +207,9 @@ testRun(void)
walPath(STRDEF("relative/path"), pgPathLink, STRDEF("test")), strNewFmt("%s/relative/path", strZ(pgPathLink)), walPath(STRDEF("relative/path"), pgPathLink, STRDEF("test")), strNewFmt("%s/relative/path", strZ(pgPathLink)),
"relative path"); "relative path");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("path errors");
THROW_ON_SYS_ERROR(chdir("/") != 0, PathMissingError, "unable to chdir()"); THROW_ON_SYS_ERROR(chdir("/") != 0, PathMissingError, "unable to chdir()");
TEST_ERROR( TEST_ERROR(
walPath(STRDEF("relative/path"), pgPathLink, STRDEF("test")), OptionInvalidValueError, walPath(STRDEF("relative/path"), pgPathLink, STRDEF("test")), OptionInvalidValueError,
@ -234,15 +228,17 @@ testRun(void)
{ {
// Load configuration to set repo-path and stanza // Load configuration to set repo-path and stanza
StringList *argList = strLstNew(); StringList *argList = strLstNew();
strLstAddZ(argList, "--stanza=db"); hrnCfgArgRawZ(argList, cfgOptStanza, "db");
hrnCfgArgRawZ(argList, cfgOptPgPath, "/path/to/pg"); hrnCfgArgRawZ(argList, cfgOptPgPath, "/path/to/pg");
strLstAddZ(argList, "--repo-path=" TEST_PATH); hrnCfgArgRawZ(argList, cfgOptRepoPath, TEST_PATH);
strLstAddZ(argList, "archive-get");
HRN_CFG_LOAD(cfgCmdArchiveGet, argList); HRN_CFG_LOAD(cfgCmdArchiveGet, argList);
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("no path or segment");
TEST_RESULT_STR(walSegmentFind(storageRepo(), STRDEF("9.6-2"), STRDEF("123456781234567812345678"), 0), NULL, "no path"); TEST_RESULT_STR(walSegmentFind(storageRepo(), STRDEF("9.6-2"), STRDEF("123456781234567812345678"), 0), NULL, "no path");
storagePathCreateP(storageTest, STRDEF("archive/db/9.6-2/1234567812345678")); HRN_STORAGE_PATH_CREATE(storageTest, "archive/db/9.6-2/1234567812345678");
TEST_RESULT_STR( TEST_RESULT_STR(
walSegmentFind(storageRepo(), STRDEF("9.6-2"), STRDEF("123456781234567812345678"), 0), NULL, "no segment"); walSegmentFind(storageRepo(), STRDEF("9.6-2"), STRDEF("123456781234567812345678"), 0), NULL, "no segment");
TEST_ERROR( TEST_ERROR(
@ -252,6 +248,9 @@ testRun(void)
"HINT: check the PostgreSQL server log for errors.\n" "HINT: check the PostgreSQL server log for errors.\n"
"HINT: run the 'start' command if the stanza was previously stopped."); "HINT: run the 'start' command if the stanza was previously stopped.");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("timeout");
// Check timeout by making the wal segment appear after 250ms // Check timeout by making the wal segment appear after 250ms
HARNESS_FORK_BEGIN() HARNESS_FORK_BEGIN()
{ {
@ -259,12 +258,9 @@ testRun(void)
{ {
sleepMSec(250); sleepMSec(250);
storagePutP( HRN_STORAGE_PUT_EMPTY(
storageNewWriteP(
storageTest, storageTest,
STRDEF( "archive/db/9.6-2/1234567812345678/123456781234567812345678-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa");
"archive/db/9.6-2/1234567812345678/123456781234567812345678-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")),
NULL);
} }
HARNESS_FORK_CHILD_END(); HARNESS_FORK_CHILD_END();
@ -278,11 +274,11 @@ testRun(void)
} }
HARNESS_FORK_END(); HARNESS_FORK_END();
storagePutP( // -------------------------------------------------------------------------------------------------------------------------
storageNewWriteP( TEST_TITLE("duplicate");
storageTest,
STRDEF("archive/db/9.6-2/1234567812345678/123456781234567812345678-bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb.gz")), HRN_STORAGE_PUT_EMPTY(
NULL); storageTest, "archive/db/9.6-2/1234567812345678/123456781234567812345678-bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb.gz");
TEST_ERROR( TEST_ERROR(
walSegmentFind(storageRepo(), STRDEF("9.6-2"), STRDEF("123456781234567812345678"), 0), walSegmentFind(storageRepo(), STRDEF("9.6-2"), STRDEF("123456781234567812345678"), 0),
@ -292,6 +288,9 @@ testRun(void)
", 123456781234567812345678-bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb.gz" ", 123456781234567812345678-bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb.gz"
"\nHINT: are multiple primaries archiving to this stanza?"); "\nHINT: are multiple primaries archiving to this stanza?");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("partial not found");
TEST_RESULT_STR( TEST_RESULT_STR(
walSegmentFind(storageRepo(), STRDEF("9.6-2"), STRDEF("123456781234567812345678.partial"), 0), NULL, walSegmentFind(storageRepo(), STRDEF("9.6-2"), STRDEF("123456781234567812345678.partial"), 0), NULL,
"did not find partial segment"); "did not find partial segment");
@ -300,12 +299,19 @@ testRun(void)
// ***************************************************************************************************************************** // *****************************************************************************************************************************
if (testBegin("walSegmentNext()")) if (testBegin("walSegmentNext()"))
{ {
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("beginning and end range");
TEST_RESULT_STR_Z( TEST_RESULT_STR_Z(
walSegmentNext(STRDEF("000000010000000100000001"), 16 * 1024 * 1024, PG_VERSION_10), "000000010000000100000002", walSegmentNext(STRDEF("000000010000000100000001"), 16 * 1024 * 1024, PG_VERSION_10), "000000010000000100000002",
"get next"); "get next");
TEST_RESULT_STR_Z( TEST_RESULT_STR_Z(
walSegmentNext(STRDEF("0000000100000001000000FE"), 16 * 1024 * 1024, PG_VERSION_93), "0000000100000001000000FF", walSegmentNext(STRDEF("0000000100000001000000FE"), 16 * 1024 * 1024, PG_VERSION_93), "0000000100000001000000FF",
"get next"); "get next");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("check overflow by version");
TEST_RESULT_STR_Z( TEST_RESULT_STR_Z(
walSegmentNext(STRDEF("0000009900000001000000FF"), 16 * 1024 * 1024, PG_VERSION_93), "000000990000000200000000", walSegmentNext(STRDEF("0000009900000001000000FF"), 16 * 1024 * 1024, PG_VERSION_93), "000000990000000200000000",
"get next overflow >= 9.3"); "get next overflow >= 9.3");
@ -323,9 +329,16 @@ testRun(void)
// ***************************************************************************************************************************** // *****************************************************************************************************************************
if (testBegin("walSegmentRange()")) if (testBegin("walSegmentRange()"))
{ {
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("single segment");
TEST_RESULT_STRLST_Z( TEST_RESULT_STRLST_Z(
walSegmentRange(STRDEF("000000010000000100000000"), 16 * 1024 * 1024, PG_VERSION_92, 1), "000000010000000100000000\n", walSegmentRange(STRDEF("000000010000000100000000"), 16 * 1024 * 1024, PG_VERSION_92, 1), "000000010000000100000000\n",
"get single"); "get single");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("check range by version");
TEST_RESULT_STRLST_Z( TEST_RESULT_STRLST_Z(
walSegmentRange(STRDEF("0000000100000001000000FD"), 16 * 1024 * 1024, PG_VERSION_92, 4), walSegmentRange(STRDEF("0000000100000001000000FD"), 16 * 1024 * 1024, PG_VERSION_92, 4),
"0000000100000001000000FD\n0000000100000001000000FE\n000000010000000200000000\n000000010000000200000001\n", "0000000100000001000000FD\n0000000100000001000000FE\n000000010000000200000000\n000000010000000200000001\n",

152
test/src/module/command/archiveGetTest.c
View File

@ -23,20 +23,25 @@ testRun(void)
if (testBegin("queueNeed()")) if (testBegin("queueNeed()"))
{ {
StringList *argList = strLstNew(); StringList *argList = strLstNew();
strLstAddZ(argList, "--stanza=test1"); hrnCfgArgRawZ(argList, cfgOptStanza, "test1");
strLstAddZ(argList, "--archive-async"); hrnCfgArgRawBool(argList, cfgOptArchiveAsync, true);
hrnCfgArgRawZ(argList, cfgOptPgPath, "/unused"); hrnCfgArgRawZ(argList, cfgOptPgPath, "/unused");
strLstAddZ(argList, "--spool-path=" TEST_PATH "/spool"); hrnCfgArgRawZ(argList, cfgOptSpoolPath, TEST_PATH "/spool");
HRN_CFG_LOAD(cfgCmdArchiveGet, argList); HRN_CFG_LOAD(cfgCmdArchiveGet, argList);
size_t queueSize = 16 * 1024 * 1024; size_t queueSize = 16 * 1024 * 1024;
size_t walSegmentSize = 16 * 1024 * 1024; size_t walSegmentSize = 16 * 1024 * 1024;
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("path missing");
TEST_ERROR( TEST_ERROR(
queueNeed(STRDEF("000000010000000100000001"), false, queueSize, walSegmentSize, PG_VERSION_92), queueNeed(STRDEF("000000010000000100000001"), false, queueSize, walSegmentSize, PG_VERSION_92),
PathMissingError, "unable to list file info for missing path '" TEST_PATH "/spool/archive/test1/in'"); PathMissingError, "unable to list file info for missing path '" TEST_PATH "/spool/archive/test1/in'");
// ------------------------------------------------------------------------------------------------------------------------- // -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("queue size too small");
HRN_STORAGE_PATH_CREATE(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_IN); HRN_STORAGE_PATH_CREATE(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_IN);
TEST_RESULT_STRLST_Z( TEST_RESULT_STRLST_Z(
@ -44,6 +49,8 @@ testRun(void)
"000000010000000100000001\n000000010000000100000002\n", "queue size smaller than min"); "000000010000000100000001\n000000010000000100000002\n", "queue size smaller than min");
// ------------------------------------------------------------------------------------------------------------------------- // -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("queue empty");
queueSize = (16 * 1024 * 1024) * 3; queueSize = (16 * 1024 * 1024) * 3;
TEST_RESULT_STRLST_Z( TEST_RESULT_STRLST_Z(
@ -51,6 +58,8 @@ testRun(void)
"000000010000000100000001\n000000010000000100000002\n000000010000000100000003\n", "empty queue"); "000000010000000100000001\n000000010000000100000002\n000000010000000100000003\n", "empty queue");
// ------------------------------------------------------------------------------------------------------------------------- // -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("pg version earlier than 9.3");
Buffer *walSegmentBuffer = bufNew(walSegmentSize); Buffer *walSegmentBuffer = bufNew(walSegmentSize);
memset(bufPtr(walSegmentBuffer), 0, walSegmentSize); memset(bufPtr(walSegmentBuffer), 0, walSegmentSize);
@ -61,8 +70,7 @@ testRun(void)
queueNeed(STRDEF("0000000100000001000000FE"), false, queueSize, walSegmentSize, PG_VERSION_92), queueNeed(STRDEF("0000000100000001000000FE"), false, queueSize, walSegmentSize, PG_VERSION_92),
"000000010000000200000000\n000000010000000200000001\n", "queue has wal < 9.3"); "000000010000000200000000\n000000010000000200000001\n", "queue has wal < 9.3");
TEST_RESULT_STRLST_Z( TEST_STORAGE_LIST(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_IN, "0000000100000001000000FE\n", .comment = "check queue");
storageListP(storageSpoolWrite(), STRDEF(STORAGE_SPOOL_ARCHIVE_IN)), "0000000100000001000000FE\n", "check queue");
// ------------------------------------------------------------------------------------------------------------------------- // -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("pg >= 9.3 and ok/junk status files"); TEST_TITLE("pg >= 9.3 and ok/junk status files");
@ -341,25 +349,28 @@ testRun(void)
TEST_RESULT_VOID(cmdArchiveGetAsync(), "archive async"); TEST_RESULT_VOID(cmdArchiveGetAsync(), "archive async");
#define TEST_WARN \
"repo2: [ArchiveMismatchError] unable to retrieve the archive id for database version '10' and system-id" \
" '18072658121562454734'"
TEST_RESULT_LOG( TEST_RESULT_LOG(
"P00 INFO: get 3 WAL file(s) from archive: 0000000100000001000000FE...000000010000000200000000\n" "P00 INFO: get 3 WAL file(s) from archive: 0000000100000001000000FE...000000010000000200000000\n"
"P00 WARN: " TEST_WARN "\n" "P00 WARN: repo2: [ArchiveMismatchError] unable to retrieve the archive id for database version '10' and system-id"
" '18072658121562454734'\n"
"P01 DETAIL: found 0000000100000001000000FE in the repo1: 10-1 archive\n" "P01 DETAIL: found 0000000100000001000000FE in the repo1: 10-1 archive\n"
"P00 DETAIL: unable to find 0000000100000001000000FF in the archive"); "P00 DETAIL: unable to find 0000000100000001000000FF in the archive");
TEST_STORAGE_GET( TEST_STORAGE_GET(
storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_IN "/0000000100000001000000FE.ok", "0\n" TEST_WARN, .remove = true); storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_IN "/0000000100000001000000FE.ok",
"0\n"
"repo2: [ArchiveMismatchError] unable to retrieve the archive id for database version '10' and system-id"
" '18072658121562454734'",
.remove = true);
TEST_STORAGE_GET( TEST_STORAGE_GET(
storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_IN "/0000000100000001000000FF.ok", "0\n" TEST_WARN, .remove = true); storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_IN "/0000000100000001000000FF.ok",
"0\n"
"repo2: [ArchiveMismatchError] unable to retrieve the archive id for database version '10' and system-id"
" '18072658121562454734'",
.remove = true);
TEST_STORAGE_GET_EMPTY(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_IN "/0000000100000001000000FE", .remove = true); TEST_STORAGE_GET_EMPTY(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_IN "/0000000100000001000000FE", .remove = true);
TEST_STORAGE_LIST_EMPTY(storageSpool(), STORAGE_SPOOL_ARCHIVE_IN); TEST_STORAGE_LIST_EMPTY(storageSpool(), STORAGE_SPOOL_ARCHIVE_IN);
#undef TEST_WARN
// ------------------------------------------------------------------------------------------------------------------------- // -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("error on duplicates now that no segments are missing, repo with bad perms"); TEST_TITLE("error on duplicates now that no segments are missing, repo with bad perms");
@ -379,17 +390,12 @@ testRun(void)
TEST_RESULT_VOID(cmdArchiveGetAsync(), "archive async"); TEST_RESULT_VOID(cmdArchiveGetAsync(), "archive async");
#define TEST_WARN1 \
"repo2: [PathOpenError] unable to list file info for path '" TEST_PATH "/repo2/archive/test2/10-1" \
"/0000000100000001': [13] Permission denied"
#define TEST_WARN2 \
"repo2: [PathOpenError] unable to list file info for path '" TEST_PATH "/repo2/archive/test2/10-1" \
"/0000000100000002': [13] Permission denied"
TEST_RESULT_LOG( TEST_RESULT_LOG(
"P00 INFO: get 3 WAL file(s) from archive: 0000000100000001000000FE...000000010000000200000000\n" "P00 INFO: get 3 WAL file(s) from archive: 0000000100000001000000FE...000000010000000200000000\n"
"P00 WARN: " TEST_WARN1 "\n" "P00 WARN: repo2: [PathOpenError] unable to list file info for path '" TEST_PATH "/repo2/archive/test2/10-1"
"P00 WARN: " TEST_WARN2 "\n" "/0000000100000001': [13] Permission denied\n"
"P00 WARN: repo2: [PathOpenError] unable to list file info for path '" TEST_PATH "/repo2/archive/test2/10-1"
"/0000000100000002': [13] Permission denied\n"
"P01 DETAIL: found 0000000100000001000000FE in the repo1: 10-1 archive\n" "P01 DETAIL: found 0000000100000001000000FE in the repo1: 10-1 archive\n"
"P01 DETAIL: found 0000000100000001000000FF in the repo1: 10-1 archive\n" "P01 DETAIL: found 0000000100000001000000FF in the repo1: 10-1 archive\n"
"P00 WARN: [ArchiveDuplicateError] duplicates found for WAL segment 000000010000000200000000:\n" "P00 WARN: [ArchiveDuplicateError] duplicates found for WAL segment 000000010000000200000000:\n"
@ -399,10 +405,18 @@ testRun(void)
TEST_STORAGE_GET_EMPTY(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_IN "/0000000100000001000000FE", .remove = true); TEST_STORAGE_GET_EMPTY(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_IN "/0000000100000001000000FE", .remove = true);
TEST_STORAGE_GET( TEST_STORAGE_GET(
storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_IN "/0000000100000001000000FE.ok", "0\n" TEST_WARN1, .remove = true); storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_IN "/0000000100000001000000FE.ok",
"0\n"
"repo2: [PathOpenError] unable to list file info for path '" TEST_PATH "/repo2/archive/test2/10-1/0000000100000001':"
" [13] Permission denied",
.remove = true);
TEST_STORAGE_GET_EMPTY(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_IN "/0000000100000001000000FF", .remove = true); TEST_STORAGE_GET_EMPTY(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_IN "/0000000100000001000000FF", .remove = true);
TEST_STORAGE_GET( TEST_STORAGE_GET(
storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_IN "/0000000100000001000000FF.ok", "0\n" TEST_WARN1, .remove = true); storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_IN "/0000000100000001000000FF.ok",
"0\n"
"repo2: [PathOpenError] unable to list file info for path '" TEST_PATH "/repo2/archive/test2/10-1/0000000100000001':"
" [13] Permission denied",
.remove = true);
TEST_STORAGE_GET( TEST_STORAGE_GET(
storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_IN "/000000010000000200000000.error", storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_IN "/000000010000000200000000.error",
"45\n" "45\n"
@ -410,15 +424,13 @@ testRun(void)
"repo1: 10-1/0000000100000002/000000010000000200000000-bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb, 10-1/0000000100000002" "repo1: 10-1/0000000100000002/000000010000000200000000-bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb, 10-1/0000000100000002"
"/000000010000000200000000-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\n" "/000000010000000200000000-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\n"
"HINT: are multiple primaries archiving to this stanza?\n" "HINT: are multiple primaries archiving to this stanza?\n"
TEST_WARN2, "repo2: [PathOpenError] unable to list file info for path '" TEST_PATH "/repo2/archive/test2/10-1" \
"/0000000100000002': [13] Permission denied",
.remove = true); .remove = true);
TEST_STORAGE_LIST_EMPTY(storageSpool(), STORAGE_SPOOL_ARCHIVE_IN); TEST_STORAGE_LIST_EMPTY(storageSpool(), STORAGE_SPOOL_ARCHIVE_IN);
HRN_STORAGE_MODE(storageRepoIdxWrite(1), STORAGE_REPO_ARCHIVE "/10-1"); HRN_STORAGE_MODE(storageRepoIdxWrite(1), STORAGE_REPO_ARCHIVE "/10-1");
#undef TEST_WARN1
#undef TEST_WARN2
// ------------------------------------------------------------------------------------------------------------------------- // -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("error on duplicates"); TEST_TITLE("error on duplicates");
@ -479,21 +491,21 @@ testRun(void)
TEST_RESULT_VOID(cmdArchiveGetAsync(), "archive async"); TEST_RESULT_VOID(cmdArchiveGetAsync(), "archive async");
#define TEST_WARN1 \
"repo3: [ArchiveMismatchError] unable to retrieve the archive id for database version '10' and system-id" \
" '18072658121562454734'"
#define TEST_WARN2 \
"repo1: 10-1/0000000100000002/000000010000000200000000-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.gz" \
" [FormatError] unexpected eof in compressed data"
TEST_RESULT_LOG( TEST_RESULT_LOG(
"P00 INFO: get 1 WAL file(s) from archive: 000000010000000200000000\n" "P00 INFO: get 1 WAL file(s) from archive: 000000010000000200000000\n"
"P00 WARN: " TEST_WARN1 "\n" "P00 WARN: repo3: [ArchiveMismatchError] unable to retrieve the archive id for database version '10' and system-id"
"P01 WARN: " TEST_WARN2 "\n" " '18072658121562454734'\n"
"P01 WARN: repo1: 10-1/0000000100000002/000000010000000200000000-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.gz"
" [FormatError] unexpected eof in compressed data\n"
"P01 DETAIL: found 000000010000000200000000 in the repo2: 10-1 archive"); "P01 DETAIL: found 000000010000000200000000 in the repo2: 10-1 archive");
TEST_STORAGE_GET( TEST_STORAGE_GET(
storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_IN "/000000010000000200000000.ok", "0\n" TEST_WARN1 "\n" TEST_WARN2, storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_IN "/000000010000000200000000.ok",
"0\n"
"repo3: [ArchiveMismatchError] unable to retrieve the archive id for database version '10' and system-id"
" '18072658121562454734'\n"
"repo1: 10-1/0000000100000002/000000010000000200000000-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.gz"
" [FormatError] unexpected eof in compressed data",
.remove = true); .remove = true);
TEST_STORAGE_GET_EMPTY(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_IN "/000000010000000200000000", .remove = true); TEST_STORAGE_GET_EMPTY(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_IN "/000000010000000200000000", .remove = true);
TEST_STORAGE_LIST_EMPTY(storageSpool(), STORAGE_SPOOL_ARCHIVE_IN); TEST_STORAGE_LIST_EMPTY(storageSpool(), STORAGE_SPOOL_ARCHIVE_IN);
@ -511,24 +523,26 @@ testRun(void)
TEST_RESULT_VOID(cmdArchiveGetAsync(), "archive async"); TEST_RESULT_VOID(cmdArchiveGetAsync(), "archive async");
#define TEST_WARN3 \
"repo2: 10-1/0000000100000002/000000010000000200000000-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.gz" \
" [FormatError] unexpected eof in compressed data"
TEST_RESULT_LOG( TEST_RESULT_LOG(
"P00 INFO: get 1 WAL file(s) from archive: 000000010000000200000000\n" "P00 INFO: get 1 WAL file(s) from archive: 000000010000000200000000\n"
"P00 WARN: " TEST_WARN1 "\n" "P00 WARN: repo3: [ArchiveMismatchError] unable to retrieve the archive id for database version '10' and system-id"
" '18072658121562454734'\n"
"P01 WARN: [FileReadError] raised from local-1 shim protocol: unable to get 000000010000000200000000:\n" "P01 WARN: [FileReadError] raised from local-1 shim protocol: unable to get 000000010000000200000000:\n"
" " TEST_WARN2 "\n" " repo1: 10-1/0000000100000002/000000010000000200000000-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.gz"
" " TEST_WARN3); " [FormatError] unexpected eof in compressed data\n"
" repo2: 10-1/0000000100000002/000000010000000200000000-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.gz"
" [FormatError] unexpected eof in compressed data");
TEST_STORAGE_GET( TEST_STORAGE_GET(
storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_IN "/000000010000000200000000.error", storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_IN "/000000010000000200000000.error",
"42\n" "42\n"
"raised from local-1 shim protocol: unable to get 000000010000000200000000:\n" "raised from local-1 shim protocol: unable to get 000000010000000200000000:\n"
TEST_WARN2 "\n" "repo1: 10-1/0000000100000002/000000010000000200000000-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.gz"
TEST_WARN3 "\n" " [FormatError] unexpected eof in compressed data\n"
TEST_WARN1, "repo2: 10-1/0000000100000002/000000010000000200000000-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.gz"
" [FormatError] unexpected eof in compressed data\n"
"repo3: [ArchiveMismatchError] unable to retrieve the archive id for database version '10' and system-id"
" '18072658121562454734'",
.remove = true); .remove = true);
TEST_STORAGE_LIST( TEST_STORAGE_LIST(
storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_IN, "000000010000000200000000.pgbackrest.tmp\n", .remove = true); storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_IN, "000000010000000200000000.pgbackrest.tmp\n", .remove = true);
@ -571,7 +585,6 @@ testRun(void)
{ {
harnessLogLevelSet(logLevelDetail); harnessLogLevelSet(logLevelDetail);
// Arguments that must be included. Use raw config here because we need to keep the
StringList *argBaseList = strLstNew(); StringList *argBaseList = strLstNew();
hrnCfgArgRawZ(argBaseList, cfgOptPgPath, TEST_PATH "/pg"); hrnCfgArgRawZ(argBaseList, cfgOptPgPath, TEST_PATH "/pg");
hrnCfgArgRawZ(argBaseList, cfgOptRepoPath, TEST_PATH "/repo"); hrnCfgArgRawZ(argBaseList, cfgOptRepoPath, TEST_PATH "/repo");
@ -588,12 +601,16 @@ testRun(void)
TEST_ERROR(cmdArchiveGet(), HostInvalidError, "archive-get command must be run on the PostgreSQL host"); TEST_ERROR(cmdArchiveGet(), HostInvalidError, "archive-get command must be run on the PostgreSQL host");
// ------------------------------------------------------------------------------------------------------------------------- // -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("segment parameter not specified");
argList = strLstDup(argBaseList); argList = strLstDup(argBaseList);
HRN_CFG_LOAD(cfgCmdArchiveGet, argList, .exeBogus = true); HRN_CFG_LOAD(cfgCmdArchiveGet, argList, .exeBogus = true);
TEST_ERROR(cmdArchiveGet(), ParamRequiredError, "WAL segment to get required"); TEST_ERROR(cmdArchiveGet(), ParamRequiredError, "WAL segment to get required");
// ------------------------------------------------------------------------------------------------------------------------- // -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("path parameter not specified");
argList = strLstDup(argBaseList); argList = strLstDup(argBaseList);
strLstAddZ(argList, "000000010000000100000001"); strLstAddZ(argList, "000000010000000100000001");
HRN_CFG_LOAD(cfgCmdArchiveGet, argList, .exeBogus = true); HRN_CFG_LOAD(cfgCmdArchiveGet, argList, .exeBogus = true);
@ -601,6 +618,8 @@ testRun(void)
TEST_ERROR(cmdArchiveGet(), ParamRequiredError, "path to copy WAL segment required"); TEST_ERROR(cmdArchiveGet(), ParamRequiredError, "path to copy WAL segment required");
// ------------------------------------------------------------------------------------------------------------------------- // -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("no valid repo");
HRN_STORAGE_PUT( HRN_STORAGE_PUT(
storagePgWrite(), PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL, storagePgWrite(), PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL,
hrnPgControlToBuffer((PgControl){.version = PG_VERSION_10, .systemId = 0xFACEFACEFACEFACE})); hrnPgControlToBuffer((PgControl){.version = PG_VERSION_10, .systemId = 0xFACEFACEFACEFACE}));
@ -623,10 +642,12 @@ testRun(void)
" scheme."); " scheme.");
// ------------------------------------------------------------------------------------------------------------------------- // -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("no valid repo - async");
argList = strLstDup(argBaseList); argList = strLstDup(argBaseList);
hrnCfgArgRawBool(argList, cfgOptArchiveAsync, true);
strLstAddZ(argList, "00000001.history"); strLstAddZ(argList, "00000001.history");
strLstAddZ(argList, TEST_PATH "/pg/pg_wal/RECOVERYHISTORY"); strLstAddZ(argList, TEST_PATH "/pg/pg_wal/RECOVERYHISTORY");
strLstAddZ(argList, "--archive-async");
HRN_CFG_LOAD(cfgCmdArchiveGet, argList, .exeBogus = true); HRN_CFG_LOAD(cfgCmdArchiveGet, argList, .exeBogus = true);
TEST_ERROR(cmdArchiveGet(), RepoInvalidError, "unable to find a valid repository"); TEST_ERROR(cmdArchiveGet(), RepoInvalidError, "unable to find a valid repository");
@ -643,8 +664,10 @@ testRun(void)
" HINT: use --no-archive-check to disable archive checks during backup if you have an alternate archiving" " HINT: use --no-archive-check to disable archive checks during backup if you have an alternate archiving"
" scheme."); " scheme.");
// Make sure the process times out when there is nothing to get
// ------------------------------------------------------------------------------------------------------------------------- // -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("WAL not found - timeout");
// Make sure the process times out when there is nothing to get
argList = strLstDup(argBaseList); argList = strLstDup(argBaseList);
hrnCfgArgRawZ(argList, cfgOptSpoolPath, TEST_PATH "/spool"); hrnCfgArgRawZ(argList, cfgOptSpoolPath, TEST_PATH "/spool");
hrnCfgArgRawBool(argList, cfgOptArchiveAsync, true); hrnCfgArgRawBool(argList, cfgOptArchiveAsync, true);
@ -658,8 +681,9 @@ testRun(void)
cmdArchiveGet(), ArchiveTimeoutError, cmdArchiveGet(), ArchiveTimeoutError,
"unable to get WAL file '000000010000000100000001' from the archive asynchronously after 1 second(s)"); "unable to get WAL file '000000010000000100000001' from the archive asynchronously after 1 second(s)");
// Check for missing WAL
// ------------------------------------------------------------------------------------------------------------------------- // -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("check for missing WAL");
HRN_STORAGE_PUT_EMPTY(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_IN "/000000010000000100000001.ok"); HRN_STORAGE_PUT_EMPTY(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_IN "/000000010000000100000001.ok");
TEST_ERROR( TEST_ERROR(
@ -670,8 +694,9 @@ testRun(void)
storageExistsP(storageSpool(), STRDEF(STORAGE_SPOOL_ARCHIVE_IN "/000000010000000100000001.ok")), false, storageExistsP(storageSpool(), STRDEF(STORAGE_SPOOL_ARCHIVE_IN "/000000010000000100000001.ok")), false,
"check OK file was removed"); "check OK file was removed");
// Write out a WAL segment for success
// ------------------------------------------------------------------------------------------------------------------------- // -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("write WAL segment for success");
HRN_STORAGE_PATH_CREATE(storagePgWrite(), "pg_wal"); HRN_STORAGE_PATH_CREATE(storagePgWrite(), "pg_wal");
HRN_STORAGE_PUT_Z(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_IN "/000000010000000100000001", "SHOULD-BE-A-REAL-WAL-FILE"); HRN_STORAGE_PUT_Z(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_IN "/000000010000000100000001", "SHOULD-BE-A-REAL-WAL-FILE");
@ -682,11 +707,13 @@ testRun(void)
TEST_STORAGE_LIST_EMPTY(storageSpool(), STORAGE_SPOOL_ARCHIVE_IN); TEST_STORAGE_LIST_EMPTY(storageSpool(), STORAGE_SPOOL_ARCHIVE_IN);
TEST_STORAGE_LIST(storagePgWrite(), "pg_wal", "RECOVERYXLOG\n", .remove = true); TEST_STORAGE_LIST(storagePgWrite(), "pg_wal", "RECOVERYXLOG\n", .remove = true);
// Write more WAL segments (in this case queue should be full)
// ------------------------------------------------------------------------------------------------------------------------- // -------------------------------------------------------------------------------------------------------------------------
strLstAddZ(argList, "--archive-get-queue-max=48"); TEST_TITLE("write WAL segments for success - queue full");
hrnCfgArgRawZ(argList, cfgOptArchiveGetQueueMax, "48");
HRN_CFG_LOAD(cfgCmdArchiveGet, argList, .exeBogus = true); HRN_CFG_LOAD(cfgCmdArchiveGet, argList, .exeBogus = true);
// Write more WAL segments (in this case queue should be full)
HRN_STORAGE_PUT_Z(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_IN "/000000010000000100000001", "SHOULD-BE-A-REAL-WAL-FILE"); HRN_STORAGE_PUT_Z(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_IN "/000000010000000100000001", "SHOULD-BE-A-REAL-WAL-FILE");
HRN_STORAGE_PUT_Z(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_IN "/000000010000000100000001.ok", "0\nwarning about x"); HRN_STORAGE_PUT_Z(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_IN "/000000010000000100000001.ok", "0\nwarning about x");
HRN_STORAGE_PUT_Z(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_IN "/000000010000000100000002", "SHOULD-BE-A-REAL-WAL-FILE"); HRN_STORAGE_PUT_Z(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_IN "/000000010000000100000002", "SHOULD-BE-A-REAL-WAL-FILE");
@ -700,8 +727,10 @@ testRun(void)
TEST_STORAGE_LIST(storagePgWrite(), "pg_wal", "RECOVERYXLOG\n", .remove = true); TEST_STORAGE_LIST(storagePgWrite(), "pg_wal", "RECOVERYXLOG\n", .remove = true);
TEST_STORAGE_LIST(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_IN, "000000010000000100000002\n", .remove = true); TEST_STORAGE_LIST(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_IN, "000000010000000100000002\n", .remove = true);
// Make sure the process times out when it can't get a lock
// ------------------------------------------------------------------------------------------------------------------------- // -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("unable to get lock");
// Make sure the process times out when it can't get a lock
HARNESS_FORK_BEGIN() HARNESS_FORK_BEGIN()
{ {
HARNESS_FORK_CHILD_BEGIN(0, true) HARNESS_FORK_CHILD_BEGIN(0, true)
@ -749,6 +778,8 @@ testRun(void)
HARNESS_FORK_END(); HARNESS_FORK_END();
// ------------------------------------------------------------------------------------------------------------------------- // -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("too many parameters specified");
strLstAddZ(argList, BOGUS_STR); strLstAddZ(argList, BOGUS_STR);
HRN_CFG_LOAD(cfgCmdArchiveGet, argList, .exeBogus = true); HRN_CFG_LOAD(cfgCmdArchiveGet, argList, .exeBogus = true);
@ -1029,9 +1060,9 @@ testRun(void)
// ------------------------------------------------------------------------------------------------------------------------- // -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("unable to get from one repo"); TEST_TITLE("unable to get from one repo");
HRN_STORAGE_PUT( HRN_STORAGE_PUT_EMPTY(
storageRepoIdxWrite(0), storageRepoIdxWrite(0),
STORAGE_REPO_ARCHIVE "/10-2/01ABCDEF01ABCDEF01ABCDEF-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.gz", NULL); STORAGE_REPO_ARCHIVE "/10-2/01ABCDEF01ABCDEF01ABCDEF-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.gz");
TEST_RESULT_INT(cmdArchiveGet(), 0, "get"); TEST_RESULT_INT(cmdArchiveGet(), 0, "get");
@ -1070,8 +1101,7 @@ testRun(void)
TEST_RESULT_INT(cmdArchiveGet(), 0, "get"); TEST_RESULT_INT(cmdArchiveGet(), 0, "get");
TEST_RESULT_LOG( TEST_RESULT_LOG("P00 INFO: found 01ABCDEF01ABCDEF01ABCDEF in the repo2: 10-1 archive");
"P00 INFO: found 01ABCDEF01ABCDEF01ABCDEF in the repo2: 10-1 archive");
// ------------------------------------------------------------------------------------------------------------------------- // -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("no segments to find with existing ok file"); TEST_TITLE("no segments to find with existing ok file");

559
test/src/module/command/archivePushTest.c
View File

@ -28,47 +28,44 @@ testRun(void)
if (testBegin("archivePushReadyList(), archivePushProcessList(), and archivePushDrop()")) if (testBegin("archivePushReadyList(), archivePushProcessList(), and archivePushDrop()"))
{ {
StringList *argList = strLstNew(); StringList *argList = strLstNew();
strLstAddZ(argList, "--stanza=db"); hrnCfgArgRawZ(argList, cfgOptStanza, "db");
strLstAddZ(argList, "--pg1-path=" TEST_PATH "/db"); hrnCfgArgRawZ(argList, cfgOptPgPath, TEST_PATH "/db");
strLstAddZ(argList, "--spool-path=" TEST_PATH "/spool"); hrnCfgArgRawZ(argList, cfgOptSpoolPath, TEST_PATH "/spool");
strLstAddZ(argList, "--" CFGOPT_ARCHIVE_ASYNC); hrnCfgArgRawBool(argList, cfgOptArchiveAsync, true);
HRN_CFG_LOAD(cfgCmdArchivePush, argList, .role = cfgCmdRoleAsync); HRN_CFG_LOAD(cfgCmdArchivePush, argList, .role = cfgCmdRoleAsync);
storagePathCreateP(storagePgWrite(), STRDEF("pg_wal/archive_status")); HRN_STORAGE_PATH_CREATE(storagePgWrite(), "pg_wal/archive_status");
storagePathCreateP(storageTest, STRDEF("spool/archive/db/out")); HRN_STORAGE_PATH_CREATE(storageTest, "spool/archive/db/out");
// Create ok files to indicate WAL that has already been archived // Create ok files to indicate WAL that has already been archived
storagePutP( HRN_STORAGE_PUT_EMPTY(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_OUT "/000000010000000100000001.ok");
storageNewWriteP(storageSpoolWrite(), STRDEF(STORAGE_SPOOL_ARCHIVE_OUT "/000000010000000100000001.ok")), NULL); HRN_STORAGE_PUT_EMPTY(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_OUT "/000000010000000100000003.ok");
storagePutP( HRN_STORAGE_PUT_EMPTY(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_OUT "/000000010000000100000004.ok");
storageNewWriteP(storageSpoolWrite(), STRDEF(STORAGE_SPOOL_ARCHIVE_OUT "/000000010000000100000003.ok")), NULL); HRN_STORAGE_PUT_EMPTY(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_OUT "/000000010000000100000005.error");
storagePutP( HRN_STORAGE_PUT_EMPTY(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_OUT "/000000010000000100000006.error");
storageNewWriteP(storageSpoolWrite(), STRDEF(STORAGE_SPOOL_ARCHIVE_OUT "/000000010000000100000004.ok")), NULL); HRN_STORAGE_PUT_EMPTY(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_OUT "/global.error");
storagePutP(
storageNewWriteP(storageSpoolWrite(), STRDEF(STORAGE_SPOOL_ARCHIVE_OUT "/000000010000000100000005.error")), NULL);
storagePutP(
storageNewWriteP(storageSpoolWrite(), STRDEF(STORAGE_SPOOL_ARCHIVE_OUT "/000000010000000100000006.error")), NULL);
storagePutP(
storageNewWriteP(storageSpoolWrite(), STRDEF(STORAGE_SPOOL_ARCHIVE_OUT "/global.error")), NULL);
// Create ready files for wal that still needs to be archived // Create ready files for wal that still needs to be archived
storagePutP(storageNewWriteP(storagePgWrite(), STRDEF("pg_wal/archive_status/000000010000000100000002.ready")), NULL); HRN_STORAGE_PUT_EMPTY(storagePgWrite(), "pg_wal/archive_status/000000010000000100000002.ready");
storagePutP(storageNewWriteP(storagePgWrite(), STRDEF("pg_wal/archive_status/000000010000000100000003.ready")), NULL); HRN_STORAGE_PUT_EMPTY(storagePgWrite(), "pg_wal/archive_status/000000010000000100000003.ready");
storagePutP(storageNewWriteP(storagePgWrite(), STRDEF("pg_wal/archive_status/000000010000000100000005.ready")), NULL); HRN_STORAGE_PUT_EMPTY(storagePgWrite(), "pg_wal/archive_status/000000010000000100000005.ready");
storagePutP(storageNewWriteP(storagePgWrite(), STRDEF("pg_wal/archive_status/000000010000000100000006.ready")), NULL); HRN_STORAGE_PUT_EMPTY(storagePgWrite(), "pg_wal/archive_status/000000010000000100000006.ready");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("ready list");
TEST_RESULT_STRLST_Z( TEST_RESULT_STRLST_Z(
archivePushProcessList(STRDEF(TEST_PATH "/db/pg_wal")), archivePushProcessList(STRDEF(TEST_PATH "/db/pg_wal")),
"000000010000000100000002\n000000010000000100000005\n000000010000000100000006\n", "ready list"); "000000010000000100000002\n000000010000000100000005\n000000010000000100000006\n", "ready list");
TEST_RESULT_STRLST_Z( TEST_STORAGE_LIST(
strLstSort(storageListP(storageSpool(), STRDEF(STORAGE_SPOOL_ARCHIVE_OUT)), sortOrderAsc), storageSpool(), STORAGE_SPOOL_ARCHIVE_OUT, "000000010000000100000003.ok\n", .comment = "remaining status list");
"000000010000000100000003.ok\n", "remaining status list");
// Test drop
// ------------------------------------------------------------------------------------------------------------------------- // -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("WAL drop");
StringList *argListDrop = strLstDup(argList); StringList *argListDrop = strLstDup(argList);
strLstAdd(argListDrop, strNewFmt("--archive-push-queue-max=%zu", (size_t)1024 * 1024 * 1024)); hrnCfgArgRawFmt(argListDrop, cfgOptArchivePushQueueMax, "%zu", (size_t)1024 * 1024 * 1024);
HRN_CFG_LOAD(cfgCmdArchivePush, argListDrop, .role = cfgCmdRoleAsync); HRN_CFG_LOAD(cfgCmdArchivePush, argListDrop, .role = cfgCmdRoleAsync);
// Write the files that we claim are in pg_wal // Write the files that we claim are in pg_wal
@ -77,50 +74,48 @@ testRun(void)
memset(bufPtr(walBuffer), 0, bufSize(walBuffer)); memset(bufPtr(walBuffer), 0, bufSize(walBuffer));
hrnPgWalToBuffer((PgWal){.version = PG_VERSION_10, .systemId = 0xFACEFACEFACEFACE}, walBuffer); hrnPgWalToBuffer((PgWal){.version = PG_VERSION_10, .systemId = 0xFACEFACEFACEFACE}, walBuffer);
storagePutP(storageNewWriteP(storagePgWrite(), STRDEF("pg_wal/000000010000000100000002")), walBuffer); HRN_STORAGE_PUT(storagePgWrite(), "pg_wal/000000010000000100000002", walBuffer);
storagePutP(storageNewWriteP(storagePgWrite(), STRDEF("pg_wal/000000010000000100000003")), walBuffer); HRN_STORAGE_PUT(storagePgWrite(), "pg_wal/000000010000000100000003", walBuffer);
storagePutP(storageNewWriteP(storagePgWrite(), STRDEF("pg_wal/000000010000000100000005")), walBuffer); HRN_STORAGE_PUT(storagePgWrite(), "pg_wal/000000010000000100000005", walBuffer);
storagePutP(storageNewWriteP(storagePgWrite(), STRDEF("pg_wal/000000010000000100000006")), walBuffer); HRN_STORAGE_PUT(storagePgWrite(), "pg_wal/000000010000000100000006", walBuffer);
// Queue max is high enough that no WAL will be dropped // Queue max is high enough that no WAL will be dropped
TEST_RESULT_BOOL( TEST_RESULT_BOOL(
archivePushDrop(STRDEF("pg_wal"), archivePushProcessList(STRDEF(TEST_PATH "/db/pg_wal"))), false, archivePushDrop(STRDEF("pg_wal"), archivePushProcessList(STRDEF(TEST_PATH "/db/pg_wal"))), false, "wal is not dropped");
"wal is not dropped");
// Now set queue max low enough that WAL will be dropped // Now set queue max low enough that WAL will be dropped
argListDrop = strLstDup(argList); argListDrop = strLstDup(argList);
strLstAdd(argListDrop, strNewFmt("--archive-push-queue-max=%zu", (size_t)16 * 1024 * 1024 * 2)); hrnCfgArgRawFmt(argListDrop, cfgOptArchivePushQueueMax, "%zu", (size_t)16 * 1024 * 1024 * 2);
HRN_CFG_LOAD(cfgCmdArchivePush, argListDrop, .role = cfgCmdRoleAsync); HRN_CFG_LOAD(cfgCmdArchivePush, argListDrop, .role = cfgCmdRoleAsync);
TEST_RESULT_BOOL( TEST_RESULT_BOOL(
archivePushDrop(STRDEF("pg_wal"), archivePushProcessList(STRDEF(TEST_PATH "/db/pg_wal"))), true, archivePushDrop(STRDEF("pg_wal"), archivePushProcessList(STRDEF(TEST_PATH "/db/pg_wal"))), true, "wal is dropped");
"wal is dropped");
} }
// ***************************************************************************************************************************** // *****************************************************************************************************************************
if (testBegin("archivePushCheck()")) if (testBegin("archivePushCheck()"))
{ {
StringList *argList = strLstNew(); StringList *argList = strLstNew();
strLstAddZ(argList, "--stanza=test"); hrnCfgArgRawZ(argList, cfgOptStanza, "test");
strLstAddZ(argList, "--pg1-path=" TEST_PATH "/pg"); hrnCfgArgRawZ(argList, cfgOptPgPath, TEST_PATH "/pg");
strLstAddZ(argList, "--repo1-path=" TEST_PATH "/repo"); hrnCfgArgRawZ(argList, cfgOptRepoPath, TEST_PATH "/repo");
HRN_CFG_LOAD(cfgCmdArchivePush, argList); HRN_CFG_LOAD(cfgCmdArchivePush, argList);
// Check mismatched pg_control and archive.info
// ------------------------------------------------------------------------------------------------------------------------- // -------------------------------------------------------------------------------------------------------------------------
storagePutP( TEST_TITLE("mismatched pg_control and archive.info - pg version");
storageNewWriteP(storageTest, STRDEF("pg/" PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL)),
HRN_STORAGE_PUT(
storageTest, "pg/" PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL,
hrnPgControlToBuffer((PgControl){.version = PG_VERSION_96, .systemId = 0xFACEFACEFACEFACE})); hrnPgControlToBuffer((PgControl){.version = PG_VERSION_96, .systemId = 0xFACEFACEFACEFACE}));
// Create incorrect archive info // Create incorrect archive info
storagePutP( HRN_INFO_PUT(
storageNewWriteP(storageTest, STRDEF("repo/archive/test/archive.info")), storageRepoIdxWrite(0), INFO_ARCHIVE_PATH_FILE,
harnessInfoChecksumZ(
"[db]\n" "[db]\n"
"db-id=1\n" "db-id=1\n"
"\n" "\n"
"[db:history]\n" "[db:history]\n"
"1={\"db-id\":5555555555555555555,\"db-version\":\"9.4\"}\n")); "1={\"db-id\":5555555555555555555,\"db-version\":\"9.4\"}\n");
TEST_ERROR( TEST_ERROR(
archivePushCheck(true), RepoInvalidError, archivePushCheck(true), RepoInvalidError,
@ -129,15 +124,17 @@ testRun(void)
" 9.4, system-id 5555555555555555555" " 9.4, system-id 5555555555555555555"
"\nHINT: are you archiving to the correct stanza?"); "\nHINT: are you archiving to the correct stanza?");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("mismatched pg_control and archive.info - system-id");
// Fix the version // Fix the version
storagePutP( HRN_INFO_PUT(
storageNewWriteP(storageTest, STRDEF("repo/archive/test/archive.info")), storageRepoIdxWrite(0), INFO_ARCHIVE_PATH_FILE,
harnessInfoChecksumZ(
"[db]\n" "[db]\n"
"db-id=1\n" "db-id=1\n"
"\n" "\n"
"[db:history]\n" "[db:history]\n"
"1={\"db-id\":5555555555555555555,\"db-version\":\"9.6\"}\n")); "1={\"db-id\":5555555555555555555,\"db-version\":\"9.6\"}\n");
TEST_ERROR( TEST_ERROR(
archivePushCheck(true), RepoInvalidError, archivePushCheck(true), RepoInvalidError,
@ -146,15 +143,17 @@ testRun(void)
" 9.6, system-id 5555555555555555555" " 9.6, system-id 5555555555555555555"
"\nHINT: are you archiving to the correct stanza?"); "\nHINT: are you archiving to the correct stanza?");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("pg_control and archive.info match");
// Fix archive info // Fix archive info
storagePutP( HRN_INFO_PUT(
storageNewWriteP(storageTest, STRDEF("repo/archive/test/archive.info")), storageRepoIdxWrite(0), INFO_ARCHIVE_PATH_FILE,
harnessInfoChecksumZ(
"[db]\n" "[db]\n"
"db-id=1\n" "db-id=1\n"
"\n" "\n"
"[db:history]\n" "[db:history]\n"
"1={\"db-id\":18072658121562454734,\"db-version\":\"9.6\"}\n")); "1={\"db-id\":18072658121562454734,\"db-version\":\"9.6\"}\n");
ArchivePushCheckResult result = {0}; ArchivePushCheckResult result = {0};
TEST_ASSIGN(result, archivePushCheck(true), "get archive check result"); TEST_ASSIGN(result, archivePushCheck(true), "get archive check result");
@ -172,30 +171,28 @@ testRun(void)
TEST_TITLE("mismatched repos when pg-path not present"); TEST_TITLE("mismatched repos when pg-path not present");
argList = strLstNew(); argList = strLstNew();
strLstAddZ(argList, "--stanza=test"); hrnCfgArgRawZ(argList, cfgOptStanza, "test");
strLstAddZ(argList, "--repo2-path=" TEST_PATH "/repo2"); hrnCfgArgKeyRawZ(argList, cfgOptRepoPath, 2, TEST_PATH "/repo2");
strLstAddZ(argList, "--repo4-path=" TEST_PATH "/repo4"); hrnCfgArgKeyRawZ(argList, cfgOptRepoPath, 4, TEST_PATH "/repo4");
HRN_CFG_LOAD(cfgCmdArchivePush, argList); HRN_CFG_LOAD(cfgCmdArchivePush, argList);
// repo2 has correct info // repo2 has correct info
storagePutP( HRN_INFO_PUT(
storageNewWriteP(storageTest, STRDEF("repo2/archive/test/archive.info")), storageRepoIdxWrite(0), INFO_ARCHIVE_PATH_FILE,
harnessInfoChecksumZ(
"[db]\n" "[db]\n"
"db-id=1\n" "db-id=1\n"
"\n" "\n"
"[db:history]\n" "[db:history]\n"
"1={\"db-id\":18072658121562454734,\"db-version\":\"9.6\"}\n")); "1={\"db-id\":18072658121562454734,\"db-version\":\"9.6\"}\n");
// repo4 has incorrect info // repo4 has incorrect info
storagePutP( HRN_INFO_PUT(
storageNewWriteP(storageTest, STRDEF("repo4/archive/test/archive.info")), storageRepoIdxWrite(1), INFO_ARCHIVE_PATH_FILE,
harnessInfoChecksumZ(
"[db]\n" "[db]\n"
"db-id=1\n" "db-id=1\n"
"\n" "\n"
"[db:history]\n" "[db:history]\n"
"1={\"db-id\":5555555555555555555,\"db-version\":\"9.4\"}\n")); "1={\"db-id\":5555555555555555555,\"db-version\":\"9.4\"}\n");
TEST_ASSIGN(result, archivePushCheck(false), "get archive check result"); TEST_ASSIGN(result, archivePushCheck(false), "get archive check result");
@ -218,15 +215,14 @@ testRun(void)
TEST_TITLE("matched repos when pg-path not present"); TEST_TITLE("matched repos when pg-path not present");
// repo4 has correct info // repo4 has correct info
storagePutP( HRN_INFO_PUT(
storageNewWriteP(storageTest, STRDEF("repo4/archive/test/archive.info")), storageRepoIdxWrite(1), INFO_ARCHIVE_PATH_FILE,
harnessInfoChecksumZ(
"[db]\n" "[db]\n"
"db-id=2\n" "db-id=2\n"
"\n" "\n"
"[db:history]\n" "[db:history]\n"
"1={\"db-id\":5555555555555555555,\"db-version\":\"9.4\"}\n" "1={\"db-id\":5555555555555555555,\"db-version\":\"9.4\"}\n"
"2={\"db-id\":18072658121562454734,\"db-version\":\"9.6\"}\n")); "2={\"db-id\":18072658121562454734,\"db-version\":\"9.6\"}\n");
TEST_ASSIGN(result, archivePushCheck(false), "get archive check result"); TEST_ASSIGN(result, archivePushCheck(false), "get archive check result");
@ -249,24 +245,29 @@ testRun(void)
// ***************************************************************************************************************************** // *****************************************************************************************************************************
if (testBegin("Synchronous cmdArchivePush() and archivePushFile()")) if (testBegin("Synchronous cmdArchivePush() and archivePushFile()"))
{ {
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("command must be run on the pg host"); TEST_TITLE("command must be run on the pg host");
StringList *argList = strLstNew(); StringList *argList = strLstNew();
hrnCfgArgRawZ(argList, cfgOptPgHost, "host"); hrnCfgArgRawZ(argList, cfgOptPgHost, "host");
hrnCfgArgRawZ(argList, cfgOptPgPath, "/pg"); hrnCfgArgRawZ(argList, cfgOptPgPath, "/pg");
strLstAddZ(argList, "--" CFGOPT_STANZA "=test2"); hrnCfgArgRawZ(argList, cfgOptStanza, "test2");
HRN_CFG_LOAD(cfgCmdArchivePush, argList, .role = cfgCmdRoleMain); HRN_CFG_LOAD(cfgCmdArchivePush, argList, .role = cfgCmdRoleMain);
TEST_ERROR(cmdArchivePush(), HostInvalidError, "archive-push command must be run on the PostgreSQL host"); TEST_ERROR(cmdArchivePush(), HostInvalidError, "archive-push command must be run on the PostgreSQL host");
// ------------------------------------------------------------------------------------------------------------------------- // -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("WAL segment not specified");
argList = strLstNew(); argList = strLstNew();
strLstAddZ(argList, "--stanza=test"); hrnCfgArgRawZ(argList, cfgOptStanza, "test");
HRN_CFG_LOAD(cfgCmdArchivePush, argList); HRN_CFG_LOAD(cfgCmdArchivePush, argList);
TEST_ERROR(cmdArchivePush(), ParamRequiredError, "WAL segment to push required"); TEST_ERROR(cmdArchivePush(), ParamRequiredError, "WAL segment to push required");
// ------------------------------------------------------------------------------------------------------------------------- // -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("pg-path not specified");
StringList *argListTemp = strLstDup(argList); StringList *argListTemp = strLstDup(argList);
strLstAddZ(argListTemp, "pg_wal/000000010000000100000001"); strLstAddZ(argListTemp, "pg_wal/000000010000000100000001");
HRN_CFG_LOAD(cfgCmdArchivePush, argListTemp); HRN_CFG_LOAD(cfgCmdArchivePush, argListTemp);
@ -277,36 +278,36 @@ testRun(void)
"\nHINT: is %f passed to archive-push instead of %p?" "\nHINT: is %f passed to archive-push instead of %p?"
"\nHINT: PostgreSQL may pass relative paths even with %p depending on the environment."); "\nHINT: PostgreSQL may pass relative paths even with %p depending on the environment.");
// Create pg_control and archive.info
// ------------------------------------------------------------------------------------------------------------------------- // -------------------------------------------------------------------------------------------------------------------------
strLstAddZ(argList, "--pg1-path=" TEST_PATH "/pg"); TEST_TITLE("attempt to push WAL with incorrect headers");
strLstAddZ(argList, "--repo1-path=" TEST_PATH "/repo");
// Create pg_control and archive.info
hrnCfgArgRawZ(argList, cfgOptPgPath, TEST_PATH "/pg");
hrnCfgArgRawZ(argList, cfgOptRepoPath, TEST_PATH "/repo");
argListTemp = strLstDup(argList); argListTemp = strLstDup(argList);
strLstAddZ(argListTemp, "pg_wal/000000010000000100000001"); strLstAddZ(argListTemp, "pg_wal/000000010000000100000001");
HRN_CFG_LOAD(cfgCmdArchivePush, argListTemp); HRN_CFG_LOAD(cfgCmdArchivePush, argListTemp);
storagePutP( HRN_STORAGE_PUT(
storageNewWriteP(storageTest, STRDEF("pg/" PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL)), storageTest, "pg/" PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL,
hrnPgControlToBuffer((PgControl){.version = PG_VERSION_11, .systemId = 0xFACEFACEFACEFACE})); hrnPgControlToBuffer((PgControl){.version = PG_VERSION_11, .systemId = 0xFACEFACEFACEFACE}));
storagePutP( HRN_INFO_PUT(
storageNewWriteP(storageTest, STRDEF("repo/archive/test/archive.info")), storageRepoIdxWrite(0), INFO_ARCHIVE_PATH_FILE,
harnessInfoChecksumZ(
"[db]\n" "[db]\n"
"db-id=1\n" "db-id=1\n"
"\n" "\n"
"[db:history]\n" "[db:history]\n"
"1={\"db-id\":18072658121562454734,\"db-version\":\"11\"}\n")); "1={\"db-id\":18072658121562454734,\"db-version\":\"11\"}\n");
// Generate WAL with incorrect headers and try to push them // Generate WAL with incorrect headers and try to push them
// -------------------------------------------------------------------------------------------------------------------------
Buffer *walBuffer1 = bufNew((size_t)16 * 1024 * 1024); Buffer *walBuffer1 = bufNew((size_t)16 * 1024 * 1024);
bufUsedSet(walBuffer1, bufSize(walBuffer1)); bufUsedSet(walBuffer1, bufSize(walBuffer1));
memset(bufPtr(walBuffer1), 0, bufSize(walBuffer1)); memset(bufPtr(walBuffer1), 0, bufSize(walBuffer1));
hrnPgWalToBuffer((PgWal){.version = PG_VERSION_10, .systemId = 0xFACEFACEFACEFACE}, walBuffer1); hrnPgWalToBuffer((PgWal){.version = PG_VERSION_10, .systemId = 0xFACEFACEFACEFACE}, walBuffer1);
storagePutP(storageNewWriteP(storagePgWrite(), STRDEF("pg_wal/000000010000000100000001")), walBuffer1); HRN_STORAGE_PUT(storagePgWrite(), "pg_wal/000000010000000100000001", walBuffer1);
THROW_ON_SYS_ERROR(chdir(strZ(cfgOptionStr(cfgOptPgPath))) != 0, PathMissingError, "unable to chdir()"); THROW_ON_SYS_ERROR(chdir(strZ(cfgOptionStr(cfgOptPgPath))) != 0, PathMissingError, "unable to chdir()");
@ -319,7 +320,7 @@ testRun(void)
hrnPgWalToBuffer((PgWal){.version = PG_VERSION_11, .systemId = 0xECAFECAFECAFECAF}, walBuffer1); hrnPgWalToBuffer((PgWal){.version = PG_VERSION_11, .systemId = 0xECAFECAFECAFECAF}, walBuffer1);
const char *walBuffer1Sha1 = strZ(bufHex(cryptoHashOne(HASH_TYPE_SHA1_STR, walBuffer1))); const char *walBuffer1Sha1 = strZ(bufHex(cryptoHashOne(HASH_TYPE_SHA1_STR, walBuffer1)));
storagePutP(storageNewWriteP(storagePgWrite(), STRDEF("pg_wal/000000010000000100000001")), walBuffer1); HRN_STORAGE_PUT(storagePgWrite(), "pg_wal/000000010000000100000001", walBuffer1);
TEST_ERROR( TEST_ERROR(
cmdArchivePush(), ArchiveMismatchError, cmdArchivePush(), ArchiveMismatchError,
@ -330,24 +331,20 @@ testRun(void)
TEST_TITLE("push by ignoring the invalid header"); TEST_TITLE("push by ignoring the invalid header");
argListTemp = strLstDup(argList); argListTemp = strLstDup(argList);
hrnCfgArgRawNegate(argListTemp, cfgOptArchiveHeaderCheck); hrnCfgArgRawBool(argListTemp, cfgOptArchiveHeaderCheck, false);
strLstAddZ(argListTemp, "pg_wal/000000010000000100000001"); strLstAddZ(argListTemp, "pg_wal/000000010000000100000001");
HRN_CFG_LOAD(cfgCmdArchivePush, argListTemp); HRN_CFG_LOAD(cfgCmdArchivePush, argListTemp);
TEST_RESULT_VOID(cmdArchivePush(), "push the WAL segment"); TEST_RESULT_VOID(cmdArchivePush(), "push the WAL segment");
TEST_RESULT_LOG("P00 INFO: pushed WAL file '000000010000000100000001' to the archive"); TEST_RESULT_LOG("P00 INFO: pushed WAL file '000000010000000100000001' to the archive");
TEST_RESULT_BOOL(
storageExistsP(
storageRepoIdx(0),
strNewFmt(STORAGE_REPO_ARCHIVE "/11-1/000000010000000100000001-%s.gz", walBuffer1Sha1)),
true, "check repo for WAL file");
TEST_STORAGE_EXISTS( TEST_STORAGE_EXISTS(
storageRepoIdxWrite(0), strZ(strNewFmt(STORAGE_REPO_ARCHIVE "/11-1/000000010000000100000001-%s.gz", walBuffer1Sha1)), storageRepoIdxWrite(0), strZ(strNewFmt(STORAGE_REPO_ARCHIVE "/11-1/000000010000000100000001-%s.gz", walBuffer1Sha1)),
.remove = true); .remove = true, .comment = "check repo for WAL file, then remove");
// Generate valid WAL and push them
// ------------------------------------------------------------------------------------------------------------------------- // -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("generate valid WAL and push them");
argListTemp = strLstDup(argList); argListTemp = strLstDup(argList);
strLstAddZ(argListTemp, "pg_wal/000000010000000100000001"); strLstAddZ(argListTemp, "pg_wal/000000010000000100000001");
HRN_CFG_LOAD(cfgCmdArchivePush, argListTemp); HRN_CFG_LOAD(cfgCmdArchivePush, argListTemp);
@ -361,15 +358,14 @@ testRun(void)
(TEST_BIG_ENDIAN() ? "1c5f963d720bb199d7935dbd315447ea2ec3feb2" : "aae7591a1dbc58f21d0d004886075094f622e6dd") : (TEST_BIG_ENDIAN() ? "1c5f963d720bb199d7935dbd315447ea2ec3feb2" : "aae7591a1dbc58f21d0d004886075094f622e6dd") :
"28a13fd8cf6fcd9f9a8108aed4c8bcc58040863a"; "28a13fd8cf6fcd9f9a8108aed4c8bcc58040863a";
storagePutP(storageNewWriteP(storagePgWrite(), STRDEF("pg_wal/000000010000000100000001")), walBuffer1); HRN_STORAGE_PUT(storagePgWrite(), "pg_wal/000000010000000100000001", walBuffer1);
TEST_RESULT_VOID(cmdArchivePush(), "push the WAL segment"); TEST_RESULT_VOID(cmdArchivePush(), "push the WAL segment");
TEST_RESULT_LOG("P00 INFO: pushed WAL file '000000010000000100000001' to the archive"); TEST_RESULT_LOG("P00 INFO: pushed WAL file '000000010000000100000001' to the archive");
TEST_RESULT_BOOL( TEST_STORAGE_EXISTS(
storageExistsP( storageRepoIdxWrite(0), strZ(strNewFmt(STORAGE_REPO_ARCHIVE "/11-1/000000010000000100000001-%s.gz", walBuffer1Sha1)),
storageTest, strNewFmt("repo/archive/test/11-1/0000000100000001/000000010000000100000001-%s.gz", walBuffer1Sha1)), .comment = "check repo for WAL file");
true, "check repo for WAL file");
TEST_RESULT_VOID(cmdArchivePush(), "push the WAL segment again"); TEST_RESULT_VOID(cmdArchivePush(), "push the WAL segment again");
TEST_RESULT_LOG( TEST_RESULT_LOG(
@ -384,7 +380,7 @@ testRun(void)
hrnPgWalToBuffer((PgWal){.version = PG_VERSION_11, .systemId = 0xFACEFACEFACEFACE}, walBuffer2); hrnPgWalToBuffer((PgWal){.version = PG_VERSION_11, .systemId = 0xFACEFACEFACEFACE}, walBuffer2);
const char *walBuffer2Sha1 = strZ(bufHex(cryptoHashOne(HASH_TYPE_SHA1_STR, walBuffer2))); const char *walBuffer2Sha1 = strZ(bufHex(cryptoHashOne(HASH_TYPE_SHA1_STR, walBuffer2)));
storagePutP(storageNewWriteP(storagePgWrite(), STRDEF("pg_wal/000000010000000100000001")), walBuffer2); HRN_STORAGE_PUT(storagePgWrite(), "pg_wal/000000010000000100000001", walBuffer2);
TEST_ERROR( TEST_ERROR(
cmdArchivePush(), ArchiveDuplicateError, cmdArchivePush(), ArchiveDuplicateError,
@ -394,58 +390,58 @@ testRun(void)
TEST_TITLE("WAL with absolute path and no pg1-path"); TEST_TITLE("WAL with absolute path and no pg1-path");
argListTemp = strLstNew(); argListTemp = strLstNew();
strLstAddZ(argListTemp, "--" CFGOPT_STANZA "=test"); hrnCfgArgRawZ(argListTemp, cfgOptStanza, "test");
hrnCfgArgRawZ(argListTemp, cfgOptRepoPath, TEST_PATH "/repo"); hrnCfgArgRawZ(argListTemp, cfgOptRepoPath, TEST_PATH "/repo");
strLstAddZ(argListTemp, TEST_PATH "/pg/pg_wal/000000010000000100000002"); strLstAddZ(argListTemp, TEST_PATH "/pg/pg_wal/000000010000000100000002");
HRN_CFG_LOAD(cfgCmdArchivePush, argListTemp); HRN_CFG_LOAD(cfgCmdArchivePush, argListTemp);
TEST_RESULT_VOID( HRN_STORAGE_PUT(storageTest, "pg/pg_wal/000000010000000100000002", walBuffer2, .comment = "write WAL");
storagePutP(storageNewWriteP(storageTest, STRDEF("pg/pg_wal/000000010000000100000002")), walBuffer2), "write WAL");
// Create tmp file to make it look like a prior push failed partway through to ensure that retries work // Create tmp file to make it look like a prior push failed partway through to ensure that retries work
TEST_RESULT_VOID( HRN_STORAGE_PUT_Z(
storagePutP(
storageNewWriteP(
storageTest, storageTest,
strNewFmt("repo/archive/test/11-1/0000000100000001/000000010000000100000002-%s.gz.pgbackrest.tmp", strZ(
walBuffer2Sha1)), strNewFmt("repo/archive/test/11-1/0000000100000001/000000010000000100000002-%s.gz.pgbackrest.tmp", walBuffer2Sha1)),
BUFSTRDEF("PARTIAL")), "PARTIAL", .comment = "write WAL tmp file");
"write WAL tmp file");
TEST_RESULT_VOID(cmdArchivePush(), "push the WAL segment"); TEST_RESULT_VOID(cmdArchivePush(), "push the WAL segment");
TEST_RESULT_LOG("P00 INFO: pushed WAL file '000000010000000100000002' to the archive"); TEST_RESULT_LOG("P00 INFO: pushed WAL file '000000010000000100000002' to the archive");
TEST_RESULT_BOOL( TEST_STORAGE_EXISTS(
storageExistsP( storageRepoIdxWrite(0),
storageTest, strNewFmt("repo/archive/test/11-1/0000000100000001/000000010000000100000002-%s.gz", walBuffer2Sha1)), strZ(
true, "check repo for WAL file"); strNewFmt(
STORAGE_REPO_ARCHIVE "/11-1/0000000100000001/000000010000000100000002-%s.gz", walBuffer2Sha1)),
.comment = "check repo for WAL file");
TEST_RESULT_BOOL( TEST_RESULT_BOOL(
storageExistsP( storageExistsP(
storageTest, storageTest,
strNewFmt("repo/archive/test/11-1/0000000100000001/000000010000000100000002-%s.gz.pgbackrest.tmp", walBuffer2Sha1)), strNewFmt("repo/archive/test/11-1/0000000100000001/000000010000000100000002-%s.gz.pgbackrest.tmp", walBuffer2Sha1)),
false, "check WAL tmp file is gone"); false, "check WAL tmp file is gone");
// Push a history file
// ------------------------------------------------------------------------------------------------------------------------- // -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("push a history file");
argListTemp = strLstDup(argList); argListTemp = strLstDup(argList);
strLstAddZ(argListTemp, "pg_wal/00000001.history"); strLstAddZ(argListTemp, "pg_wal/00000001.history");
HRN_CFG_LOAD(cfgCmdArchivePush, argListTemp); HRN_CFG_LOAD(cfgCmdArchivePush, argListTemp);
storagePutP(storageNewWriteP(storagePgWrite(), STRDEF("pg_wal/00000001.history")), BUFSTRDEF("FAKEHISTORY")); HRN_STORAGE_PUT_Z(storagePgWrite(), "pg_wal/00000001.history", "FAKEHISTORY");
TEST_RESULT_VOID(cmdArchivePush(), "push a history file"); TEST_RESULT_VOID(cmdArchivePush(), "push a history file");
TEST_RESULT_LOG("P00 INFO: pushed WAL file '00000001.history' to the archive"); TEST_RESULT_LOG("P00 INFO: pushed WAL file '00000001.history' to the archive");
TEST_RESULT_BOOL( TEST_STORAGE_EXISTS(
storageExistsP(storageTest, STRDEF("repo/archive/test/11-1/00000001.history")), true, "check repo for history file"); storageRepoIdx(0), STORAGE_REPO_ARCHIVE "/11-1/00000001.history", .comment = "check repo for history file");
// Check drop functionality
// ------------------------------------------------------------------------------------------------------------------------- // -------------------------------------------------------------------------------------------------------------------------
storagePutP(storageNewWriteP(storagePgWrite(), STRDEF("pg_wal/archive_status/000000010000000100000001.ready")), NULL); TEST_TITLE("check drop functionality");
storagePutP(storageNewWriteP(storagePgWrite(), STRDEF("pg_wal/archive_status/000000010000000100000002.ready")), NULL);
HRN_STORAGE_PUT_EMPTY(storagePgWrite(), "pg_wal/archive_status/000000010000000100000001.ready");
HRN_STORAGE_PUT_EMPTY(storagePgWrite(), "pg_wal/archive_status/000000010000000100000002.ready");
argListTemp = strLstDup(argList); argListTemp = strLstDup(argList);
strLstAddZ(argListTemp, "--archive-push-queue-max=16m"); hrnCfgArgRawZ(argListTemp, cfgOptArchivePushQueueMax, "16m");
strLstAddZ(argListTemp, "pg_wal/000000010000000100000002"); strLstAddZ(argListTemp, "pg_wal/000000010000000100000002");
HRN_CFG_LOAD(cfgCmdArchivePush, argListTemp); HRN_CFG_LOAD(cfgCmdArchivePush, argListTemp);
@ -453,7 +449,7 @@ testRun(void)
TEST_RESULT_LOG("P00 WARN: dropped WAL file '000000010000000100000002' because archive queue exceeded 16MB"); TEST_RESULT_LOG("P00 WARN: dropped WAL file '000000010000000100000002' because archive queue exceeded 16MB");
argListTemp = strLstDup(argList); argListTemp = strLstDup(argList);
strLstAddZ(argListTemp, "--archive-push-queue-max=1GB"); hrnCfgArgRawZ(argListTemp, cfgOptArchivePushQueueMax, "1GB");
strLstAddZ(argListTemp, "pg_wal/000000010000000100000002"); strLstAddZ(argListTemp, "pg_wal/000000010000000100000002");
HRN_CFG_LOAD(cfgCmdArchivePush, argListTemp); HRN_CFG_LOAD(cfgCmdArchivePush, argListTemp);
@ -469,36 +465,6 @@ testRun(void)
// Remove old repo // Remove old repo
HRN_STORAGE_PATH_REMOVE(storageTest, "repo", .errorOnMissing = true, .recurse = true); HRN_STORAGE_PATH_REMOVE(storageTest, "repo", .errorOnMissing = true, .recurse = true);
// repo2 is encrypted
StorageWrite *infoWrite = storageNewWriteP(storageTest, STRDEF("repo2/archive/test/archive.info"));
ioFilterGroupAdd(
ioWriteFilterGroup(storageWriteIo(infoWrite)), cipherBlockNew(cipherModeEncrypt, cipherTypeAes256Cbc,
BUFSTRDEF("badpassphrase"), NULL));
storagePutP(
infoWrite,
harnessInfoChecksumZ(
"[cipher]\n"
"cipher-pass=\"badsubpassphrase\"\n"
"\n"
"[db]\n"
"db-id=1\n"
"\n"
"[db:history]\n"
"1={\"db-id\":18072658121562454734,\"db-version\":\"11\"}"));
// repo3 is not encrypted
storagePutP(
storageNewWriteP(storageTest, STRDEF("repo3/archive/test/archive.info")),
harnessInfoChecksumZ(
"[db]\n"
"db-id=1\n"
"\n"
"[db:history]\n"
"1={\"db-id\":18072658121562454734,\"db-version\":\"11\"}"));
// Push encrypted WAL segment
argListTemp = strLstNew(); argListTemp = strLstNew();
hrnCfgArgRawZ(argListTemp, cfgOptStanza, "test"); hrnCfgArgRawZ(argListTemp, cfgOptStanza, "test");
hrnCfgArgKeyRawZ(argListTemp, cfgOptPgPath, 1, TEST_PATH "/pg"); hrnCfgArgKeyRawZ(argListTemp, cfgOptPgPath, 1, TEST_PATH "/pg");
@ -511,29 +477,43 @@ testRun(void)
HRN_CFG_LOAD(cfgCmdArchivePush, argListTemp); HRN_CFG_LOAD(cfgCmdArchivePush, argListTemp);
hrnCfgEnvKeyRemoveRaw(cfgOptRepoCipherPass, 2); hrnCfgEnvKeyRemoveRaw(cfgOptRepoCipherPass, 2);
// repo2 is encrypted
HRN_INFO_PUT(
storageRepoIdxWrite(0), INFO_ARCHIVE_PATH_FILE,
"[cipher]\n"
"cipher-pass=\"badsubpassphrase\"\n"
"\n"
"[db]\n"
"db-id=1\n"
"\n"
"[db:history]\n"
"1={\"db-id\":18072658121562454734,\"db-version\":\"11\"}",
.cipherType = cipherTypeAes256Cbc, .cipherPass = "badpassphrase");
// repo3 is not encrypted
HRN_INFO_PUT(
storageRepoIdxWrite(1), INFO_ARCHIVE_PATH_FILE,
"[db]\n"
"db-id=1\n"
"\n"
"[db:history]\n"
"1={\"db-id\":18072658121562454734,\"db-version\":\"11\"}");
// Push encrypted WAL segment
TEST_RESULT_VOID(cmdArchivePush(), "push the WAL segment"); TEST_RESULT_VOID(cmdArchivePush(), "push the WAL segment");
TEST_RESULT_LOG("P00 INFO: pushed WAL file '000000010000000100000002' to the archive"); TEST_RESULT_LOG("P00 INFO: pushed WAL file '000000010000000100000002' to the archive");
TEST_RESULT_BOOL( TEST_STORAGE_EXISTS(
storageExistsP( storageTest, strZ(strNewFmt("repo2/archive/test/11-1/0000000100000001/000000010000000100000002-%s", walBuffer2Sha1)),
storageTest, strNewFmt("repo2/archive/test/11-1/0000000100000001/000000010000000100000002-%s", walBuffer2Sha1)), .remove = true, .comment = "check repo2 for WAL file then remove");
true, "check repo2 for WAL file");
TEST_RESULT_BOOL( TEST_STORAGE_EXISTS(
storageExistsP( storageTest, strZ(strNewFmt("repo3/archive/test/11-1/0000000100000001/000000010000000100000002-%s", walBuffer2Sha1)),
storageTest, strNewFmt("repo3/archive/test/11-1/0000000100000001/000000010000000100000002-%s", walBuffer2Sha1)), .remove = true, .comment = "check repo3 for WAL file then remove");
true, "check repo3 for WAL file");
// ------------------------------------------------------------------------------------------------------------------------- // -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("write error on one repo but other repo succeeds"); TEST_TITLE("write error on one repo but other repo succeeds");
storageRemoveP(
storageTest, strNewFmt("repo2/archive/test/11-1/0000000100000001/000000010000000100000002-%s", walBuffer2Sha1),
.errorOnMissing = true);
storageRemoveP(
storageTest, strNewFmt("repo3/archive/test/11-1/0000000100000001/000000010000000100000002-%s", walBuffer2Sha1),
.errorOnMissing = true);
HRN_STORAGE_MODE(storageTest, "repo2/archive/test/11-1/0000000100000001", .mode = 0500); HRN_STORAGE_MODE(storageTest, "repo2/archive/test/11-1/0000000100000001", .mode = 0500);
TEST_ERROR( TEST_ERROR(
@ -544,14 +524,10 @@ testRun(void)
"repo2: [FileOpenError] unable to open file '" TEST_PATH "/repo2/archive/test/11-1/0000000100000001" "repo2: [FileOpenError] unable to open file '" TEST_PATH "/repo2/archive/test/11-1/0000000100000001"
"/000000010000000100000002-%s' for write: [13] Permission denied", walBuffer2Sha1))); "/000000010000000100000002-%s' for write: [13] Permission denied", walBuffer2Sha1)));
TEST_RESULT_BOOL( TEST_STORAGE_LIST_EMPTY(storageTest, "repo2/archive/test/11-1/0000000100000001", .comment = "check repo2 for no WAL file");
storageExistsP( TEST_STORAGE_LIST(
storageTest, strNewFmt("repo2/archive/test/11-1/0000000100000001/000000010000000100000002-%s", walBuffer2Sha1)), storageTest, "repo3/archive/test/11-1/0000000100000001",
false, "check repo2 for no WAL file"); strZ(strNewFmt("000000010000000100000002-%s\n", walBuffer2Sha1)), .comment = "check repo3 for WAL file");
TEST_RESULT_BOOL(
storageExistsP(
storageTest, strNewFmt("repo3/archive/test/11-1/0000000100000001/000000010000000100000002-%s", walBuffer2Sha1)),
true, "check repo3 for WAL file");
HRN_STORAGE_MODE(storageTest, "repo2/archive/test/11-1/0000000100000001"); HRN_STORAGE_MODE(storageTest, "repo2/archive/test/11-1/0000000100000001");
@ -564,10 +540,9 @@ testRun(void)
" HINT: this is valid in some recovery scenarios but may also indicate a problem.\n" " HINT: this is valid in some recovery scenarios but may also indicate a problem.\n"
"P00 INFO: pushed WAL file '000000010000000100000002' to the archive"); "P00 INFO: pushed WAL file '000000010000000100000002' to the archive");
TEST_RESULT_BOOL( TEST_STORAGE_LIST(
storageExistsP( storageTest, "repo2/archive/test/11-1/0000000100000001",
storageTest, strNewFmt("repo2/archive/test/11-1/0000000100000001/000000010000000100000002-%s", walBuffer2Sha1)), strZ(strNewFmt("000000010000000100000002-%s\n", walBuffer2Sha1)), .comment = "check repo2 for WAL file");
true, "check repo2 for WAL file");
// ------------------------------------------------------------------------------------------------------------------------- // -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("WAL already exists in both repos"); TEST_TITLE("WAL already exists in both repos");
@ -639,6 +614,7 @@ testRun(void)
static const ProtocolServerHandler testLocalHandlerList[] = {PROTOCOL_SERVER_HANDLER_ARCHIVE_PUSH_LIST}; static const ProtocolServerHandler testLocalHandlerList[] = {PROTOCOL_SERVER_HANDLER_ARCHIVE_PUSH_LIST};
hrnProtocolLocalShimInstall(testLocalHandlerList, PROTOCOL_SERVER_HANDLER_LIST_SIZE(testLocalHandlerList)); hrnProtocolLocalShimInstall(testLocalHandlerList, PROTOCOL_SERVER_HANDLER_LIST_SIZE(testLocalHandlerList));
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("command must be run on the pg host"); TEST_TITLE("command must be run on the pg host");
StringList *argList = strLstNew(); StringList *argList = strLstNew();
@ -655,78 +631,80 @@ testRun(void)
TEST_TITLE("pg1-path must be set when async"); TEST_TITLE("pg1-path must be set when async");
argList = strLstNew(); argList = strLstNew();
strLstAddZ(argList, "--" CFGOPT_SPOOL_PATH "=/spool"); hrnCfgArgRawZ(argList, cfgOptSpoolPath, "/spool");
strLstAddZ(argList, "--" CFGOPT_STANZA "=test2"); hrnCfgArgRawZ(argList, cfgOptStanza, "test2");
strLstAddZ(argList, "--" CFGOPT_ARCHIVE_ASYNC); hrnCfgArgRawBool(argList, cfgOptArchiveAsync, true);
strLstAddZ(argList, "/000000010000000100000001"); strLstAddZ(argList, "/000000010000000100000001");
HRN_CFG_LOAD(cfgCmdArchivePush, argList, .role = cfgCmdRoleAsync); HRN_CFG_LOAD(cfgCmdArchivePush, argList, .role = cfgCmdRoleAsync);
TEST_ERROR(cmdArchivePush(), OptionRequiredError, "'archive-push' command in async mode requires option 'pg1-path'"); TEST_ERROR(cmdArchivePush(), OptionRequiredError, "'archive-push' command in async mode requires option 'pg1-path'");
// Call with a bogus exe name so the async process will error out and we can make sure timeouts work
// ------------------------------------------------------------------------------------------------------------------------- // -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("check timeout on async error");
// Call with a bogus exe name so the async process will error out and we can make sure timeouts work
argList = strLstNew(); argList = strLstNew();
strLstAddZ(argList, "--stanza=test"); hrnCfgArgRawZ(argList, cfgOptSpoolPath, TEST_PATH "/spool");
strLstAddZ(argList, "--archive-async"); hrnCfgArgRawZ(argList, cfgOptStanza, "test");
strLstAddZ(argList, "--archive-timeout=1"); hrnCfgArgRawBool(argList, cfgOptArchiveAsync, true);
strLstAddZ(argList, "--spool-path=" TEST_PATH " /spool"); hrnCfgArgRawZ(argList, cfgOptPgPath, TEST_PATH "/pg");
strLstAddZ(argList, "--pg1-path=" TEST_PATH "/pg"); hrnCfgArgRawZ(argList, cfgOptRepoPath, TEST_PATH "/repo");
strLstAddZ(argList, "--repo1-path=" TEST_PATH "/repo"); hrnCfgArgRawZ(argList, cfgOptArchiveTimeout, "1");
strLstAddZ(argList, "pg_wal/bogus"); strLstAddZ(argList, "pg_wal/bogus");
HRN_CFG_LOAD(cfgCmdArchivePush, argList, .exeBogus = true); HRN_CFG_LOAD(cfgCmdArchivePush, argList, .exeBogus = true);
storagePathCreateP(storageTest, cfgOptionStr(cfgOptPgPath)); HRN_STORAGE_PATH_CREATE(storageTest, strZ(cfgOptionStr(cfgOptPgPath)));
THROW_ON_SYS_ERROR(chdir(strZ(cfgOptionStr(cfgOptPgPath))) != 0, PathMissingError, "unable to chdir()"); THROW_ON_SYS_ERROR(chdir(strZ(cfgOptionStr(cfgOptPgPath))) != 0, PathMissingError, "unable to chdir()");
TEST_ERROR( TEST_ERROR(
cmdArchivePush(), ArchiveTimeoutError, cmdArchivePush(), ArchiveTimeoutError,
"unable to push WAL file 'bogus' to the archive asynchronously after 1 second(s)"); "unable to push WAL file 'bogus' to the archive asynchronously after 1 second(s)");
// Create pg_control and archive.info // Create pg_control and archive.info for next set of tests
// ------------------------------------------------------------------------------------------------------------------------- // -------------------------------------------------------------------------------------------------------------------------
argList = strLstNew(); argList = strLstNew();
strLstAddZ(argList, "--stanza=test"); hrnCfgArgRawZ(argList, cfgOptStanza, "test");
strLstAddZ(argList, "--archive-async"); hrnCfgArgRawBool(argList, cfgOptArchiveAsync, true);
strLstAddZ(argList, "--no-compress"); hrnCfgArgRawBool(argList, cfgOptCompress, false);
strLstAddZ(argList, "--spool-path=" TEST_PATH "/spool"); hrnCfgArgRawZ(argList, cfgOptSpoolPath, TEST_PATH "/spool");
strLstAddZ(argList, "--pg1-path=" TEST_PATH "/pg"); hrnCfgArgRawZ(argList, cfgOptPgPath, TEST_PATH "/pg");
strLstAddZ(argList, "--repo1-path=" TEST_PATH "/repo"); hrnCfgArgRawZ(argList, cfgOptRepoPath, TEST_PATH "/repo");
strLstAddZ(argList, "--log-subprocess"); hrnCfgArgRawBool(argList, cfgOptLogSubprocess, true);
storagePutP( HRN_STORAGE_PUT(storageTest, "pg/" PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL,
storageNewWriteP(storageTest, STRDEF("pg/" PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL)),
hrnPgControlToBuffer((PgControl){.version = PG_VERSION_94, .systemId = 0xAAAABBBBCCCCDDDD})); hrnPgControlToBuffer((PgControl){.version = PG_VERSION_94, .systemId = 0xAAAABBBBCCCCDDDD}));
storagePutP( HRN_INFO_PUT(
storageNewWriteP(storageTest, STRDEF("repo/archive/test/archive.info")), storageTest, "repo/archive/test/archive.info",
harnessInfoChecksumZ(
"[db]\n" "[db]\n"
"db-id=1\n" "db-id=1\n"
"\n" "\n"
"[db:history]\n" "[db:history]\n"
"1={\"db-id\":12297848147757817309,\"db-version\":\"9.4\"}\n")); "1={\"db-id\":12297848147757817309,\"db-version\":\"9.4\"}\n");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("async, ignore error file on first pass");
// Write out an error file that will be ignored on the first pass, then the async process will write a new one // Write out an error file that will be ignored on the first pass, then the async process will write a new one
// -------------------------------------------------------------------------------------------------------------------------
StringList *argListTemp = strLstDup(argList); StringList *argListTemp = strLstDup(argList);
strLstAddZ(argListTemp, TEST_PATH "/pg/pg_xlog/000000010000000100000001"); strLstAddZ(argListTemp, TEST_PATH "/pg/pg_xlog/000000010000000100000001");
HRN_CFG_LOAD(cfgCmdArchivePush, argListTemp); HRN_CFG_LOAD(cfgCmdArchivePush, argListTemp);
storagePathCreateP(storagePgWrite(), STRDEF("pg_xlog/archive_status")); HRN_STORAGE_PATH_CREATE(storagePgWrite(), "pg_xlog/archive_status");
storagePutP( HRN_STORAGE_PUT_Z(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_OUT "/000000010000000100000001.error", "25\nbogus error");
storageNewWriteP(storageSpoolWrite(), STRDEF(STORAGE_SPOOL_ARCHIVE_OUT "/000000010000000100000001.error")),
BUFSTRDEF("25\nbogus error"));
TEST_ERROR(cmdArchivePush(), AssertError, "no WAL files to process"); TEST_ERROR(cmdArchivePush(), AssertError, "no WAL files to process");
storageRemoveP(storageSpoolWrite(), STRDEF(STORAGE_SPOOL_ARCHIVE_OUT "/global.error"), .errorOnMissing = true); TEST_STORAGE_EXISTS(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_OUT "/global.error", .remove = true);
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("with lock, prevent async from running");
// Acquire a lock so the async process will not be able to run -- this will result in a timeout // Acquire a lock so the async process will not be able to run -- this will result in a timeout
// -------------------------------------------------------------------------------------------------------------------------
argListTemp = strLstDup(argList); argListTemp = strLstDup(argList);
strLstAddZ(argListTemp, TEST_PATH "/pg/pg_xlog/000000010000000100000001"); strLstAddZ(argListTemp, TEST_PATH "/pg/pg_xlog/000000010000000100000001");
strLstAddZ(argListTemp, "--archive-timeout=1"); hrnCfgArgRawZ(argListTemp, cfgOptArchiveTimeout, "1");
HRN_CFG_LOAD(cfgCmdArchivePush, argListTemp); HRN_CFG_LOAD(cfgCmdArchivePush, argListTemp);
THROW_ON_SYS_ERROR(chdir(strZ(cfgOptionStr(cfgOptPgPath))) != 0, PathMissingError, "unable to chdir()"); THROW_ON_SYS_ERROR(chdir(strZ(cfgOptionStr(cfgOptPgPath))) != 0, PathMissingError, "unable to chdir()");
@ -774,13 +752,15 @@ testRun(void)
} }
HARNESS_FORK_END(); HARNESS_FORK_END();
// Actually push a WAL file
// ------------------------------------------------------------------------------------------------------------------------- // -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("async WAL push");
// Actually push a WAL file
argListTemp = strLstDup(argList); argListTemp = strLstDup(argList);
strLstAddZ(argListTemp, TEST_PATH "/pg/pg_xlog/000000010000000100000001"); strLstAddZ(argListTemp, TEST_PATH "/pg/pg_xlog/000000010000000100000001");
HRN_CFG_LOAD(cfgCmdArchivePush, argListTemp); HRN_CFG_LOAD(cfgCmdArchivePush, argListTemp);
storagePutP(storageNewWriteP(storagePgWrite(), STRDEF("pg_xlog/archive_status/000000010000000100000001.ready")), NULL); HRN_STORAGE_PUT_EMPTY(storagePgWrite(), "pg_xlog/archive_status/000000010000000100000001.ready");
Buffer *walBuffer1 = bufNew((size_t)16 * 1024 * 1024); Buffer *walBuffer1 = bufNew((size_t)16 * 1024 * 1024);
bufUsedSet(walBuffer1, bufSize(walBuffer1)); bufUsedSet(walBuffer1, bufSize(walBuffer1));
@ -788,51 +768,50 @@ testRun(void)
hrnPgWalToBuffer((PgWal){.version = PG_VERSION_94, .systemId = 0xAAAABBBBCCCCDDDD}, walBuffer1); hrnPgWalToBuffer((PgWal){.version = PG_VERSION_94, .systemId = 0xAAAABBBBCCCCDDDD}, walBuffer1);
const char *walBuffer1Sha1 = strZ(bufHex(cryptoHashOne(HASH_TYPE_SHA1_STR, walBuffer1))); const char *walBuffer1Sha1 = strZ(bufHex(cryptoHashOne(HASH_TYPE_SHA1_STR, walBuffer1)));
storagePutP(storageNewWriteP(storagePgWrite(), STRDEF("pg_xlog/000000010000000100000001")), walBuffer1); HRN_STORAGE_PUT(storagePgWrite(),"pg_xlog/000000010000000100000001", walBuffer1);
TEST_RESULT_VOID(cmdArchivePush(), "push the WAL segment"); TEST_RESULT_VOID(cmdArchivePush(), "push the WAL segment");
TEST_RESULT_LOG("P00 INFO: pushed WAL file '000000010000000100000001' to the archive asynchronously"); TEST_RESULT_LOG("P00 INFO: pushed WAL file '000000010000000100000001' to the archive asynchronously");
TEST_RESULT_BOOL( TEST_STORAGE_EXISTS(
storageExistsP( storageTest, strZ(strNewFmt("repo/archive/test/9.4-1/0000000100000001/000000010000000100000001-%s", walBuffer1Sha1)),
storageTest, strNewFmt("repo/archive/test/9.4-1/0000000100000001/000000010000000100000001-%s", walBuffer1Sha1)), .comment = "check repo for WAL file");
true, "check repo for WAL file");
// Direct tests of the async function
// ------------------------------------------------------------------------------------------------------------------------- // -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("direct tests of the async function");
argList = strLstNew(); argList = strLstNew();
strLstAddZ(argList, "--stanza=test"); hrnCfgArgRawZ(argList, cfgOptSpoolPath, TEST_PATH "/spool");
hrnCfgArgRawZ(argList, cfgOptStanza, "test");
hrnCfgArgRawBool(argList, cfgOptArchiveAsync, true);
hrnCfgArgRawZ(argList, cfgOptPgPath, TEST_PATH "/pg");
hrnCfgArgRawZ(argList, cfgOptRepoPath, TEST_PATH "/repo");
hrnCfgArgRawBool(argList, cfgOptLogSubprocess, true);
hrnCfgArgRawZ(argList, cfgOptCompressType, "none"); hrnCfgArgRawZ(argList, cfgOptCompressType, "none");
strLstAddZ(argList, "--spool-path=" TEST_PATH "/spool");
strLstAddZ(argList, "--" CFGOPT_ARCHIVE_ASYNC);
strLstAddZ(argList, "--pg1-path=" TEST_PATH "/pg");
strLstAddZ(argList, "--repo1-path=" TEST_PATH "/repo");
strLstAddZ(argList, "--log-subprocess");
HRN_CFG_LOAD(cfgCmdArchivePush, argList, .role = cfgCmdRoleAsync); HRN_CFG_LOAD(cfgCmdArchivePush, argList, .role = cfgCmdRoleAsync);
TEST_ERROR(cmdArchivePushAsync(), ParamRequiredError, "WAL path to push required"); TEST_ERROR(cmdArchivePushAsync(), ParamRequiredError, "WAL path to push required");
// Check that global.error is created
// ------------------------------------------------------------------------------------------------------------------------- // -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("async, check that global.error is created");
// Remove data from prior tests // Remove data from prior tests
HRN_STORAGE_PATH_REMOVE(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_OUT, .recurse = true); HRN_STORAGE_PATH_REMOVE(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_OUT, .recurse = true);
storagePathCreateP(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_OUT_STR); HRN_STORAGE_PATH_CREATE(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_OUT);
HRN_STORAGE_PATH_REMOVE(storagePgWrite(), "pg_xlog/archive_status", .recurse = true); HRN_STORAGE_PATH_REMOVE(storagePgWrite(), "pg_xlog/archive_status", .recurse = true);
storagePathCreateP(storagePgWrite(), STRDEF("pg_xlog/archive_status")); HRN_STORAGE_PATH_CREATE(storagePgWrite(), "pg_xlog/archive_status");
strLstAddZ(argList, TEST_PATH "/pg/pg_xlog"); strLstAddZ(argList, TEST_PATH "/pg/pg_xlog");
HRN_CFG_LOAD(cfgCmdArchivePush, argList, .role = cfgCmdRoleAsync); HRN_CFG_LOAD(cfgCmdArchivePush, argList, .role = cfgCmdRoleAsync);
TEST_ERROR(cmdArchivePushAsync(), AssertError, "no WAL files to process"); TEST_ERROR(cmdArchivePushAsync(), AssertError, "no WAL files to process");
TEST_RESULT_STR_Z( TEST_STORAGE_GET(
strNewBuf(storageGetP(storageNewReadP(storageSpool(), STRDEF(STORAGE_SPOOL_ARCHIVE_OUT "/global.error")))), storageSpool(), STORAGE_SPOOL_ARCHIVE_OUT "/global.error", "25\nno WAL files to process",
"25\nno WAL files to process", "check global.error"); .comment = "check global.error");
TEST_RESULT_STRLST_Z( TEST_STORAGE_LIST(storageSpool(), STORAGE_SPOOL_ARCHIVE_OUT, "global.error\n", .comment = "check status files");
strLstSort(storageListP(storageSpool(), STRDEF(STORAGE_SPOOL_ARCHIVE_OUT)), sortOrderAsc),
"global.error\n", "check status files");
// ------------------------------------------------------------------------------------------------------------------------- // -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("add repo, push already pushed WAL and new WAL"); TEST_TITLE("add repo, push already pushed WAL and new WAL");
@ -841,25 +820,23 @@ testRun(void)
hrnCfgArgKeyRawZ(argList, cfgOptRepoPath, 3, TEST_PATH "/repo3"); hrnCfgArgKeyRawZ(argList, cfgOptRepoPath, 3, TEST_PATH "/repo3");
HRN_CFG_LOAD(cfgCmdArchivePush, argList, .role = cfgCmdRoleAsync); HRN_CFG_LOAD(cfgCmdArchivePush, argList, .role = cfgCmdRoleAsync);
storagePutP( HRN_INFO_PUT(
storageNewWriteP(storageTest, STRDEF("repo3/archive/test/archive.info")), storageTest, "repo3/archive/test/archive.info",
harnessInfoChecksumZ(
"[db]\n" "[db]\n"
"db-id=1\n" "db-id=1\n"
"\n" "\n"
"[db:history]\n" "[db:history]\n"
"1={\"db-id\":12297848147757817309,\"db-version\":\"9.4\"}\n")); "1={\"db-id\":12297848147757817309,\"db-version\":\"9.4\"}\n");
// Recreate ready file for WAL 1 // Recreate ready file for WAL 1
storagePutP(storageNewWriteP(storagePgWrite(), STRDEF("pg_xlog/archive_status/000000010000000100000001.ready")), NULL); HRN_STORAGE_PUT_EMPTY(storagePgWrite(), "pg_xlog/archive_status/000000010000000100000001.ready");
TEST_RESULT_BOOL( TEST_STORAGE_EXISTS(
storageExistsP( storageTest, strZ(strNewFmt("repo/archive/test/9.4-1/0000000100000001/000000010000000100000001-%s", walBuffer1Sha1)),
storageTest, strNewFmt("repo/archive/test/9.4-1/0000000100000001/000000010000000100000001-%s", walBuffer1Sha1)), .comment = "check repo1 for WAL 1 file");
true, "check repo1 for WAL 1 file");
// Create a ready file for WAL 2 but don't create the segment yet -- this will test the file error // Create a ready file for WAL 2 but don't create the segment yet -- this will test the file error
storagePutP(storageNewWriteP(storagePgWrite(), STRDEF("pg_xlog/archive_status/000000010000000100000002.ready")), NULL); HRN_STORAGE_PUT_EMPTY(storagePgWrite(), "pg_xlog/archive_status/000000010000000100000002.ready");
TEST_RESULT_VOID(cmdArchivePushAsync(), "push WAL segments"); TEST_RESULT_VOID(cmdArchivePushAsync(), "push WAL segments");
TEST_RESULT_LOG_FMT( TEST_RESULT_LOG_FMT(
@ -871,19 +848,19 @@ testRun(void)
"[55] raised from local-1 shim protocol: " STORAGE_ERROR_READ_MISSING, "[55] raised from local-1 shim protocol: " STORAGE_ERROR_READ_MISSING,
TEST_PATH "/pg/pg_xlog/000000010000000100000002"); TEST_PATH "/pg/pg_xlog/000000010000000100000002");
TEST_RESULT_BOOL( TEST_STORAGE_EXISTS(
storageExistsP( storageTest, strZ(strNewFmt("repo/archive/test/9.4-1/0000000100000001/000000010000000100000001-%s", walBuffer1Sha1)),
storageTest, strNewFmt("repo/archive/test/9.4-1/0000000100000001/000000010000000100000001-%s", walBuffer1Sha1)), .comment = "check repo1 for WAL 1 file");
true, "check repo1 for WAL 1 file");
TEST_RESULT_BOOL( TEST_STORAGE_EXISTS(
storageExistsP( storageTest, strZ(strNewFmt("repo3/archive/test/9.4-1/0000000100000001/000000010000000100000001-%s", walBuffer1Sha1)),
storageTest, strNewFmt("repo/archive/test/9.4-1/0000000100000001/000000010000000100000001-%s", walBuffer1Sha1)), .comment = "check repo3 for WAL 1 file");
true, "check repo3 for WAL 1 file");
TEST_RESULT_STRLST_Z( TEST_STORAGE_LIST(
strLstSort(storageListP(storageSpool(), STRDEF(STORAGE_SPOOL_ARCHIVE_OUT)), sortOrderAsc), storageSpool(), STORAGE_SPOOL_ARCHIVE_OUT,
"000000010000000100000001.ok\n000000010000000100000002.error\n", "check status files"); "000000010000000100000001.ok\n"
"000000010000000100000002.error\n",
.comment = "check status files");
// ------------------------------------------------------------------------------------------------------------------------- // -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("create and push previously missing WAL"); TEST_TITLE("create and push previously missing WAL");
@ -895,10 +872,10 @@ testRun(void)
hrnPgWalToBuffer((PgWal){.version = PG_VERSION_94, .systemId = 0xAAAABBBBCCCCDDDD}, walBuffer2); hrnPgWalToBuffer((PgWal){.version = PG_VERSION_94, .systemId = 0xAAAABBBBCCCCDDDD}, walBuffer2);
const char *walBuffer2Sha1 = strZ(bufHex(cryptoHashOne(HASH_TYPE_SHA1_STR, walBuffer2))); const char *walBuffer2Sha1 = strZ(bufHex(cryptoHashOne(HASH_TYPE_SHA1_STR, walBuffer2)));
storagePutP(storageNewWriteP(storagePgWrite(), STRDEF("pg_xlog/000000010000000100000002")), walBuffer2); HRN_STORAGE_PUT(storagePgWrite(), "pg_xlog/000000010000000100000002", walBuffer2);
argListTemp = strLstDup(argList); argListTemp = strLstDup(argList);
strLstAddZ(argListTemp, "--archive-push-queue-max=1gb"); hrnCfgArgRawZ(argListTemp, cfgOptArchivePushQueueMax, "1gb");
HRN_CFG_LOAD(cfgCmdArchivePush, argListTemp, .role = cfgCmdRoleAsync); HRN_CFG_LOAD(cfgCmdArchivePush, argListTemp, .role = cfgCmdRoleAsync);
TEST_RESULT_VOID(cmdArchivePushAsync(), "push WAL segments"); TEST_RESULT_VOID(cmdArchivePushAsync(), "push WAL segments");
@ -906,24 +883,24 @@ testRun(void)
"P00 INFO: push 1 WAL file(s) to archive: 000000010000000100000002\n" "P00 INFO: push 1 WAL file(s) to archive: 000000010000000100000002\n"
"P01 DETAIL: pushed WAL file '000000010000000100000002' to the archive"); "P01 DETAIL: pushed WAL file '000000010000000100000002' to the archive");
TEST_RESULT_BOOL( TEST_STORAGE_EXISTS(
storageExistsP( storageTest, strZ(strNewFmt("repo/archive/test/9.4-1/0000000100000001/000000010000000100000002-%s", walBuffer2Sha1)),
storageTest, strNewFmt("repo/archive/test/9.4-1/0000000100000001/000000010000000100000002-%s", walBuffer2Sha1)), .comment = "check repo1 for WAL 2 file");
true, "check repo1 for WAL 2 file"); TEST_STORAGE_EXISTS(
TEST_RESULT_BOOL( storageTest, strZ(strNewFmt("repo3/archive/test/9.4-1/0000000100000001/000000010000000100000002-%s", walBuffer2Sha1)),
storageExistsP( .comment = "check repo3 for WAL 2 file");
storageTest, strNewFmt("repo3/archive/test/9.4-1/0000000100000001/000000010000000100000002-%s", walBuffer2Sha1)),
true, "check repo3 for WAL 2 file");
TEST_RESULT_STRLST_Z( TEST_STORAGE_LIST(
strLstSort(storageListP(storageSpool(), STRDEF(STORAGE_SPOOL_ARCHIVE_OUT)), sortOrderAsc), storageSpool(), STORAGE_SPOOL_ARCHIVE_OUT,
"000000010000000100000001.ok\n000000010000000100000002.ok\n", "check status files"); "000000010000000100000001.ok\n"
"000000010000000100000002.ok\n",
.comment = "check status files");
// ------------------------------------------------------------------------------------------------------------------------- // -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("push wal 2 again to get warnings from both repos"); TEST_TITLE("push wal 2 again to get warnings from both repos");
// Remove the OK file so the WAL gets pushed again // Remove the OK file so the WAL gets pushed again
storageRemoveP(storageSpoolWrite(), STRDEF(STORAGE_SPOOL_ARCHIVE_OUT "/000000010000000100000002.ok")); HRN_STORAGE_REMOVE(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_OUT "/000000010000000100000002.ok");
TEST_RESULT_VOID(cmdArchivePushAsync(), "push WAL segments"); TEST_RESULT_VOID(cmdArchivePushAsync(), "push WAL segments");
TEST_RESULT_LOG( TEST_RESULT_LOG(
@ -944,36 +921,34 @@ testRun(void)
hrnPgWalToBuffer((PgWal){.version = PG_VERSION_94, .systemId = 0xAAAABBBBCCCCDDDD}, walBuffer3); hrnPgWalToBuffer((PgWal){.version = PG_VERSION_94, .systemId = 0xAAAABBBBCCCCDDDD}, walBuffer3);
const char *walBuffer3Sha1 = strZ(bufHex(cryptoHashOne(HASH_TYPE_SHA1_STR, walBuffer3))); const char *walBuffer3Sha1 = strZ(bufHex(cryptoHashOne(HASH_TYPE_SHA1_STR, walBuffer3)));
storagePutP(storageNewWriteP(storagePgWrite(), STRDEF("pg_xlog/000000010000000100000003")), walBuffer3); HRN_STORAGE_PUT(storagePgWrite(), "pg_xlog/000000010000000100000003", walBuffer3);
// Create ready file // Create ready file
storagePutP(storageNewWriteP(storagePgWrite(), STRDEF("pg_xlog/archive_status/000000010000000100000003.ready")), NULL); HRN_STORAGE_PUT_EMPTY(storagePgWrite(), "pg_xlog/archive_status/000000010000000100000003.ready");
TEST_RESULT_VOID(cmdArchivePushAsync(), "push WAL segment"); TEST_RESULT_VOID(cmdArchivePushAsync(), "push WAL segment");
TEST_RESULT_LOG( TEST_RESULT_LOG(
"P00 INFO: push 1 WAL file(s) to archive: 000000010000000100000003\n" "P00 INFO: push 1 WAL file(s) to archive: 000000010000000100000003\n"
"P01 DETAIL: pushed WAL file '000000010000000100000003' to the archive"); "P01 DETAIL: pushed WAL file '000000010000000100000003' to the archive");
TEST_RESULT_BOOL( TEST_STORAGE_EXISTS(
storageExistsP( storageTest, strZ(strNewFmt("repo/archive/test/9.4-1/0000000100000001/000000010000000100000003-%s", walBuffer3Sha1)),
storageTest, strNewFmt("repo/archive/test/9.4-1/0000000100000001/000000010000000100000003-%s", walBuffer3Sha1)), .comment = "check repo1 for WAL 3 file");
true, "check repo1 for WAL 3 file"); TEST_STORAGE_EXISTS(
TEST_RESULT_BOOL( storageTest, strZ(strNewFmt("repo3/archive/test/9.4-1/0000000100000001/000000010000000100000003-%s", walBuffer3Sha1)),
storageExistsP( .comment = "check repo3 for WAL 3 file");
storageTest, strNewFmt("repo3/archive/test/9.4-1/0000000100000001/000000010000000100000003-%s", walBuffer3Sha1)),
true, "check repo3 for WAL 3 file");
// Remove the ready file to prevent WAL 3 from being considered for the next test // Remove the ready file to prevent WAL 3 from being considered for the next test
storageRemoveP(storagePgWrite(), STRDEF("pg_xlog/archive_status/000000010000000100000003.ready"), .errorOnMissing = true); HRN_STORAGE_REMOVE(storagePgWrite(), "pg_xlog/archive_status/000000010000000100000003.ready", .errorOnMissing = true);
// Check that drop functionality works // Check that drop functionality works
// ------------------------------------------------------------------------------------------------------------------------- // -------------------------------------------------------------------------------------------------------------------------
// Remove status files // Remove status files
HRN_STORAGE_PATH_REMOVE(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_OUT, .recurse = true); HRN_STORAGE_PATH_REMOVE(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_OUT, .recurse = true);
storagePathCreateP(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_OUT_STR); HRN_STORAGE_PATH_CREATE(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_OUT);
argListTemp = strLstDup(argList); argListTemp = strLstDup(argList);
strLstAddZ(argListTemp, "--archive-push-queue-max=16m"); hrnCfgArgRawZ(argListTemp, cfgOptArchivePushQueueMax, "16m");
HRN_CFG_LOAD(cfgCmdArchivePush, argListTemp, .role = cfgCmdRoleAsync); HRN_CFG_LOAD(cfgCmdArchivePush, argListTemp, .role = cfgCmdRoleAsync);
TEST_RESULT_VOID(cmdArchivePushAsync(), "push WAL segments"); TEST_RESULT_VOID(cmdArchivePushAsync(), "push WAL segments");
@ -982,19 +957,19 @@ testRun(void)
"P00 WARN: dropped WAL file '000000010000000100000001' because archive queue exceeded 16MB\n" "P00 WARN: dropped WAL file '000000010000000100000001' because archive queue exceeded 16MB\n"
"P00 WARN: dropped WAL file '000000010000000100000002' because archive queue exceeded 16MB"); "P00 WARN: dropped WAL file '000000010000000100000002' because archive queue exceeded 16MB");
TEST_RESULT_STR_Z( TEST_STORAGE_GET(
strNewBuf( storageSpool(), STORAGE_SPOOL_ARCHIVE_OUT "/000000010000000100000001.ok",
storageGetP(storageNewReadP(storageSpool(), STRDEF(STORAGE_SPOOL_ARCHIVE_OUT "/000000010000000100000001.ok")))), "0\ndropped WAL file '000000010000000100000001' because archive queue exceeded 16MB", .comment = "check WAL 1 warning");
"0\ndropped WAL file '000000010000000100000001' because archive queue exceeded 16MB", "check WAL 1 warning");
TEST_RESULT_STR_Z( TEST_STORAGE_GET(
strNewBuf( storageSpool(), STORAGE_SPOOL_ARCHIVE_OUT "/000000010000000100000002.ok",
storageGetP(storageNewReadP(storageSpool(), STRDEF(STORAGE_SPOOL_ARCHIVE_OUT "/000000010000000100000002.ok")))), "0\ndropped WAL file '000000010000000100000002' because archive queue exceeded 16MB", .comment = "check WAL 2 warning");
"0\ndropped WAL file '000000010000000100000002' because archive queue exceeded 16MB", "check WAL 2 warning");
TEST_RESULT_STRLST_Z( TEST_STORAGE_LIST(
strLstSort(storageListP(storageSpool(), STRDEF(STORAGE_SPOOL_ARCHIVE_OUT)), sortOrderAsc), storageSpool(), STORAGE_SPOOL_ARCHIVE_OUT,
"000000010000000100000001.ok\n000000010000000100000002.ok\n", "check status files"); "000000010000000100000001.ok\n"
"000000010000000100000002.ok\n",
.comment = "check status files");
// Uninstall local command handler shim // Uninstall local command handler shim
hrnProtocolLocalShimUninstall(); hrnProtocolLocalShimUninstall();