Refactor common/archiveGet unit test.

The test was pretty old and written in stages during the migration, so storage use was a bit archaic and the organization was poor.

Update using the new storage macros and reorganize the tests to provide better coverage.

test/define.yaml

@@ -565,7 +565,7 @@ unit:
 
       # ----------------------------------------------------------------------------------------------------------------------------
       - name: archive-get
-        total: 5
+        total: 4
         binReq: true
 
         coverage:

test/src/module/command/archiveGetTest.c

@@ -11,6 +11,7 @@ Test Archive Get Command
 #include "storage/posix/storage.h"
 
 #include "common/harnessInfo.h"
+#include "common/harnessStorage.h"
 
 /***********************************************************************************************************************************
 Test Run
@@ -22,16 +23,6 @@ testRun(void)
 
     Storage *storageTest = storagePosixNewP(strNew(testPath()), .write = true);
 
-    // Start a protocol server to test the protocol directly
-    Buffer *serverWrite = bufNew(8192);
-    IoWrite *serverWriteIo = ioBufferWriteNew(serverWrite);
-    ioWriteOpen(serverWriteIo);
-
-    ProtocolServer *server = protocolServerNew(
-        strNew("test"), strNew("test"), ioBufferReadNew(bufNew(0)), serverWriteIo);
-
-    bufUsedSet(serverWrite, 0);
-
     // *****************************************************************************************************************************
     if (testBegin("archiveGetCheck()"))
     {
@@ -39,12 +30,12 @@ testRun(void)
         StringList *argList = strLstNew();
         strLstAddZ(argList, "--stanza=test1");
         strLstAdd(argList, strNewFmt("--repo1-path=%s/repo", testPath()));
-        strLstAdd(argList, strNewFmt("--pg1-path=%s/db", testPath()));
+        strLstAdd(argList, strNewFmt("--pg1-path=%s/pg", testPath()));
         harnessCfgLoad(cfgCmdArchiveGet, argList);
 
         // Create pg_control file
         storagePutP(
-            storageNewWriteP(storageTest, strNew("db/" PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL)),
+            storageNewWriteP(storageTest, strNew("pg/" PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL)),
             pgControlTestToBuffer((PgControl){.version = PG_VERSION_10, .systemId = 0xFACEFACEFACEFACE}));
 
         // Control and archive info mismatch
@@ -117,132 +108,6 @@ testRun(void)
             "history file found");
     }
 
-    // *****************************************************************************************************************************
-    if (testBegin("archiveGetFile()"))
-    {
-        // Load Parameters
-        StringList *argList = strLstNew();
-        strLstAddZ(argList, "--stanza=test1");
-        strLstAdd(argList, strNewFmt("--repo1-path=%s/repo", testPath()));
-        strLstAdd(argList, strNewFmt("--pg1-path=%s/db", testPath()));
-        harnessCfgLoad(cfgCmdArchiveGet, argList);
-
-        // Create pg_control file
-        storagePutP(
-            storageNewWriteP(storageTest, strNew("db/" PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL)),
-            pgControlTestToBuffer((PgControl){.version = PG_VERSION_10, .systemId = 0xFACEFACEFACEFACE}));
-
-        // Create archive.info
-        storagePutP(
-            storageNewWriteP(storageTest, strNew("repo/archive/test1/archive.info")),
-            harnessInfoChecksumZ(
-                "[db]\n"
-                "db-id=1\n"
-                "\n"
-                "[db:history]\n"
-                "1={\"db-id\":18072658121562454734,\"db-version\":\"10\"}"));
-
-        // Nothing to copy
-        // -------------------------------------------------------------------------------------------------------------------------
-        String *archiveFile = strNew("01ABCDEF01ABCDEF01ABCDEF");
-        String *walDestination = strNewFmt("%s/db/pg_wal/RECOVERYXLOG", testPath());
-        storagePathCreateP(storageTest, strPath(walDestination));
-
-        TEST_RESULT_INT(
-            archiveGetFile(storageTest, archiveFile, walDestination, false, cipherTypeNone, NULL), 1, "WAL segment missing");
-
-        // Create a WAL segment to copy
-        // -------------------------------------------------------------------------------------------------------------------------
-        Buffer *buffer = bufNew(16 * 1024 * 1024);
-        memset(bufPtr(buffer), 0, bufSize(buffer));
-        bufUsedSet(buffer, bufSize(buffer));
-
-        storagePutP(
-            storageNewWriteP(
-                storageTest,
-                strNew(
-                    "repo/archive/test1/10-1/01ABCDEF01ABCDEF/01ABCDEF01ABCDEF01ABCDEF-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")),
-            buffer);
-
-        TEST_RESULT_INT(
-            archiveGetFile(storageTest, archiveFile, walDestination, false, cipherTypeNone, NULL), 0, "WAL segment copied");
-        TEST_RESULT_BOOL(storageExistsP(storageTest, walDestination), true, "  check exists");
-        TEST_RESULT_UINT(storageInfoP(storageTest, walDestination).size, 16 * 1024 * 1024, "  check size");
-
-        storageRemoveP(
-            storageTest,
-            strNew("repo/archive/test1/10-1/01ABCDEF01ABCDEF/01ABCDEF01ABCDEF01ABCDEF-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"),
-            .errorOnMissing = true);
-        storageRemoveP(storageTest, walDestination, .errorOnMissing = true);
-
-        // Create a compressed WAL segment to copy
-        // -------------------------------------------------------------------------------------------------------------------------
-        StorageWrite *infoWrite = storageNewWriteP(storageTest, strNew("repo/archive/test1/archive.info"));
-
-        ioFilterGroupAdd(
-            ioWriteFilterGroup(storageWriteIo(infoWrite)), cipherBlockNew(cipherModeEncrypt, cipherTypeAes256Cbc,
-            BUFSTRDEF("12345678"), NULL));
-
-        storagePutP(
-            infoWrite,
-            harnessInfoChecksumZ(
-                "[cipher]\n"
-                "cipher-pass=\"worstpassphraseever\"\n"
-                "\n"
-                "[db]\n"
-                "db-id=1\n"
-                "\n"
-                "[db:history]\n"
-                "1={\"db-id\":18072658121562454734,\"db-version\":\"10\"}"));
-
-        StorageWrite *destination = storageNewWriteP(
-            storageTest,
-            strNew(
-                "repo/archive/test1/10-1/01ABCDEF01ABCDEF/01ABCDEF01ABCDEF01ABCDEF-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.gz"));
-
-        IoFilterGroup *filterGroup = ioWriteFilterGroup(storageWriteIo(destination));
-        ioFilterGroupAdd(filterGroup, compressFilter(compressTypeGz, 3));
-        ioFilterGroupAdd(
-            filterGroup, cipherBlockNew(cipherModeEncrypt, cipherTypeAes256Cbc, BUFSTRDEF("worstpassphraseever"), NULL));
-        storagePutP(destination, buffer);
-
-        TEST_RESULT_INT(
-            archiveGetFile(
-                storageTest, archiveFile, walDestination, false, cipherTypeAes256Cbc, strNew("12345678")), 0, "WAL segment copied");
-        TEST_RESULT_BOOL(storageExistsP(storageTest, walDestination), true, "  check exists");
-        TEST_RESULT_UINT(storageInfoP(storageTest, walDestination).size, 16 * 1024 * 1024, "  check size");
-
-        // Check protocol function directly
-        // -------------------------------------------------------------------------------------------------------------------------
-        argList = strLstNew();
-        strLstAddZ(argList, "--stanza=test1");
-        strLstAdd(argList, strNewFmt("--repo1-path=%s/repo", testPath()));
-        strLstAdd(argList, strNewFmt("--pg1-path=%s/db", testPath()));
-        strLstAdd(argList, strNewFmt("--spool-path=%s/spool", testPath()));
-        strLstAddZ(argList, "--" CFGOPT_ARCHIVE_ASYNC);
-        strLstAddZ(argList, "--repo1-cipher-type=aes-256-cbc");
-        setenv("PGBACKREST_REPO1_CIPHER_PASS", "12345678", true);
-        harnessCfgLoadRole(cfgCmdArchiveGet, cfgCmdRoleAsync, argList);
-        unsetenv("PGBACKREST_REPO1_CIPHER_PASS");
-
-        storagePathCreateP(storageTest, strNew("spool/archive/test1/in"));
-
-        VariantList *paramList = varLstNew();
-        varLstAdd(paramList, varNewStr(archiveFile));
-
-        TEST_RESULT_BOOL(
-            archiveGetProtocol(PROTOCOL_COMMAND_ARCHIVE_GET_STR, paramList, server), true, "protocol archive get");
-        TEST_RESULT_STR_Z(strNewBuf(serverWrite), "{\"out\":0}\n", "check result");
-        TEST_RESULT_BOOL(
-            storageExistsP(storageTest, strNewFmt("spool/archive/test1/in/%s", strZ(archiveFile))), true, "  check exists");
-
-        bufUsedSet(serverWrite, 0);
-
-        // Check invalid protocol function
-        // -------------------------------------------------------------------------------------------------------------------------
-        TEST_RESULT_BOOL(archiveGetProtocol(strNew(BOGUS_STR), paramList, server), false, "invalid function");
-    }
-
     // *****************************************************************************************************************************
     if (testBegin("queueNeed()"))
     {
@@ -303,124 +168,120 @@ testRun(void)
         TEST_STORAGE_LIST(storageSpool(), STORAGE_SPOOL_ARCHIVE_IN, "000000010000000A00000FFE\n000000010000000A00000FFF\n");
     }
 
-
     // *****************************************************************************************************************************
     if (testBegin("cmdArchiveGetAsync()"))
     {
         harnessLogLevelSet(logLevelDetail);
 
-        StringList *argCleanList = strLstNew();
+        // Arguments that must be included
-        strLstAdd(argCleanList, strNewFmt("--pg1-path=%s/pg", testPath()));
+        StringList *argBaseList = strLstNew();
-        strLstAdd(argCleanList, strNewFmt("--repo1-path=%s/repo", testPath()));
+        hrnCfgArgRawZ(argBaseList, cfgOptPgPath, TEST_PATH_PG);
-        strLstAdd(argCleanList, strNewFmt("--spool-path=%s/spool", testPath()));
+        hrnCfgArgRawZ(argBaseList, cfgOptRepoPath, TEST_PATH_REPO);
-        strLstAddZ(argCleanList, "--" CFGOPT_ARCHIVE_ASYNC);
+        hrnCfgArgRawZ(argBaseList, cfgOptSpoolPath, TEST_PATH_SPOOL);
-        strLstAddZ(argCleanList, "--stanza=test2");
+        hrnCfgArgRawBool(argBaseList, cfgOptArchiveAsync, true);
-        harnessCfgLoadRole(cfgCmdArchiveGet, cfgCmdRoleAsync, argCleanList);
+        hrnCfgArgRawZ(argBaseList, cfgOptStanza, "test2");
-
-        TEST_ERROR(cmdArchiveGetAsync(), ParamInvalidError, "at least one wal segment is required");
-
-        TEST_STORAGE_LIST(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_IN, "global.error\n", .remove = true);
 
         // -------------------------------------------------------------------------------------------------------------------------
         TEST_TITLE("command must be run on the pg host");
 
-        StringList *argList = strLstNew();
+        StringList *argList = strLstDup(argBaseList);
-        hrnCfgArgRawZ(argList, cfgOptPgHost, "host");
+        hrnCfgArgRawZ(argList, cfgOptPgHost, BOGUS_STR);
-        hrnCfgArgRawZ(argList, cfgOptPgPath, "/pg");
-        hrnCfgArgRawZ(argList, cfgOptRepoPath, "/repo");
-        hrnCfgArgRawZ(argList, cfgOptSpoolPath, TEST_PATH "/spool");
-        strLstAddZ(argList, "--" CFGOPT_ARCHIVE_ASYNC);
-        strLstAddZ(argList, "--" CFGOPT_STANZA "=test2");
-        strLstAddZ(argList, "000000010000000100000001");
         harnessCfgLoadRole(cfgCmdArchiveGet, cfgCmdRoleAsync, argList);
 
         TEST_ERROR(cmdArchiveGetAsync(), HostInvalidError, "archive-get command must be run on the PostgreSQL host");
 
         TEST_STORAGE_LIST(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_IN, "global.error\n", .remove = true);
 
-        // Create pg_control file and archive.info
         // -------------------------------------------------------------------------------------------------------------------------
-        storagePutP(
+        TEST_TITLE("error on no segments");
-            storageNewWriteP(storageTest, strNew("pg/" PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL)),
+
+        argList = strLstDup(argBaseList);
+        harnessCfgLoadRole(cfgCmdArchiveGet, cfgCmdRoleAsync, argList);
+
+        TEST_ERROR(cmdArchiveGetAsync(), ParamInvalidError, "at least one wal segment is required");
+
+        TEST_STORAGE_LIST(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_IN, "global.error\n", .remove = true);
+
+        // -------------------------------------------------------------------------------------------------------------------------
+        TEST_TITLE("no segments to find");
+
+        HRN_STORAGE_PUT(
+            storagePgWrite(), PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL,
             pgControlTestToBuffer((PgControl){.version = PG_VERSION_10, .systemId = 0xFACEFACEFACEFACE}));
 
-        storagePutP(
+        HRN_INFO_PUT(
-            storageNewWriteP(storageTest, strNew("repo/archive/test2/archive.info")),
+            storageRepoWrite(), INFO_ARCHIVE_PATH_FILE,
-            harnessInfoChecksumZ(
+            "[db]\n"
-                "[db]\n"
+            "db-id=1\n"
-                "db-id=1\n"
+            "\n"
-                "\n"
+            "[db:history]\n"
-                "[db:history]\n"
+            "1={\"db-id\":18072658121562454734,\"db-version\":\"10\"}\n");
-                "1={\"db-id\":18072658121562454734,\"db-version\":\"10\"}\n"));
 
-        // Get a single segment
-        // -------------------------------------------------------------------------------------------------------------------------
-        argList = strLstDup(argCleanList);
         strLstAddZ(argList, "000000010000000100000001");
         harnessCfgLoadRole(cfgCmdArchiveGet, cfgCmdRoleAsync, argList);
 
-        storagePathCreateP(storageSpoolWrite(), strNew(STORAGE_SPOOL_ARCHIVE_IN));
+        TEST_RESULT_VOID(cmdArchiveGetAsync(), "get async");
 
-        TEST_RESULT_VOID(
+        harnessLogResult(
-            storagePutP(
+            "P00   INFO: get 1 WAL file(s) from archive: 000000010000000100000001\n"
-                storageNewWriteP(
+            "P01 DETAIL: unable to find 000000010000000100000001 in the archive");
-                    storageTest,
-                    strNew(
-                        "repo/archive/test2/10-1/0000000100000001/"
-                            "000000010000000100000001-abcdabcdabcdabcdabcdabcdabcdabcdabcdabcd")),
-                NULL),
-            "normal WAL segment");
 
-        // Create tmp file to make it look like a prior async get failed partway through to ensure that retries work
+        TEST_STORAGE_LIST(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_IN, "000000010000000100000001.ok\n", .remove = true);
-        TEST_RESULT_VOID(
+
-            storagePutP(
+        // -------------------------------------------------------------------------------------------------------------------------
-                storageNewWriteP(storageSpoolWrite(), strNew(STORAGE_SPOOL_ARCHIVE_IN "/000000010000000100000001.pgbackrest.tmp")),
+        TEST_TITLE("error on invalid compressed segment");
-                NULL),
+
-            "normal WAL segment");
+        HRN_STORAGE_PUT_EMPTY(
+            storageRepoWrite(), STORAGE_REPO_ARCHIVE "/10-1/000000010000000100000001-abcdabcdabcdabcdabcdabcdabcdabcdabcdabcd.gz");
+
+        TEST_RESULT_VOID(cmdArchiveGetAsync(), "get async");
+
+        harnessLogResult(
+            "P00   INFO: get 1 WAL file(s) from archive: 000000010000000100000001\n"
+            "P01   WARN: could not get 000000010000000100000001 from the archive (will be retried):"
+                " [29] raised from local-1 protocol: unexpected eof in compressed data");
+
+        TEST_STORAGE_LIST(
+            storageSpool(), STORAGE_SPOOL_ARCHIVE_IN, "000000010000000100000001.error\n000000010000000100000001.pgbackrest.tmp\n");
+        TEST_STORAGE_REMOVE(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_IN "/000000010000000100000001.error");
+
+        TEST_STORAGE_REMOVE(
+            storageRepoWrite(), STORAGE_REPO_ARCHIVE "/10-1/000000010000000100000001-abcdabcdabcdabcdabcdabcdabcdabcdabcdabcd.gz");
+
+        // -------------------------------------------------------------------------------------------------------------------------
+        TEST_TITLE("single segment");
+
+        HRN_STORAGE_PUT_EMPTY(
+            storageRepoWrite(), STORAGE_REPO_ARCHIVE "/10-1/000000010000000100000001-abcdabcdabcdabcdabcdabcdabcdabcdabcdabcd");
+
+        // There should be a temp file left over. Make sure it still exists to test that temp files are removed on retry.
+        TEST_STORAGE_EXISTS(storageSpool(), STORAGE_SPOOL_ARCHIVE_IN "/000000010000000100000001.pgbackrest.tmp");
 
         TEST_RESULT_VOID(cmdArchiveGetAsync(), "archive async");
+
         harnessLogResult(
             "P00   INFO: get 1 WAL file(s) from archive: 000000010000000100000001\n"
             "P01 DETAIL: found 000000010000000100000001 in the archive");
 
-        TEST_RESULT_BOOL(
+        TEST_STORAGE_LIST(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_IN, "000000010000000100000001\n", .remove = true);
-            storageExistsP(storageSpool(), strNew(STORAGE_SPOOL_ARCHIVE_IN "/000000010000000100000001")), true,
-            "check 000000010000000100000001 in spool");
-        TEST_RESULT_BOOL(
-            storageExistsP(storageSpool(), strNew(STORAGE_SPOOL_ARCHIVE_IN "/000000010000000100000001.pgbackrest.tmp")), false,
-            "check 000000010000000100000001 tmp not in spool");
 
-        // Get multiple segments where some are missing or errored
         // -------------------------------------------------------------------------------------------------------------------------
-        argList = strLstDup(argCleanList);
+        TEST_TITLE("multiple segments where some are missing or errored");
+
+        argList = strLstDup(argBaseList);
         strLstAddZ(argList, "000000010000000100000001");
         strLstAddZ(argList, "000000010000000100000002");
         strLstAddZ(argList, "000000010000000100000003");
         harnessCfgLoadRole(cfgCmdArchiveGet, cfgCmdRoleAsync, argList);
 
-        storagePathCreateP(storageSpoolWrite(), strNew(STORAGE_SPOOL_ARCHIVE_IN));
+        // Create segment duplicates
-
+        HRN_STORAGE_PUT_EMPTY(
-        TEST_RESULT_VOID(
+            storageRepoWrite(), STORAGE_REPO_ARCHIVE "/10-1/000000010000000100000003-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa");
-            storagePutP(
+        HRN_STORAGE_PUT_EMPTY(
-                storageNewWriteP(
+            storageRepoWrite(), STORAGE_REPO_ARCHIVE "/10-1/000000010000000100000003-bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb");
-                    storageTest,
-                    strNew(
-                        "repo/archive/test2/10-1/0000000100000001/"
-                            "000000010000000100000003-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")),
-                NULL),
-            "normal WAL segment");
-
-        TEST_RESULT_VOID(
-            storagePutP(
-                storageNewWriteP(
-                    storageTest,
-                    strNew(
-                        "repo/archive/test2/10-1/0000000100000001/"
-                            "000000010000000100000003-bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb")),
-                NULL),
-            "duplicate WAL segment");
 
         TEST_RESULT_VOID(cmdArchiveGetAsync(), "archive async");
+
         harnessLogResult(
             "P00   INFO: get 3 WAL file(s) from archive: 000000010000000100000001...000000010000000100000003\n"
             "P01 DETAIL: found 000000010000000100000001 in the archive\n"
@@ -431,35 +292,21 @@ testRun(void)
                 "000000010000000100000003-bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\n"
             "            HINT: are multiple primaries archiving to this stanza?");
 
-        TEST_RESULT_BOOL(
+        TEST_STORAGE_LIST(
-            storageExistsP(storageSpool(), strNew(STORAGE_SPOOL_ARCHIVE_IN "/000000010000000100000001")), true,
+            storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_IN,
-            "check 000000010000000100000001 in spool");
+            "000000010000000100000001\n000000010000000100000002.ok\n000000010000000100000003.error\n",
-        TEST_RESULT_BOOL(
+            .remove = true);
-            storageExistsP(storageSpool(), strNew(STORAGE_SPOOL_ARCHIVE_IN "/000000010000000100000002")), false,
-            "check 000000010000000100000002 not in spool");
-        TEST_RESULT_BOOL(
-            storageExistsP(storageSpool(), strNew(STORAGE_SPOOL_ARCHIVE_IN "/000000010000000100000002.ok")), true,
-            "check 000000010000000100000002.ok in spool");
-        TEST_RESULT_BOOL(
-            storageExistsP(storageSpool(), strNew(STORAGE_SPOOL_ARCHIVE_IN "/000000010000000100000003")), false,
-            "check 000000010000000100000003 not in spool");
-        TEST_RESULT_BOOL(
-            storageExistsP(storageSpool(), strNew(STORAGE_SPOOL_ARCHIVE_IN "/000000010000000100000003.error")), true,
-            "check 000000010000000100000003.error in spool");
-
-        protocolFree();
 
         // -------------------------------------------------------------------------------------------------------------------------
-        storageRemoveP(
+        TEST_TITLE("global error on invalid executable");
-            storageSpoolWrite(), strNew(STORAGE_SPOOL_ARCHIVE_IN "/000000010000000100000003.error"), .errorOnMissing = true);
 
         argList = strLstNew();
         strLstAddZ(argList, "pgbackrest-bogus");
-        strLstAdd(argList, strNewFmt("--pg1-path=%s/pg", testPath()));
+        hrnCfgArgRawZ(argList, cfgOptPgPath, TEST_PATH_PG);
-        strLstAdd(argList, strNewFmt("--repo1-path=%s/repo", testPath()));
+        hrnCfgArgRawZ(argList, cfgOptRepoPath, TEST_PATH_REPO);
-        strLstAdd(argList, strNewFmt("--spool-path=%s/spool", testPath()));
+        hrnCfgArgRawZ(argList, cfgOptSpoolPath, TEST_PATH_SPOOL);
-        strLstAddZ(argList, "--" CFGOPT_ARCHIVE_ASYNC);
+        hrnCfgArgRawBool(argList, cfgOptArchiveAsync, true);
-        strLstAddZ(argList, "--stanza=test2");
+        hrnCfgArgRawZ(argList, cfgOptStanza, "test2");
         strLstAddZ(argList, CFGCMD_ARCHIVE_GET ":" CONFIG_COMMAND_ROLE_ASYNC);
         strLstAddZ(argList, "000000010000000100000001");
         strLstAddZ(argList, "000000010000000100000002");
@@ -473,71 +320,60 @@ testRun(void)
         harnessLogResult(
             "P00   INFO: get 3 WAL file(s) from archive: 000000010000000100000001...000000010000000100000003");
 
-        TEST_RESULT_BOOL(
-            storageExistsP(storageSpool(), strNew(STORAGE_SPOOL_ARCHIVE_IN "/000000010000000100000001.error")), false,
-            "check 000000010000000100000001.error not in spool");
-        TEST_RESULT_BOOL(
-            storageExistsP(storageSpool(), strNew(STORAGE_SPOOL_ARCHIVE_IN "/000000010000000100000002.error")), false,
-            "check 000000010000000100000002.error not in spool");
-        TEST_RESULT_BOOL(
-            storageExistsP(storageSpool(), strNew(STORAGE_SPOOL_ARCHIVE_IN "/000000010000000100000003.error")), false,
-            "check 000000010000000100000003.error not in spool");
         TEST_RESULT_STR_Z(
             strNewBuf(storageGetP(storageNewReadP(storageSpool(), strNew(STORAGE_SPOOL_ARCHIVE_IN "/global.error")))),
             "102\nlocal-1 process terminated unexpectedly [102]: unable to execute 'pgbackrest-bogus': "
                 "[2] No such file or directory",
             "check global error");
+
+        TEST_STORAGE_LIST(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_IN, "global.error\n", .remove = true);
     }
 
     // *****************************************************************************************************************************
     if (testBegin("cmdArchiveGet()"))
     {
+        harnessLogLevelSet(logLevelDetail);
+
+        // Arguments that must be included. Use raw config here because we need to keep the
+        StringList *argBaseList = strLstNew();
+        strLstAddZ(argBaseList, "pgbackrest-bogus");
+        hrnCfgArgRawZ(argBaseList, cfgOptPgPath, TEST_PATH_PG);
+        hrnCfgArgRawZ(argBaseList, cfgOptRepoPath, TEST_PATH_REPO);
+        hrnCfgArgRawZ(argBaseList, cfgOptStanza, "test1");
+        hrnCfgArgRawZ(argBaseList, cfgOptArchiveTimeout, "1");
+        strLstAddZ(argBaseList, CFGCMD_ARCHIVE_GET);
+
+        // -------------------------------------------------------------------------------------------------------------------------
         TEST_TITLE("command must be run on the pg host");
 
-        StringList *argList = strLstNew();
+        StringList *argList = strLstDup(argBaseList);
-        hrnCfgArgRawZ(argList, cfgOptPgHost, "host");
+        hrnCfgArgRawZ(argList, cfgOptPgHost, BOGUS_STR);
-        hrnCfgArgRawZ(argList, cfgOptPgPath, "/pg");
+        harnessCfgLoadRaw(strLstSize(argList), strLstPtr(argList));
-        hrnCfgArgRawZ(argList, cfgOptRepoPath, "/repo");
-        strLstAddZ(argList, "--" CFGOPT_STANZA "=test2");
-        strLstAddZ(argList, "000000010000000100000001");
-        strLstAddZ(argList, "pg_wal/000000010000000100000001");
-        harnessCfgLoadRole(cfgCmdArchiveGet, cfgCmdRoleDefault, argList);
 
         TEST_ERROR(cmdArchiveGet(), HostInvalidError, "archive-get command must be run on the PostgreSQL host");
 
         // -------------------------------------------------------------------------------------------------------------------------
-        argList = strLstNew();
+        argList = strLstDup(argBaseList);
-        strLstAddZ(argList, "pgbackrest-bogus");                    // Break this until async tests are setup correctly
-        strLstAddZ(argList, "--archive-timeout=1");
-        strLstAdd(argList, strNewFmt("--lock-path=%s/lock", testPath()));
-        strLstAdd(argList, strNewFmt("--log-path=%s", testPath()));
-        strLstAdd(argList, strNewFmt("--log-level-file=debug"));
-        strLstAdd(argList, strNewFmt("--repo1-path=%s/repo", testPath()));
-        hrnCfgArgRawFmt(argList, cfgOptPgPath, "%s/db", testPath());
-        strLstAddZ(argList, "--stanza=test1");
-        strLstAddZ(argList, "archive-get");
         harnessCfgLoadRaw(strLstSize(argList), strLstPtr(argList));
 
         TEST_ERROR(cmdArchiveGet(), ParamRequiredError, "WAL segment to get required");
 
         // -------------------------------------------------------------------------------------------------------------------------
-        StringList *argListTemp = strLstDup(argList);
+        argList = strLstDup(argBaseList);
-        String *walSegment = strNew("000000010000000100000001");
+        strLstAddZ(argList, "000000010000000100000001");
-        strLstAdd(argListTemp, walSegment);
+        harnessCfgLoadRaw(strLstSize(argList), strLstPtr(argList));
-        harnessCfgLoadRaw(strLstSize(argListTemp), strLstPtr(argListTemp));
 
         TEST_ERROR(cmdArchiveGet(), ParamRequiredError, "path to copy WAL segment required");
 
         // -------------------------------------------------------------------------------------------------------------------------
-        storagePutP(
+        HRN_STORAGE_PUT(
-            storageNewWriteP(storageTest, strNew("db/" PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL)),
+            storagePgWrite(), PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL,
             pgControlTestToBuffer((PgControl){.version = PG_VERSION_10, .systemId = 0xFACEFACEFACEFACE}));
 
-        storagePathCreateP(storageTest, strNewFmt("%s/db/pg_wal", testPath()));
+        storagePathCreateP(storageTest, strNewFmt("%s/pg/pg_wal", testPath()));
 
-        String *walFile = strNewFmt("%s/db/pg_wal/RECOVERYXLOG", testPath());
+        strLstAddZ(argList, TEST_PATH_PG "/pg_wal/RECOVERYXLOG");
-        strLstAdd(argListTemp, walFile);
+        harnessCfgLoadRaw(strLstSize(argList), strLstPtr(argList));
-        harnessCfgLoadRaw(strLstSize(argListTemp), strLstPtr(argListTemp));
 
         TEST_ERROR_FMT(
             cmdArchiveGet(), FileMissingError,
@@ -554,11 +390,12 @@ testRun(void)
             strZ(strNewFmt("%s/archive/test1/archive.info.copy", strZ(cfgOptionStr(cfgOptRepoPath)))));
 
         // -------------------------------------------------------------------------------------------------------------------------
-        argListTemp = strLstDup(argList);
+        // !!! IS THIS TEST NEEDED
-        strLstAddZ(argListTemp, "00000001.history");
+        argList = strLstDup(argBaseList);
-        strLstAdd(argListTemp, walFile);
+        strLstAddZ(argList, "00000001.history");
-        strLstAddZ(argListTemp, "--archive-async");
+        strLstAddZ(argList, TEST_PATH_PG "/pg_wal/RECOVERYHISTORY");
-        harnessCfgLoadRaw(strLstSize(argListTemp), strLstPtr(argListTemp));
+        strLstAddZ(argList, "--archive-async");
+        harnessCfgLoadRaw(strLstSize(argList), strLstPtr(argList));
 
         TEST_ERROR_FMT(
             cmdArchiveGet(), FileMissingError,
@@ -576,9 +413,10 @@ testRun(void)
 
         // Make sure the process times out when there is nothing to get
         // -------------------------------------------------------------------------------------------------------------------------
-        strLstAdd(argList, strNewFmt("--spool-path=%s/spool", testPath()));
+        argList = strLstDup(argBaseList);
-        strLstAddZ(argList, "--archive-async");
+        hrnCfgArgRawZ(argList, cfgOptSpoolPath, TEST_PATH_SPOOL);
-        strLstAdd(argList, walSegment);
+        hrnCfgArgRawBool(argList, cfgOptArchiveAsync, true);
+        strLstAddZ(argList, "000000010000000100000001");
         strLstAddZ(argList, "pg_wal/RECOVERYXLOG");
         harnessCfgLoadRaw(strLstSize(argList), strLstPtr(argList));
 
@@ -590,51 +428,40 @@ testRun(void)
 
         // Check for missing WAL
         // -------------------------------------------------------------------------------------------------------------------------
-        storagePutP(storageNewWriteP(storageSpoolWrite(), strNewFmt(STORAGE_SPOOL_ARCHIVE_IN "/%s.ok", strZ(walSegment))), NULL);
+        HRN_STORAGE_PUT_EMPTY(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_IN "/000000010000000100000001.ok");
 
         TEST_RESULT_INT(cmdArchiveGet(), 1, "successful get of missing WAL");
 
         harnessLogResult("P00   INFO: unable to find 000000010000000100000001 in the archive");
 
         TEST_RESULT_BOOL(
-            storageExistsP(storageSpool(), strNewFmt(STORAGE_SPOOL_ARCHIVE_IN "/%s.ok", strZ(walSegment))), false,
+            storageExistsP(storageSpool(), STRDEF(STORAGE_SPOOL_ARCHIVE_IN "/000000010000000100000001.ok")), false,
             "check OK file was removed");
 
         // Write out a WAL segment for success
         // -------------------------------------------------------------------------------------------------------------------------
-        storagePutP(
+        HRN_STORAGE_PUT_Z(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_IN "/000000010000000100000001", "SHOULD-BE-A-REAL-WAL-FILE");
-            storageNewWriteP(storageSpoolWrite(), strNewFmt(STORAGE_SPOOL_ARCHIVE_IN "/%s", strZ(walSegment))),
-            BUFSTRDEF("SHOULD-BE-A-REAL-WAL-FILE"));
 
         TEST_RESULT_INT(cmdArchiveGet(), 0, "successful get");
 
         TEST_RESULT_VOID(harnessLogResult("P00   INFO: found 000000010000000100000001 in the archive"), "check log");
 
-        TEST_RESULT_BOOL(
+        TEST_STORAGE_LIST_EMPTY(storageSpool(), STORAGE_SPOOL_ARCHIVE_IN);
-            storageExistsP(storageSpoolWrite(), strNewFmt(STORAGE_SPOOL_ARCHIVE_IN "/%s", strZ(walSegment))), false,
+        TEST_STORAGE_LIST(storageTest, TEST_PATH_PG "/pg_wal", "RECOVERYXLOG\n", .remove = true);
-            "check WAL segment was removed from source");
-        TEST_RESULT_BOOL(storageExistsP(storageTest, walFile), true, "check WAL segment was moved to destination");
-        storageRemoveP(storageTest, walFile, .errorOnMissing = true);
 
         // Write more WAL segments (in this case queue should be full)
         // -------------------------------------------------------------------------------------------------------------------------
         strLstAddZ(argList, "--archive-get-queue-max=48");
         harnessCfgLoadRaw(strLstSize(argList), strLstPtr(argList));
 
-        String *walSegment2 = strNew("000000010000000100000002");
+        HRN_STORAGE_PUT_Z(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_IN "/000000010000000100000001", "SHOULD-BE-A-REAL-WAL-FILE");
-
+        HRN_STORAGE_PUT_Z(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_IN "/000000010000000100000002", "SHOULD-BE-A-REAL-WAL-FILE");
-        storagePutP(
-            storageNewWriteP(storageSpoolWrite(), strNewFmt(STORAGE_SPOOL_ARCHIVE_IN "/%s", strZ(walSegment))),
-            BUFSTRDEF("SHOULD-BE-A-REAL-WAL-FILE"));
-        storagePutP(
-            storageNewWriteP(storageSpoolWrite(), strNewFmt(STORAGE_SPOOL_ARCHIVE_IN "/%s", strZ(walSegment2))),
-            BUFSTRDEF("SHOULD-BE-A-REAL-WAL-FILE"));
 
         TEST_RESULT_INT(cmdArchiveGet(), 0, "successful get");
 
         TEST_RESULT_VOID(harnessLogResult("P00   INFO: found 000000010000000100000001 in the archive"), "check log");
 
-        TEST_RESULT_BOOL(storageExistsP(storageTest, walFile), true, "check WAL segment was moved");
+        TEST_STORAGE_LIST(storageTest, TEST_PATH_PG "/pg_wal", "RECOVERYXLOG\n", .remove = true);
 
         // Make sure the process times out when it can't get a lock
         // -------------------------------------------------------------------------------------------------------------------------
@@ -653,6 +480,132 @@ testRun(void)
         harnessCfgLoadRaw(strLstSize(argList), strLstPtr(argList));
 
         TEST_ERROR(cmdArchiveGet(), ParamInvalidError, "extra parameters found");
+
+        // -------------------------------------------------------------------------------------------------------------------------
+        TEST_TITLE("file is missing");
+
+        HRN_STORAGE_PUT(
+            storagePgWrite(), PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL,
+            pgControlTestToBuffer((PgControl){.version = PG_VERSION_10, .systemId = 0xFACEFACEFACEFACE}));
+
+        HRN_INFO_PUT(
+            storageRepoWrite(), INFO_ARCHIVE_PATH_FILE,
+            "[db]\n"
+            "db-id=1\n"
+            "\n"
+            "[db:history]\n"
+            "1={\"db-id\":18072658121562454734,\"db-version\":\"10\"}");
+
+        argList = strLstNew();
+        hrnCfgArgRawZ(argList, cfgOptPgPath, TEST_PATH_PG);
+        hrnCfgArgRawZ(argList, cfgOptRepoPath, TEST_PATH_REPO);
+        hrnCfgArgRawZ(argList, cfgOptStanza, "test1");
+        strLstAddZ(argList, "01ABCDEF01ABCDEF01ABCDEF");
+        strLstAddZ(argList, TEST_PATH_PG "/pg_wal/RECOVERYXLOG");
+        harnessCfgLoad(cfgCmdArchiveGet, argList);
+
+        TEST_RESULT_INT(cmdArchiveGet(), 1, "get");
+
+        harnessLogResult("P00   INFO: unable to find 01ABCDEF01ABCDEF01ABCDEF in the archive");
+
+        TEST_STORAGE_LIST_EMPTY(storageTest, TEST_PATH_PG "/pg_wal");
+
+        // -------------------------------------------------------------------------------------------------------------------------
+        TEST_TITLE("get WAL segment");
+
+        Buffer *buffer = bufNew(16 * 1024 * 1024);
+        memset(bufPtr(buffer), 0, bufSize(buffer));
+        bufUsedSet(buffer, bufSize(buffer));
+
+        HRN_STORAGE_PUT(
+            storageRepoWrite(), STORAGE_REPO_ARCHIVE "/10-1/01ABCDEF01ABCDEF01ABCDEF-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+            buffer);
+
+        TEST_RESULT_INT(cmdArchiveGet(), 0, "get");
+
+        harnessLogResult("P00   INFO: found 01ABCDEF01ABCDEF01ABCDEF in the archive");
+
+        TEST_RESULT_UINT(
+            storageInfoP(storageTest, STRDEF(TEST_PATH_PG "/pg_wal/RECOVERYXLOG")).size, 16 * 1024 * 1024, "check size");
+        TEST_STORAGE_LIST(storageTest, TEST_PATH_PG "/pg_wal", "RECOVERYXLOG\n", .remove = true);
+
+        TEST_STORAGE_REMOVE(
+            storageRepoWrite(), STORAGE_REPO_ARCHIVE "/10-1/01ABCDEF01ABCDEF01ABCDEF-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa");
+
+        // -------------------------------------------------------------------------------------------------------------------------
+        TEST_TITLE("get compressed and encrypted WAL segment");
+
+        HRN_INFO_PUT(
+            storageRepoWrite(), INFO_ARCHIVE_PATH_FILE,
+            "[cipher]\n"
+            "cipher-pass=\"" TEST_CIPHER_PASS_ARCHIVE "\"\n"
+            "\n"
+            "[db]\n"
+            "db-id=1\n"
+            "\n"
+            "[db:history]\n"
+            "1={\"db-id\":18072658121562454734,\"db-version\":\"10\"}",
+            .cipherType = cipherTypeAes256Cbc);
+
+        HRN_STORAGE_PUT(
+            storageRepoWrite(), STORAGE_REPO_ARCHIVE "/10-1/01ABCDEF01ABCDEF01ABCDEF-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+            buffer, .compressType = compressTypeGz, .cipherType = cipherTypeAes256Cbc, .cipherPass = TEST_CIPHER_PASS_ARCHIVE);
+
+        // Add encryption options
+        hrnCfgArgRawZ(argList, cfgOptRepoCipherType, CIPHER_TYPE_AES_256_CBC);
+        hrnCfgEnvRawZ(cfgOptRepoCipherPass, TEST_CIPHER_PASS);
+        harnessCfgLoad(cfgCmdArchiveGet, argList);
+        hrnCfgEnvRemoveRaw(cfgOptRepoCipherPass);
+
+        TEST_RESULT_INT(cmdArchiveGet(), 0, "get");
+
+        harnessLogResult("P00   INFO: found 01ABCDEF01ABCDEF01ABCDEF in the archive");
+
+        TEST_STORAGE_LIST(storageTest, TEST_PATH_PG "/pg_wal", "RECOVERYXLOG\n");
+        TEST_RESULT_UINT(
+            storageInfoP(storageTest, STRDEF(TEST_PATH_PG "/pg_wal/RECOVERYXLOG")).size, 16 * 1024 * 1024, "check size");
+
+        // -------------------------------------------------------------------------------------------------------------------------
+        TEST_TITLE("call protocol function directly");
+
+        // Start a protocol server
+        Buffer *serverWrite = bufNew(8192);
+        IoWrite *serverWriteIo = ioBufferWriteNew(serverWrite);
+        ioWriteOpen(serverWriteIo);
+
+        ProtocolServer *server = protocolServerNew(
+            strNew("test"), strNew("test"), ioBufferReadNew(bufNew(0)), serverWriteIo);
+
+        bufUsedSet(serverWrite, 0);
+
+        // Add spool path
+        argList = strLstNew();
+        hrnCfgArgRawZ(argList, cfgOptPgPath, TEST_PATH_PG);
+        hrnCfgArgRawZ(argList, cfgOptRepoPath, TEST_PATH_REPO);
+        hrnCfgArgRawZ(argList, cfgOptStanza, "test1");
+        hrnCfgArgRawZ(argList, cfgOptRepoCipherType, CIPHER_TYPE_AES_256_CBC);
+        hrnCfgArgRawZ(argList, cfgOptSpoolPath, TEST_PATH_SPOOL);
+        hrnCfgArgRawBool(argList, cfgOptArchiveAsync, true);
+        hrnCfgEnvRawZ(cfgOptRepoCipherPass, TEST_CIPHER_PASS);
+        harnessCfgLoadRole(cfgCmdArchiveGet, cfgCmdRoleLocal, argList);
+        hrnCfgEnvRemoveRaw(cfgOptRepoCipherPass);
+
+        // Setup protocol command
+        VariantList *paramList = varLstNew();
+        varLstAdd(paramList, varNewStrZ("01ABCDEF01ABCDEF01ABCDEF"));
+
+        TEST_RESULT_BOOL(
+            archiveGetProtocol(PROTOCOL_COMMAND_ARCHIVE_GET_STR, paramList, server), true, "protocol archive get");
+
+        TEST_RESULT_STR_Z(strNewBuf(serverWrite), "{\"out\":0}\n", "check result");
+        TEST_STORAGE_LIST(storageSpool(), STORAGE_SPOOL_ARCHIVE_IN, "000000010000000100000002\n01ABCDEF01ABCDEF01ABCDEF\n");
+
+        bufUsedSet(serverWrite, 0);
+
+        // -------------------------------------------------------------------------------------------------------------------------
+        TEST_TITLE("invalid protocol command");
+
+        TEST_RESULT_BOOL(archiveGetProtocol(strNew(BOGUS_STR), paramList, server), false, "invalid function");
     }
 
     FUNCTION_HARNESS_RESULT_VOID();