Add storage test macros for common operations.

The macros should make it much easier to write complex tests, especially when compression and encryption are involved.

Update the command/archiveGet test to show how the new macros are used.

test/src/common/harnessInfo.c

@@ -65,6 +65,8 @@ harnessInfoChecksum(const String *info)
         FUNCTION_HARNESS_PARAM(STRING, info);
     FUNCTION_HARNESS_END();
 
+    ASSERT(info != NULL);
+
     Buffer *result = NULL;
 
     MEM_CONTEXT_TEMP_BEGIN()
@@ -110,6 +112,8 @@ harnessInfoChecksumZ(const char *info)
         FUNCTION_HARNESS_PARAM(STRINGZ, info);
     FUNCTION_HARNESS_END();
 
+    ASSERT(info != NULL);
+
     FUNCTION_HARNESS_RESULT(BUFFER, harnessInfoChecksum(STR(info)));
 }
 

test/src/common/harnessInfo.h

@@ -4,6 +4,16 @@ Harness for Generating Test Info Files
 #include "common/type/buffer.h"
 #include "info/info.h"
 
+#include "common/harnessStorage.h"
+
+/***********************************************************************************************************************************
+Write info to a file and add the checksum
+***********************************************************************************************************************************/
+#define HRN_INFO_PUT(storage, file, info, ...)                                                                                     \
+    TEST_RESULT_VOID(                                                                                                              \
+        hrnStoragePut(storage, file, harnessInfoChecksumZ(info), (HrnStoragePutParam){VAR_PARAM_INIT, __VA_ARGS__}), "put info %s",\
+        hrnStoragePutLog(storage, file, LF_BUF, (HrnStoragePutParam){VAR_PARAM_INIT, __VA_ARGS__}))
+
 /***********************************************************************************************************************************
 Functions
 ***********************************************************************************************************************************/

test/src/common/harnessStorage.c

@@ -2,14 +2,19 @@
 Storage Test Harness
 ***********************************************************************************************************************************/
 #include <inttypes.h>
+#include <stdio.h>
+#include <string.h>
 
+#include "common/crypto/cipherBlock.h"
 #include "common/debug.h"
-#include "common/compress/helper.h"
 #include "common/type/object.h"
+#include "common/type/param.h"
 #include "common/user.h"
 #include "storage/storage.h"
 
+#include "common/harnessConfig.h"
 #include "common/harnessStorage.h"
+#include "common/harnessTest.h"
 
 /***********************************************************************************************************************************
 Dummy functions and interface for constructing test storage drivers
@@ -169,3 +174,96 @@ hrnStorageInfoListCallback(void *callbackData, const StorageInfo *info)
 
     strCatZ(data->content, "}\n");
 }
+
+/**********************************************************************************************************************************/
+StringList *
+hrnStorageList(const Storage *storage, const char *path, HrnStorageListParam param)
+{
+    StringList *list = strLstSort(storageListP(storage, STR(path)), sortOrderAsc);
+
+    // Remove files if requested
+    if (param.remove)
+    {
+        for (unsigned int listIdx = 0; listIdx < strLstSize(list); listIdx++)
+            storageRemoveP(storage, strNewFmt("%s/%s", path, strZ(strLstGet(list, listIdx))), .errorOnMissing = true);
+    }
+
+    // Return list for comparison
+    return list;
+}
+
+const char *
+hrnStorageListLog(const Storage *storage, const char *path, HrnStorageListParam param)
+{
+    StringList *list = strLstSort(storageListP(storage, STR(path)), sortOrderAsc);
+
+    return strZ(
+        strNewFmt(
+            "list%s %u file%s in '%s'", param.remove ? "/remove": "", strLstSize(list), strLstSize(list) == 1 ? "" : "s",
+            strZ(storagePathP(storage, STR(path)))));
+}
+
+/**********************************************************************************************************************************/
+void
+hrnStoragePut(const Storage *storage, const char *file, const Buffer *buffer, HrnStoragePutParam param)
+{
+    // Add compression extension to file name
+    String *fileStr = strNew(file);
+    compressExtCat(fileStr, param.compressType);
+
+    // Create file
+    StorageWrite *destination = storageNewWriteP(storage, fileStr);
+    IoFilterGroup *filterGroup = ioWriteFilterGroup(storageWriteIo(destination));
+
+    // Add compression filter
+    if (param.compressType != compressTypeNone)
+    {
+        ASSERT(param.compressType == compressTypeGz || param.compressType == compressTypeBz2);
+        ioFilterGroupAdd(filterGroup, compressFilter(param.compressType, 1));
+    }
+
+    // Add encrypted filter
+    if (param.cipherType != cipherTypeNone)
+    {
+        // Default to main cipher pass
+        if (param.cipherPass == NULL)
+            param.cipherPass = TEST_CIPHER_PASS;
+
+        ioFilterGroupAdd(filterGroup, cipherBlockNew(cipherModeEncrypt, param.cipherType, BUFSTRZ(param.cipherPass), NULL));
+    }
+
+    // Put file
+    storagePutP(destination, buffer);
+}
+
+const char *
+hrnStoragePutLog(const Storage *storage, const char *file, const Buffer *buffer, HrnStoragePutParam param)
+{
+    // Empty if buffer is NULL
+    String *log = strNew(buffer == NULL || bufUsed(buffer) == 0 ? "(empty) " : "");
+
+    // Add compression detail
+    if (param.compressType != compressTypeNone)
+        strCatFmt(log, "cmp[%s]", strZ(compressTypeStr(param.compressType)));
+
+    // Add encryption detail
+    if (param.cipherType != cipherTypeNone)
+    {
+        if (param.cipherPass == NULL)
+            param.cipherPass = TEST_CIPHER_PASS;
+
+        if (param.compressType != compressTypeNone)
+            strCatZ(log, "/");
+
+        strCatFmt(log, "enc[%s,%s]", strZ(cipherTypeName(param.cipherType)), param.cipherPass);
+    }
+
+    // Add a space if compression/encryption defined
+    if (param.compressType != compressTypeNone || param.cipherType != cipherTypeNone)
+        strCatZ(log, " ");
+
+    // Add file name
+    strCatFmt(log, "'%s%s'", strZ(storagePathP(storage, STR(file))), strZ(compressExtStr(param.compressType)));
+
+    return strZ(log);
+}

test/src/common/harnessStorage.h

@@ -6,8 +6,68 @@ Helper functions for testing storage and related functions.
 #ifndef TEST_COMMON_HARNESS_STORAGE_H
 #define TEST_COMMON_HARNESS_STORAGE_H
 
+#include "common/compress/helper.h"
+#include "common/crypto/common.h"
 #include "storage/storage.intern.h"
 
+/***********************************************************************************************************************************
+Check file exists
+***********************************************************************************************************************************/
+#define TEST_STORAGE_EXISTS(storage, file)                                                                                         \
+    TEST_RESULT_BOOL(storageExistsP(storage, STR(file)), true, "file exists '%s'", strZ(storagePathP(storage, STR(file))))
+
+/***********************************************************************************************************************************
+List files in a path and optionally remove them
+***********************************************************************************************************************************/
+typedef struct HrnStorageListParam
+{
+    VAR_PARAM_HEADER;
+    bool remove;
+} HrnStorageListParam;
+
+#define TEST_STORAGE_LIST(storage, path, list, ...)                                                                                \
+    TEST_RESULT_STRLST_Z(                                                                                                          \
+        hrnStorageList(storage, path, (HrnStorageListParam){VAR_PARAM_INIT, __VA_ARGS__}), list, "%s",                             \
+        hrnStorageListLog(storage, path, (HrnStorageListParam){VAR_PARAM_INIT, __VA_ARGS__}))
+
+#define TEST_STORAGE_LIST_EMPTY(storage, path, ...)                                                                                \
+    TEST_STORAGE_LIST(storage, path, NULL, __VA_ARGS__)
+
+StringList *hrnStorageList(const Storage *storage, const char *path, HrnStorageListParam param);
+const char *hrnStorageListLog(const Storage *storage, const char *path, HrnStorageListParam param);
+
+/***********************************************************************************************************************************
+Put a file with optional compression and/or encryption
+***********************************************************************************************************************************/
+typedef struct HrnStoragePutParam
+{
+    VAR_PARAM_HEADER;
+    CompressType compressType;
+    CipherType cipherType;
+    const char *cipherPass;
+} HrnStoragePutParam;
+
+#define HRN_STORAGE_PUT(storage, file, buffer, ...)                                                                                \
+    TEST_RESULT_VOID(                                                                                                              \
+        hrnStoragePut(storage, file, buffer, (HrnStoragePutParam){VAR_PARAM_INIT, __VA_ARGS__}), "put file %s",                    \
+        hrnStoragePutLog(storage, file, buffer, (HrnStoragePutParam){VAR_PARAM_INIT, __VA_ARGS__}))
+
+#define HRN_STORAGE_PUT_EMPTY(storage, file, ...)                                                                                  \
+    HRN_STORAGE_PUT(storage, file, NULL, __VA_ARGS__)
+
+#define HRN_STORAGE_PUT_Z(storage, file, stringz, ...)                                                                             \
+    HRN_STORAGE_PUT(storage, file, BUFSTRZ(stringz), __VA_ARGS__)
+
+void hrnStoragePut(const Storage *storage, const char *file, const Buffer *buffer, HrnStoragePutParam param);
+const char *hrnStoragePutLog(const Storage *storage, const char *file, const Buffer *buffer, HrnStoragePutParam param);
+
+/***********************************************************************************************************************************
+Remove a file and error if it does not exist
+***********************************************************************************************************************************/
+#define TEST_STORAGE_REMOVE(storage, path)                                                                                         \
+    TEST_RESULT_VOID(storageRemoveP(storage, STR(path), .errorOnMissing = true), "remove file '%s'",                               \
+    strZ(storagePathP(storage, STR(path))))
+
 /***********************************************************************************************************************************
 Dummy interface for constructing test storage drivers. All required functions are stubbed out so this interface can be copied and
 specific functions replaced for testing.

test/src/module/command/archiveGetTest.c

@@ -278,12 +278,8 @@ testRun(void)
         Buffer *walSegmentBuffer = bufNew(walSegmentSize);
         memset(bufPtr(walSegmentBuffer), 0, walSegmentSize);
 
-        storagePutP(
+        HRN_STORAGE_PUT(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_IN "/0000000100000001000000FE", walSegmentBuffer);
-            storageNewWriteP(
+        HRN_STORAGE_PUT(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_IN "/0000000100000001000000FF", walSegmentBuffer);
-                storageSpoolWrite(), strNew(STORAGE_SPOOL_ARCHIVE_IN "/0000000100000001000000FE")), walSegmentBuffer);
-        storagePutP(
-            storageNewWriteP(
-                storageSpoolWrite(), strNew(STORAGE_SPOOL_ARCHIVE_IN "/0000000100000001000000FF")), walSegmentBuffer);
 
         TEST_RESULT_STRLST_Z(
             queueNeed(strNew("0000000100000001000000FE"), false, queueSize, walSegmentSize, PG_VERSION_92),
@@ -296,21 +292,15 @@ testRun(void)
         walSegmentSize = 1024 * 1024;
         queueSize = walSegmentSize * 5;
 
-        storagePutP(storageNewWriteP(storageSpoolWrite(), strNew(STORAGE_SPOOL_ARCHIVE_IN "/junk")), BUFSTRDEF("JUNK"));
+        HRN_STORAGE_PUT_Z(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_IN "/junk", "JUNK");
-        storagePutP(
+        HRN_STORAGE_PUT(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_IN "/000000010000000A00000FFE", walSegmentBuffer);
-            storageNewWriteP(
+        HRN_STORAGE_PUT(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_IN "/000000010000000A00000FFF", walSegmentBuffer);
-                storageSpoolWrite(), strNew(STORAGE_SPOOL_ARCHIVE_IN "/000000010000000A00000FFE")), walSegmentBuffer);
-        storagePutP(
-            storageNewWriteP(
-                storageSpoolWrite(), strNew(STORAGE_SPOOL_ARCHIVE_IN "/000000010000000A00000FFF")), walSegmentBuffer);
 
         TEST_RESULT_STRLST_Z(
             queueNeed(strNew("000000010000000A00000FFD"), true, queueSize, walSegmentSize, PG_VERSION_11),
             "000000010000000B00000000\n000000010000000B00000001\n000000010000000B00000002\n", "queue has wal >= 9.3");
 
-        TEST_RESULT_STRLST_Z(
+        TEST_STORAGE_LIST(storageSpool(), STORAGE_SPOOL_ARCHIVE_IN, "000000010000000A00000FFE\n000000010000000A00000FFF\n");
-            strLstSort(storageListP(storageSpool(), strNew(STORAGE_SPOOL_ARCHIVE_IN)), sortOrderAsc),
-            "000000010000000A00000FFE\n000000010000000A00000FFF\n", "check queue");
     }