From 7d3d6ecbe1e04d577687e8d4da7465c40f211344 Mon Sep 17 00:00:00 2001 From: David Steele Date: Fri, 8 Jan 2021 16:14:26 -0500 Subject: [PATCH] Add storage test macros for common operations. The macros should make it much easier to write complex tests, especially when compression and encryption are involved. Update the command/archiveGet test to show how the new macros are used. --- test/src/common/harnessInfo.c | 4 + test/src/common/harnessInfo.h | 10 +++ test/src/common/harnessStorage.c | 100 ++++++++++++++++++++++- test/src/common/harnessStorage.h | 60 ++++++++++++++ test/src/module/command/archiveGetTest.c | 22 ++--- 5 files changed, 179 insertions(+), 17 deletions(-) diff --git a/test/src/common/harnessInfo.c b/test/src/common/harnessInfo.c index 59f294174..da6461f15 100644 --- a/test/src/common/harnessInfo.c +++ b/test/src/common/harnessInfo.c @@ -65,6 +65,8 @@ harnessInfoChecksum(const String *info) FUNCTION_HARNESS_PARAM(STRING, info); FUNCTION_HARNESS_END(); + ASSERT(info != NULL); + Buffer *result = NULL; MEM_CONTEXT_TEMP_BEGIN() @@ -110,6 +112,8 @@ harnessInfoChecksumZ(const char *info) FUNCTION_HARNESS_PARAM(STRINGZ, info); FUNCTION_HARNESS_END(); + ASSERT(info != NULL); + FUNCTION_HARNESS_RESULT(BUFFER, harnessInfoChecksum(STR(info))); } diff --git a/test/src/common/harnessInfo.h b/test/src/common/harnessInfo.h index 551c0eb36..68aeb3923 100644 --- a/test/src/common/harnessInfo.h +++ b/test/src/common/harnessInfo.h @@ -4,6 +4,16 @@ Harness for Generating Test Info Files #include "common/type/buffer.h" #include "info/info.h" +#include "common/harnessStorage.h" + +/*********************************************************************************************************************************** +Write info to a file and add the checksum +***********************************************************************************************************************************/ +#define HRN_INFO_PUT(storage, file, info, ...) \ + TEST_RESULT_VOID( \ + hrnStoragePut(storage, file, harnessInfoChecksumZ(info), (HrnStoragePutParam){VAR_PARAM_INIT, __VA_ARGS__}), "put info %s",\ + hrnStoragePutLog(storage, file, LF_BUF, (HrnStoragePutParam){VAR_PARAM_INIT, __VA_ARGS__})) + /*********************************************************************************************************************************** Functions ***********************************************************************************************************************************/ diff --git a/test/src/common/harnessStorage.c b/test/src/common/harnessStorage.c index c2a0edfff..89fe16e84 100644 --- a/test/src/common/harnessStorage.c +++ b/test/src/common/harnessStorage.c @@ -2,14 +2,19 @@ Storage Test Harness ***********************************************************************************************************************************/ #include +#include +#include +#include "common/crypto/cipherBlock.h" #include "common/debug.h" -#include "common/compress/helper.h" #include "common/type/object.h" +#include "common/type/param.h" #include "common/user.h" #include "storage/storage.h" +#include "common/harnessConfig.h" #include "common/harnessStorage.h" +#include "common/harnessTest.h" /*********************************************************************************************************************************** Dummy functions and interface for constructing test storage drivers @@ -169,3 +174,96 @@ hrnStorageInfoListCallback(void *callbackData, const StorageInfo *info) strCatZ(data->content, "}\n"); } + +/**********************************************************************************************************************************/ +StringList * +hrnStorageList(const Storage *storage, const char *path, HrnStorageListParam param) +{ + StringList *list = strLstSort(storageListP(storage, STR(path)), sortOrderAsc); + + // Remove files if requested + if (param.remove) + { + for (unsigned int listIdx = 0; listIdx < strLstSize(list); listIdx++) + storageRemoveP(storage, strNewFmt("%s/%s", path, strZ(strLstGet(list, listIdx))), .errorOnMissing = true); + } + + // Return list for comparison + return list; +} + +const char * +hrnStorageListLog(const Storage *storage, const char *path, HrnStorageListParam param) +{ + StringList *list = strLstSort(storageListP(storage, STR(path)), sortOrderAsc); + + return strZ( + strNewFmt( + "list%s %u file%s in '%s'", param.remove ? "/remove": "", strLstSize(list), strLstSize(list) == 1 ? "" : "s", + strZ(storagePathP(storage, STR(path))))); +} + +/**********************************************************************************************************************************/ +void +hrnStoragePut(const Storage *storage, const char *file, const Buffer *buffer, HrnStoragePutParam param) +{ + // Add compression extension to file name + String *fileStr = strNew(file); + compressExtCat(fileStr, param.compressType); + + // Create file + StorageWrite *destination = storageNewWriteP(storage, fileStr); + IoFilterGroup *filterGroup = ioWriteFilterGroup(storageWriteIo(destination)); + + // Add compression filter + if (param.compressType != compressTypeNone) + { + ASSERT(param.compressType == compressTypeGz || param.compressType == compressTypeBz2); + ioFilterGroupAdd(filterGroup, compressFilter(param.compressType, 1)); + } + + // Add encrypted filter + if (param.cipherType != cipherTypeNone) + { + // Default to main cipher pass + if (param.cipherPass == NULL) + param.cipherPass = TEST_CIPHER_PASS; + + ioFilterGroupAdd(filterGroup, cipherBlockNew(cipherModeEncrypt, param.cipherType, BUFSTRZ(param.cipherPass), NULL)); + } + + // Put file + storagePutP(destination, buffer); +} + +const char * +hrnStoragePutLog(const Storage *storage, const char *file, const Buffer *buffer, HrnStoragePutParam param) +{ + // Empty if buffer is NULL + String *log = strNew(buffer == NULL || bufUsed(buffer) == 0 ? "(empty) " : ""); + + // Add compression detail + if (param.compressType != compressTypeNone) + strCatFmt(log, "cmp[%s]", strZ(compressTypeStr(param.compressType))); + + // Add encryption detail + if (param.cipherType != cipherTypeNone) + { + if (param.cipherPass == NULL) + param.cipherPass = TEST_CIPHER_PASS; + + if (param.compressType != compressTypeNone) + strCatZ(log, "/"); + + strCatFmt(log, "enc[%s,%s]", strZ(cipherTypeName(param.cipherType)), param.cipherPass); + } + + // Add a space if compression/encryption defined + if (param.compressType != compressTypeNone || param.cipherType != cipherTypeNone) + strCatZ(log, " "); + + // Add file name + strCatFmt(log, "'%s%s'", strZ(storagePathP(storage, STR(file))), strZ(compressExtStr(param.compressType))); + + return strZ(log); +} diff --git a/test/src/common/harnessStorage.h b/test/src/common/harnessStorage.h index 31dd3808d..8bcb82d25 100644 --- a/test/src/common/harnessStorage.h +++ b/test/src/common/harnessStorage.h @@ -6,8 +6,68 @@ Helper functions for testing storage and related functions. #ifndef TEST_COMMON_HARNESS_STORAGE_H #define TEST_COMMON_HARNESS_STORAGE_H +#include "common/compress/helper.h" +#include "common/crypto/common.h" #include "storage/storage.intern.h" +/*********************************************************************************************************************************** +Check file exists +***********************************************************************************************************************************/ +#define TEST_STORAGE_EXISTS(storage, file) \ + TEST_RESULT_BOOL(storageExistsP(storage, STR(file)), true, "file exists '%s'", strZ(storagePathP(storage, STR(file)))) + +/*********************************************************************************************************************************** +List files in a path and optionally remove them +***********************************************************************************************************************************/ +typedef struct HrnStorageListParam +{ + VAR_PARAM_HEADER; + bool remove; +} HrnStorageListParam; + +#define TEST_STORAGE_LIST(storage, path, list, ...) \ + TEST_RESULT_STRLST_Z( \ + hrnStorageList(storage, path, (HrnStorageListParam){VAR_PARAM_INIT, __VA_ARGS__}), list, "%s", \ + hrnStorageListLog(storage, path, (HrnStorageListParam){VAR_PARAM_INIT, __VA_ARGS__})) + +#define TEST_STORAGE_LIST_EMPTY(storage, path, ...) \ + TEST_STORAGE_LIST(storage, path, NULL, __VA_ARGS__) + +StringList *hrnStorageList(const Storage *storage, const char *path, HrnStorageListParam param); +const char *hrnStorageListLog(const Storage *storage, const char *path, HrnStorageListParam param); + +/*********************************************************************************************************************************** +Put a file with optional compression and/or encryption +***********************************************************************************************************************************/ +typedef struct HrnStoragePutParam +{ + VAR_PARAM_HEADER; + CompressType compressType; + CipherType cipherType; + const char *cipherPass; +} HrnStoragePutParam; + +#define HRN_STORAGE_PUT(storage, file, buffer, ...) \ + TEST_RESULT_VOID( \ + hrnStoragePut(storage, file, buffer, (HrnStoragePutParam){VAR_PARAM_INIT, __VA_ARGS__}), "put file %s", \ + hrnStoragePutLog(storage, file, buffer, (HrnStoragePutParam){VAR_PARAM_INIT, __VA_ARGS__})) + +#define HRN_STORAGE_PUT_EMPTY(storage, file, ...) \ + HRN_STORAGE_PUT(storage, file, NULL, __VA_ARGS__) + +#define HRN_STORAGE_PUT_Z(storage, file, stringz, ...) \ + HRN_STORAGE_PUT(storage, file, BUFSTRZ(stringz), __VA_ARGS__) + +void hrnStoragePut(const Storage *storage, const char *file, const Buffer *buffer, HrnStoragePutParam param); +const char *hrnStoragePutLog(const Storage *storage, const char *file, const Buffer *buffer, HrnStoragePutParam param); + +/*********************************************************************************************************************************** +Remove a file and error if it does not exist +***********************************************************************************************************************************/ +#define TEST_STORAGE_REMOVE(storage, path) \ + TEST_RESULT_VOID(storageRemoveP(storage, STR(path), .errorOnMissing = true), "remove file '%s'", \ + strZ(storagePathP(storage, STR(path)))) + /*********************************************************************************************************************************** Dummy interface for constructing test storage drivers. All required functions are stubbed out so this interface can be copied and specific functions replaced for testing. diff --git a/test/src/module/command/archiveGetTest.c b/test/src/module/command/archiveGetTest.c index 409c5d87d..87edb21e1 100644 --- a/test/src/module/command/archiveGetTest.c +++ b/test/src/module/command/archiveGetTest.c @@ -278,12 +278,8 @@ testRun(void) Buffer *walSegmentBuffer = bufNew(walSegmentSize); memset(bufPtr(walSegmentBuffer), 0, walSegmentSize); - storagePutP( - storageNewWriteP( - storageSpoolWrite(), strNew(STORAGE_SPOOL_ARCHIVE_IN "/0000000100000001000000FE")), walSegmentBuffer); - storagePutP( - storageNewWriteP( - storageSpoolWrite(), strNew(STORAGE_SPOOL_ARCHIVE_IN "/0000000100000001000000FF")), walSegmentBuffer); + HRN_STORAGE_PUT(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_IN "/0000000100000001000000FE", walSegmentBuffer); + HRN_STORAGE_PUT(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_IN "/0000000100000001000000FF", walSegmentBuffer); TEST_RESULT_STRLST_Z( queueNeed(strNew("0000000100000001000000FE"), false, queueSize, walSegmentSize, PG_VERSION_92), @@ -296,21 +292,15 @@ testRun(void) walSegmentSize = 1024 * 1024; queueSize = walSegmentSize * 5; - storagePutP(storageNewWriteP(storageSpoolWrite(), strNew(STORAGE_SPOOL_ARCHIVE_IN "/junk")), BUFSTRDEF("JUNK")); - storagePutP( - storageNewWriteP( - storageSpoolWrite(), strNew(STORAGE_SPOOL_ARCHIVE_IN "/000000010000000A00000FFE")), walSegmentBuffer); - storagePutP( - storageNewWriteP( - storageSpoolWrite(), strNew(STORAGE_SPOOL_ARCHIVE_IN "/000000010000000A00000FFF")), walSegmentBuffer); + HRN_STORAGE_PUT_Z(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_IN "/junk", "JUNK"); + HRN_STORAGE_PUT(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_IN "/000000010000000A00000FFE", walSegmentBuffer); + HRN_STORAGE_PUT(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_IN "/000000010000000A00000FFF", walSegmentBuffer); TEST_RESULT_STRLST_Z( queueNeed(strNew("000000010000000A00000FFD"), true, queueSize, walSegmentSize, PG_VERSION_11), "000000010000000B00000000\n000000010000000B00000001\n000000010000000B00000002\n", "queue has wal >= 9.3"); - TEST_RESULT_STRLST_Z( - strLstSort(storageListP(storageSpool(), strNew(STORAGE_SPOOL_ARCHIVE_IN)), sortOrderAsc), - "000000010000000A00000FFE\n000000010000000A00000FFF\n", "check queue"); + TEST_STORAGE_LIST(storageSpool(), STORAGE_SPOOL_ARCHIVE_IN, "000000010000000A00000FFE\n000000010000000A00000FFF\n"); }