db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2024-12-12 10:04:14 +02:00
Code Issues Releases Activity

Convert filter param/result to Pack type.

Browse Source 
The Pack type is more compact and flexible than the Variant type. The Pack type also allows binary data to be stored, which is useful for transferring the passphrase in the CipherBlock filter.

The primary purpose is to allow more (and more complex) result data to be returned efficiently from the PageChecksum filter. For now the PageChecksum filter still returns the original Variant. Converting the result data will be the subject of a future commit.

Also convert filter types to StringId.
This commit is contained in:
David Steele 2021-09-22 10:48:21 -04:00
parent 802373cb9d
commit 0e76ccb5b7
56 changed files with 616 additions and 402 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
src/command/archive/push/file.c
View File

@ -160,7 +160,8 @@ archivePushFile(
ioFilterGroupAdd(ioReadFilterGroup(read), cryptoHashNew(HASH_TYPE_SHA1_STR));
ioReadDrain(read);
const String *walSegmentChecksum = varStr(ioFilterGroupResult(ioReadFilterGroup(read), CRYPTO_HASH_FILTER_TYPE_STR));
const String *walSegmentChecksum = pckReadStrP(
ioFilterGroupResultP(ioReadFilterGroup(read), CRYPTO_HASH_FILTER_TYPE));
// Check each repo for the WAL segment
for (unsigned int repoListIdx = 0; repoListIdx < lstSize(repoList); repoListIdx++)

6
src/command/backup/backup.c
View File

@ -944,12 +944,12 @@ backupFilePut(BackupData *backupData, Manifest *manifest, const String *name, ti
.user = basePath->user,
.group = basePath->group,
.size = strSize(content),
.sizeRepo = varUInt64Force(ioFilterGroupResult(filterGroup, SIZE_FILTER_TYPE_STR)),
.sizeRepo = pckReadU64P(ioFilterGroupResultP(filterGroup, SIZE_FILTER_TYPE)),
.timestamp = timestamp,
};
memcpy(
file.checksumSha1, strZ(varStr(ioFilterGroupResult(filterGroup, CRYPTO_HASH_FILTER_TYPE_STR))),
file.checksumSha1, strZ(pckReadStrP(ioFilterGroupResultP(filterGroup, CRYPTO_HASH_FILTER_TYPE))),
HASH_TYPE_SHA1_SIZE_HEX + 1);
manifestFileAdd(manifest, &file);
@ -1799,7 +1799,7 @@ backupArchiveCheckCopy(Manifest *manifest, unsigned int walSegmentSize, const St
.user = basePath->user,
.group = basePath->group,
.size = walSegmentSize,
.sizeRepo = varUInt64Force(ioFilterGroupResult(filterGroup, SIZE_FILTER_TYPE_STR)),
.sizeRepo = pckReadU64P(ioFilterGroupResultP(filterGroup, SIZE_FILTER_TYPE)),
.timestamp = manifestData(manifest)->backupTimestampStop,
};

26
src/command/backup/file.c
View File

@ -16,6 +16,7 @@ Backup File
#include "common/log.h"
#include "common/regExp.h"
#include "common/type/convert.h"
#include "common/type/json.h"
#include "postgres/interface.h"
#include "storage/helper.h"
@ -97,9 +98,9 @@ backupFile(
// If the pg file exists check the checksum/size
if (ioReadDrain(read))
{
const String *pgTestChecksum = varStr(
ioFilterGroupResult(ioReadFilterGroup(read), CRYPTO_HASH_FILTER_TYPE_STR));
uint64_t pgTestSize = varUInt64Force(ioFilterGroupResult(ioReadFilterGroup(read), SIZE_FILTER_TYPE_STR));
const String *pgTestChecksum = pckReadStrP(
ioFilterGroupResultP(ioReadFilterGroup(read), CRYPTO_HASH_FILTER_TYPE));
uint64_t pgTestSize = pckReadU64P(ioFilterGroupResultP(ioReadFilterGroup(read), SIZE_FILTER_TYPE));
// Does the pg file match?
if (pgFileSize == pgTestSize && strEq(pgFileChecksum, pgTestChecksum))
@ -160,9 +161,9 @@ backupFile(
ioReadDrain(read);
// Test checksum/size
const String *pgTestChecksum = varStr(
ioFilterGroupResult(ioReadFilterGroup(read), CRYPTO_HASH_FILTER_TYPE_STR));
uint64_t pgTestSize = varUInt64Force(ioFilterGroupResult(ioReadFilterGroup(read), SIZE_FILTER_TYPE_STR));
const String *pgTestChecksum = pckReadStrP(
ioFilterGroupResultP(ioReadFilterGroup(read), CRYPTO_HASH_FILTER_TYPE));
uint64_t pgTestSize = pckReadU64P(ioFilterGroupResultP(ioReadFilterGroup(read), SIZE_FILTER_TYPE));
// No need to recopy if checksum/size match
if (pgFileSize == pgTestSize && strEq(pgFileChecksum, pgTestChecksum))
@ -240,18 +241,19 @@ backupFile(
MEM_CONTEXT_PRIOR_BEGIN()
{
// Get sizes and checksum
result.copySize = varUInt64Force(
ioFilterGroupResult(ioReadFilterGroup(storageReadIo(read)), SIZE_FILTER_TYPE_STR));
result.copySize = pckReadU64P(
ioFilterGroupResultP(ioReadFilterGroup(storageReadIo(read)), SIZE_FILTER_TYPE));
result.copyChecksum = strDup(
varStr(ioFilterGroupResult(ioReadFilterGroup(storageReadIo(read)), CRYPTO_HASH_FILTER_TYPE_STR)));
pckReadStrP(ioFilterGroupResultP(ioReadFilterGroup(storageReadIo(read)), CRYPTO_HASH_FILTER_TYPE)));
result.repoSize =
varUInt64Force(ioFilterGroupResult(ioWriteFilterGroup(storageWriteIo(write)), SIZE_FILTER_TYPE_STR));
pckReadU64P(ioFilterGroupResultP(ioWriteFilterGroup(storageWriteIo(write)), SIZE_FILTER_TYPE));
// Get results of page checksum validation
if (pgFileChecksumPage)
{
result.pageChecksumResult = kvDup(
varKv(ioFilterGroupResult(ioReadFilterGroup(storageReadIo(read)), PAGE_CHECKSUM_FILTER_TYPE_STR)));
result.pageChecksumResult = jsonToKv(
pckReadStrP(
ioFilterGroupResultP(ioReadFilterGroup(storageReadIo(read)), PAGE_CHECKSUM_FILTER_TYPE)));
}
}
MEM_CONTEXT_PRIOR_END();

132
src/command/backup/pageChecksum.c
View File

@ -8,15 +8,11 @@ Page Checksum Filter
#include "command/backup/pageChecksum.h"
#include "common/log.h"
#include "common/macro.h"
#include "common/type/json.h"
#include "common/type/object.h"
#include "postgres/interface.h"
#include "postgres/interface/static.vendor.h"
/***********************************************************************************************************************************
Filter type constant
***********************************************************************************************************************************/
STRING_EXTERN(PAGE_CHECKSUM_FILTER_TYPE_STR, PAGE_CHECKSUM_FILTER_TYPE);
/***********************************************************************************************************************************
Object type
***********************************************************************************************************************************/
@ -150,7 +146,7 @@ pageChecksumProcess(THIS_VOID, const Buffer *input)
/***********************************************************************************************************************************
Return filter result
***********************************************************************************************************************************/
static Variant *
static Buffer *
pageChecksumResult(THIS_VOID)
{
THIS(PageChecksum);
@ -161,67 +157,81 @@ pageChecksumResult(THIS_VOID)
ASSERT(this != NULL);
KeyValue *result = kvNew();
Buffer *result = NULL;
if (this->error != NULL)
MEM_CONTEXT_TEMP_BEGIN()
{
VariantList *errorList = varLstNew();
unsigned int errorIdx = 0;
KeyValue *error = kvNew();
// Convert the full list to an abbreviated list. In the future we want to return the entire list so pages can be verified
// in the WAL.
do
if (this->error != NULL)
{
unsigned int pageId = varUInt(varLstGet(varVarLst(varLstGet(this->error, errorIdx)), 0));
VariantList *errorList = varLstNew();
unsigned int errorIdx = 0;
if (errorIdx == varLstSize(this->error) - 1)
// Convert the full list to an abbreviated list. In the future we want to return the entire list so pages can be verified
// in the WAL.
do
{
varLstAdd(errorList, varNewUInt(pageId));
errorIdx++;
}
else
{
unsigned int pageIdNext = varUInt(varLstGet(varVarLst(varLstGet(this->error, errorIdx + 1)), 0));
unsigned int pageId = varUInt(varLstGet(varVarLst(varLstGet(this->error, errorIdx)), 0));
if (pageIdNext > pageId + 1)
if (errorIdx == varLstSize(this->error) - 1)
{
varLstAdd(errorList, varNewUInt(pageId));
errorIdx++;
}
else
{
unsigned int pageIdLast = pageIdNext;
errorIdx++;
unsigned int pageIdNext = varUInt(varLstGet(varVarLst(varLstGet(this->error, errorIdx + 1)), 0));
while (errorIdx < varLstSize(this->error) - 1)
if (pageIdNext > pageId + 1)
{
pageIdNext = varUInt(varLstGet(varVarLst(varLstGet(this->error, errorIdx + 1)), 0));
if (pageIdNext > pageIdLast + 1)
break;
pageIdLast = pageIdNext;
varLstAdd(errorList, varNewUInt(pageId));
errorIdx++;
}
else
{
unsigned int pageIdLast = pageIdNext;
errorIdx++;
VariantList *errorListSub = varLstNew();
varLstAdd(errorListSub, varNewUInt(pageId));
varLstAdd(errorListSub, varNewUInt(pageIdLast));
varLstAdd(errorList, varNewVarLst(errorListSub));
errorIdx++;
while (errorIdx < varLstSize(this->error) - 1)
{
pageIdNext = varUInt(varLstGet(varVarLst(varLstGet(this->error, errorIdx + 1)), 0));
if (pageIdNext > pageIdLast + 1)
break;
pageIdLast = pageIdNext;
errorIdx++;
}
VariantList *errorListSub = varLstNew();
varLstAdd(errorListSub, varNewUInt(pageId));
varLstAdd(errorListSub, varNewUInt(pageIdLast));
varLstAdd(errorList, varNewVarLst(errorListSub));
errorIdx++;
}
}
}
while (errorIdx < varLstSize(this->error));
this->valid = false;
kvPut(error, varNewStrZ("error"), varNewVarLst(errorList));
}
while (errorIdx < varLstSize(this->error));
this->valid = false;
kvPut(result, varNewStrZ("error"), varNewVarLst(errorList));
kvPut(error, VARSTRDEF("valid"), VARBOOL(this->valid));
kvPut(error, VARSTRDEF("align"), VARBOOL(this->align));
result = bufNew(PACK_EXTRA_MIN);
PackWrite *const write = pckWriteNewBuf(result);
pckWriteStrP(write, jsonFromKv(error));
pckWriteEndP(write);
bufMove(result, memContextPrior());
}
MEM_CONTEXT_TEMP_END();
kvPut(result, VARSTRDEF("valid"), VARBOOL(this->valid));
kvPut(result, VARSTRDEF("align"), VARBOOL(this->align));
FUNCTION_LOG_RETURN(VARIANT, varNewKv(result));
FUNCTION_LOG_RETURN(BUFFER, result);
}
/**********************************************************************************************************************************/
@ -251,13 +261,21 @@ pageChecksumNew(unsigned int segmentNo, unsigned int segmentPageTotal, uint64_t
};
// Create param list
VariantList *paramList = varLstNew();
varLstAdd(paramList, varNewUInt(segmentNo));
varLstAdd(paramList, varNewUInt(segmentPageTotal));
varLstAdd(paramList, varNewUInt64(lsnLimit));
Buffer *const paramList = bufNew(PACK_EXTRA_MIN);
MEM_CONTEXT_TEMP_BEGIN()
{
PackWrite *const packWrite = pckWriteNewBuf(paramList);
pckWriteU32P(packWrite, segmentNo);
pckWriteU32P(packWrite, segmentPageTotal);
pckWriteU64P(packWrite, lsnLimit);
pckWriteEndP(packWrite);
}
MEM_CONTEXT_TEMP_END();
this = ioFilterNewP(
PAGE_CHECKSUM_FILTER_TYPE_STR, driver, paramList, .in = pageChecksumProcess, .result = pageChecksumResult);
PAGE_CHECKSUM_FILTER_TYPE, driver, paramList, .in = pageChecksumProcess, .result = pageChecksumResult);
}
OBJ_NEW_END();
@ -265,8 +283,20 @@ pageChecksumNew(unsigned int segmentNo, unsigned int segmentPageTotal, uint64_t
}
IoFilter *
pageChecksumNewVar(const VariantList *paramList)
pageChecksumNewPack(const Buffer *const paramList)
{
return pageChecksumNew(
varUIntForce(varLstGet(paramList, 0)), varUIntForce(varLstGet(paramList, 1)), varUInt64(varLstGet(paramList, 2)));
IoFilter *result = NULL;
MEM_CONTEXT_TEMP_BEGIN()
{
PackRead *const paramListPack = pckReadNewBuf(paramList);
const unsigned int segmentNo = pckReadU32P(paramListPack);
const unsigned int segmentPageTotal = pckReadU32P(paramListPack);
const uint64_t lsnLimit = pckReadU64P(paramListPack);
result = objMoveContext(pageChecksumNew(segmentNo, segmentPageTotal, lsnLimit), memContextPrior());
}
MEM_CONTEXT_TEMP_END();
return result;
}

5
src/command/backup/pageChecksum.h
View File

@ -11,13 +11,12 @@ Check all pages in a PostgreSQL relation to ensure the checksums are valid.
/***********************************************************************************************************************************
Filter type constant
***********************************************************************************************************************************/
#define PAGE_CHECKSUM_FILTER_TYPE "pageChecksum"
STRING_DECLARE(PAGE_CHECKSUM_FILTER_TYPE_STR);
#define PAGE_CHECKSUM_FILTER_TYPE STRID5("pg-chksum", 0xdacd681ecf00)
/***********************************************************************************************************************************
Constructors
***********************************************************************************************************************************/
IoFilter *pageChecksumNew(unsigned int segmentNo, unsigned int segmentPageTotal, uint64_t lsnLimit);
IoFilter *pageChecksumNewVar(const VariantList *paramList);
IoFilter *pageChecksumNewPack(const Buffer *paramList);
#endif

7
src/command/restore/file.c
View File

@ -91,7 +91,8 @@ restoreFile(
// If size and checksum are equal then no need to copy the file
if (pgFileSize == 0 ||
strEq(
pgFileChecksum, varStr(ioFilterGroupResult(ioReadFilterGroup(read), CRYPTO_HASH_FILTER_TYPE_STR))))
pgFileChecksum,
pckReadStrP(ioFilterGroupResultP(ioReadFilterGroup(read), CRYPTO_HASH_FILTER_TYPE))))
{
// Even if hash/size are the same set the time back to backup time. This helps with unit testing, but
// also presents a pristine version of the database after restore.
@ -173,12 +174,12 @@ restoreFile(
pgFileWrite);
// Validate checksum
if (!strEq(pgFileChecksum, varStr(ioFilterGroupResult(filterGroup, CRYPTO_HASH_FILTER_TYPE_STR))))
if (!strEq(pgFileChecksum, pckReadStrP(ioFilterGroupResultP(filterGroup, CRYPTO_HASH_FILTER_TYPE))))
{
THROW_FMT(
ChecksumError,
"error restoring '%s': actual checksum '%s' does not match expected checksum '%s'", strZ(pgFile),
strZ(varStr(ioFilterGroupResult(filterGroup, CRYPTO_HASH_FILTER_TYPE_STR))), strZ(pgFileChecksum));
strZ(pckReadStrP(ioFilterGroupResultP(filterGroup, CRYPTO_HASH_FILTER_TYPE))), strZ(pgFileChecksum));
}
}
}

4
src/command/verify/file.c
View File

@ -59,12 +59,12 @@ verifyFile(
if (ioReadDrain(read))
{
// Validate checksum
if (!strEq(fileChecksum, varStr(ioFilterGroupResult(filterGroup, CRYPTO_HASH_FILTER_TYPE_STR))))
if (!strEq(fileChecksum, pckReadStrP(ioFilterGroupResultP(filterGroup, CRYPTO_HASH_FILTER_TYPE))))
{
result = verifyChecksumMismatch;
}
// If the size can be checked, do so
else if (fileSize != varUInt64Force(ioFilterGroupResult(ioReadFilterGroup(read), SIZE_FILTER_TYPE_STR)))
else if (fileSize != pckReadU64P(ioFilterGroupResultP(ioReadFilterGroup(read), SIZE_FILTER_TYPE)))
result = verifySizeInvalid;
}
else

3
src/command/verify/verify.c
View File

@ -219,7 +219,8 @@ verifyInfoFile(const String *pathFileName, bool keepFile, const String *cipherPa
MEM_CONTEXT_PRIOR_BEGIN()
{
result.checksum = strDup(varStr(ioFilterGroupResult(ioReadFilterGroup(infoRead), CRYPTO_HASH_FILTER_TYPE_STR)));
result.checksum = strDup(
pckReadStrP(ioFilterGroupResultP(ioReadFilterGroup(infoRead), CRYPTO_HASH_FILTER_TYPE)));
}
MEM_CONTEXT_PRIOR_END();
}

20
src/common/compress/bz2/compress.c
View File

@ -13,11 +13,7 @@ BZ2 Compress
#include "common/log.h"
#include "common/macro.h"
#include "common/type/object.h"
/***********************************************************************************************************************************
Filter type constant
***********************************************************************************************************************************/
STRING_EXTERN(BZ2_COMPRESS_FILTER_TYPE_STR, BZ2_COMPRESS_FILTER_TYPE);
#include "common/type/pack.h"
/***********************************************************************************************************************************
Object type
@ -187,12 +183,20 @@ bz2CompressNew(int level)
memContextCallbackSet(objMemContext(driver), bz2CompressFreeResource, driver);
// Create param list
VariantList *paramList = varLstNew();
varLstAdd(paramList, varNewInt(level));
Buffer *const paramList = bufNew(PACK_EXTRA_MIN);
MEM_CONTEXT_TEMP_BEGIN()
{
PackWrite *const packWrite = pckWriteNewBuf(paramList);
pckWriteI32P(packWrite, level);
pckWriteEndP(packWrite);
}
MEM_CONTEXT_TEMP_END();
// Create filter interface
this = ioFilterNewP(
BZ2_COMPRESS_FILTER_TYPE_STR, driver, paramList, .done = bz2CompressDone, .inOut = bz2CompressProcess,
BZ2_COMPRESS_FILTER_TYPE, driver, paramList, .done = bz2CompressDone, .inOut = bz2CompressProcess,
.inputSame = bz2CompressInputSame);
}
OBJ_NEW_END();

3
src/common/compress/bz2/compress.h
View File

@ -11,8 +11,7 @@ Compress IO to the bz2 format.
/***********************************************************************************************************************************
Filter type constant
***********************************************************************************************************************************/
#define BZ2_COMPRESS_FILTER_TYPE "bz2Compress"
STRING_DECLARE(BZ2_COMPRESS_FILTER_TYPE_STR);
#define BZ2_COMPRESS_FILTER_TYPE STRID5("bz2-cmp", 0x41a3df3420)
/***********************************************************************************************************************************
Constructors

7
src/common/compress/bz2/decompress.c
View File

@ -14,11 +14,6 @@ BZ2 Decompress
#include "common/macro.h"
#include "common/type/object.h"
/***********************************************************************************************************************************
Filter type constant
***********************************************************************************************************************************/
STRING_EXTERN(BZ2_DECOMPRESS_FILTER_TYPE_STR, BZ2_DECOMPRESS_FILTER_TYPE);
/***********************************************************************************************************************************
Object type
***********************************************************************************************************************************/
@ -173,7 +168,7 @@ bz2DecompressNew(void)
// Create filter interface
this = ioFilterNewP(
BZ2_DECOMPRESS_FILTER_TYPE_STR, driver, NULL, .done = bz2DecompressDone, .inOut = bz2DecompressProcess,
BZ2_DECOMPRESS_FILTER_TYPE, driver, NULL, .done = bz2DecompressDone, .inOut = bz2DecompressProcess,
.inputSame = bz2DecompressInputSame);
}
OBJ_NEW_END();

3
src/common/compress/bz2/decompress.h
View File

@ -11,8 +11,7 @@ Decompress IO from the bz2 format.
/***********************************************************************************************************************************
Filter type constant
***********************************************************************************************************************************/
#define BZ2_DECOMPRESS_FILTER_TYPE "bz2Decompress"
STRING_DECLARE(BZ2_DECOMPRESS_FILTER_TYPE_STR);
#define BZ2_DECOMPRESS_FILTER_TYPE STRID5("bz2-dcmp", 0x83464df3420)
/***********************************************************************************************************************************
Constructors

20
src/common/compress/gz/compress.c
View File

@ -13,11 +13,7 @@ Gz Compress
#include "common/log.h"
#include "common/macro.h"
#include "common/type/object.h"
/***********************************************************************************************************************************
Filter type constant
***********************************************************************************************************************************/
STRING_EXTERN(GZ_COMPRESS_FILTER_TYPE_STR, GZ_COMPRESS_FILTER_TYPE);
#include "common/type/pack.h"
/***********************************************************************************************************************************
Object type
@ -192,12 +188,20 @@ gzCompressNew(int level)
memContextCallbackSet(objMemContext(driver), gzCompressFreeResource, driver);
// Create param list
VariantList *paramList = varLstNew();
varLstAdd(paramList, varNewInt(level));
Buffer *const paramList = bufNew(PACK_EXTRA_MIN);
MEM_CONTEXT_TEMP_BEGIN()
{
PackWrite *const packWrite = pckWriteNewBuf(paramList);
pckWriteI32P(packWrite, level);
pckWriteEndP(packWrite);
}
MEM_CONTEXT_TEMP_END();
// Create filter interface
this = ioFilterNewP(
GZ_COMPRESS_FILTER_TYPE_STR, driver, paramList, .done = gzCompressDone, .inOut = gzCompressProcess,
GZ_COMPRESS_FILTER_TYPE, driver, paramList, .done = gzCompressDone, .inOut = gzCompressProcess,
.inputSame = gzCompressInputSame);
}
OBJ_NEW_END();

3
src/common/compress/gz/compress.h
View File

@ -11,8 +11,7 @@ Compress IO using the gz format.
/***********************************************************************************************************************************
Filter type constant
***********************************************************************************************************************************/
#define GZ_COMPRESS_FILTER_TYPE "gzCompress"
STRING_DECLARE(GZ_COMPRESS_FILTER_TYPE_STR);
#define GZ_COMPRESS_FILTER_TYPE STRID5("gz-cmp", 0x20d1ef470)
/***********************************************************************************************************************************
Constructors

7
src/common/compress/gz/decompress.c
View File

@ -14,11 +14,6 @@ Gz Decompress
#include "common/macro.h"
#include "common/type/object.h"
/***********************************************************************************************************************************
Filter type constant
***********************************************************************************************************************************/
STRING_EXTERN(GZ_DECOMPRESS_FILTER_TYPE_STR, GZ_DECOMPRESS_FILTER_TYPE);
/***********************************************************************************************************************************
Object type
***********************************************************************************************************************************/
@ -173,7 +168,7 @@ gzDecompressNew(void)
// Create filter interface
this = ioFilterNewP(
GZ_DECOMPRESS_FILTER_TYPE_STR, driver, NULL, .done = gzDecompressDone, .inOut = gzDecompressProcess,
GZ_DECOMPRESS_FILTER_TYPE, driver, NULL, .done = gzDecompressDone, .inOut = gzDecompressProcess,
.inputSame = gzDecompressInputSame);
}
OBJ_NEW_END();

3
src/common/compress/gz/decompress.h
View File

@ -11,8 +11,7 @@ Decompress IO from the gz format.
/***********************************************************************************************************************************
Filter type constant
***********************************************************************************************************************************/
#define GZ_DECOMPRESS_FILTER_TYPE "gzDecompress"
STRING_DECLARE(GZ_DECOMPRESS_FILTER_TYPE_STR);
#define GZ_DECOMPRESS_FILTER_TYPE STRID5("gz-dcmp", 0x41a326f470)
/***********************************************************************************************************************************
Constructors

38
src/common/compress/helper.c
View File

@ -37,9 +37,9 @@ static const struct CompressHelperLocal
{
const String *const type; // Compress type -- must be extension without period prefixed
const String *const ext; // File extension with period prefixed
const char *compressType; // Type of the compression filter
StringId compressType; // Type of the compression filter
IoFilter *(*compressNew)(int); // Function to create new compression filter
const char *decompressType; // Type of the decompression filter
StringId decompressType; // Type of the decompression filter
IoFilter *(*decompressNew)(void); // Function to create new decompression filter
int levelDefault; // Default compression level
} compressHelperLocal[] =
@ -208,32 +208,38 @@ compressFilter(CompressType type, int level)
/**********************************************************************************************************************************/
IoFilter *
compressFilterVar(const String *filterType, const VariantList *filterParamList)
compressFilterPack(const StringId filterType, const Buffer *const filterParamList)
{
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(STRING, filterType);
FUNCTION_LOG_PARAM(STRING_ID, filterType);
FUNCTION_LOG_PARAM(VARIANT_LIST, filterParamList);
FUNCTION_LOG_END();
ASSERT(filterType != NULL);
ASSERT(filterType != 0);
IoFilter *result = NULL;
for (CompressType compressIdx = compressTypeNone + 1; compressIdx < COMPRESS_LIST_SIZE; compressIdx++)
MEM_CONTEXT_TEMP_BEGIN()
{
const struct CompressHelperLocal *compress = &compressHelperLocal[compressIdx];
for (CompressType compressIdx = compressTypeNone + 1; compressIdx < COMPRESS_LIST_SIZE; compressIdx++)
{
const struct CompressHelperLocal *compress = &compressHelperLocal[compressIdx];
if (compress->compressType != NULL && strEqZ(filterType, compress->compressType))
{
result = compress->compressNew(varIntForce(varLstGet(filterParamList, 0)));
break;
}
else if (compress->decompressType != NULL && strEqZ(filterType, compress->decompressType))
{
result = compress->decompressNew();
break;
if (filterType == compress->compressType)
{
ASSERT(filterParamList != NULL);
result = objMoveContext(compress->compressNew(pckReadI32P(pckReadNewBuf(filterParamList))), memContextPrior());
break;
}
else if (filterType == compress->decompressType)
{
result = objMoveContext(compress->decompressNew(), memContextPrior());
break;
}
}
}
MEM_CONTEXT_TEMP_END();
FUNCTION_LOG_RETURN(IO_FILTER, result);
}

2
src/common/compress/helper.h
View File

@ -57,7 +57,7 @@ IoFilter *compressFilter(CompressType type, int level);
// Compression/decompression filter based on string type and a parameter list. This is useful when a filter must be created on a
// remote system since the filter type and parameters can be passed through a protocol.
IoFilter *compressFilterVar(const String *filterType, const VariantList *filterParamList);
IoFilter *compressFilterPack(StringId filterType, const Buffer *filterParamList);
// Decompression filter for the specified type. Error when compress type is none or invalid.
IoFilter *decompressFilter(CompressType type);

20
src/common/compress/lz4/compress.c
View File

@ -17,6 +17,7 @@ Developed against version r131 using the documentation in https://github.com/lz4
#include "common/io/filter/filter.h"
#include "common/log.h"
#include "common/type/object.h"
#include "common/type/pack.h"
/***********************************************************************************************************************************
Older versions of lz4 do not define the max header size. This seems to be the max for any version.
@ -25,11 +26,6 @@ Older versions of lz4 do not define the max header size. This seems to be the m
#define LZ4F_HEADER_SIZE_MAX 19
#endif
/***********************************************************************************************************************************
Filter type constant
***********************************************************************************************************************************/
STRING_EXTERN(LZ4_COMPRESS_FILTER_TYPE_STR, LZ4_COMPRESS_FILTER_TYPE);
/***********************************************************************************************************************************
Object type
***********************************************************************************************************************************/
@ -275,12 +271,20 @@ lz4CompressNew(int level)
memContextCallbackSet(objMemContext(driver), lz4CompressFreeResource, driver);
// Create param list
VariantList *paramList = varLstNew();
varLstAdd(paramList, varNewInt(level));
Buffer *const paramList = bufNew(PACK_EXTRA_MIN);
MEM_CONTEXT_TEMP_BEGIN()
{
PackWrite *const packWrite = pckWriteNewBuf(paramList);
pckWriteI32P(packWrite, level);
pckWriteEndP(packWrite);
}
MEM_CONTEXT_TEMP_END();
// Create filter interface
this = ioFilterNewP(
LZ4_COMPRESS_FILTER_TYPE_STR, driver, paramList, .done = lz4CompressDone, .inOut = lz4CompressProcess,
LZ4_COMPRESS_FILTER_TYPE, driver, paramList, .done = lz4CompressDone, .inOut = lz4CompressProcess,
.inputSame = lz4CompressInputSame);
}
OBJ_NEW_END();

3
src/common/compress/lz4/compress.h
View File

@ -13,8 +13,7 @@ Compress IO to the lz4 format.
/***********************************************************************************************************************************
Filter type constant
***********************************************************************************************************************************/
#define LZ4_COMPRESS_FILTER_TYPE "lz4Compress"
STRING_DECLARE(LZ4_COMPRESS_FILTER_TYPE_STR);
#define LZ4_COMPRESS_FILTER_TYPE STRID6("lz4-cmp", 0x103436e068c1)
/***********************************************************************************************************************************
Constructors

7
src/common/compress/lz4/decompress.c
View File

@ -15,11 +15,6 @@ LZ4 Decompress
#include "common/log.h"
#include "common/type/object.h"
/***********************************************************************************************************************************
Filter type constant
***********************************************************************************************************************************/
STRING_EXTERN(LZ4_DECOMPRESS_FILTER_TYPE_STR, LZ4_DECOMPRESS_FILTER_TYPE);
/***********************************************************************************************************************************
Object type
***********************************************************************************************************************************/
@ -181,7 +176,7 @@ lz4DecompressNew(void)
// Create filter interface
this = ioFilterNewP(
LZ4_DECOMPRESS_FILTER_TYPE_STR, driver, NULL, .done = lz4DecompressDone, .inOut = lz4DecompressProcess,
LZ4_DECOMPRESS_FILTER_TYPE, driver, NULL, .done = lz4DecompressDone, .inOut = lz4DecompressProcess,
.inputSame = lz4DecompressInputSame);
}
OBJ_NEW_END();

3
src/common/compress/lz4/decompress.h
View File

@ -13,8 +13,7 @@ Decompress IO from the lz4 format.
/***********************************************************************************************************************************
Filter type constant
***********************************************************************************************************************************/
#define LZ4_DECOMPRESS_FILTER_TYPE "lz4Decompress"
STRING_DECLARE(LZ4_DECOMPRESS_FILTER_TYPE_STR);
#define LZ4_DECOMPRESS_FILTER_TYPE STRID6("lz4-dcmp", 0x40d0c46e068c1)
/***********************************************************************************************************************************
Constructors

20
src/common/compress/zst/compress.c
View File

@ -13,11 +13,7 @@ ZST Compress
#include "common/io/filter/filter.h"
#include "common/log.h"
#include "common/type/object.h"
/***********************************************************************************************************************************
Filter type constant
***********************************************************************************************************************************/
STRING_EXTERN(ZST_COMPRESS_FILTER_TYPE_STR, ZST_COMPRESS_FILTER_TYPE);
#include "common/type/pack.h"
/***********************************************************************************************************************************
Object type
@ -195,12 +191,20 @@ zstCompressNew(int level)
zstError(ZSTD_initCStream(driver->context, driver->level));
// Create param list
VariantList *paramList = varLstNew();
varLstAdd(paramList, varNewInt(level));
Buffer *const paramList = bufNew(PACK_EXTRA_MIN);
MEM_CONTEXT_TEMP_BEGIN()
{
PackWrite *const packWrite = pckWriteNewBuf(paramList);
pckWriteI32P(packWrite, level);
pckWriteEndP(packWrite);
}
MEM_CONTEXT_TEMP_END();
// Create filter interface
this = ioFilterNewP(
ZST_COMPRESS_FILTER_TYPE_STR, driver, paramList, .done = zstCompressDone, .inOut = zstCompressProcess,
ZST_COMPRESS_FILTER_TYPE, driver, paramList, .done = zstCompressDone, .inOut = zstCompressProcess,
.inputSame = zstCompressInputSame);
}
OBJ_NEW_END();

3
src/common/compress/zst/compress.h
View File

@ -13,8 +13,7 @@ Compress IO to the zst format.
/***********************************************************************************************************************************
Filter type constant
***********************************************************************************************************************************/
#define ZST_COMPRESS_FILTER_TYPE "zstCompress"
STRING_DECLARE(ZST_COMPRESS_FILTER_TYPE_STR);
#define ZST_COMPRESS_FILTER_TYPE STRID5("zst-cmp", 0x41a3dd27a0)
/***********************************************************************************************************************************
Constructors

7
src/common/compress/zst/decompress.c
View File

@ -14,11 +14,6 @@ ZST Decompress
#include "common/log.h"
#include "common/type/object.h"
/***********************************************************************************************************************************
Filter type constant
***********************************************************************************************************************************/
STRING_EXTERN(ZST_DECOMPRESS_FILTER_TYPE_STR, ZST_DECOMPRESS_FILTER_TYPE);
/***********************************************************************************************************************************
Object type
***********************************************************************************************************************************/
@ -184,7 +179,7 @@ zstDecompressNew(void)
// Create filter interface
this = ioFilterNewP(
ZST_DECOMPRESS_FILTER_TYPE_STR, driver, NULL, .done = zstDecompressDone, .inOut = zstDecompressProcess,
ZST_DECOMPRESS_FILTER_TYPE, driver, NULL, .done = zstDecompressDone, .inOut = zstDecompressProcess,
.inputSame = zstDecompressInputSame);
}
OBJ_NEW_END();

3
src/common/compress/zst/decompress.h
View File

@ -13,8 +13,7 @@ Decompress IO from the zst format.
/***********************************************************************************************************************************
Filter type constant
***********************************************************************************************************************************/
#define ZST_DECOMPRESS_FILTER_TYPE "zstDecompress"
STRING_DECLARE(ZST_DECOMPRESS_FILTER_TYPE_STR);
#define ZST_DECOMPRESS_FILTER_TYPE STRID5("zst-dcmp", 0x83464dd27a0)
/***********************************************************************************************************************************
Constructors

47
src/common/crypto/cipherBlock.c
View File

@ -15,11 +15,6 @@ Block Cipher
#include "common/log.h"
#include "common/type/object.h"
/***********************************************************************************************************************************
Filter type constant
***********************************************************************************************************************************/
STRING_EXTERN(CIPHER_BLOCK_FILTER_TYPE_STR, CIPHER_BLOCK_FILTER_TYPE);
/***********************************************************************************************************************************
Header constants and sizes
***********************************************************************************************************************************/
@ -440,17 +435,23 @@ cipherBlockNew(CipherMode mode, CipherType cipherType, const Buffer *pass, const
memcpy(driver->pass, bufPtrConst(pass), driver->passSize);
// Create param list
VariantList *paramList = varLstNew();
varLstAdd(paramList, varNewUInt64(mode));
varLstAdd(paramList, varNewUInt64(cipherType));
// ??? Using a string here is not correct since the passphrase is being passed as a buffer so may contain null characters.
// However, since strings are used to hold the passphrase in the rest of the code this is currently valid.
varLstAdd(paramList, varNewStr(strNewBuf(pass)));
varLstAdd(paramList, digestName ? varNewStr(digestName) : NULL);
Buffer *const paramList = bufNew(PACK_EXTRA_MIN);
MEM_CONTEXT_TEMP_BEGIN()
{
PackWrite *const packWrite = pckWriteNewBuf(paramList);
pckWriteU64P(packWrite, mode);
pckWriteU64P(packWrite, cipherType);
pckWriteBinP(packWrite, pass);
pckWriteStrP(packWrite, digestName);
pckWriteEndP(packWrite);
}
MEM_CONTEXT_TEMP_END();
// Create filter interface
this = ioFilterNewP(
CIPHER_BLOCK_FILTER_TYPE_STR, driver, paramList, .done = cipherBlockDone, .inOut = cipherBlockProcess,
CIPHER_BLOCK_FILTER_TYPE, driver, paramList, .done = cipherBlockDone, .inOut = cipherBlockProcess,
.inputSame = cipherBlockInputSame);
}
OBJ_NEW_END();
@ -459,11 +460,23 @@ cipherBlockNew(CipherMode mode, CipherType cipherType, const Buffer *pass, const
}
IoFilter *
cipherBlockNewVar(const VariantList *paramList)
cipherBlockNewPack(const Buffer *const paramList)
{
return cipherBlockNew(
(CipherMode)varUInt64(varLstGet(paramList, 0)), (CipherType)varUInt64(varLstGet(paramList, 1)),
BUFSTR(varStr(varLstGet(paramList, 2))), varLstGet(paramList, 3) == NULL ? NULL : varStr(varLstGet(paramList, 3)));
IoFilter *result = NULL;
MEM_CONTEXT_TEMP_BEGIN()
{
PackRead *const paramListPack = pckReadNewBuf(paramList);
const CipherMode cipherMode = (CipherMode)pckReadU64P(paramListPack);
const CipherType cipherType = (CipherType)pckReadU64P(paramListPack);
const Buffer *const pass = pckReadBinP(paramListPack);
const String *const digestName = pckReadStrP(paramListPack);
result = objMoveContext(cipherBlockNew(cipherMode, cipherType, pass, digestName), memContextPrior());
}
MEM_CONTEXT_TEMP_END();
return result;
}
/**********************************************************************************************************************************/

5
src/common/crypto/cipherBlock.h
View File

@ -10,14 +10,13 @@ Block Cipher Header
/***********************************************************************************************************************************
Filter type constant
***********************************************************************************************************************************/
#define CIPHER_BLOCK_FILTER_TYPE "cipherBlock"
STRING_DECLARE(CIPHER_BLOCK_FILTER_TYPE_STR);
#define CIPHER_BLOCK_FILTER_TYPE STRID5("cipher-blk", 0x16c16e45441230)
/***********************************************************************************************************************************
Constructors
***********************************************************************************************************************************/
IoFilter *cipherBlockNew(CipherMode mode, CipherType cipherType, const Buffer *pass, const String *digestName);
IoFilter *cipherBlockNewVar(const VariantList *paramList);
IoFilter *cipherBlockNewPack(const Buffer *paramList);
/***********************************************************************************************************************************
Helper functions

47
src/common/crypto/hash.c
View File

@ -14,13 +14,9 @@ Cryptographic Hash
#include "common/io/filter/filter.h"
#include "common/log.h"
#include "common/type/object.h"
#include "common/type/pack.h"
#include "common/crypto/common.h"
/***********************************************************************************************************************************
Filter type constant
***********************************************************************************************************************************/
STRING_EXTERN(CRYPTO_HASH_FILTER_TYPE_STR, CRYPTO_HASH_FILTER_TYPE);
/***********************************************************************************************************************************
Hash types
***********************************************************************************************************************************/
@ -146,7 +142,7 @@ cryptoHash(CryptoHash *this)
/***********************************************************************************************************************************
Get string representation of the hash as a filter result
***********************************************************************************************************************************/
static Variant *
static Buffer *
cryptoHashResult(THIS_VOID)
{
THIS(CryptoHash);
@ -157,7 +153,18 @@ cryptoHashResult(THIS_VOID)
ASSERT(this != NULL);
FUNCTION_LOG_RETURN(VARIANT, varNewStr(bufHex(cryptoHash(this))));
Buffer *const result = bufNew(PACK_EXTRA_MIN);
MEM_CONTEXT_TEMP_BEGIN()
{
PackWrite *const pack = pckWriteNewBuf(result);
pckWriteStrP(pack, bufHex(cryptoHash(this)));
pckWriteEndP(pack);
}
MEM_CONTEXT_TEMP_END();
FUNCTION_LOG_RETURN(BUFFER, result);
}
/**********************************************************************************************************************************/
@ -208,11 +215,19 @@ cryptoHashNew(const String *type)
}
// Create param list
VariantList *paramList = varLstNew();
varLstAdd(paramList, varNewStr(type));
Buffer *const paramList = bufNew(PACK_EXTRA_MIN);
MEM_CONTEXT_TEMP_BEGIN()
{
PackWrite *const packWrite = pckWriteNewBuf(paramList);
pckWriteStrP(packWrite, type);
pckWriteEndP(packWrite);
}
MEM_CONTEXT_TEMP_END();
// Create filter interface
this = ioFilterNewP(CRYPTO_HASH_FILTER_TYPE_STR, driver, paramList, .in = cryptoHashProcess, .result = cryptoHashResult);
this = ioFilterNewP(CRYPTO_HASH_FILTER_TYPE, driver, paramList, .in = cryptoHashProcess, .result = cryptoHashResult);
}
OBJ_NEW_END();
@ -220,9 +235,17 @@ cryptoHashNew(const String *type)
}
IoFilter *
cryptoHashNewVar(const VariantList *paramList)
cryptoHashNewPack(const Buffer *const paramList)
{
return cryptoHashNew(varStr(varLstGet(paramList, 0)));
IoFilter *result = NULL;
MEM_CONTEXT_TEMP_BEGIN()
{
result = objMoveContext(cryptoHashNew(pckReadStrP(pckReadNewBuf(paramList))), memContextPrior());
}
MEM_CONTEXT_TEMP_END();
return result;
}
/**********************************************************************************************************************************/

5
src/common/crypto/hash.h
View File

@ -12,8 +12,7 @@ Generate a hash (sha1, md5, etc.) from a string, Buffer, or using an IoFilter.
/***********************************************************************************************************************************
Filter type constant
***********************************************************************************************************************************/
#define CRYPTO_HASH_FILTER_TYPE "hash"
STRING_DECLARE(CRYPTO_HASH_FILTER_TYPE_STR);
#define CRYPTO_HASH_FILTER_TYPE STRID5("hash", 0x44c280)
/***********************************************************************************************************************************
Hash types
@ -51,7 +50,7 @@ Hash type sizes
Constructors
***********************************************************************************************************************************/
IoFilter *cryptoHashNew(const String *type);
IoFilter *cryptoHashNewVar(const VariantList *paramList);
IoFilter *cryptoHashNewPack(const Buffer *paramList);
/***********************************************************************************************************************************
Helper functions

5
src/common/io/filter/buffer.c
View File

@ -14,8 +14,7 @@ IO Buffer Filter
/***********************************************************************************************************************************
Filter type constant
***********************************************************************************************************************************/
#define BUFFER_FILTER_TYPE "buffer"
STRING_STATIC(BUFFER_FILTER_TYPE_STR, BUFFER_FILTER_TYPE);
#define BUFFER_FILTER_TYPE STRID5("buffer", 0x24531aa20)
/***********************************************************************************************************************************
Object type
@ -116,7 +115,7 @@ ioBufferNew(void)
IoBuffer *driver = OBJ_NEW_ALLOC();
*driver = (IoBuffer){0};
this = ioFilterNewP(BUFFER_FILTER_TYPE_STR, driver, NULL, .inOut = ioBufferProcess, .inputSame = ioBufferInputSame);
this = ioFilterNewP(BUFFER_FILTER_TYPE, driver, NULL, .inOut = ioBufferProcess, .inputSame = ioBufferInputSame);
}
OBJ_NEW_END();

12
src/common/io/filter/filter.c
View File

@ -21,16 +21,16 @@ struct IoFilter
Allocations will be in the memory context of the caller.
***********************************************************************************************************************************/
IoFilter *
ioFilterNew(const String *type, void *driver, VariantList *paramList, IoFilterInterface interface)
ioFilterNew(const StringId type, void *const driver, Buffer *const paramList, const IoFilterInterface interface)
{
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(STRING, type);
FUNCTION_LOG_PARAM(STRING_ID, type);
FUNCTION_LOG_PARAM_P(VOID, driver);
FUNCTION_LOG_PARAM(VARIANT_LIST, paramList);
FUNCTION_LOG_PARAM(BUFFER, paramList);
FUNCTION_LOG_PARAM(IO_FILTER_INTERFACE, interface);
FUNCTION_LOG_END();
ASSERT(type != NULL);
ASSERT(type != 0);
ASSERT(driver != NULL);
// One of processIn or processInOut must be set
ASSERT(interface.in != NULL || interface.inOut != NULL);
@ -139,7 +139,7 @@ ioFilterInputSame(const IoFilter *this)
}
/**********************************************************************************************************************************/
Variant *
Buffer *
ioFilterResult(const IoFilter *this)
{
FUNCTION_TEST_BEGIN();
@ -155,5 +155,5 @@ ioFilterResult(const IoFilter *this)
String *
ioFilterToLog(const IoFilter *this)
{
return strNewFmt("{type: %s}", strZ(ioFilterType(this)));
return strNewFmt("{type: %s}", strZ(strIdToStr(ioFilterType(this))));
}

4
src/common/io/filter/filter.h
View File

@ -25,10 +25,10 @@ typedef struct IoFilter IoFilter;
Getters/Setters
***********************************************************************************************************************************/
// Get filter result
Variant *ioFilterResult(const IoFilter *this);
Buffer *ioFilterResult(const IoFilter *this);
// Identifies the filter and is used when pulling results from the filter group
__attribute__((always_inline)) static inline const String *
__attribute__((always_inline)) static inline StringId
ioFilterType(const IoFilter *const this)
{
return THIS_PUB(IoFilter)->type;

17
src/common/io/filter/filter.intern.h
View File

@ -18,7 +18,7 @@ Each filter has a type that allows it to be identified in the filter list.
#ifndef COMMON_IO_FILTER_FILTER_INTERN_H
#define COMMON_IO_FILTER_FILTER_INTERN_H
#include "common/type/variantList.h"
#include "common/type/stringId.h"
/***********************************************************************************************************************************
Constructors
@ -43,15 +43,16 @@ typedef struct IoFilterInterface
// call to inOut the same input will be passed along with a fresh output buffer with space for more processed output.
bool (*inputSame)(const void *driver);
// If the filter produces a result then this function must be implemented to return the result. A result can be anything that
// is not processed output, e.g. a count of total bytes or a cryptographic hash.
Variant *(*result)(void *driver);
// If the filter produces a result then this function must be implemented to return the result. A result can be anything that
// is not processed output, e.g. a count of total bytes or a cryptographic hash. The returned buffer must be a pack containing
// the result.
Buffer *(*result)(void *driver);
} IoFilterInterface;
#define ioFilterNewP(type, driver, paramList, ...) \
ioFilterNew(type, driver, paramList, (IoFilterInterface){__VA_ARGS__})
IoFilter *ioFilterNew(const String *type, void *driver, VariantList *paramList, IoFilterInterface);
IoFilter *ioFilterNew(StringId type, void *driver, Buffer *paramList, IoFilterInterface);
/***********************************************************************************************************************************
Getters/Setters
@ -59,10 +60,10 @@ Getters/Setters
typedef struct IoFilterPub
{
MemContext *memContext; // Mem context
const String *type; // Filter type
StringId type; // Filter type
IoFilterInterface interface; // Filter interface
void *driver; // Filter driver
const VariantList *paramList; // Filter parameters
const Buffer *paramList; // Filter parameters
} IoFilterPub;
// Is the filter done?
@ -94,7 +95,7 @@ ioFilterOutput(const IoFilter *const this)
}
// List of filter parameters
__attribute__((always_inline)) static inline const VariantList *
__attribute__((always_inline)) static inline const Buffer *
ioFilterParamList(const IoFilter *const this)
{
return THIS_PUB(IoFilter)->paramList;

165
src/common/io/filter/group.c
View File

@ -32,6 +32,15 @@ typedef struct IoFilterData
#define FUNCTION_LOG_IO_FILTER_DATA_FORMAT(value, buffer, bufferSize) \
objToLog(value, "IoFilterData", buffer, bufferSize)
/***********************************************************************************************************************************
Filter results
***********************************************************************************************************************************/
typedef struct IoFilterResult
{
StringId type; // Filter type
Buffer *result; // Filter result
} IoFilterResult;
/***********************************************************************************************************************************
Object type
***********************************************************************************************************************************/
@ -39,7 +48,7 @@ struct IoFilterGroup
{
IoFilterGroupPub pub; // Publicly accessible variables
const Buffer *input; // Input buffer passed in for processing
KeyValue *filterResult; // Filter results (if any)
List *filterResult; // Filter results (if any)
#ifdef DEBUG
bool flushing; // Is output being flushed?
@ -65,6 +74,8 @@ ioFilterGroupNew(void)
.done = false,
.filterList = lstNewP(sizeof(IoFilterData)),
},
.filterResult = lstNewP(sizeof(IoFilterResult)),
};
}
OBJ_NEW_END();
@ -353,25 +364,16 @@ ioFilterGroupClose(IoFilterGroup *this)
ASSERT(this != NULL);
ASSERT(this->pub.opened && !this->pub.closed);
// Gather results from the filters
for (unsigned int filterIdx = 0; filterIdx < ioFilterGroupSize(this); filterIdx++)
{
IoFilterData *filterData = ioFilterGroupGet(this, filterIdx);
const Variant *filterResult = ioFilterResult(filterData->filter);
IoFilter *const filter = ioFilterGroupGet(this, filterIdx)->filter;
if (this->filterResult == NULL)
MEM_CONTEXT_BEGIN(lstMemContext(this->filterResult))
{
MEM_CONTEXT_BEGIN(objMemContext(this))
{
this->filterResult = kvNew();
}
MEM_CONTEXT_END();
lstAdd(this->filterResult, &(IoFilterResult){.type = ioFilterType(filter), .result = ioFilterResult(filter)});
}
MEM_CONTEXT_TEMP_BEGIN()
{
kvAdd(this->filterResult, VARSTR(ioFilterType(filterData->filter)), filterResult);
}
MEM_CONTEXT_TEMP_END();
MEM_CONTEXT_END();
}
// Filter group is open
@ -383,7 +385,7 @@ ioFilterGroupClose(IoFilterGroup *this)
}
/**********************************************************************************************************************************/
Variant *
Buffer *
ioFilterGroupParamAll(const IoFilterGroup *this)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
@ -394,48 +396,72 @@ ioFilterGroupParamAll(const IoFilterGroup *this)
ASSERT(!this->pub.opened);
ASSERT(this->pub.filterList != NULL);
VariantList *result = varLstNew();
for (unsigned int filterIdx = 0; filterIdx < ioFilterGroupSize(this); filterIdx++)
{
IoFilter *filter = ioFilterGroupGet(this, filterIdx)->filter;
const VariantList *paramList = ioFilterParamList(filter);
KeyValue *filterParam = kvNew();
kvPut(filterParam, VARSTR(ioFilterType(filter)), paramList ? varNewVarLst(paramList) : NULL);
varLstAdd(result, varNewKv(filterParam));
}
FUNCTION_LOG_RETURN(VARIANT, varNewVarLst(result));
}
/**********************************************************************************************************************************/
const Variant *
ioFilterGroupResult(const IoFilterGroup *this, const String *filterType)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(IO_FILTER_GROUP, this);
FUNCTION_LOG_PARAM(STRING, filterType);
FUNCTION_LOG_END();
ASSERT(this->pub.opened);
ASSERT(filterType != NULL);
const Variant *result = NULL;
Buffer *result = NULL;
MEM_CONTEXT_TEMP_BEGIN()
{
result = kvGet(this->filterResult, VARSTR(filterType));
result = bufNew(PACK_EXTRA_MIN);
PackWrite *const pack = pckWriteNewBuf(result);
for (unsigned int filterIdx = 0; filterIdx < ioFilterGroupSize(this); filterIdx++)
{
IoFilter *filter = ioFilterGroupGet(this, filterIdx)->filter;
pckWriteStrIdP(pack, ioFilterType(filter));
pckWritePackBufP(pack, ioFilterParamList(filter));
}
pckWriteEndP(pack);
bufMove(result, memContextPrior());
}
MEM_CONTEXT_TEMP_END();
FUNCTION_LOG_RETURN_CONST(VARIANT, result);
FUNCTION_LOG_RETURN(BUFFER, result);
}
/**********************************************************************************************************************************/
const Variant *
ioFilterGroupResultAll(const IoFilterGroup *this)
PackRead *
ioFilterGroupResult(const IoFilterGroup *const this, const StringId filterType, const IoFilterGroupResultParam param)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(IO_FILTER_GROUP, this);
FUNCTION_LOG_PARAM(STRING_ID, filterType);
FUNCTION_LOG_PARAM(UINT, param.idx);
FUNCTION_LOG_END();
ASSERT(this->pub.opened);
ASSERT(filterType != 0);
PackRead *result = NULL;
// Search for the result
unsigned int foundIdx = 0;
for (unsigned int filterResultIdx = 0; filterResultIdx < lstSize(this->filterResult); filterResultIdx++)
{
const IoFilterResult *const filterResult = lstGet(this->filterResult, filterResultIdx);
// If the filter matches check the index
if (filterResult->type == filterType)
{
// If the index matches return the result
if (foundIdx == param.idx)
{
result = pckReadNewBuf(filterResult->result);
break;
}
// Increment the index and keep searching
foundIdx++;
}
}
FUNCTION_LOG_RETURN(PACK_READ, result);
}
/**********************************************************************************************************************************/
PackWrite *
ioFilterGroupResultAll(const IoFilterGroup *const this)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(IO_FILTER_GROUP, this);
@ -444,24 +470,55 @@ ioFilterGroupResultAll(const IoFilterGroup *this)
ASSERT(this != NULL);
ASSERT(this->pub.closed);
FUNCTION_LOG_RETURN_CONST(VARIANT, varNewKv(this->filterResult));
PackWrite *result = NULL;
// Pack the result list
MEM_CONTEXT_TEMP_BEGIN()
{
Buffer *const buffer = bufNew(0);
result = pckWriteNewBuf(buffer);
for (unsigned int filterResultIdx = 0; filterResultIdx < lstSize(this->filterResult); filterResultIdx++)
{
const IoFilterResult *const filterResult = lstGet(this->filterResult, filterResultIdx);
pckWriteStrIdP(result, filterResult->type);
pckWritePackBufP(result, filterResult->result);
}
pckWriteEndP(result);
pckWriteMove(result, memContextPrior());
bufMove(buffer, memContextPrior());
}
MEM_CONTEXT_TEMP_END();
FUNCTION_LOG_RETURN(PACK_WRITE, result);
}
/**********************************************************************************************************************************/
void
ioFilterGroupResultAllSet(IoFilterGroup *this, const Variant *filterResult)
ioFilterGroupResultAllSet(IoFilterGroup *const this, PackRead *const filterResultPack)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(IO_FILTER_GROUP, this);
FUNCTION_LOG_PARAM(PACK_READ, filterResultPack);
FUNCTION_LOG_END();
ASSERT(this != NULL);
if (filterResult != NULL)
if (filterResultPack != NULL)
{
MEM_CONTEXT_BEGIN(objMemContext(this))
// Unpack the results into a list
MEM_CONTEXT_BEGIN(lstMemContext(this->filterResult))
{
this->filterResult = kvDup(varKv(filterResult));
while (!pckReadNullP(filterResultPack))
{
const StringId type = pckReadStrIdP(filterResultPack);
Buffer *const result = pckReadPackBufP(filterResultPack);
lstAdd(this->filterResult, &(IoFilterResult){.type = type, .result = result});
}
}
MEM_CONTEXT_END();
}

20
src/common/io/filter/group.h
View File

@ -17,6 +17,7 @@ typedef struct IoFilterGroup IoFilterGroup;
#include "common/io/filter/filter.h"
#include "common/type/object.h"
#include "common/type/pack.h"
#include "common/type/string.h"
/***********************************************************************************************************************************
@ -57,14 +58,23 @@ ioFilterGroupInputSame(const IoFilterGroup *const this)
}
// Get all filters and their parameters so they can be passed to a remote
Variant *ioFilterGroupParamAll(const IoFilterGroup *this);
Buffer *ioFilterGroupParamAll(const IoFilterGroup *this);
// Get filter results
const Variant *ioFilterGroupResult(const IoFilterGroup *this, const String *filterType);
// Get filter results. If the same filter was used more than once then idx can be used to specify which one to get.
typedef struct IoFilterGroupResultParam
{
VAR_PARAM_HEADER;
unsigned int idx;
} IoFilterGroupResultParam;
#define ioFilterGroupResultP(this, filterType, ...) \
ioFilterGroupResult(this, filterType, (IoFilterGroupResultParam){VAR_PARAM_INIT, __VA_ARGS__})
PackRead *ioFilterGroupResult(const IoFilterGroup *this, StringId filterType, IoFilterGroupResultParam param);
// Get/set all filter results
const Variant *ioFilterGroupResultAll(const IoFilterGroup *this);
void ioFilterGroupResultAllSet(IoFilterGroup *this, const Variant *filterResult);
PackWrite *ioFilterGroupResultAll(const IoFilterGroup *this);
void ioFilterGroupResultAllSet(IoFilterGroup *this, PackRead *filterResult);
// Return total number of filters
__attribute__((always_inline)) static inline unsigned int

7
src/common/io/filter/sink.c
View File

@ -9,11 +9,6 @@ IO Sink Filter
#include "common/log.h"
#include "common/type/object.h"
/***********************************************************************************************************************************
Filter type constant
***********************************************************************************************************************************/
STRING_EXTERN(SINK_FILTER_TYPE_STR, SINK_FILTER_TYPE);
/***********************************************************************************************************************************
Object type
***********************************************************************************************************************************/
@ -62,7 +57,7 @@ ioSinkNew(void)
OBJ_NEW_BEGIN(IoSink)
{
IoSink *driver = OBJ_NEW_ALLOC();
this = ioFilterNewP(SINK_FILTER_TYPE_STR, driver, NULL, .inOut = ioSinkProcess);
this = ioFilterNewP(SINK_FILTER_TYPE, driver, NULL, .inOut = ioSinkProcess);
}
OBJ_NEW_END();

3
src/common/io/filter/sink.h
View File

@ -12,8 +12,7 @@ no data should be returned.
/***********************************************************************************************************************************
Filter type constant
***********************************************************************************************************************************/
#define SINK_FILTER_TYPE "sink"
STRING_DECLARE(SINK_FILTER_TYPE_STR);
#define SINK_FILTER_TYPE STRID5("sink", 0x5b9330)
/***********************************************************************************************************************************
Constructors

23
src/common/io/filter/size.c
View File

@ -10,11 +10,7 @@ IO Size Filter
#include "common/io/filter/size.h"
#include "common/log.h"
#include "common/type/object.h"
/***********************************************************************************************************************************
Filter type constant
***********************************************************************************************************************************/
STRING_EXTERN(SIZE_FILTER_TYPE_STR, SIZE_FILTER_TYPE);
#include "common/type/pack.h"
/***********************************************************************************************************************************
Object type
@ -62,7 +58,7 @@ ioSizeProcess(THIS_VOID, const Buffer *input)
/***********************************************************************************************************************************
Return filter result
***********************************************************************************************************************************/
static Variant *
static Buffer *
ioSizeResult(THIS_VOID)
{
THIS(IoSize);
@ -73,7 +69,18 @@ ioSizeResult(THIS_VOID)
ASSERT(this != NULL);
FUNCTION_LOG_RETURN(VARIANT, varNewUInt64(this->size));
Buffer *const result = bufNew(PACK_EXTRA_MIN);
MEM_CONTEXT_TEMP_BEGIN()
{
PackWrite *const pack = pckWriteNewBuf(result);
pckWriteU64P(pack, this->size);
pckWriteEndP(pack);
}
MEM_CONTEXT_TEMP_END();
FUNCTION_LOG_RETURN(BUFFER, result);
}
/**********************************************************************************************************************************/
@ -89,7 +96,7 @@ ioSizeNew(void)
IoSize *driver = OBJ_NEW_ALLOC();
*driver = (IoSize){0};
this = ioFilterNewP(SIZE_FILTER_TYPE_STR, driver, NULL, .in = ioSizeProcess, .result = ioSizeResult);
this = ioFilterNewP(SIZE_FILTER_TYPE, driver, NULL, .in = ioSizeProcess, .result = ioSizeResult);
}
OBJ_NEW_END();

3
src/common/io/filter/size.h
View File

@ -12,8 +12,7 @@ in a FilterGroup with IoWrite.
/***********************************************************************************************************************************
Filter type constant
***********************************************************************************************************************************/
#define SIZE_FILTER_TYPE "size"
STRING_DECLARE(SIZE_FILTER_TYPE_STR);
#define SIZE_FILTER_TYPE STRID5("size", 0x2e9330)
/***********************************************************************************************************************************
Constructors

27
src/common/type/pack.c
View File

@ -1620,18 +1620,31 @@ pckWritePack(PackWrite *this, const PackWrite *value, PckWritePackParam param)
ASSERT(this != NULL);
pckWritePackBuf(this, value != NULL ? pckWriteBuf(value) : NULL, param);
FUNCTION_TEST_RETURN(this);
}
/**********************************************************************************************************************************/
PackWrite *
pckWritePackBuf(PackWrite *this, const Buffer *value, PckWritePackParam param)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(PACK_WRITE, this);
FUNCTION_TEST_PARAM(BUFFER, value);
FUNCTION_TEST_PARAM(UINT, param.id);
FUNCTION_TEST_END();
ASSERT(this != NULL);
if (!pckWriteDefaultNull(this, false, value == NULL))
{
ASSERT(value != NULL);
// Write pack size
// Write pack buffer
pckWriteTag(this, pckTypeMapPack, param.id, 0);
// Write pack data
const Buffer *packBuffer = pckWriteBuf(value);
pckWriteU64Internal(this, bufUsed(packBuffer));
pckWriteBuffer(this, packBuffer);
pckWriteU64Internal(this, bufUsed(value));
pckWriteBuffer(this, value);
}
FUNCTION_TEST_RETURN(this);

6
src/common/type/pack.h
View File

@ -505,6 +505,12 @@ typedef struct PckWritePackParam
PackWrite *pckWritePack(PackWrite *this, const PackWrite *value, PckWritePackParam param);
// Write pack buffer
#define pckWritePackBufP(this, value, ...) \
pckWritePackBuf(this, value, (PckWritePackParam){VAR_PARAM_INIT, __VA_ARGS__})
PackWrite *pckWritePackBuf(PackWrite *this, const Buffer *value, PckWritePackParam param);
// Write pointer. Use with extreme caution. Pointers cannot be sent to another host -- they must only be used locally.
typedef struct PckWritePtrParam
{

4
src/info/info.c
View File

@ -300,7 +300,7 @@ infoNewLoad(IoRead *read, InfoLoadNewCallback *callbackFunction, void *callbackD
INFO_CHECKSUM_END(data.checksumActual);
// Verify the checksum
const String *checksumActual = varStr(ioFilterResult(data.checksumActual));
const String *checksumActual = pckReadStrP(pckReadNewBuf(ioFilterResult(data.checksumActual)));
if (data.checksumExpected == NULL)
THROW_FMT(ChecksumError, "invalid checksum, actual '%s' but no checksum found", strZ(checksumActual));
@ -435,7 +435,7 @@ infoSave(Info *this, IoWrite *write, InfoSaveCallback *callbackFunction, void *c
INFO_CHECKSUM_END(data.checksum);
ioWrite(data.write, BUFSTRDEF("\n[" INFO_SECTION_BACKREST "]\n" INFO_KEY_CHECKSUM "="));
ioWriteLine(data.write, BUFSTR(jsonFromVar(ioFilterResult(data.checksum))));
ioWriteLine(data.write, BUFSTR(jsonFromStr(pckReadStrP(pckReadNewBuf(ioFilterResult(data.checksum))))));
// Close the file
ioWriteClose(data.write);

2
src/storage/gcs/write.c
View File

@ -88,7 +88,7 @@ storageWriteGcsVerify(StorageWriteGcs *this, HttpResponse *response)
CHECK(md5base64 != NULL);
const String *md5actual = bufHex(bufNewDecode(encodeBase64, md5base64));
const String *md5expected = varStr(ioFilterResult(this->md5hash));
const String *md5expected = pckReadStrP(pckReadNewBuf(ioFilterResult(this->md5hash)));
if (!strEq(md5actual, md5expected))
{

64
src/storage/remote/protocol.c
View File

@ -34,38 +34,55 @@ static struct
Set filter group based on passed filters
***********************************************************************************************************************************/
static void
storageRemoteFilterGroup(IoFilterGroup *filterGroup, const Variant *filterList)
storageRemoteFilterGroup(IoFilterGroup *filterGroup, const Buffer *const filterPack)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(IO_FILTER_GROUP, filterGroup);
FUNCTION_TEST_PARAM(VARIANT, filterList);
FUNCTION_TEST_PARAM(BUFFER, filterPack);
FUNCTION_TEST_END();
ASSERT(filterGroup != NULL);
ASSERT(filterList != NULL);
ASSERT(filterPack != NULL);
for (unsigned int filterIdx = 0; filterIdx < varLstSize(varVarLst(filterList)); filterIdx++)
PackRead *const filterList = pckReadNewBuf(filterPack);
while (!pckReadNullP(filterList))
{
const KeyValue *filterKv = varKv(varLstGet(varVarLst(filterList), filterIdx));
const String *filterKey = varStr(varLstGet(kvKeyList(filterKv), 0));
const VariantList *filterParam = varVarLst(kvGet(filterKv, VARSTR(filterKey)));
const StringId filterKey = pckReadStrIdP(filterList);
const Buffer *const filterParam = pckReadPackBufP(filterList);
IoFilter *filter = compressFilterVar(filterKey, filterParam);
IoFilter *filter = compressFilterPack(filterKey, filterParam);
if (filter != NULL)
ioFilterGroupAdd(filterGroup, filter);
else if (strEq(filterKey, CIPHER_BLOCK_FILTER_TYPE_STR))
ioFilterGroupAdd(filterGroup, cipherBlockNewVar(filterParam));
else if (strEq(filterKey, CRYPTO_HASH_FILTER_TYPE_STR))
ioFilterGroupAdd(filterGroup, cryptoHashNewVar(filterParam));
else if (strEq(filterKey, PAGE_CHECKSUM_FILTER_TYPE_STR))
ioFilterGroupAdd(filterGroup, pageChecksumNewVar(filterParam));
else if (strEq(filterKey, SINK_FILTER_TYPE_STR))
ioFilterGroupAdd(filterGroup, ioSinkNew());
else if (strEq(filterKey, SIZE_FILTER_TYPE_STR))
ioFilterGroupAdd(filterGroup, ioSizeNew());
else
THROW_FMT(AssertError, "unable to add filter '%s'", strZ(filterKey));
{
switch (filterKey)
{
case CIPHER_BLOCK_FILTER_TYPE:
ioFilterGroupAdd(filterGroup, cipherBlockNewPack(filterParam));
break;
case CRYPTO_HASH_FILTER_TYPE:
ioFilterGroupAdd(filterGroup, cryptoHashNewPack(filterParam));
break;
case PAGE_CHECKSUM_FILTER_TYPE:
ioFilterGroupAdd(filterGroup, pageChecksumNewPack(filterParam));
break;
case SINK_FILTER_TYPE:
ioFilterGroupAdd(filterGroup, ioSinkNew());
break;
case SIZE_FILTER_TYPE:
ioFilterGroupAdd(filterGroup, ioSizeNew());
break;
default:
THROW_FMT(AssertError, "unable to add filter '%s'", strZ(strIdToStr(filterKey)));
}
}
}
FUNCTION_TEST_RETURN_VOID();
@ -339,7 +356,7 @@ storageRemoteOpenReadProtocol(PackRead *const param, ProtocolServer *const serve
const String *file = pckReadStrP(param);
bool ignoreMissing = pckReadBoolP(param);
const Variant *limit = jsonToVar(pckReadStrP(param));
const Variant *filter = jsonToVar(pckReadStrP(param));
const Buffer *const filter = pckReadPackBufP(param);
// Create the read object
IoRead *fileRead = storageReadIo(
@ -380,8 +397,7 @@ storageRemoteOpenReadProtocol(PackRead *const param, ProtocolServer *const serve
ioReadClose(fileRead);
// Write filter results
protocolServerDataPut(
server, pckWriteStrP(protocolPackNew(), jsonFromVar(ioFilterGroupResultAll(ioReadFilterGroup(fileRead)))));
protocolServerDataPut(server, pckWritePackP(protocolPackNew(), ioFilterGroupResultAll(ioReadFilterGroup(fileRead))));
}
protocolServerDataEndPut(server);
@ -417,7 +433,7 @@ storageRemoteOpenWriteProtocol(PackRead *const param, ProtocolServer *const serv
bool syncFile = pckReadBoolP(param);
bool syncPath = pckReadBoolP(param);
bool atomic = pckReadBoolP(param);
const Variant *filter = jsonToVar(pckReadStrP(param));
const Buffer *const filter = pckReadPackBufP(param);
IoWrite *fileWrite = storageWriteIo(
storageInterfaceNewWriteP(
@ -444,7 +460,7 @@ storageRemoteOpenWriteProtocol(PackRead *const param, ProtocolServer *const serv
// Push filter results
protocolServerDataPut(
server, pckWriteStrP(protocolPackNew(), jsonFromVar(ioFilterGroupResultAll(ioWriteFilterGroup(fileWrite)))));
server, pckWritePackP(protocolPackNew(), ioFilterGroupResultAll(ioWriteFilterGroup(fileWrite))));
break;
}
// Else more data to write

4
src/storage/remote/read.c
View File

@ -75,7 +75,7 @@ storageReadRemoteOpen(THIS_VOID)
pckWriteStrP(param, this->interface.name);
pckWriteBoolP(param, this->interface.ignoreMissing);
pckWriteStrP(param, jsonFromVar(this->interface.limit));
pckWriteStrP(param, jsonFromVar(ioFilterGroupParamAll(ioReadFilterGroup(storageReadIo(this->read)))));
pckWritePackBufP(param, ioFilterGroupParamAll(ioReadFilterGroup(storageReadIo(this->read))));
protocolClientCommandPut(this->client, command);
@ -147,7 +147,7 @@ storageReadRemote(THIS_VOID, Buffer *buffer, bool block)
{
bufFree(this->block);
ioFilterGroupResultAllSet(ioReadFilterGroup(storageReadIo(this->read)), jsonToVar(pckReadStrP(read)));
ioFilterGroupResultAllSet(ioReadFilterGroup(storageReadIo(this->read)), pckReadPackP(read));
this->eof = true;
protocolClientDataEndGet(this->client);

4
src/storage/remote/write.c
View File

@ -91,7 +91,7 @@ storageWriteRemoteOpen(THIS_VOID)
pckWriteBoolP(param, this->interface.syncFile);
pckWriteBoolP(param, this->interface.syncPath);
pckWriteBoolP(param, this->interface.atomic);
pckWriteStrP(param, jsonFromVar(ioFilterGroupParamAll(ioWriteFilterGroup(storageWriteIo(this->write)))));
pckWritePackBufP(param, ioFilterGroupParamAll(ioWriteFilterGroup(storageWriteIo(this->write))));
protocolClientCommandPut(this->client, command);
protocolClientDataGet(this->client);
@ -166,7 +166,7 @@ storageWriteRemoteClose(THIS_VOID)
{
protocolClientDataPut(this->client, NULL);
ioFilterGroupResultAllSet(
ioWriteFilterGroup(storageWriteIo(this->write)), jsonToVar(pckReadStrP(protocolClientDataGet(this->client))));
ioWriteFilterGroup(storageWriteIo(this->write)), pckReadPackP(protocolClientDataGet(this->client)));
protocolClientDataEndGet(this->client);
}
MEM_CONTEXT_TEMP_END();

5
test/define.yaml
View File

@ -260,6 +260,7 @@ unit:
- name: io
total: 4
feature: IO
harness: pack
coverage:
- common/io/bufferRead
@ -276,10 +277,12 @@ unit:
- common/io/read
- common/io/write
depend:
- common/type/pack
# ----------------------------------------------------------------------------------------------------------------------------
- name: type-pack
total: 1
harness: pack
coverage:
- common/type/pack

2
test/src/common/harnessInfo.c
View File

@ -97,7 +97,7 @@ harnessInfoChecksum(const String *info)
// Append checksum to buffer
bufCat(result, BUFSTRDEF("\n[backrest]\nbackrest-checksum="));
bufCat(result, BUFSTR(jsonFromVar(ioFilterResult(data.checksum))));
bufCat(result, BUFSTR(jsonFromStr(pckReadStrP(pckReadNewBuf(ioFilterResult(data.checksum))))));
bufCat(result, BUFSTRDEF("\n"));
bufMove(result, memContextPrior());

16
test/src/module/command/backupCommonTest.c
View File

@ -140,7 +140,7 @@ testRun(void)
ioWriteClose(write);
TEST_RESULT_STR_Z(
jsonFromVar(ioFilterGroupResult(ioWriteFilterGroup(write), PAGE_CHECKSUM_FILTER_TYPE_STR)),
pckReadStrP(ioFilterGroupResultP(ioWriteFilterGroup(write), PAGE_CHECKSUM_FILTER_TYPE)),
"{\"align\":true,\"valid\":true}", "all zero pages");
// -------------------------------------------------------------------------------------------------------------------------
@ -168,14 +168,13 @@ testRun(void)
ioFilterGroupAdd(
ioWriteFilterGroup(write),
pageChecksumNewVar(varVarLst(jsonToVar(
strNewFmt("[0,%u,%" PRIu64 "]", PG_SEGMENT_PAGE_DEFAULT, (uint64_t)0xFACEFACE00000000)))));
pageChecksumNewPack(ioFilterParamList(pageChecksumNew(0, PG_SEGMENT_PAGE_DEFAULT, 0xFACEFACE00000000))));
ioWriteOpen(write);
ioWrite(write, buffer);
ioWriteClose(write);
TEST_RESULT_STR_Z(
jsonFromVar(ioFilterGroupResult(ioWriteFilterGroup(write), PAGE_CHECKSUM_FILTER_TYPE_STR)),
pckReadStrP(ioFilterGroupResultP(ioWriteFilterGroup(write), PAGE_CHECKSUM_FILTER_TYPE)),
"{\"align\":true,\"valid\":true}", "single valid page");
// -------------------------------------------------------------------------------------------------------------------------
@ -200,14 +199,13 @@ testRun(void)
ioFilterGroupAdd(
ioWriteFilterGroup(write),
pageChecksumNewVar(varVarLst(jsonToVar(
strNewFmt("[0,%u,%" PRIu64 "]", PG_SEGMENT_PAGE_DEFAULT, (uint64_t)0xFACEFACE00000000)))));
pageChecksumNewPack(ioFilterParamList(pageChecksumNew(0, PG_SEGMENT_PAGE_DEFAULT, 0xFACEFACE00000000))));
ioWriteOpen(write);
ioWrite(write, buffer);
ioWriteClose(write);
TEST_RESULT_STR_Z(
jsonFromVar(ioFilterGroupResult(ioWriteFilterGroup(write), PAGE_CHECKSUM_FILTER_TYPE_STR)),
pckReadStrP(ioFilterGroupResultP(ioWriteFilterGroup(write), PAGE_CHECKSUM_FILTER_TYPE)),
"{\"align\":true,\"error\":[0],\"valid\":false}", "single checksum error");
// -------------------------------------------------------------------------------------------------------------------------
@ -299,7 +297,7 @@ testRun(void)
ioWriteClose(write);
TEST_RESULT_STR_Z(
jsonFromVar(ioFilterGroupResult(ioWriteFilterGroup(write), PAGE_CHECKSUM_FILTER_TYPE_STR)),
pckReadStrP(ioFilterGroupResultP(ioWriteFilterGroup(write), PAGE_CHECKSUM_FILTER_TYPE)),
"{\"align\":false,\"error\":[0,[2,4],[6,7]],\"valid\":false}", "various checksum errors");
// -------------------------------------------------------------------------------------------------------------------------
@ -318,7 +316,7 @@ testRun(void)
ioWriteClose(write);
TEST_RESULT_STR_Z(
jsonFromVar(ioFilterGroupResult(ioWriteFilterGroup(write), PAGE_CHECKSUM_FILTER_TYPE_STR)),
pckReadStrP(ioFilterGroupResultP(ioWriteFilterGroup(write), PAGE_CHECKSUM_FILTER_TYPE)),
"{\"align\":false,\"valid\":false}", "misalignment");
// -------------------------------------------------------------------------------------------------------------------------

4
test/src/module/command/backupTest.c
View File

@ -68,8 +68,8 @@ testBackupValidateCallback(void *callbackData, const StorageInfo *info)
ioFilterGroupAdd(ioReadFilterGroup(storageReadIo(read)), cryptoHashNew(HASH_TYPE_SHA1_STR));
uint64_t size = bufUsed(storageGetP(read));
const String *checksum = varStr(
ioFilterGroupResult(ioReadFilterGroup(storageReadIo(read)), CRYPTO_HASH_FILTER_TYPE_STR));
const String *checksum = pckReadStrP(
ioFilterGroupResultP(ioReadFilterGroup(storageReadIo(read)), CRYPTO_HASH_FILTER_TYPE));
strCatFmt(data->content, ", s=%" PRIu64, size);

19
test/src/module/common/compressTest.c
View File

@ -79,8 +79,9 @@ testSuite(CompressType type, const char *decompressCmd)
Buffer *compressed = NULL;
Buffer *decompressed = bufNewC(simpleData, strlen(simpleData));
VariantList *compressParamList = varLstNew();
varLstAdd(compressParamList, varNewUInt(1));
PackWrite *packWrite = pckWriteNewBuf(bufNew(PACK_EXTRA_MIN));
pckWriteI32P(packWrite, 1);
pckWriteEndP(packWrite);
// Create default storage object for testing
Storage *storageTest = storagePosixNewP(TEST_PATH_STR, .write = true);
@ -90,7 +91,7 @@ testSuite(CompressType type, const char *decompressCmd)
TEST_ASSIGN(
compressed,
testCompress(
compressFilterVar(strNewFmt("%sCompress", strZ(compressTypeStr(type))), compressParamList), decompressed, 1024,
compressFilterPack(compressHelperLocal[type].compressType, pckWriteBuf(packWrite)), decompressed, 1024,
256 * 1024 * 1024),
"simple data - compress large in/large out buffer");
@ -116,8 +117,7 @@ testSuite(CompressType type, const char *decompressCmd)
TEST_RESULT_BOOL(
bufEq(
decompressed,
testDecompress(
compressFilterVar(strNewFmt("%sDecompress", strZ(compressTypeStr(type))), NULL), compressed, 1024, 1024)),
testDecompress(compressFilterPack(compressHelperLocal[type].decompressType, NULL), compressed, 1024, 1024)),
true, "simple data - decompress large in/large out buffer");
TEST_RESULT_BOOL(
@ -339,6 +339,11 @@ testRun(void)
TEST_RESULT_UINT(compressTypeEnum(STRDEF("gz")), compressTypeGz, "gz enum");
TEST_ERROR(compressTypeEnum(STRDEF(BOGUS_STR)), AssertError, "invalid compression type 'BOGUS'");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("compressTypeStr()");
TEST_RESULT_STR_Z(compressTypeStr(compressTypeGz), "gz", "gz str");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("compressTypePresent()");
@ -352,9 +357,9 @@ testRun(void)
TEST_RESULT_UINT(compressTypeFromName(STRDEF("file.gz")), compressTypeGz, "type from name");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("compressFilterVar()");
TEST_TITLE("compressFilterPack()");
TEST_RESULT_PTR(compressFilterVar(STRDEF("BOGUS"), 0), NULL, "no filter match");
TEST_RESULT_PTR(compressFilterPack(STRID5("bogus", 0x13a9de20), NULL), NULL, "no filter match");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("compressExtStr()");

21
test/src/module/common/cryptoTest.c
View File

@ -96,7 +96,7 @@ testRun(void)
Buffer *encryptBuffer = bufNew(TEST_BUFFER_SIZE);
IoFilter *blockEncryptFilter = cipherBlockNew(cipherModeEncrypt, cipherTypeAes256Cbc, testPass, NULL);
blockEncryptFilter = cipherBlockNewVar(ioFilterParamList(blockEncryptFilter));
blockEncryptFilter = cipherBlockNewPack(ioFilterParamList(blockEncryptFilter));
CipherBlock *blockEncrypt = (CipherBlock *)ioFilterDriver(blockEncryptFilter);
TEST_RESULT_UINT(
@ -147,7 +147,7 @@ testRun(void)
Buffer *decryptBuffer = bufNew(TEST_BUFFER_SIZE);
IoFilter *blockDecryptFilter = cipherBlockNew(cipherModeDecrypt, cipherTypeAes256Cbc, testPass, HASH_TYPE_SHA1_STR);
blockDecryptFilter = cipherBlockNewVar(ioFilterParamList(blockDecryptFilter));
blockDecryptFilter = cipherBlockNewPack(ioFilterParamList(blockDecryptFilter));
CipherBlock *blockDecrypt = (CipherBlock *)ioFilterDriver(blockDecryptFilter);
TEST_RESULT_UINT(
@ -301,7 +301,11 @@ testRun(void)
TEST_RESULT_VOID(ioFilterFree(hash), " free hash");
// -------------------------------------------------------------------------------------------------------------------------
TEST_ASSIGN(hash, cryptoHashNewVar(varVarLst(jsonToVar(strNewFmt("[\"%s\"]", HASH_TYPE_SHA1)))), "create sha1 hash");
PackWrite *packWrite = pckWriteNewBuf(bufNew(PACK_EXTRA_MIN));
pckWriteStrP(packWrite, HASH_TYPE_SHA1_STR);
pckWriteEndP(packWrite);
TEST_ASSIGN(hash, cryptoHashNewPack(pckWriteBuf(packWrite)), "create sha1 hash");
TEST_RESULT_STR_Z(bufHex(cryptoHash((CryptoHash *)ioFilterDriver(hash))), HASH_TYPE_SHA1_ZERO, " check empty hash");
TEST_RESULT_STR_Z(
bufHex(cryptoHash((CryptoHash *)ioFilterDriver(hash))), HASH_TYPE_SHA1_ZERO, " check empty hash again");
@ -315,14 +319,15 @@ testRun(void)
TEST_RESULT_VOID(ioFilterProcessIn(hash, BUFSTRDEF("4")), " add 4");
TEST_RESULT_VOID(ioFilterProcessIn(hash, BUFSTRDEF("5")), " add 5");
TEST_RESULT_STR_Z(varStr(ioFilterResult(hash)), "8cb2237d0679ca88db6464eac60da96345513964", " check small hash");
TEST_RESULT_STR_Z(
pckReadStrP(pckReadNewBuf(ioFilterResult(hash))), "8cb2237d0679ca88db6464eac60da96345513964", " check small hash");
TEST_RESULT_VOID(ioFilterFree(hash), " free hash");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("md5 hash - zero bytes");
TEST_ASSIGN(hash, cryptoHashNew(STRDEF(HASH_TYPE_MD5)), "create md5 hash");
TEST_RESULT_STR_Z(varStr(ioFilterResult(hash)), HASH_TYPE_MD5_ZERO, "check empty hash");
TEST_RESULT_STR_Z(pckReadStrP(pckReadNewBuf(ioFilterResult(hash))), HASH_TYPE_MD5_ZERO, "check empty hash");
// Exercise most of the conditions in the local MD5 code
// -------------------------------------------------------------------------------------------------------------------------
@ -341,7 +346,7 @@ testRun(void)
TEST_RESULT_VOID(
ioFilterProcessIn(hash, BUFSTRZ("12345678901234567890123456789001234567890012345678901234")), "add 58 bytes");
TEST_RESULT_STR_Z(varStr(ioFilterResult(hash)), "3318600bc9c1d379e91e4bae90721243", "check hash");
TEST_RESULT_STR_Z(pckReadStrP(pckReadNewBuf(ioFilterResult(hash))), "3318600bc9c1d379e91e4bae90721243", "check hash");
// Full coverage of local MD5 requires processing > 511MB of data but that makes the test run too long. Instead we'll cheat
// a bit and initialize the context at 511MB to start. This does not produce a valid MD5 hash but does provide coverage of
@ -353,11 +358,11 @@ testRun(void)
((CryptoHash *)ioFilterDriver(hash))->md5Context->lo = 0x1fffffff;
TEST_RESULT_VOID(ioFilterProcessIn(hash, BUFSTRZ("1")), "add 1");
TEST_RESULT_STR_Z(varStr(ioFilterResult(hash)), "5c99876f9cafa7f485eac9c7a8a2764c", "check hash");
TEST_RESULT_STR_Z(pckReadStrP(pckReadNewBuf(ioFilterResult(hash))), "5c99876f9cafa7f485eac9c7a8a2764c", "check hash");
// -------------------------------------------------------------------------------------------------------------------------
TEST_ASSIGN(hash, cryptoHashNew(STRDEF(HASH_TYPE_SHA256)), "create sha256 hash");
TEST_RESULT_STR_Z(varStr(ioFilterResult(hash)), HASH_TYPE_SHA256_ZERO, " check empty hash");
TEST_RESULT_STR_Z(pckReadStrP(pckReadNewBuf(ioFilterResult(hash))), HASH_TYPE_SHA256_ZERO, " check empty hash");
// -------------------------------------------------------------------------------------------------------------------------
TEST_RESULT_STR_Z(

92
test/src/module/common/ioTest.c
View File

@ -7,6 +7,7 @@ Test IO
#include "common/type/json.h"
#include "common/harnessFork.h"
#include "common/harnessPack.h"
/***********************************************************************************************************************************
Test functions for IoRead that are not covered by testing the IoBufferRead object
@ -95,16 +96,22 @@ ioTestFilterSizeProcess(THIS_VOID, const Buffer *buffer)
FUNCTION_LOG_RETURN_VOID();
}
static Variant *
static Buffer *
ioTestFilterSizeResult(THIS_VOID)
{
THIS(IoTestFilterSize);
return varNewUInt64(this->size);
Buffer *const result = bufNew(16);
PackWrite *const pack = pckWriteNewBuf(result);
pckWriteU64P(pack, this->size);
pckWriteEndP(pack);
return result;
}
static IoFilter *
ioTestFilterSizeNew(const char *type)
ioTestFilterSizeNew(const StringId type)
{
IoFilter *this = NULL;
@ -113,7 +120,7 @@ ioTestFilterSizeNew(const char *type)
IoTestFilterSize *driver = OBJ_NEW_ALLOC();
*driver = (IoTestFilterSize){0};
this = ioFilterNewP(strNewZ(type), driver, NULL, .in = ioTestFilterSizeProcess, .result = ioTestFilterSizeResult);
this = ioFilterNewP(type, driver, NULL, .in = ioTestFilterSizeProcess, .result = ioTestFilterSizeResult);
}
OBJ_NEW_END();
@ -204,7 +211,7 @@ ioTestFilterMultiplyInputSame(const THIS_VOID)
}
static IoFilter *
ioTestFilterMultiplyNew(const char *type, unsigned int multiplier, unsigned int flushTotal, char flushChar)
ioTestFilterMultiplyNew(const StringId type, unsigned int multiplier, unsigned int flushTotal, char flushChar)
{
IoFilter *this = NULL;
@ -220,13 +227,21 @@ ioTestFilterMultiplyNew(const char *type, unsigned int multiplier, unsigned int
.flushChar = flushChar,
};
VariantList *paramList = varLstNew();
varLstAdd(paramList, varNewStrZ(type));
varLstAdd(paramList, varNewUInt(multiplier));
varLstAdd(paramList, varNewUInt(flushTotal));
Buffer *const paramList = bufNew(PACK_EXTRA_MIN);
MEM_CONTEXT_TEMP_BEGIN()
{
PackWrite *const packWrite = pckWriteNewBuf(paramList);
pckWriteStrIdP(packWrite, type);
pckWriteU32P(packWrite, multiplier);
pckWriteU32P(packWrite, flushTotal);
pckWriteEndP(packWrite);
}
MEM_CONTEXT_TEMP_END();
this = ioFilterNewP(
strNewZ(type), driver, paramList, .done = ioTestFilterMultiplyDone, .inOut = ioTestFilterMultiplyProcess,
type, driver, paramList, .done = ioTestFilterMultiplyDone, .inOut = ioTestFilterMultiplyProcess,
.inputSame = ioTestFilterMultiplyInputSame);
}
OBJ_NEW_END();
@ -306,7 +321,7 @@ testRun(void)
IoFilter *sizeFilter = ioSizeNew();
TEST_RESULT_VOID(
ioFilterGroupAdd(ioReadFilterGroup(bufferRead), ioTestFilterMultiplyNew("double", 2, 3, 'X')),
ioFilterGroupAdd(ioReadFilterGroup(bufferRead), ioTestFilterMultiplyNew(STRID5("double", 0xac155e40), 2, 3, 'X')),
" add filter to filter group");
TEST_RESULT_PTR(
ioFilterGroupInsert(ioReadFilterGroup(bufferRead), 0, sizeFilter), bufferRead->pub.filterGroup,
@ -316,8 +331,9 @@ testRun(void)
TEST_RESULT_VOID(ioFilterGroupAdd(ioReadFilterGroup(bufferRead), bufferFilter), " add filter to filter group");
TEST_RESULT_PTR(ioFilterMove(NULL, memContextTop()), NULL, " move NULL filter to top context");
TEST_RESULT_STR_Z(
jsonFromVar(ioFilterGroupParamAll(ioReadFilterGroup(bufferRead))),
"[{\"size\":null},{\"double\":[\"double\",2,3]},{\"size\":null},{\"buffer\":null}]", " check filter params");
hrnPackBufToStr(ioFilterGroupParamAll(ioReadFilterGroup(bufferRead))),
"1:strid:size, 3:strid:double, 4:pack:<1:strid:double, 2:u32:2, 3:u32:3>, 5:strid:size, 7:strid:buffer",
" check filter params");
TEST_RESULT_BOOL(ioReadOpen(bufferRead), true, " open");
TEST_RESULT_INT(ioReadFd(bufferRead), -1, " fd invalid");
@ -325,7 +341,7 @@ testRun(void)
TEST_RESULT_UINT(ioRead(bufferRead, buffer), 2, " read 2 bytes");
TEST_RESULT_UINT(ioRead(bufferRead, buffer), 0, " read 0 bytes (full buffer)");
TEST_RESULT_STR_Z(strNewBuf(buffer), "11", " check read");
TEST_RESULT_STR_Z(ioFilterType(sizeFilter), "size", "check filter type");
TEST_RESULT_STR_Z(strIdToStr(ioFilterType(sizeFilter)), "size", "check filter type");
TEST_RESULT_BOOL(ioReadEof(bufferRead), false, " not eof");
TEST_RESULT_VOID(bufUsedZero(buffer), " zero buffer");
@ -344,19 +360,23 @@ testRun(void)
TEST_RESULT_UINT(ioRead(bufferRead, buffer), 0, " read 0 bytes");
TEST_RESULT_VOID(ioReadClose(bufferRead), " close buffer read object");
TEST_RESULT_STR_Z(
jsonFromVar(ioFilterGroupResultAll(ioReadFilterGroup(bufferRead))),
"{\"buffer\":null,\"double\":null,\"size\":[3,9]}",
hrnPackBufToStr(pckWriteBuf(ioFilterGroupResultAll(ioReadFilterGroup(bufferRead)))),
"1:strid:size, 2:pack:<1:u64:3>, 3:strid:double, 5:strid:size, 6:pack:<1:u64:9>, 7:strid:buffer",
" check filter result all");
TEST_RESULT_PTR(ioReadFilterGroup(bufferRead), ioReadFilterGroup(bufferRead), " check filter group");
TEST_RESULT_UINT(
varUInt64(varLstGet(varVarLst(ioFilterGroupResult(ioReadFilterGroup(bufferRead), ioFilterType(sizeFilter))), 0)), 3,
pckReadU64P(ioFilterGroupResultP(ioReadFilterGroup(bufferRead), ioFilterType(sizeFilter))), 3,
" check filter result");
TEST_RESULT_PTR(
ioFilterGroupResult(ioReadFilterGroup(bufferRead), STRDEF("double")), NULL, " check filter result is NULL");
ioFilterGroupResultP(ioReadFilterGroup(bufferRead), STRID5("double", 0xac155e40)), NULL,
" check filter result is NULL");
TEST_RESULT_UINT(
varUInt64(varLstGet(varVarLst(ioFilterGroupResult(ioReadFilterGroup(bufferRead), ioFilterType(sizeFilter))), 1)), 9,
pckReadU64P(ioFilterGroupResultP(ioReadFilterGroup(bufferRead), ioFilterType(sizeFilter), .idx = 1)), 9,
" check filter result");
TEST_RESULT_PTR(
ioFilterGroupResultP(ioReadFilterGroup(bufferRead), STRID5("bogus", 0x13a9de20)), NULL,
" check missing filter result");
TEST_RESULT_PTR(ioFilterDriver(bufferFilter), bufferFilter->pub.driver, " check filter driver");
TEST_RESULT_PTR(ioFilterInterface(bufferFilter), &bufferFilter->pub.interface, " check filter interface");
@ -369,9 +389,19 @@ testRun(void)
IoFilterGroup *filterGroup = ioFilterGroupNew();
filterGroup->pub.opened = true;
TEST_RESULT_VOID(ioFilterGroupResultAllSet(filterGroup, NULL), "null result");
TEST_RESULT_VOID(ioFilterGroupResultAllSet(filterGroup, jsonToVar(STRDEF("{\"test\":777}"))), "add result");
PackWrite *filterResult = pckWriteNewBuf(bufNew(256));
pckWriteU64P(filterResult, 777);
pckWriteEndP(filterResult);
PackWrite *filterResultAll = pckWriteNewBuf(bufNew(256));
pckWriteStrIdP(filterResultAll, STRID5("test", 0xa4cb40));
pckWritePackP(filterResultAll, filterResult);
pckWriteEndP(filterResultAll);
TEST_RESULT_VOID(ioFilterGroupResultAllSet(filterGroup, pckReadNewBuf(pckWriteBuf(filterResultAll))), "add result");
filterGroup->pub.closed = true;
TEST_RESULT_UINT(varUInt64(ioFilterGroupResult(filterGroup, STRDEF("test"))), 777, " check filter result");
TEST_RESULT_UINT(pckReadU64P(ioFilterGroupResultP(filterGroup, STRID5("test", 0xa4cb40))), 777, " check filter result");
// Read a zero-size buffer to ensure filters are still processed even when there is no input. Some filters (e.g. encryption
// and compression) will produce output even if there is no input.
@ -382,7 +412,7 @@ testRun(void)
TEST_ASSIGN(bufferRead, ioBufferReadNew(bufferOriginal), "create buffer read object");
TEST_RESULT_VOID(
ioFilterGroupAdd(ioReadFilterGroup(bufferRead), ioTestFilterMultiplyNew("double", 2, 5, 'Y')),
ioFilterGroupAdd(ioReadFilterGroup(bufferRead), ioTestFilterMultiplyNew(STRID5("double", 0xac155e40), 2, 5, 'Y')),
" add filter that produces output with no input");
TEST_RESULT_BOOL(ioReadOpen(bufferRead), true, " open read");
TEST_RESULT_UINT(ioRead(bufferRead, buffer), 5, " read 5 chars");
@ -461,7 +491,8 @@ testRun(void)
ioFilterGroupAdd(ioReadFilterGroup(bufferRead), ioSizeNew());
TEST_RESULT_BOOL(ioReadDrain(bufferRead), true, "drain read io");
TEST_RESULT_UINT(varUInt64(ioFilterGroupResult(ioReadFilterGroup(bufferRead), SIZE_FILTER_TYPE_STR)), 20, "check length");
TEST_RESULT_UINT(
pckReadU64P(ioFilterGroupResultP(ioReadFilterGroup(bufferRead), SIZE_FILTER_TYPE)), 20, "check length");
// Cannot open file
TEST_ASSIGN(
@ -499,11 +530,13 @@ testRun(void)
IoFilter *sizeFilter = ioSizeNew();
TEST_RESULT_VOID(ioFilterGroupAdd(filterGroup, sizeFilter), " add filter to filter group");
TEST_RESULT_VOID(
ioFilterGroupAdd(filterGroup, ioTestFilterMultiplyNew("double", 2, 3, 'X')), " add filter to filter group");
TEST_RESULT_VOID(
ioFilterGroupAdd(filterGroup, ioTestFilterMultiplyNew("single", 1, 1, 'Y')),
ioFilterGroupAdd(filterGroup, ioTestFilterMultiplyNew(STRID5("double", 0xac155e40), 2, 3, 'X')),
" add filter to filter group");
TEST_RESULT_VOID(ioFilterGroupAdd(filterGroup, ioTestFilterSizeNew("size2")), " add filter to filter group");
TEST_RESULT_VOID(
ioFilterGroupAdd(filterGroup, ioTestFilterMultiplyNew(STRID5("single", 0xac3b9330), 1, 1, 'Y')),
" add filter to filter group");
TEST_RESULT_VOID(
ioFilterGroupAdd(filterGroup, ioTestFilterSizeNew(STRID5("size2", 0x1c2e9330))), " add filter to filter group");
TEST_RESULT_VOID(ioWriteOpen(bufferWrite), " open buffer write object");
TEST_RESULT_INT(ioWriteFd(bufferWrite), -1, " fd invalid");
@ -522,8 +555,9 @@ testRun(void)
TEST_RESULT_PTR(ioWriteFilterGroup(bufferWrite), filterGroup, " check filter group");
TEST_RESULT_UINT(
varUInt64(ioFilterGroupResult(filterGroup, ioFilterType(sizeFilter))), 9, " check filter result");
TEST_RESULT_UINT(varUInt64(ioFilterGroupResult(filterGroup, STRDEF("size2"))), 22, " check filter result");
pckReadU64P(ioFilterGroupResultP(filterGroup, ioFilterType(sizeFilter))), 9, " check filter result");
TEST_RESULT_UINT(
pckReadU64P(ioFilterGroupResultP(filterGroup, STRID5("size2", 0x1c2e9330))), 22, " check filter result");
}
// *****************************************************************************************************************************

2
test/src/module/performance/storageTest.c
View File

@ -128,7 +128,7 @@ testIoRateNew(uint64_t bytesPerSec)
.bytesPerSec = bytesPerSec,
};
this = ioFilterNewP(STRDEF("TestIoRate"), driver, NULL, .in = testIoRateProcess);
this = ioFilterNewP(STRID5("test-io-rate", 0x2d032dbd3ba4cb40), driver, NULL, .in = testIoRateProcess);
}
OBJ_NEW_END();

25
test/src/module/storage/remoteTest.c
View File

@ -9,6 +9,7 @@ Test Remote Storage
#include "postgres/interface.h"
#include "common/harnessConfig.h"
#include "common/harnessPack.h"
#include "common/harnessProtocol.h"
#include "common/harnessStorage.h"
@ -304,10 +305,10 @@ testRun(void)
TEST_RESULT_STR_Z(strNewBuf(storageGetP(fileRead)), "TESTDATA", "check contents");
TEST_RESULT_STR_Z(
jsonFromVar(ioFilterGroupResultAll(filterGroup)),
"{\"buffer\":null,\"cipherBlock\":null,\"gzCompress\":null,\"gzDecompress\":null"
",\"hash\":\"bbbcf2c59433f68f22376cd2439d6cd309378df6\",\"pageChecksum\":{\"align\":false,\"valid\":false}"
",\"size\":8}",
hrnPackBufToStr(pckWriteBuf(ioFilterGroupResultAll(filterGroup))),
"1:strid:size, 2:pack:<1:u64:8>, 3:strid:hash, 4:pack:<1:str:bbbcf2c59433f68f22376cd2439d6cd309378df6>,"
" 5:strid:pg-chksum, 6:pack:<1:str:{\"align\":false,\"valid\":false}>, 7:strid:cipher-blk, 9:strid:cipher-blk,"
" 11:strid:gz-cmp, 13:strid:gz-dcmp, 15:strid:buffer",
"filter results");
// Check protocol function directly (file exists but all data goes to sink)
@ -327,16 +328,20 @@ testRun(void)
TEST_RESULT_STR_Z(strNewBuf(storageGetP(fileRead)), "", "no content");
TEST_RESULT_STR_Z(
jsonFromVar(ioFilterGroupResultAll(filterGroup)),
"{\"buffer\":null,\"hash\":\"bbbcf2c59433f68f22376cd2439d6cd309378df6\",\"sink\":null,\"size\":8}", "filter results");
hrnPackBufToStr(pckWriteBuf(ioFilterGroupResultAll(filterGroup))),
"1:strid:size, 2:pack:<1:u64:8>, 3:strid:hash, 4:pack:<1:str:bbbcf2c59433f68f22376cd2439d6cd309378df6>, 5:strid:sink,"
" 7:strid:buffer",
"filter results");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("error on invalid filter");
TEST_ERROR(
storageRemoteFilterGroup(
ioFilterGroupNew(), varNewVarLst(varLstAdd(varLstNew(), varNewKv(kvPut(kvNew(), VARSTRDEF("bogus"), NULL))))),
AssertError, "unable to add filter 'bogus'");
Buffer *filterPack = bufNew(PACK_EXTRA_MIN);
PackWrite *filterWrite = pckWriteNewBuf(filterPack);
pckWriteStrIdP(filterWrite, STRID5("bogus", 0x13a9de20));
pckWriteEndP(filterWrite);
TEST_ERROR(storageRemoteFilterGroup(ioFilterGroupNew(), filterPack), AssertError, "unable to add filter 'bogus'");
}
// *****************************************************************************************************************************