db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2024-12-12 10:04:14 +02:00
Code Issues Releases Activity

Add optional raw format for compression types.

Browse Source 
Raw format saves 12 bytes of header for gzip and 4 bytes of checksum for lz4 (plus CPU overhead). This may not seem like much, but over millions of small files or incremental blocks can really add up. Even though it may be a relatively small percentage of the overall backup size it is still objectively a large amount of data.

Use raw format for protocol compression to exercise the feature.

Raw compression format will be added to bundling and block incremental in a followup commit.
This commit is contained in:
David Steele 2023-03-07 18:31:17 +07:00
parent f6e307365f
commit da91858702
35 changed files with 184 additions and 85 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
doc/xml/release.xml
View File

@ -25,6 +25,7 @@
<commit subject="Rename DeltaMap to BlockHash."/>
<commit subject="Add repo-block-age-map and repo-block-size-map options."/>
<commit subject="Rename block incremental manifest keys."/>
<commit subject="Add optional raw format for compression types."/>
<release-item-contributor-list>
<release-item-contributor id="david.steele"/>

2
src/build/help/render.c
View File

@ -270,7 +270,7 @@ bldHlpRenderHelpAutoCCmp(const BldCfg bldCfg, const BldHlp bldHlp)
IoRead *const source = ioBufferReadNewOpen(packBuf);
IoWrite *const destination = ioBufferWriteNew(result);
ioFilterGroupAdd(ioWriteFilterGroup(destination), bz2CompressNew(9));
ioFilterGroupAdd(ioWriteFilterGroup(destination), bz2CompressNew(9, false));
ioWriteOpen(destination);
// Copy data from source to destination

2
src/command/archive/get/file.c
View File

@ -70,7 +70,7 @@ archiveGetFile(
if (compressType != compressTypeNone)
{
ioFilterGroupAdd(ioWriteFilterGroup(storageWriteIo(destination)), decompressFilter(compressType));
ioFilterGroupAdd(ioWriteFilterGroup(storageWriteIo(destination)), decompressFilterP(compressType));
compressible = false;
}

2
src/command/archive/push/file.c
View File

@ -241,7 +241,7 @@ archivePushFile(
if (isSegment && compressType != compressTypeNone)
{
compressExtCat(archiveDestination, compressType);
ioFilterGroupAdd(ioReadFilterGroup(storageReadIo(source)), compressFilter(compressType, compressLevel));
ioFilterGroupAdd(ioReadFilterGroup(storageReadIo(source)), compressFilterP(compressType, compressLevel));
compressible = false;
}

8
src/command/backup/backup.c
View File

@ -1085,7 +1085,7 @@ backupFilePut(BackupData *backupData, Manifest *manifest, const String *name, ti
if (compressType != compressTypeNone)
{
ioFilterGroupAdd(
ioWriteFilterGroup(storageWriteIo(write)), compressFilter(compressType, cfgOptionInt(cfgOptCompressLevel)));
ioWriteFilterGroup(storageWriteIo(write)), compressFilterP(compressType, cfgOptionInt(cfgOptCompressLevel)));
repoChecksum = true;
}
@ -2263,12 +2263,12 @@ backupArchiveCheckCopy(const BackupData *const backupData, Manifest *const manif
if (archiveCompressType != backupCompressType)
{
if (archiveCompressType != compressTypeNone)
ioFilterGroupAdd(filterGroup, decompressFilter(archiveCompressType));
ioFilterGroupAdd(filterGroup, decompressFilterP(archiveCompressType));
if (backupCompressType != compressTypeNone)
{
ioFilterGroupAdd(
filterGroup, compressFilter(backupCompressType, cfgOptionInt(cfgOptCompressLevel)));
filterGroup, compressFilterP(backupCompressType, cfgOptionInt(cfgOptCompressLevel)));
}
}
@ -2365,7 +2365,7 @@ backupComplete(InfoBackup *const infoBackup, Manifest *const manifest)
STORAGE_REPO_BACKUP "/" BACKUP_PATH_HISTORY "/%s/%s.manifest%s", strZ(strSubN(backupLabel, 0, 4)),
strZ(backupLabel), strZ(compressExtStr(compressTypeGz))));
ioFilterGroupAdd(ioWriteFilterGroup(storageWriteIo(manifestWrite)), compressFilter(compressTypeGz, 9));
ioFilterGroupAdd(ioWriteFilterGroup(storageWriteIo(manifestWrite)), compressFilterP(compressTypeGz, 9));
cipherBlockFilterGroupAdd(
ioWriteFilterGroup(storageWriteIo(manifestWrite)), cfgOptionStrId(cfgOptRepoCipherType), cipherModeEncrypt,

2
src/command/backup/file.c
View File

@ -215,7 +215,7 @@ backupFile(
// Compress filter
IoFilter *const compress =
repoFileCompressType != compressTypeNone ?
compressFilter(repoFileCompressType, repoFileCompressLevel) : NULL;
compressFilterP(repoFileCompressType, repoFileCompressLevel) : NULL;
// Encrypt filter
IoFilter *const encrypt =

2
src/command/help/help.c
View File

@ -255,7 +255,7 @@ helpRender(const Buffer *const helpData)
// Read pack from compressed buffer
IoRead *const helpRead = ioBufferReadNew(helpData);
ioFilterGroupAdd(ioReadFilterGroup(helpRead), bz2DecompressNew());
ioFilterGroupAdd(ioReadFilterGroup(helpRead), bz2DecompressNew(false));
ioReadOpen(helpRead);
PackRead *pckHelp = pckReadNewIo(helpRead);

4
src/command/restore/file.c
View File

@ -403,7 +403,7 @@ restoreFile(
if (repoFileCompressType != compressTypeNone)
{
ioFilterGroupAdd(
ioReadFilterGroup(chunkedRead), decompressFilter(repoFileCompressType));
ioReadFilterGroup(chunkedRead), decompressFilterP(repoFileCompressType));
}
// Open chunked read
@ -454,7 +454,7 @@ restoreFile(
// Add decompression filter
if (repoFileCompressType != compressTypeNone)
ioFilterGroupAdd(filterGroup, decompressFilter(repoFileCompressType));
ioFilterGroupAdd(filterGroup, decompressFilterP(repoFileCompressType));
// Add sha1 filter
ioFilterGroupAdd(filterGroup, cryptoHashNew(hashTypeSha1));

2
src/command/verify/file.c
View File

@ -50,7 +50,7 @@ verifyFile(
// Add decompression filter
if (compressType != compressTypeNone)
ioFilterGroupAdd(filterGroup, decompressFilter(compressType));
ioFilterGroupAdd(filterGroup, decompressFilterP(compressType));
// Add sha1 filter
ioFilterGroupAdd(filterGroup, cryptoHashNew(hashTypeSha1));

2
src/command/verify/verify.c
View File

@ -184,7 +184,7 @@ verifyFileLoad(const String *pathFileName, const String *cipherPass)
// If the file is compressed, add a decompression filter
if (compressTypeFromName(pathFileName) != compressTypeNone)
ioFilterGroupAdd(ioReadFilterGroup(read), decompressFilter(compressTypeFromName(pathFileName)));
ioFilterGroupAdd(ioReadFilterGroup(read), decompressFilterP(compressTypeFromName(pathFileName)));
FUNCTION_TEST_RETURN(STORAGE_READ, result);
}

3
src/common/compress/bz2/compress.c
View File

@ -157,10 +157,11 @@ bz2CompressInputSame(const THIS_VOID)
/**********************************************************************************************************************************/
FN_EXTERN IoFilter *
bz2CompressNew(int level)
bz2CompressNew(const int level, const bool raw)
{
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(INT, level);
(void)raw; // Raw unsupported
FUNCTION_LOG_END();
ASSERT(level >= BZ2_COMPRESS_LEVEL_MIN && level <= BZ2_COMPRESS_LEVEL_MAX);

2
src/common/compress/bz2/compress.h
View File

@ -23,6 +23,6 @@ Level constants
/***********************************************************************************************************************************
Constructors
***********************************************************************************************************************************/
FN_EXTERN IoFilter *bz2CompressNew(int level);
FN_EXTERN IoFilter *bz2CompressNew(int level, bool raw);
#endif

6
src/common/compress/bz2/decompress.c
View File

@ -144,9 +144,11 @@ bz2DecompressInputSame(const THIS_VOID)
/**********************************************************************************************************************************/
FN_EXTERN IoFilter *
bz2DecompressNew(void)
bz2DecompressNew(const bool raw)
{
FUNCTION_LOG_VOID(logLevelTrace);
FUNCTION_LOG_BEGIN(logLevelTrace);
(void)raw; // Raw unsupported
FUNCTION_LOG_END();
IoFilter *this = NULL;

2
src/common/compress/bz2/decompress.h
View File

@ -16,6 +16,6 @@ Filter type constant
/***********************************************************************************************************************************
Constructors
***********************************************************************************************************************************/
FN_EXTERN IoFilter *bz2DecompressNew(void);
FN_EXTERN IoFilter *bz2DecompressNew(bool raw);
#endif

8
src/common/compress/gz/compress.c
View File

@ -1,5 +1,7 @@
/***********************************************************************************************************************************
Gz Compress
Based on the documentation at https://github.com/madler/zlib/blob/master/zlib.h
***********************************************************************************************************************************/
#include "build.auto.h"
@ -162,10 +164,11 @@ gzCompressInputSame(const THIS_VOID)
/**********************************************************************************************************************************/
FN_EXTERN IoFilter *
gzCompressNew(int level)
gzCompressNew(const int level, const bool raw)
{
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(INT, level);
FUNCTION_LOG_PARAM(BOOL, raw);
FUNCTION_LOG_END();
ASSERT(level >= GZ_COMPRESS_LEVEL_MIN && level <= GZ_COMPRESS_LEVEL_MAX);
@ -182,7 +185,7 @@ gzCompressNew(int level)
};
// Create gz stream
gzError(deflateInit2(&driver->stream, level, Z_DEFLATED, WANT_GZ | WINDOW_BITS, MEM_LEVEL, Z_DEFAULT_STRATEGY));
gzError(deflateInit2(&driver->stream, level, Z_DEFLATED, (raw ? 0 : WANT_GZ) | WINDOW_BITS, MEM_LEVEL, Z_DEFAULT_STRATEGY));
// Set free callback to ensure gz context is freed
memContextCallbackSet(objMemContext(driver), gzCompressFreeResource, driver);
@ -195,6 +198,7 @@ gzCompressNew(int level)
PackWrite *const packWrite = pckWriteNewP();
pckWriteI32P(packWrite, level);
pckWriteBoolP(packWrite, raw);
pckWriteEndP(packWrite);
paramList = pckMove(pckWriteResult(packWrite), memContextPrior());

2
src/common/compress/gz/compress.h
View File

@ -23,6 +23,6 @@ Level constants
/***********************************************************************************************************************************
Constructors
***********************************************************************************************************************************/
FN_EXTERN IoFilter *gzCompressNew(int level);
FN_EXTERN IoFilter *gzCompressNew(int level, bool raw);
#endif

26
src/common/compress/gz/decompress.c
View File

@ -1,5 +1,7 @@
/***********************************************************************************************************************************
Gz Decompress
Based on the documentation at https://github.com/madler/zlib/blob/master/zlib.h
***********************************************************************************************************************************/
#include "build.auto.h"
@ -144,9 +146,11 @@ gzDecompressInputSame(const THIS_VOID)
/**********************************************************************************************************************************/
FN_EXTERN IoFilter *
gzDecompressNew(void)
gzDecompressNew(const bool raw)
{
FUNCTION_LOG_VOID(logLevelTrace);
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(BOOL, raw);
FUNCTION_LOG_END();
IoFilter *this = NULL;
@ -161,14 +165,28 @@ gzDecompressNew(void)
};
// Create gz stream
gzError(driver->result = inflateInit2(&driver->stream, WANT_GZ | WINDOW_BITS));
gzError(driver->result = inflateInit2(&driver->stream, (raw ? 0 : WANT_GZ) | WINDOW_BITS));
// Set free callback to ensure gz context is freed
memContextCallbackSet(objMemContext(driver), gzDecompressFreeResource, driver);
// Create param list
Pack *paramList = NULL;
MEM_CONTEXT_TEMP_BEGIN()
{
PackWrite *const packWrite = pckWriteNewP();
pckWriteBoolP(packWrite, raw);
pckWriteEndP(packWrite);
paramList = pckMove(pckWriteResult(packWrite), memContextPrior());
}
MEM_CONTEXT_TEMP_END();
// Create filter interface
this = ioFilterNewP(
GZ_DECOMPRESS_FILTER_TYPE, driver, NULL, .done = gzDecompressDone, .inOut = gzDecompressProcess,
GZ_DECOMPRESS_FILTER_TYPE, driver, paramList, .done = gzDecompressDone, .inOut = gzDecompressProcess,
.inputSame = gzDecompressInputSame);
}
OBJ_NEW_END();

2
src/common/compress/gz/decompress.h
View File

@ -16,6 +16,6 @@ Filter type constant
/***********************************************************************************************************************************
Constructors
***********************************************************************************************************************************/
FN_EXTERN IoFilter *gzDecompressNew(void);
FN_EXTERN IoFilter *gzDecompressNew(bool raw);
#endif

22
src/common/compress/helper.c
View File

@ -39,9 +39,9 @@ static const struct CompressHelperLocal
const String *const type; // Compress type -- must be extension without period prefixed
const String *const ext; // File extension with period prefixed
StringId compressType; // Type of the compression filter
IoFilter *(*compressNew)(int); // Function to create new compression filter
IoFilter *(*compressNew)(int, bool); // Function to create new compression filter
StringId decompressType; // Type of the decompression filter
IoFilter *(*decompressNew)(void); // Function to create new decompression filter
IoFilter *(*decompressNew)(bool); // Function to create new decompression filter
int levelDefault : 8; // Default compression level
int levelMin : 8; // Minimum compression level
int levelMax : 8; // Maximum compression level
@ -230,18 +230,19 @@ compressLevelMax(CompressType type)
/**********************************************************************************************************************************/
FN_EXTERN IoFilter *
compressFilter(CompressType type, int level)
compressFilter(const CompressType type, const int level, const CompressFilterParam param)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(ENUM, type);
FUNCTION_TEST_PARAM(INT, level);
FUNCTION_TEST_PARAM(BOOL, param.raw);
FUNCTION_TEST_END();
ASSERT(type < LENGTH_OF(compressHelperLocal));
ASSERT(type != compressTypeNone);
compressTypePresent(type);
FUNCTION_TEST_RETURN(IO_FILTER, compressHelperLocal[type].compressNew(level));
FUNCTION_TEST_RETURN(IO_FILTER, compressHelperLocal[type].compressNew(level, param.raw));
}
/**********************************************************************************************************************************/
@ -267,12 +268,16 @@ compressFilterPack(const StringId filterType, const Pack *const filterParam)
{
ASSERT(filterParam != NULL);
result = ioFilterMove(compress->compressNew(pckReadI32P(pckReadNew(filterParam))), memContextPrior());
PackRead *const paramRead = pckReadNew(filterParam);
const int level = pckReadI32P(paramRead);
const bool raw = pckReadBoolP(paramRead);
result = ioFilterMove(compress->compressNew(level, raw), memContextPrior());
break;
}
else if (filterType == compress->decompressType)
{
result = ioFilterMove(compress->decompressNew(), memContextPrior());
result = ioFilterMove(compress->decompressNew(pckReadBoolP(pckReadNew(filterParam))), memContextPrior());
break;
}
}
@ -284,17 +289,18 @@ compressFilterPack(const StringId filterType, const Pack *const filterParam)
/**********************************************************************************************************************************/
FN_EXTERN IoFilter *
decompressFilter(CompressType type)
decompressFilter(const CompressType type, const DecompressFilterParam param)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(ENUM, type);
FUNCTION_TEST_PARAM(BOOL, param.raw);
FUNCTION_TEST_END();
ASSERT(type < LENGTH_OF(compressHelperLocal));
ASSERT(type != compressTypeNone);
compressTypePresent(type);
FUNCTION_TEST_RETURN(IO_FILTER, compressHelperLocal[type].decompressNew());
FUNCTION_TEST_RETURN(IO_FILTER, compressHelperLocal[type].decompressNew(param.raw));
}
/**********************************************************************************************************************************/

22
src/common/compress/helper.h
View File

@ -53,14 +53,32 @@ FN_EXTERN const String *compressTypeStr(CompressType type);
FN_EXTERN CompressType compressTypeFromName(const String *name);
// Compression filter for the specified type. Error when compress type is none or invalid.
FN_EXTERN IoFilter *compressFilter(CompressType type, int level);
typedef struct CompressFilterParam
{
VAR_PARAM_HEADER;
bool raw; // Omit headers, checksum, etc. when possible
} CompressFilterParam;
#define compressFilterP(type, level, ...) \
compressFilter(type, level, (CompressFilterParam){VAR_PARAM_INIT, __VA_ARGS__})
FN_EXTERN IoFilter *compressFilter(CompressType type, int level, CompressFilterParam param);
// Compression/decompression filter based on string type and a parameter list. This is useful when a filter must be created on a
// remote system since the filter type and parameters can be passed through a protocol.
FN_EXTERN IoFilter *compressFilterPack(StringId filterType, const Pack *filterParam);
// Decompression filter for the specified type. Error when compress type is none or invalid.
FN_EXTERN IoFilter *decompressFilter(CompressType type);
typedef struct DecompressFilterParam
{
VAR_PARAM_HEADER;
bool raw; // Omit headers, checksum, etc. when possible
} DecompressFilterParam;
#define decompressFilterP(type, ...) \
decompressFilter(type, (DecompressFilterParam){VAR_PARAM_INIT, __VA_ARGS__})
FN_EXTERN IoFilter *decompressFilter(CompressType type, DecompressFilterParam param);
// Get extension for the current compression type
FN_EXTERN const String *compressExtStr(CompressType type);

10
src/common/compress/lz4/compress.c
View File

@ -243,10 +243,11 @@ lz4CompressInputSame(const THIS_VOID)
/**********************************************************************************************************************************/
FN_EXTERN IoFilter *
lz4CompressNew(int level)
lz4CompressNew(const int level, const bool raw)
{
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(INT, level);
FUNCTION_LOG_PARAM(BOOL, raw);
FUNCTION_LOG_END();
ASSERT(level >= LZ4_COMPRESS_LEVEL_MIN && level <= LZ4_COMPRESS_LEVEL_MAX);
@ -259,7 +260,11 @@ lz4CompressNew(int level)
*driver = (Lz4Compress)
{
.prefs = {.compressionLevel = level, .frameInfo = {.contentChecksumFlag = LZ4F_contentChecksumEnabled}},
.prefs =
{
.compressionLevel = level,
.frameInfo = {.contentChecksumFlag = raw ? LZ4F_noContentChecksum : LZ4F_contentChecksumEnabled},
},
.first = true,
.buffer = bufNew(0),
};
@ -278,6 +283,7 @@ lz4CompressNew(int level)
PackWrite *const packWrite = pckWriteNewP();
pckWriteI32P(packWrite, level);
pckWriteBoolP(packWrite, raw);
pckWriteEndP(packWrite);
paramList = pckMove(pckWriteResult(packWrite), memContextPrior());

2
src/common/compress/lz4/compress.h
View File

@ -25,7 +25,7 @@ Level constants
/***********************************************************************************************************************************
Constructors
***********************************************************************************************************************************/
FN_EXTERN IoFilter *lz4CompressNew(int level);
FN_EXTERN IoFilter *lz4CompressNew(int level, bool raw);
#endif

6
src/common/compress/lz4/decompress.c
View File

@ -157,9 +157,11 @@ lz4DecompressInputSame(const THIS_VOID)
/**********************************************************************************************************************************/
FN_EXTERN IoFilter *
lz4DecompressNew(void)
lz4DecompressNew(const bool raw)
{
FUNCTION_LOG_VOID(logLevelTrace);
FUNCTION_LOG_BEGIN(logLevelTrace);
(void)raw; // Not required for decompress
FUNCTION_LOG_END();
IoFilter *this = NULL;

2
src/common/compress/lz4/decompress.h
View File

@ -18,7 +18,7 @@ Filter type constant
/***********************************************************************************************************************************
Constructors
***********************************************************************************************************************************/
FN_EXTERN IoFilter *lz4DecompressNew(void);
FN_EXTERN IoFilter *lz4DecompressNew(bool raw);
#endif

3
src/common/compress/zst/compress.c
View File

@ -164,10 +164,11 @@ zstCompressInputSame(const THIS_VOID)
/**********************************************************************************************************************************/
FN_EXTERN IoFilter *
zstCompressNew(int level)
zstCompressNew(const int level, const bool raw)
{
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(INT, level);
(void)raw; // Raw unsupported
FUNCTION_LOG_END();
ASSERT(level >= ZST_COMPRESS_LEVEL_MIN && level <= ZST_COMPRESS_LEVEL_MAX);

2
src/common/compress/zst/compress.h
View File

@ -25,7 +25,7 @@ Level constants
/***********************************************************************************************************************************
Constructors
***********************************************************************************************************************************/
FN_EXTERN IoFilter *zstCompressNew(int level);
FN_EXTERN IoFilter *zstCompressNew(int level, bool raw);
#endif

6
src/common/compress/zst/decompress.c
View File

@ -156,9 +156,11 @@ zstDecompressInputSame(const THIS_VOID)
/**********************************************************************************************************************************/
FN_EXTERN IoFilter *
zstDecompressNew(void)
zstDecompressNew(const bool raw)
{
FUNCTION_LOG_VOID(logLevelTrace);
FUNCTION_LOG_BEGIN(logLevelTrace);
(void)raw; // Raw unsupported
FUNCTION_LOG_END();
IoFilter *this = NULL;

2
src/common/compress/zst/decompress.h
View File

@ -18,7 +18,7 @@ Filter type constant
/***********************************************************************************************************************************
Constructors
***********************************************************************************************************************************/
FN_EXTERN IoFilter *zstDecompressNew(void);
FN_EXTERN IoFilter *zstDecompressNew(bool raw);
#endif

5
src/storage/remote/read.c
View File

@ -217,7 +217,8 @@ storageReadRemoteOpen(THIS_VOID)
if (this->interface.compressible)
{
ioFilterGroupAdd(
ioReadFilterGroup(storageReadIo(this->read)), compressFilter(compressTypeGz, (int)this->interface.compressLevel));
ioReadFilterGroup(storageReadIo(this->read)),
compressFilterP(compressTypeGz, (int)this->interface.compressLevel, .raw = true));
}
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_STORAGE_OPEN_READ);
@ -246,7 +247,7 @@ storageReadRemoteOpen(THIS_VOID)
// If the file is compressible add decompression filter locally
if (this->interface.compressible)
ioFilterGroupAdd(ioReadFilterGroup(storageReadIo(this->read)), decompressFilter(compressTypeGz));
ioFilterGroupAdd(ioReadFilterGroup(storageReadIo(this->read)), decompressFilterP(compressTypeGz, .raw = true));
// Set free callback to ensure the protocol is cleared on a short read
memContextCallbackSet(objMemContext(this), storageReadRemoteFreeResource, this);

4
src/storage/remote/write.c
View File

@ -78,7 +78,7 @@ storageWriteRemoteOpen(THIS_VOID)
{
// If the file is compressible add decompression filter on the remote
if (this->interface.compressible)
ioFilterGroupInsert(ioWriteFilterGroup(storageWriteIo(this->write)), 0, decompressFilter(compressTypeGz));
ioFilterGroupInsert(ioWriteFilterGroup(storageWriteIo(this->write)), 0, decompressFilterP(compressTypeGz));
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_STORAGE_OPEN_WRITE);
PackWrite *const param = protocolCommandParam(command);
@ -106,7 +106,7 @@ storageWriteRemoteOpen(THIS_VOID)
{
ioFilterGroupAdd(
ioWriteFilterGroup(storageWriteIo(this->write)),
compressFilter(compressTypeGz, (int)this->interface.compressLevel));
compressFilterP(compressTypeGz, (int)this->interface.compressLevel));
}
// Set free callback to ensure remote file is freed

4
test/src/common/harnessStorage.c
View File

@ -131,7 +131,7 @@ testStorageGet(const Storage *const storage, const char *const file, const char
if (param.compressType != compressTypeNone)
{
ASSERT(param.compressType == compressTypeGz || param.compressType == compressTypeBz2);
ioFilterGroupAdd(filterGroup, decompressFilter(param.compressType));
ioFilterGroupAdd(filterGroup, decompressFilterP(param.compressType));
}
printf("test content of %s'%s'", strEmpty(filter) ? "" : strZ(filter), strZ(fileFull));
@ -429,7 +429,7 @@ hrnStoragePut(
if (param.compressType != compressTypeNone)
{
ASSERT(param.compressType == compressTypeGz || param.compressType == compressTypeBz2);
ioFilterGroupAdd(filterGroup, compressFilter(param.compressType, 1));
ioFilterGroupAdd(filterGroup, compressFilterP(param.compressType, 1));
strCatFmt(filter, "%scmp[%s]", strEmpty(filter) ? "" : "/", strZ(compressTypeStr(param.compressType)));
}

8
test/src/module/command/backupTest.c
View File

@ -191,7 +191,7 @@ testBackupValidateList(
if (manifestData->backupOptionCompressType != compressTypeNone)
{
ioFilterGroupAdd(
ioReadFilterGroup(chunkRead), decompressFilter(manifestData->backupOptionCompressType));
ioReadFilterGroup(chunkRead), decompressFilterP(manifestData->backupOptionCompressType));
}
ioReadOpen(chunkRead);
@ -243,7 +243,7 @@ testBackupValidateList(
if (manifestData->backupOptionCompressType != compressTypeNone)
{
ioFilterGroupAdd(
ioReadFilterGroup(storageReadIo(read)), decompressFilter(manifestData->backupOptionCompressType));
ioReadFilterGroup(storageReadIo(read)), decompressFilterP(manifestData->backupOptionCompressType));
}
ioFilterGroupAdd(ioReadFilterGroup(storageReadIo(read)), cryptoHashNew(hashTypeSha1));
@ -530,7 +530,7 @@ testBackupPqScript(unsigned int pgVersion, time_t backupTimeStart, TestBackupPqS
strZ(walChecksum), strZ(compressExtStr(param.walCompressType))));
if (param.walCompressType != compressTypeNone)
ioFilterGroupAdd(ioWriteFilterGroup(storageWriteIo(write)), compressFilter(param.walCompressType, 1));
ioFilterGroupAdd(ioWriteFilterGroup(storageWriteIo(write)), compressFilterP(param.walCompressType, 1));
storagePutP(write, walBuffer);
}
@ -1252,7 +1252,7 @@ testRun(void)
blockIncrNewPack(
ioFilterParamList(
blockIncrNew(
3, 2, 4, 5, NULL, compressFilter(compressTypeGz, 1),
3, 2, 4, 5, NULL, compressFilterP(compressTypeGz, 1),
cipherBlockNewP(cipherModeEncrypt, cipherTypeAes256Cbc, BUFSTRDEF(TEST_CIPHER_PASS), .raw = true)))),
"block incr pack");
}

85
test/src/module/common/compressTest.c
View File

@ -73,14 +73,16 @@ testDecompress(IoFilter *decompress, Buffer *compressed, size_t inputSize, size_
Standard test suite to be applied to all compression types
***********************************************************************************************************************************/
static void
testSuite(CompressType type, const char *decompressCmd)
testSuite(CompressType type, const char *decompressCmd, size_t rawDelta)
{
const char *simpleData = "A simple string";
Buffer *compressed = NULL;
Buffer *compressedRaw = NULL;
Buffer *decompressed = bufNewC(simpleData, strlen(simpleData));
PackWrite *packWrite = pckWriteNewP();
pckWriteI32P(packWrite, 1);
pckWriteBoolP(packWrite, false);
pckWriteEndP(packWrite);
// Create default storage object for testing
@ -95,6 +97,20 @@ testSuite(CompressType type, const char *decompressCmd)
256 * 1024 * 1024),
"simple data - compress large in/large out buffer");
packWrite = pckWriteNewP();
pckWriteI32P(packWrite, 1);
pckWriteBoolP(packWrite, true);
pckWriteEndP(packWrite);
TEST_ASSIGN(
compressedRaw,
testCompress(
compressFilterPack(compressHelperLocal[type].compressType, pckWriteResult(packWrite)), decompressed, 1024,
1024),
"simple data - compress large in/large out buffer (raw)");
TEST_RESULT_UINT(bufUsed(compressed) - rawDelta, bufUsed(compressedRaw), "compare to raw");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("compressed output can be decompressed with command-line tool");
@ -103,39 +119,60 @@ testSuite(CompressType type, const char *decompressCmd)
TEST_RESULT_BOOL(bufEq(decompressed, storageGetP(storageNewReadP(storageTest, STRDEF("test.out")))), true, "check output");
TEST_RESULT_BOOL(
bufEq(compressed, testCompress(compressFilter(type, 1), decompressed, 1024, 1)), true,
bufEq(compressed, testCompress(compressFilterP(type, 1), decompressed, 1024, 1)), true,
"simple data - compress large in/small out buffer");
TEST_RESULT_BOOL(
bufEq(compressed, testCompress(compressFilter(type, 1), decompressed, 1, 1024)), true,
bufEq(compressed, testCompress(compressFilterP(type, 1), decompressed, 1, 1024)), true,
"simple data - compress small in/large out buffer");
TEST_RESULT_BOOL(
bufEq(compressed, testCompress(compressFilter(type, 1), decompressed, 1, 1)), true,
bufEq(compressed, testCompress(compressFilterP(type, 1), decompressed, 1, 1)), true,
"simple data - compress small in/small out buffer");
TEST_RESULT_BOOL(
bufEq(compressedRaw, testCompress(compressFilterP(type, 1, .raw = true), decompressed, 1, 1)), true,
"simple data - compress small in/small out buffer (raw)");
packWrite = pckWriteNewP();
pckWriteBoolP(packWrite, false);
pckWriteEndP(packWrite);
TEST_RESULT_BOOL(
bufEq(
decompressed,
testDecompress(compressFilterPack(compressHelperLocal[type].decompressType, NULL), compressed, 1024, 1024)),
true, "simple data - decompress large in/large out buffer");
testDecompress(
compressFilterPack(compressHelperLocal[type].decompressType, pckWriteResult(packWrite)), compressed, 1024, 1024)),
true, "simple data - decompress large in/small out buffer");
packWrite = pckWriteNewP();
pckWriteBoolP(packWrite, true);
pckWriteEndP(packWrite);
TEST_RESULT_BOOL(
bufEq(decompressed, testDecompress(decompressFilter(type), compressed, 1024, 1)), true,
bufEq(
decompressed,
testDecompress(
compressFilterPack(
compressHelperLocal[type].decompressType, pckWriteResult(packWrite)), compressedRaw, 1024, 1024)),
true, "simple data - decompress large in/large out buffer (raw)");
TEST_RESULT_BOOL(
bufEq(decompressed, testDecompress(decompressFilterP(type), compressed, 1024, 1)), true,
"simple data - decompress large in/small out buffer");
TEST_RESULT_BOOL(
bufEq(decompressed, testDecompress(decompressFilter(type), compressed, 1, 1024)), true,
bufEq(decompressed, testDecompress(decompressFilterP(type), compressed, 1, 1024)), true,
"simple data - decompress small in/large out buffer");
TEST_RESULT_BOOL(
bufEq(decompressed, testDecompress(decompressFilter(type), compressed, 1, 1)), true,
bufEq(decompressed, testDecompress(decompressFilterP(type), compressed, 1, 1)), true,
"simple data - decompress small in/small out buffer");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("error on no compression data");
TEST_ERROR(testDecompress(decompressFilter(type), bufNew(0), 1, 1), FormatError, "unexpected eof in compressed data");
TEST_ERROR(testDecompress(decompressFilterP(type), bufNew(0), 1, 1), FormatError, "unexpected eof in compressed data");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("error on truncated compression data");
@ -144,7 +181,7 @@ testSuite(CompressType type, const char *decompressCmd)
bufCatSub(truncated, compressed, 0, bufUsed(compressed) - 1);
TEST_RESULT_UINT(bufUsed(truncated), bufUsed(compressed) - 1, "check truncated buffer size");
TEST_ERROR(testDecompress(decompressFilter(type), truncated, 512, 512), FormatError, "unexpected eof in compressed data");
TEST_ERROR(testDecompress(decompressFilterP(type), truncated, 512, 512), FormatError, "unexpected eof in compressed data");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("compress a large non-zero input buffer into small output buffer");
@ -160,11 +197,11 @@ testSuite(CompressType type, const char *decompressCmd)
bufUsedSet(decompressed, bufSize(decompressed));
TEST_ASSIGN(
compressed, testCompress(compressFilter(type, 3), decompressed, bufSize(decompressed), 32),
compressed, testCompress(compressFilterP(type, 3), decompressed, bufSize(decompressed), 32),
"non-zero data - compress large in/small out buffer");
TEST_RESULT_BOOL(
bufEq(decompressed, testDecompress(decompressFilter(type), compressed, bufSize(compressed), 1024 * 256)), true,
bufEq(decompressed, testDecompress(decompressFilterP(type), compressed, bufSize(compressed), 1024 * 256)), true,
"non-zero data - decompress large in/small out buffer");
}
@ -180,7 +217,7 @@ testRun(void)
if (testBegin("gz"))
{
// Run standard test suite
testSuite(compressTypeGz, "gzip -dc");
testSuite(compressTypeGz, "gzip -dc", 12);
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("gzError()");
@ -208,7 +245,7 @@ testRun(void)
char buffer[STACK_TRACE_PARAM_MAX];
GzDecompress *decompress = (GzDecompress *)ioFilterDriver(gzDecompressNew());
GzDecompress *decompress = (GzDecompress *)ioFilterDriver(gzDecompressNew(false));
TEST_RESULT_VOID(FUNCTION_LOG_OBJECT_FORMAT(decompress, gzDecompressToLog, buffer, sizeof(buffer)), "gzDecompressToLog");
TEST_RESULT_Z(buffer, "{inputSame: false, done: false, availIn: 0}", "check log");
@ -224,7 +261,7 @@ testRun(void)
if (testBegin("bz2"))
{
// Run standard test suite
testSuite(compressTypeBz2, "bzip2 -dc");
testSuite(compressTypeBz2, "bzip2 -dc", 0);
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("bz2Error()");
@ -257,14 +294,14 @@ testRun(void)
char buffer[STACK_TRACE_PARAM_MAX];
Bz2Compress *compress = (Bz2Compress *)ioFilterDriver(bz2CompressNew(1));
Bz2Compress *compress = (Bz2Compress *)ioFilterDriver(bz2CompressNew(1, false));
compress->stream.avail_in = 999;
TEST_RESULT_VOID(FUNCTION_LOG_OBJECT_FORMAT(compress, bz2CompressToLog, buffer, sizeof(buffer)), "bz2CompressToLog");
TEST_RESULT_Z(buffer, "{inputSame: false, done: false, flushing: false, avail_in: 999}", "check log");
Bz2Decompress *decompress = (Bz2Decompress *)ioFilterDriver(bz2DecompressNew());
Bz2Decompress *decompress = (Bz2Decompress *)ioFilterDriver(bz2DecompressNew(false));
decompress->inputSame = true;
decompress->done = true;
@ -278,7 +315,7 @@ testRun(void)
{
#ifdef HAVE_LIBLZ4
// Run standard test suite
testSuite(compressTypeLz4, "lz4 -dc");
testSuite(compressTypeLz4, "lz4 -dc", 4);
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("lz4Error()");
@ -298,7 +335,7 @@ testRun(void)
char buffer[STACK_TRACE_PARAM_MAX];
Lz4Compress *compress = (Lz4Compress *)ioFilterDriver(lz4CompressNew(7));
Lz4Compress *compress = (Lz4Compress *)ioFilterDriver(lz4CompressNew(7, false));
compress->inputSame = true;
compress->flushing = true;
@ -306,7 +343,7 @@ testRun(void)
TEST_RESULT_VOID(FUNCTION_LOG_OBJECT_FORMAT(compress, lz4CompressToLog, buffer, sizeof(buffer)), "lz4CompressToLog");
TEST_RESULT_Z(buffer, "{level: 7, first: true, inputSame: true, flushing: true}", "check log");
Lz4Decompress *decompress = (Lz4Decompress *)ioFilterDriver(lz4DecompressNew());
Lz4Decompress *decompress = (Lz4Decompress *)ioFilterDriver(lz4DecompressNew(false));
decompress->inputSame = true;
decompress->done = true;
@ -324,7 +361,7 @@ testRun(void)
{
#ifdef HAVE_LIBZST
// Run standard test suite
testSuite(compressTypeZst, "zstd -dc");
testSuite(compressTypeZst, "zstd -dc", 0);
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("zstError()");
@ -344,7 +381,7 @@ testRun(void)
char buffer[STACK_TRACE_PARAM_MAX];
ZstCompress *compress = (ZstCompress *)ioFilterDriver(zstCompressNew(14));
ZstCompress *compress = (ZstCompress *)ioFilterDriver(zstCompressNew(14, false));
compress->inputSame = true;
compress->inputOffset = 49;
@ -353,7 +390,7 @@ testRun(void)
TEST_RESULT_VOID(FUNCTION_LOG_OBJECT_FORMAT(compress, zstCompressToLog, buffer, sizeof(buffer)), "zstCompressToLog");
TEST_RESULT_Z(buffer, "{level: 14, inputSame: true, inputOffset: 49, flushing: true}", "check log");
ZstDecompress *decompress = (ZstDecompress *)ioFilterDriver(zstDecompressNew());
ZstDecompress *decompress = (ZstDecompress *)ioFilterDriver(zstDecompressNew(false));
decompress->inputSame = true;
decompress->done = true;

4
test/src/module/performance/storageTest.c
View File

@ -334,7 +334,7 @@ testRun(void)
MEM_CONTEXT_TEMP_BEGIN()
{
BENCHMARK_BEGIN();
BENCHMARK_FILTER_ADD(gzCompressNew(6));
BENCHMARK_FILTER_ADD(gzCompressNew(6, false));
BENCHMARK_END(gzip6Total);
}
MEM_CONTEXT_TEMP_END();
@ -346,7 +346,7 @@ testRun(void)
MEM_CONTEXT_TEMP_BEGIN()
{
BENCHMARK_BEGIN();
BENCHMARK_FILTER_ADD(lz4CompressNew(1));
BENCHMARK_FILTER_ADD(lz4CompressNew(1, false));
BENCHMARK_END(lz41Total);
}
MEM_CONTEXT_TEMP_END();

4
test/src/module/storage/remoteTest.c
View File

@ -346,8 +346,8 @@ testRun(void)
ioFilterGroupAdd(filterGroup, cryptoHashNew(hashTypeSha1));
ioFilterGroupAdd(filterGroup, cipherBlockNewP(cipherModeEncrypt, cipherTypeAes256Cbc, BUFSTRZ("x")));
ioFilterGroupAdd(filterGroup, cipherBlockNewP(cipherModeDecrypt, cipherTypeAes256Cbc, BUFSTRZ("x")));
ioFilterGroupAdd(filterGroup, compressFilter(compressTypeGz, 3));
ioFilterGroupAdd(filterGroup, decompressFilter(compressTypeGz));
ioFilterGroupAdd(filterGroup, compressFilterP(compressTypeGz, 3));
ioFilterGroupAdd(filterGroup, decompressFilterP(compressTypeGz));
TEST_RESULT_STR_Z(strNewBuf(storageGetP(fileRead)), "TESTDATA", "check contents");