You've already forked pgbackrest
mirror of
https://github.com/pgbackrest/pgbackrest.git
synced 2025-07-13 01:00:23 +02:00
Remove raw option for gz compression.
This was a minor optimization used in protocol layer compression. Even though it was slightly faster, it omitted the crc-32 that is generated during normal compression which could lead to corrupt data after a bad network transmission. This would be caught on restore by our checksum but it seems better to catch an issue like this early. The raw option also made the function signature different than future compression formats which may not support raw, or require different code to support raw. In general, it doesn't seem worth the extra testing to support a format that has minimal benefit and is seldom used, since protocol compression is only enabled when the transmitted data is uncompressed.
This commit is contained in:
@ -135,14 +135,9 @@ storageFilterXsAdd(IoFilterGroup *filterGroup, const String *filter, const Strin
|
|||||||
else if (strEqZ(filter, "pgBackRest::Storage::Filter::Gz"))
|
else if (strEqZ(filter, "pgBackRest::Storage::Filter::Gz"))
|
||||||
{
|
{
|
||||||
if (strEqZ(varStr(varLstGet(paramList, 0)), "compress"))
|
if (strEqZ(varStr(varLstGet(paramList, 0)), "compress"))
|
||||||
{
|
ioFilterGroupAdd(filterGroup, gzCompressNew(varUIntForce(varLstGet(paramList, 1))));
|
||||||
ioFilterGroupAdd(
|
|
||||||
filterGroup, gzCompressNew(varUIntForce(varLstGet(paramList, 2)), varBoolForce(varLstGet(paramList, 1))));
|
|
||||||
}
|
|
||||||
else
|
else
|
||||||
{
|
ioFilterGroupAdd(filterGroup, gzDecompressNew());
|
||||||
ioFilterGroupAdd(filterGroup, gzDecompressNew(varBoolForce(varLstGet(paramList, 1))));
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
THROW_FMT(AssertError, "unable to add invalid filter '%s'", strPtr(filter));
|
THROW_FMT(AssertError, "unable to add invalid filter '%s'", strPtr(filter));
|
||||||
|
@ -162,7 +162,7 @@ archiveGetFile(
|
|||||||
// If file is compressed then add the decompression filter
|
// If file is compressed then add the decompression filter
|
||||||
if (strEndsWithZ(archiveGetCheckResult.archiveFileActual, "." GZ_EXT))
|
if (strEndsWithZ(archiveGetCheckResult.archiveFileActual, "." GZ_EXT))
|
||||||
{
|
{
|
||||||
ioFilterGroupAdd(ioWriteFilterGroup(storageWriteIo(destination)), gzDecompressNew(false));
|
ioFilterGroupAdd(ioWriteFilterGroup(storageWriteIo(destination)), gzDecompressNew());
|
||||||
compressible = false;
|
compressible = false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -117,7 +117,7 @@ archivePushFile(
|
|||||||
if (isSegment && compress)
|
if (isSegment && compress)
|
||||||
{
|
{
|
||||||
strCat(archiveDestination, "." GZ_EXT);
|
strCat(archiveDestination, "." GZ_EXT);
|
||||||
ioFilterGroupAdd(ioReadFilterGroup(storageReadIo(source)), gzCompressNew(compressLevel, false));
|
ioFilterGroupAdd(ioReadFilterGroup(storageReadIo(source)), gzCompressNew(compressLevel));
|
||||||
compressible = false;
|
compressible = false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -936,10 +936,7 @@ backupFilePut(BackupData *backupData, Manifest *manifest, const String *name, ti
|
|||||||
|
|
||||||
// Add compression
|
// Add compression
|
||||||
if (compress)
|
if (compress)
|
||||||
{
|
ioFilterGroupAdd(ioWriteFilterGroup(storageWriteIo(write)), gzCompressNew((int)cfgOptionUInt(cfgOptCompressLevel)));
|
||||||
ioFilterGroupAdd(
|
|
||||||
ioWriteFilterGroup(storageWriteIo(write)), gzCompressNew((int)cfgOptionUInt(cfgOptCompressLevel), false));
|
|
||||||
}
|
|
||||||
|
|
||||||
// Add encryption filter if required
|
// Add encryption filter if required
|
||||||
cipherBlockFilterGroupAdd(
|
cipherBlockFilterGroupAdd(
|
||||||
@ -1774,9 +1771,9 @@ backupArchiveCheckCopy(Manifest *manifest, unsigned int walSegmentSize, const St
|
|||||||
if (archiveCompressed != cfgOptionBool(cfgOptCompress))
|
if (archiveCompressed != cfgOptionBool(cfgOptCompress))
|
||||||
{
|
{
|
||||||
if (archiveCompressed)
|
if (archiveCompressed)
|
||||||
ioFilterGroupAdd(ioReadFilterGroup(storageReadIo(read)), gzDecompressNew(false));
|
ioFilterGroupAdd(ioReadFilterGroup(storageReadIo(read)), gzDecompressNew());
|
||||||
else
|
else
|
||||||
ioFilterGroupAdd(filterGroup, gzCompressNew(cfgOptionInt(cfgOptCompressLevel), false));
|
ioFilterGroupAdd(filterGroup, gzCompressNew(cfgOptionInt(cfgOptCompressLevel)));
|
||||||
}
|
}
|
||||||
|
|
||||||
// Encrypt with backup key if encrypted
|
// Encrypt with backup key if encrypted
|
||||||
@ -1870,7 +1867,7 @@ backupComplete(InfoBackup *const infoBackup, Manifest *const manifest)
|
|||||||
STORAGE_REPO_BACKUP "/" BACKUP_PATH_HISTORY "/%s/%s.manifest." GZ_EXT, strPtr(strSubN(backupLabel, 0, 4)),
|
STORAGE_REPO_BACKUP "/" BACKUP_PATH_HISTORY "/%s/%s.manifest." GZ_EXT, strPtr(strSubN(backupLabel, 0, 4)),
|
||||||
strPtr(backupLabel)));
|
strPtr(backupLabel)));
|
||||||
|
|
||||||
ioFilterGroupAdd(ioWriteFilterGroup(storageWriteIo(manifestWrite)), gzCompressNew(9, false));
|
ioFilterGroupAdd(ioWriteFilterGroup(storageWriteIo(manifestWrite)), gzCompressNew(9));
|
||||||
|
|
||||||
cipherBlockFilterGroupAdd(
|
cipherBlockFilterGroupAdd(
|
||||||
ioWriteFilterGroup(storageWriteIo(manifestWrite)), cipherType(cfgOptionStr(cfgOptRepoCipherType)), cipherModeEncrypt,
|
ioWriteFilterGroup(storageWriteIo(manifestWrite)), cipherType(cfgOptionStr(cfgOptRepoCipherType)), cipherModeEncrypt,
|
||||||
|
@ -149,7 +149,7 @@ backupFile(
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (repoFileCompress)
|
if (repoFileCompress)
|
||||||
ioFilterGroupAdd(ioReadFilterGroup(read), gzDecompressNew(false));
|
ioFilterGroupAdd(ioReadFilterGroup(read), gzDecompressNew());
|
||||||
|
|
||||||
ioFilterGroupAdd(ioReadFilterGroup(read), cryptoHashNew(HASH_TYPE_SHA1_STR));
|
ioFilterGroupAdd(ioReadFilterGroup(read), cryptoHashNew(HASH_TYPE_SHA1_STR));
|
||||||
ioFilterGroupAdd(ioReadFilterGroup(read), ioSizeNew());
|
ioFilterGroupAdd(ioReadFilterGroup(read), ioSizeNew());
|
||||||
@ -208,7 +208,7 @@ backupFile(
|
|||||||
|
|
||||||
// Add compression
|
// Add compression
|
||||||
if (repoFileCompress)
|
if (repoFileCompress)
|
||||||
ioFilterGroupAdd(ioReadFilterGroup(storageReadIo(read)), gzCompressNew((int)repoFileCompressLevel, false));
|
ioFilterGroupAdd(ioReadFilterGroup(storageReadIo(read)), gzCompressNew((int)repoFileCompressLevel));
|
||||||
|
|
||||||
// If there is a cipher then add the encrypt filter
|
// If there is a cipher then add the encrypt filter
|
||||||
if (cipherType != cipherTypeNone)
|
if (cipherType != cipherTypeNone)
|
||||||
|
@ -155,7 +155,7 @@ restoreFile(
|
|||||||
// Add decompression filter
|
// Add decompression filter
|
||||||
if (repoFileCompressed)
|
if (repoFileCompressed)
|
||||||
{
|
{
|
||||||
ioFilterGroupAdd(filterGroup, gzDecompressNew(false));
|
ioFilterGroupAdd(filterGroup, gzDecompressNew());
|
||||||
compressible = false;
|
compressible = false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -9,12 +9,6 @@ Gz Common
|
|||||||
#include "common/debug.h"
|
#include "common/debug.h"
|
||||||
#include "common/memContext.h"
|
#include "common/memContext.h"
|
||||||
|
|
||||||
/***********************************************************************************************************************************
|
|
||||||
Constants
|
|
||||||
***********************************************************************************************************************************/
|
|
||||||
#define WINDOW_BITS 15
|
|
||||||
#define WANT_GZ 16
|
|
||||||
|
|
||||||
/***********************************************************************************************************************************
|
/***********************************************************************************************************************************
|
||||||
Process gz errors
|
Process gz errors
|
||||||
***********************************************************************************************************************************/
|
***********************************************************************************************************************************/
|
||||||
@ -89,16 +83,3 @@ gzError(int error)
|
|||||||
|
|
||||||
return error;
|
return error;
|
||||||
}
|
}
|
||||||
|
|
||||||
/***********************************************************************************************************************************
|
|
||||||
Get gz window bits
|
|
||||||
|
|
||||||
Window bits define how large the compression window is. Larger window sizes generally result in better compression so we'll always
|
|
||||||
use the largest size. When raw is specified disable the gz header and produce raw compressed output (this is indicated by passing
|
|
||||||
negative window bits).
|
|
||||||
***********************************************************************************************************************************/
|
|
||||||
int
|
|
||||||
gzWindowBits(bool raw)
|
|
||||||
{
|
|
||||||
return raw ? -WINDOW_BITS : WANT_GZ | WINDOW_BITS;
|
|
||||||
}
|
|
||||||
|
@ -4,17 +4,20 @@ Gz Common
|
|||||||
#ifndef COMMON_COMPRESS_GZ_COMMON_H
|
#ifndef COMMON_COMPRESS_GZ_COMMON_H
|
||||||
#define COMMON_COMPRESS_GZ_COMMON_H
|
#define COMMON_COMPRESS_GZ_COMMON_H
|
||||||
|
|
||||||
#include <stdbool.h>
|
|
||||||
|
|
||||||
/***********************************************************************************************************************************
|
/***********************************************************************************************************************************
|
||||||
Gz extension
|
Gz extension
|
||||||
***********************************************************************************************************************************/
|
***********************************************************************************************************************************/
|
||||||
#define GZ_EXT "gz"
|
#define GZ_EXT "gz"
|
||||||
|
|
||||||
|
/***********************************************************************************************************************************
|
||||||
|
Constants
|
||||||
|
***********************************************************************************************************************************/
|
||||||
|
#define WINDOW_BITS 15
|
||||||
|
#define WANT_GZ 16
|
||||||
|
|
||||||
/***********************************************************************************************************************************
|
/***********************************************************************************************************************************
|
||||||
Functions
|
Functions
|
||||||
***********************************************************************************************************************************/
|
***********************************************************************************************************************************/
|
||||||
int gzError(int error);
|
int gzError(int error);
|
||||||
int gzWindowBits(bool raw);
|
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
|
@ -160,11 +160,10 @@ gzCompressInputSame(const THIS_VOID)
|
|||||||
New object
|
New object
|
||||||
***********************************************************************************************************************************/
|
***********************************************************************************************************************************/
|
||||||
IoFilter *
|
IoFilter *
|
||||||
gzCompressNew(int level, bool raw)
|
gzCompressNew(int level)
|
||||||
{
|
{
|
||||||
FUNCTION_LOG_BEGIN(logLevelTrace);
|
FUNCTION_LOG_BEGIN(logLevelTrace);
|
||||||
FUNCTION_LOG_PARAM(INT, level);
|
FUNCTION_LOG_PARAM(INT, level);
|
||||||
FUNCTION_LOG_PARAM(BOOL, raw);
|
|
||||||
FUNCTION_LOG_END();
|
FUNCTION_LOG_END();
|
||||||
|
|
||||||
ASSERT(level >= -1 && level <= 9);
|
ASSERT(level >= -1 && level <= 9);
|
||||||
@ -182,7 +181,7 @@ gzCompressNew(int level, bool raw)
|
|||||||
};
|
};
|
||||||
|
|
||||||
// Create gz stream
|
// Create gz stream
|
||||||
gzError(deflateInit2(&driver->stream, level, Z_DEFLATED, gzWindowBits(raw), MEM_LEVEL, Z_DEFAULT_STRATEGY));
|
gzError(deflateInit2(&driver->stream, level, Z_DEFLATED, WANT_GZ | WINDOW_BITS, MEM_LEVEL, Z_DEFAULT_STRATEGY));
|
||||||
|
|
||||||
// Set free callback to ensure gz context is freed
|
// Set free callback to ensure gz context is freed
|
||||||
memContextCallbackSet(driver->memContext, gzCompressFreeResource, driver);
|
memContextCallbackSet(driver->memContext, gzCompressFreeResource, driver);
|
||||||
@ -190,7 +189,6 @@ gzCompressNew(int level, bool raw)
|
|||||||
// Create param list
|
// Create param list
|
||||||
VariantList *paramList = varLstNew();
|
VariantList *paramList = varLstNew();
|
||||||
varLstAdd(paramList, varNewInt(level));
|
varLstAdd(paramList, varNewInt(level));
|
||||||
varLstAdd(paramList, varNewBool(raw));
|
|
||||||
|
|
||||||
// Create filter interface
|
// Create filter interface
|
||||||
this = ioFilterNewP(
|
this = ioFilterNewP(
|
||||||
@ -205,5 +203,5 @@ gzCompressNew(int level, bool raw)
|
|||||||
IoFilter *
|
IoFilter *
|
||||||
gzCompressNewVar(const VariantList *paramList)
|
gzCompressNewVar(const VariantList *paramList)
|
||||||
{
|
{
|
||||||
return gzCompressNew(varIntForce(varLstGet(paramList, 0)), varBool(varLstGet(paramList, 1)));
|
return gzCompressNew(varIntForce(varLstGet(paramList, 0)));
|
||||||
}
|
}
|
||||||
|
@ -17,7 +17,7 @@ Filter type constant
|
|||||||
/***********************************************************************************************************************************
|
/***********************************************************************************************************************************
|
||||||
Constructor
|
Constructor
|
||||||
***********************************************************************************************************************************/
|
***********************************************************************************************************************************/
|
||||||
IoFilter *gzCompressNew(int level, bool raw);
|
IoFilter *gzCompressNew(int level);
|
||||||
IoFilter *gzCompressNewVar(const VariantList *paramList);
|
IoFilter *gzCompressNewVar(const VariantList *paramList);
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
|
@ -143,11 +143,9 @@ gzDecompressInputSame(const THIS_VOID)
|
|||||||
New object
|
New object
|
||||||
***********************************************************************************************************************************/
|
***********************************************************************************************************************************/
|
||||||
IoFilter *
|
IoFilter *
|
||||||
gzDecompressNew(bool raw)
|
gzDecompressNew(void)
|
||||||
{
|
{
|
||||||
FUNCTION_LOG_BEGIN(logLevelTrace);
|
FUNCTION_LOG_VOID(logLevelTrace);
|
||||||
FUNCTION_LOG_PARAM(BOOL, raw);
|
|
||||||
FUNCTION_LOG_END();
|
|
||||||
|
|
||||||
IoFilter *this = NULL;
|
IoFilter *this = NULL;
|
||||||
|
|
||||||
@ -163,27 +161,17 @@ gzDecompressNew(bool raw)
|
|||||||
};
|
};
|
||||||
|
|
||||||
// Create gz stream
|
// Create gz stream
|
||||||
gzError(driver->result = inflateInit2(&driver->stream, gzWindowBits(raw)));
|
gzError(driver->result = inflateInit2(&driver->stream, WANT_GZ | WINDOW_BITS));
|
||||||
|
|
||||||
// Set free callback to ensure gz context is freed
|
// Set free callback to ensure gz context is freed
|
||||||
memContextCallbackSet(driver->memContext, gzDecompressFreeResource, driver);
|
memContextCallbackSet(driver->memContext, gzDecompressFreeResource, driver);
|
||||||
|
|
||||||
// Create param list
|
|
||||||
VariantList *paramList = varLstNew();
|
|
||||||
varLstAdd(paramList, varNewBool(raw));
|
|
||||||
|
|
||||||
// Create filter interface
|
// Create filter interface
|
||||||
this = ioFilterNewP(
|
this = ioFilterNewP(
|
||||||
GZ_DECOMPRESS_FILTER_TYPE_STR, driver, paramList, .done = gzDecompressDone, .inOut = gzDecompressProcess,
|
GZ_DECOMPRESS_FILTER_TYPE_STR, driver, NULL, .done = gzDecompressDone, .inOut = gzDecompressProcess,
|
||||||
.inputSame = gzDecompressInputSame);
|
.inputSame = gzDecompressInputSame);
|
||||||
}
|
}
|
||||||
MEM_CONTEXT_NEW_END();
|
MEM_CONTEXT_NEW_END();
|
||||||
|
|
||||||
FUNCTION_LOG_RETURN(IO_FILTER, this);
|
FUNCTION_LOG_RETURN(IO_FILTER, this);
|
||||||
}
|
}
|
||||||
|
|
||||||
IoFilter *
|
|
||||||
gzDecompressNewVar(const VariantList *paramList)
|
|
||||||
{
|
|
||||||
return gzDecompressNew(varBool(varLstGet(paramList, 0)));
|
|
||||||
}
|
|
||||||
|
@ -17,7 +17,6 @@ Filter type constant
|
|||||||
/***********************************************************************************************************************************
|
/***********************************************************************************************************************************
|
||||||
Constructor
|
Constructor
|
||||||
***********************************************************************************************************************************/
|
***********************************************************************************************************************************/
|
||||||
IoFilter *gzDecompressNew(bool raw);
|
IoFilter *gzDecompressNew(void);
|
||||||
IoFilter *gzDecompressNewVar(const VariantList *paramList);
|
|
||||||
|
|
||||||
#endif
|
#endif
|
||||||
|
@ -75,7 +75,7 @@ storageRemoteFilterGroup(IoFilterGroup *filterGroup, const Variant *filterList)
|
|||||||
if (strEq(filterKey, GZ_COMPRESS_FILTER_TYPE_STR))
|
if (strEq(filterKey, GZ_COMPRESS_FILTER_TYPE_STR))
|
||||||
ioFilterGroupAdd(filterGroup, gzCompressNewVar(filterParam));
|
ioFilterGroupAdd(filterGroup, gzCompressNewVar(filterParam));
|
||||||
else if (strEq(filterKey, GZ_DECOMPRESS_FILTER_TYPE_STR))
|
else if (strEq(filterKey, GZ_DECOMPRESS_FILTER_TYPE_STR))
|
||||||
ioFilterGroupAdd(filterGroup, gzDecompressNewVar(filterParam));
|
ioFilterGroupAdd(filterGroup, gzDecompressNew());
|
||||||
else if (strEq(filterKey, CIPHER_BLOCK_FILTER_TYPE_STR))
|
else if (strEq(filterKey, CIPHER_BLOCK_FILTER_TYPE_STR))
|
||||||
ioFilterGroupAdd(filterGroup, cipherBlockNewVar(filterParam));
|
ioFilterGroupAdd(filterGroup, cipherBlockNewVar(filterParam));
|
||||||
else if (strEq(filterKey, CRYPTO_HASH_FILTER_TYPE_STR))
|
else if (strEq(filterKey, CRYPTO_HASH_FILTER_TYPE_STR))
|
||||||
|
@ -65,10 +65,7 @@ storageReadRemoteOpen(THIS_VOID)
|
|||||||
{
|
{
|
||||||
// If the file is compressible add compression filter on the remote
|
// If the file is compressible add compression filter on the remote
|
||||||
if (this->interface.compressible)
|
if (this->interface.compressible)
|
||||||
{
|
ioFilterGroupAdd(ioReadFilterGroup(storageReadIo(this->read)), gzCompressNew((int)this->interface.compressLevel));
|
||||||
ioFilterGroupAdd(
|
|
||||||
ioReadFilterGroup(storageReadIo(this->read)), gzCompressNew((int)this->interface.compressLevel, true));
|
|
||||||
}
|
|
||||||
|
|
||||||
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_STORAGE_OPEN_READ_STR);
|
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_STORAGE_OPEN_READ_STR);
|
||||||
protocolCommandParamAdd(command, VARSTR(this->interface.name));
|
protocolCommandParamAdd(command, VARSTR(this->interface.name));
|
||||||
@ -82,7 +79,7 @@ storageReadRemoteOpen(THIS_VOID)
|
|||||||
|
|
||||||
// If the file is compressible add decompression filter locally
|
// If the file is compressible add decompression filter locally
|
||||||
if (this->interface.compressible)
|
if (this->interface.compressible)
|
||||||
ioFilterGroupAdd(ioReadFilterGroup(storageReadIo(this->read)), gzDecompressNew(true));
|
ioFilterGroupAdd(ioReadFilterGroup(storageReadIo(this->read)), gzDecompressNew());
|
||||||
}
|
}
|
||||||
MEM_CONTEXT_TEMP_END();
|
MEM_CONTEXT_TEMP_END();
|
||||||
|
|
||||||
|
@ -70,7 +70,7 @@ storageWriteRemoteOpen(THIS_VOID)
|
|||||||
{
|
{
|
||||||
// If the file is compressible add decompression filter on the remote
|
// If the file is compressible add decompression filter on the remote
|
||||||
if (this->interface.compressible)
|
if (this->interface.compressible)
|
||||||
ioFilterGroupInsert(ioWriteFilterGroup(storageWriteIo(this->write)), 0, gzDecompressNew(true));
|
ioFilterGroupInsert(ioWriteFilterGroup(storageWriteIo(this->write)), 0, gzDecompressNew());
|
||||||
|
|
||||||
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_STORAGE_OPEN_WRITE_STR);
|
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_STORAGE_OPEN_WRITE_STR);
|
||||||
protocolCommandParamAdd(command, VARSTR(this->interface.name));
|
protocolCommandParamAdd(command, VARSTR(this->interface.name));
|
||||||
@ -92,10 +92,7 @@ storageWriteRemoteOpen(THIS_VOID)
|
|||||||
|
|
||||||
// If the file is compressible add compression filter locally
|
// If the file is compressible add compression filter locally
|
||||||
if (this->interface.compressible)
|
if (this->interface.compressible)
|
||||||
{
|
ioFilterGroupAdd(ioWriteFilterGroup(storageWriteIo(this->write)), gzCompressNew((int)this->interface.compressLevel));
|
||||||
ioFilterGroupAdd(
|
|
||||||
ioWriteFilterGroup(storageWriteIo(this->write)), gzCompressNew((int)this->interface.compressLevel, true));
|
|
||||||
}
|
|
||||||
|
|
||||||
// Set free callback to ensure remote file is freed
|
// Set free callback to ensure remote file is freed
|
||||||
memContextCallbackSet(this->memContext, storageWriteRemoteFreeResource, this);
|
memContextCallbackSet(this->memContext, storageWriteRemoteFreeResource, this);
|
||||||
|
@ -258,7 +258,7 @@ unit:
|
|||||||
|
|
||||||
# ----------------------------------------------------------------------------------------------------------------------------
|
# ----------------------------------------------------------------------------------------------------------------------------
|
||||||
- name: compress
|
- name: compress
|
||||||
total: 4
|
total: 3
|
||||||
|
|
||||||
coverage:
|
coverage:
|
||||||
common/compress/gz/common: full
|
common/compress/gz/common: full
|
||||||
|
@ -39,7 +39,7 @@ hrnStorageInfoListCallback(void *callbackData, const StorageInfo *info)
|
|||||||
StorageRead *read = storageNewReadP(
|
StorageRead *read = storageNewReadP(
|
||||||
data->storage,
|
data->storage,
|
||||||
data->path != NULL ? strNewFmt("%s/%s", strPtr(data->path), strPtr(info->name)) : info->name);
|
data->path != NULL ? strNewFmt("%s/%s", strPtr(data->path), strPtr(info->name)) : info->name);
|
||||||
ioFilterGroupAdd(ioReadFilterGroup(storageReadIo(read)), gzDecompressNew(false));
|
ioFilterGroupAdd(ioReadFilterGroup(storageReadIo(read)), gzDecompressNew());
|
||||||
size = bufUsed(storageGetP(read));
|
size = bufUsed(storageGetP(read));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -202,7 +202,7 @@ testRun(void)
|
|||||||
"repo/archive/test1/10-1/01ABCDEF01ABCDEF/01ABCDEF01ABCDEF01ABCDEF-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.gz"));
|
"repo/archive/test1/10-1/01ABCDEF01ABCDEF/01ABCDEF01ABCDEF01ABCDEF-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.gz"));
|
||||||
|
|
||||||
IoFilterGroup *filterGroup = ioWriteFilterGroup(storageWriteIo(destination));
|
IoFilterGroup *filterGroup = ioWriteFilterGroup(storageWriteIo(destination));
|
||||||
ioFilterGroupAdd(filterGroup, gzCompressNew(3, false));
|
ioFilterGroupAdd(filterGroup, gzCompressNew(3));
|
||||||
ioFilterGroupAdd(
|
ioFilterGroupAdd(
|
||||||
filterGroup, cipherBlockNew(cipherModeEncrypt, cipherTypeAes256Cbc, BUFSTRDEF("worstpassphraseever"), NULL));
|
filterGroup, cipherBlockNew(cipherModeEncrypt, cipherTypeAes256Cbc, BUFSTRDEF("worstpassphraseever"), NULL));
|
||||||
storagePutP(destination, buffer);
|
storagePutP(destination, buffer);
|
||||||
|
@ -77,7 +77,7 @@ testBackupValidateCallback(void *callbackData, const StorageInfo *info)
|
|||||||
StorageRead *read = storageNewReadP(
|
StorageRead *read = storageNewReadP(
|
||||||
data->storage,
|
data->storage,
|
||||||
data->path != NULL ? strNewFmt("%s/%s", strPtr(data->path), strPtr(info->name)) : info->name);
|
data->path != NULL ? strNewFmt("%s/%s", strPtr(data->path), strPtr(info->name)) : info->name);
|
||||||
ioFilterGroupAdd(ioReadFilterGroup(storageReadIo(read)), gzDecompressNew(false));
|
ioFilterGroupAdd(ioReadFilterGroup(storageReadIo(read)), gzDecompressNew());
|
||||||
size = bufUsed(storageGetP(read));
|
size = bufUsed(storageGetP(read));
|
||||||
|
|
||||||
manifestName = strSubN(info->name, 0, strSize(info->name) - strlen("." GZ_EXT));
|
manifestName = strSubN(info->name, 0, strSize(info->name) - strlen("." GZ_EXT));
|
||||||
@ -244,7 +244,7 @@ testBackupPqScript(unsigned int pgVersion, time_t backupTimeStart, TestBackupPqS
|
|||||||
strPtr(walChecksum), param.walCompress ? "." GZ_EXT : ""));
|
strPtr(walChecksum), param.walCompress ? "." GZ_EXT : ""));
|
||||||
|
|
||||||
if (param.walCompress)
|
if (param.walCompress)
|
||||||
ioFilterGroupAdd(ioWriteFilterGroup(storageWriteIo(write)), gzCompressNew(1, false));
|
ioFilterGroupAdd(ioWriteFilterGroup(storageWriteIo(write)), gzCompressNew(1));
|
||||||
|
|
||||||
storagePutP(write, walBuffer);
|
storagePutP(write, walBuffer);
|
||||||
}
|
}
|
||||||
|
@ -196,7 +196,7 @@ testRun(void)
|
|||||||
StorageWrite *ceRepoFile = storageNewWriteP(
|
StorageWrite *ceRepoFile = storageNewWriteP(
|
||||||
storageRepoWrite(), strNewFmt(STORAGE_REPO_BACKUP "/%s/%s.gz", strPtr(repoFileReferenceFull), strPtr(repoFile1)));
|
storageRepoWrite(), strNewFmt(STORAGE_REPO_BACKUP "/%s/%s.gz", strPtr(repoFileReferenceFull), strPtr(repoFile1)));
|
||||||
IoFilterGroup *filterGroup = ioWriteFilterGroup(storageWriteIo(ceRepoFile));
|
IoFilterGroup *filterGroup = ioWriteFilterGroup(storageWriteIo(ceRepoFile));
|
||||||
ioFilterGroupAdd(filterGroup, gzCompressNew(3, false));
|
ioFilterGroupAdd(filterGroup, gzCompressNew(3));
|
||||||
ioFilterGroupAdd(filterGroup, cipherBlockNew(cipherModeEncrypt, cipherTypeAes256Cbc, BUFSTRDEF("badpass"), NULL));
|
ioFilterGroupAdd(filterGroup, cipherBlockNew(cipherModeEncrypt, cipherTypeAes256Cbc, BUFSTRDEF("badpass"), NULL));
|
||||||
|
|
||||||
storagePutP(ceRepoFile, BUFSTRDEF("acefile"));
|
storagePutP(ceRepoFile, BUFSTRDEF("acefile"));
|
||||||
|
@ -91,14 +91,6 @@ testRun(void)
|
|||||||
TEST_ERROR(gzError(999), AssertError, "zlib threw error: [999] unknown error");
|
TEST_ERROR(gzError(999), AssertError, "zlib threw error: [999] unknown error");
|
||||||
}
|
}
|
||||||
|
|
||||||
// *****************************************************************************************************************************
|
|
||||||
if (testBegin("gzWindowBits"))
|
|
||||||
{
|
|
||||||
|
|
||||||
TEST_RESULT_INT(gzWindowBits(true), -15, "raw window bits");
|
|
||||||
TEST_RESULT_INT(gzWindowBits(false), 31, "gz window bits");
|
|
||||||
}
|
|
||||||
|
|
||||||
// *****************************************************************************************************************************
|
// *****************************************************************************************************************************
|
||||||
if (testBegin("GzCompress and GzDecompress"))
|
if (testBegin("GzCompress and GzDecompress"))
|
||||||
{
|
{
|
||||||
@ -110,45 +102,42 @@ testRun(void)
|
|||||||
varLstAdd(compressParamList, varNewUInt(3));
|
varLstAdd(compressParamList, varNewUInt(3));
|
||||||
varLstAdd(compressParamList, varNewBool(false));
|
varLstAdd(compressParamList, varNewBool(false));
|
||||||
|
|
||||||
VariantList *decompressParamList = varLstNew();
|
|
||||||
varLstAdd(decompressParamList, varNewBool(false));
|
|
||||||
|
|
||||||
TEST_ASSIGN(
|
TEST_ASSIGN(
|
||||||
compressed, testCompress(gzCompressNewVar(compressParamList), decompressed, 1024, 1024),
|
compressed, testCompress(gzCompressNewVar(compressParamList), decompressed, 1024, 1024),
|
||||||
"simple data - compress large in/large out buffer");
|
"simple data - compress large in/large out buffer");
|
||||||
|
|
||||||
TEST_RESULT_BOOL(
|
TEST_RESULT_BOOL(
|
||||||
bufEq(compressed, testCompress(gzCompressNew(3, false), decompressed, 1024, 1)), true,
|
bufEq(compressed, testCompress(gzCompressNew(3), decompressed, 1024, 1)), true,
|
||||||
"simple data - compress large in/small out buffer");
|
"simple data - compress large in/small out buffer");
|
||||||
|
|
||||||
TEST_RESULT_BOOL(
|
TEST_RESULT_BOOL(
|
||||||
bufEq(compressed, testCompress(gzCompressNew(3, false), decompressed, 1, 1024)), true,
|
bufEq(compressed, testCompress(gzCompressNew(3), decompressed, 1, 1024)), true,
|
||||||
"simple data - compress small in/large out buffer");
|
"simple data - compress small in/large out buffer");
|
||||||
|
|
||||||
TEST_RESULT_BOOL(
|
TEST_RESULT_BOOL(
|
||||||
bufEq(compressed, testCompress(gzCompressNew(3, false), decompressed, 1, 1)), true,
|
bufEq(compressed, testCompress(gzCompressNew(3), decompressed, 1, 1)), true,
|
||||||
"simple data - compress small in/small out buffer");
|
"simple data - compress small in/small out buffer");
|
||||||
|
|
||||||
TEST_RESULT_BOOL(
|
TEST_RESULT_BOOL(
|
||||||
bufEq(decompressed, testDecompress(gzDecompressNewVar(decompressParamList), compressed, 1024, 1024)), true,
|
bufEq(decompressed, testDecompress(gzDecompressNew(), compressed, 1024, 1024)), true,
|
||||||
"simple data - decompress large in/large out buffer");
|
"simple data - decompress large in/large out buffer");
|
||||||
|
|
||||||
TEST_RESULT_BOOL(
|
TEST_RESULT_BOOL(
|
||||||
bufEq(decompressed, testDecompress(gzDecompressNew(false), compressed, 1024, 1)), true,
|
bufEq(decompressed, testDecompress(gzDecompressNew(), compressed, 1024, 1)), true,
|
||||||
"simple data - decompress large in/small out buffer");
|
"simple data - decompress large in/small out buffer");
|
||||||
|
|
||||||
TEST_RESULT_BOOL(
|
TEST_RESULT_BOOL(
|
||||||
bufEq(decompressed, testDecompress(gzDecompressNew(false), compressed, 1, 1024)), true,
|
bufEq(decompressed, testDecompress(gzDecompressNew(), compressed, 1, 1024)), true,
|
||||||
"simple data - decompress small in/large out buffer");
|
"simple data - decompress small in/large out buffer");
|
||||||
|
|
||||||
TEST_RESULT_BOOL(
|
TEST_RESULT_BOOL(
|
||||||
bufEq(decompressed, testDecompress(gzDecompressNew(false), compressed, 1, 1)), true,
|
bufEq(decompressed, testDecompress(gzDecompressNew(), compressed, 1, 1)), true,
|
||||||
"simple data - decompress small in/small out buffer");
|
"simple data - decompress small in/small out buffer");
|
||||||
|
|
||||||
// -------------------------------------------------------------------------------------------------------------------------
|
// -------------------------------------------------------------------------------------------------------------------------
|
||||||
TEST_TITLE("error on no compression data");
|
TEST_TITLE("error on no compression data");
|
||||||
|
|
||||||
TEST_ERROR(testDecompress(gzDecompressNew(true), bufNew(0), 1, 1), FormatError, "unexpected eof in compressed data");
|
TEST_ERROR(testDecompress(gzDecompressNew(), bufNew(0), 1, 1), FormatError, "unexpected eof in compressed data");
|
||||||
|
|
||||||
// -------------------------------------------------------------------------------------------------------------------------
|
// -------------------------------------------------------------------------------------------------------------------------
|
||||||
TEST_TITLE("error on truncated compression data");
|
TEST_TITLE("error on truncated compression data");
|
||||||
@ -157,7 +146,7 @@ testRun(void)
|
|||||||
bufCatSub(truncated, compressed, 0, bufUsed(compressed) - 1);
|
bufCatSub(truncated, compressed, 0, bufUsed(compressed) - 1);
|
||||||
|
|
||||||
TEST_RESULT_UINT(bufUsed(truncated), bufUsed(compressed) - 1, "check truncated buffer size");
|
TEST_RESULT_UINT(bufUsed(truncated), bufUsed(compressed) - 1, "check truncated buffer size");
|
||||||
TEST_ERROR(testDecompress(gzDecompressNew(false), truncated, 512, 512), FormatError, "unexpected eof in compressed data");
|
TEST_ERROR(testDecompress(gzDecompressNew(), truncated, 512, 512), FormatError, "unexpected eof in compressed data");
|
||||||
|
|
||||||
// Compress a large zero input buffer into small output buffer
|
// Compress a large zero input buffer into small output buffer
|
||||||
// -------------------------------------------------------------------------------------------------------------------------
|
// -------------------------------------------------------------------------------------------------------------------------
|
||||||
@ -166,18 +155,18 @@ testRun(void)
|
|||||||
bufUsedSet(decompressed, bufSize(decompressed));
|
bufUsedSet(decompressed, bufSize(decompressed));
|
||||||
|
|
||||||
TEST_ASSIGN(
|
TEST_ASSIGN(
|
||||||
compressed, testCompress(gzCompressNew(3, true), decompressed, bufSize(decompressed), 1024),
|
compressed, testCompress(gzCompressNew(3), decompressed, bufSize(decompressed), 1024),
|
||||||
"zero data - compress large in/small out buffer");
|
"zero data - compress large in/small out buffer");
|
||||||
|
|
||||||
TEST_RESULT_BOOL(
|
TEST_RESULT_BOOL(
|
||||||
bufEq(decompressed, testDecompress(gzDecompressNew(true), compressed, bufSize(compressed), 1024 * 256)), true,
|
bufEq(decompressed, testDecompress(gzDecompressNew(), compressed, bufSize(compressed), 1024 * 256)), true,
|
||||||
"zero data - decompress large in/small out buffer");
|
"zero data - decompress large in/small out buffer");
|
||||||
}
|
}
|
||||||
|
|
||||||
// *****************************************************************************************************************************
|
// *****************************************************************************************************************************
|
||||||
if (testBegin("gzDecompressToLog() and gzCompressToLog()"))
|
if (testBegin("gzDecompressToLog() and gzCompressToLog()"))
|
||||||
{
|
{
|
||||||
GzDecompress *decompress = (GzDecompress *)ioFilterDriver(gzDecompressNew(false));
|
GzDecompress *decompress = (GzDecompress *)ioFilterDriver(gzDecompressNew());
|
||||||
|
|
||||||
TEST_RESULT_STR_Z(gzDecompressToLog(decompress), "{inputSame: false, done: false, availIn: 0}", "format object");
|
TEST_RESULT_STR_Z(gzDecompressToLog(decompress), "{inputSame: false, done: false, availIn: 0}", "format object");
|
||||||
|
|
||||||
|
@ -447,8 +447,8 @@ testRun(void)
|
|||||||
ioFilterGroupAdd(filterGroup, pageChecksumNew(0, PG_SEGMENT_PAGE_DEFAULT, PG_PAGE_SIZE_DEFAULT, 0));
|
ioFilterGroupAdd(filterGroup, pageChecksumNew(0, PG_SEGMENT_PAGE_DEFAULT, PG_PAGE_SIZE_DEFAULT, 0));
|
||||||
ioFilterGroupAdd(filterGroup, cipherBlockNew(cipherModeEncrypt, cipherTypeAes256Cbc, BUFSTRZ("x"), NULL));
|
ioFilterGroupAdd(filterGroup, cipherBlockNew(cipherModeEncrypt, cipherTypeAes256Cbc, BUFSTRZ("x"), NULL));
|
||||||
ioFilterGroupAdd(filterGroup, cipherBlockNew(cipherModeDecrypt, cipherTypeAes256Cbc, BUFSTRZ("x"), NULL));
|
ioFilterGroupAdd(filterGroup, cipherBlockNew(cipherModeDecrypt, cipherTypeAes256Cbc, BUFSTRZ("x"), NULL));
|
||||||
ioFilterGroupAdd(filterGroup, gzCompressNew(3, false));
|
ioFilterGroupAdd(filterGroup, gzCompressNew(3));
|
||||||
ioFilterGroupAdd(filterGroup, gzDecompressNew(false));
|
ioFilterGroupAdd(filterGroup, gzDecompressNew());
|
||||||
varLstAdd(paramList, ioFilterGroupParamAll(filterGroup));
|
varLstAdd(paramList, ioFilterGroupParamAll(filterGroup));
|
||||||
|
|
||||||
TEST_RESULT_BOOL(
|
TEST_RESULT_BOOL(
|
||||||
|
Reference in New Issue
Block a user