2018-08-14 14:56:59 -04:00
|
|
|
/***********************************************************************************************************************************
|
2020-02-27 12:09:05 -05:00
|
|
|
Test Compression
|
2018-08-14 14:56:59 -04:00
|
|
|
***********************************************************************************************************************************/
|
|
|
|
#include "common/io/filter/group.h"
|
|
|
|
#include "common/io/bufferRead.h"
|
|
|
|
#include "common/io/bufferWrite.h"
|
|
|
|
#include "common/io/io.h"
|
2020-03-06 14:41:03 -05:00
|
|
|
#include "storage/posix/storage.h"
|
2018-08-14 14:56:59 -04:00
|
|
|
|
|
|
|
/***********************************************************************************************************************************
|
|
|
|
Compress data
|
|
|
|
***********************************************************************************************************************************/
|
|
|
|
static Buffer *
|
2019-05-02 17:52:24 -04:00
|
|
|
testCompress(IoFilter *compress, Buffer *decompressed, size_t inputSize, size_t outputSize)
|
2018-08-14 14:56:59 -04:00
|
|
|
{
|
|
|
|
Buffer *compressed = bufNew(1024 * 1024);
|
|
|
|
size_t inputTotal = 0;
|
|
|
|
ioBufferSizeSet(outputSize);
|
|
|
|
|
2019-05-02 17:52:24 -04:00
|
|
|
IoWrite *write = ioBufferWriteNew(compressed);
|
2019-06-24 10:20:47 -04:00
|
|
|
ioFilterGroupAdd(ioWriteFilterGroup(write), compress);
|
2018-08-14 14:56:59 -04:00
|
|
|
ioWriteOpen(write);
|
|
|
|
|
|
|
|
// Compress input data
|
|
|
|
while (inputTotal < bufSize(decompressed))
|
|
|
|
{
|
|
|
|
// Generate the input buffer based on input size. This breaks the data up into chunks as it would be in a real scenario.
|
|
|
|
Buffer *input = bufNewC(
|
2019-05-02 12:43:09 -04:00
|
|
|
bufPtr(decompressed) + inputTotal,
|
|
|
|
inputSize > bufSize(decompressed) - inputTotal ? bufSize(decompressed) - inputTotal : inputSize);
|
2018-08-14 14:56:59 -04:00
|
|
|
|
|
|
|
ioWrite(write, input);
|
|
|
|
|
|
|
|
inputTotal += bufUsed(input);
|
|
|
|
bufFree(input);
|
|
|
|
}
|
|
|
|
|
|
|
|
ioWriteClose(write);
|
2020-03-06 14:41:03 -05:00
|
|
|
ioFilterFree(compress);
|
2018-08-14 14:56:59 -04:00
|
|
|
|
|
|
|
return compressed;
|
|
|
|
}
|
|
|
|
|
|
|
|
/***********************************************************************************************************************************
|
|
|
|
Decompress data
|
|
|
|
***********************************************************************************************************************************/
|
|
|
|
static Buffer *
|
2019-05-02 17:52:24 -04:00
|
|
|
testDecompress(IoFilter *decompress, Buffer *compressed, size_t inputSize, size_t outputSize)
|
2018-08-14 14:56:59 -04:00
|
|
|
{
|
|
|
|
Buffer *decompressed = bufNew(1024 * 1024);
|
|
|
|
Buffer *output = bufNew(outputSize);
|
|
|
|
ioBufferSizeSet(inputSize);
|
|
|
|
|
2019-05-02 17:52:24 -04:00
|
|
|
IoRead *read = ioBufferReadNew(compressed);
|
2019-06-24 10:20:47 -04:00
|
|
|
ioFilterGroupAdd(ioReadFilterGroup(read), decompress);
|
2018-08-14 14:56:59 -04:00
|
|
|
ioReadOpen(read);
|
|
|
|
|
|
|
|
while (!ioReadEof(read))
|
|
|
|
{
|
|
|
|
ioRead(read, output);
|
|
|
|
bufCat(decompressed, output);
|
|
|
|
bufUsedZero(output);
|
|
|
|
}
|
|
|
|
|
|
|
|
ioReadClose(read);
|
|
|
|
bufFree(output);
|
2020-03-06 14:41:03 -05:00
|
|
|
ioFilterFree(decompress);
|
2018-08-14 14:56:59 -04:00
|
|
|
|
|
|
|
return decompressed;
|
|
|
|
}
|
|
|
|
|
2020-03-06 14:41:03 -05:00
|
|
|
/***********************************************************************************************************************************
|
|
|
|
Standard test suite to be applied to all compression types
|
|
|
|
***********************************************************************************************************************************/
|
|
|
|
static void
|
|
|
|
testSuite(CompressType type, const char *decompressCmd)
|
|
|
|
{
|
|
|
|
const char *simpleData = "A simple string";
|
|
|
|
Buffer *compressed = NULL;
|
|
|
|
Buffer *decompressed = bufNewC(simpleData, strlen(simpleData));
|
|
|
|
|
|
|
|
VariantList *compressParamList = varLstNew();
|
|
|
|
varLstAdd(compressParamList, varNewUInt(1));
|
|
|
|
|
|
|
|
// Create default storage object for testing
|
2020-04-30 11:01:38 -04:00
|
|
|
Storage *storageTest = storagePosixNewP(strNew(testPath()), .write = true);
|
2020-03-06 14:41:03 -05:00
|
|
|
|
|
|
|
TEST_TITLE("simple data");
|
|
|
|
|
|
|
|
TEST_ASSIGN(
|
|
|
|
compressed,
|
|
|
|
testCompress(
|
|
|
|
compressFilterVar(strNewFmt("%sCompress", strPtr(compressTypeStr(type))), compressParamList), decompressed, 1024,
|
|
|
|
256 * 1024 * 1024),
|
|
|
|
"simple data - compress large in/large out buffer");
|
|
|
|
|
2020-04-29 13:55:05 -04:00
|
|
|
// -------------------------------------------------------------------------------------------------------------------------
|
|
|
|
TEST_TITLE("compressed output can be decompressed with command-line tool");
|
|
|
|
|
|
|
|
storagePutP(storageNewWriteP(storageTest, STRDEF("test.cmp")), compressed);
|
|
|
|
TEST_SYSTEM_FMT("%s {[path]}/test.cmp > {[path]}/test.out", decompressCmd);
|
|
|
|
TEST_RESULT_BOOL(bufEq(decompressed, storageGetP(storageNewReadP(storageTest, STRDEF("test.out")))), true, "check output");
|
|
|
|
|
2020-03-06 14:41:03 -05:00
|
|
|
TEST_RESULT_BOOL(
|
|
|
|
bufEq(compressed, testCompress(compressFilter(type, 1), decompressed, 1024, 1)), true,
|
|
|
|
"simple data - compress large in/small out buffer");
|
|
|
|
|
|
|
|
TEST_RESULT_BOOL(
|
|
|
|
bufEq(compressed, testCompress(compressFilter(type, 1), decompressed, 1, 1024)), true,
|
|
|
|
"simple data - compress small in/large out buffer");
|
|
|
|
|
|
|
|
TEST_RESULT_BOOL(
|
|
|
|
bufEq(compressed, testCompress(compressFilter(type, 1), decompressed, 1, 1)), true,
|
|
|
|
"simple data - compress small in/small out buffer");
|
|
|
|
|
|
|
|
TEST_RESULT_BOOL(
|
|
|
|
bufEq(
|
|
|
|
decompressed,
|
|
|
|
testDecompress(
|
|
|
|
compressFilterVar(strNewFmt("%sDecompress", strPtr(compressTypeStr(type))), NULL), compressed, 1024, 1024)),
|
|
|
|
true, "simple data - decompress large in/large out buffer");
|
|
|
|
|
|
|
|
TEST_RESULT_BOOL(
|
|
|
|
bufEq(decompressed, testDecompress(decompressFilter(type), compressed, 1024, 1)), true,
|
|
|
|
"simple data - decompress large in/small out buffer");
|
|
|
|
|
|
|
|
TEST_RESULT_BOOL(
|
|
|
|
bufEq(decompressed, testDecompress(decompressFilter(type), compressed, 1, 1024)), true,
|
|
|
|
"simple data - decompress small in/large out buffer");
|
|
|
|
|
|
|
|
TEST_RESULT_BOOL(
|
|
|
|
bufEq(decompressed, testDecompress(decompressFilter(type), compressed, 1, 1)), true,
|
|
|
|
"simple data - decompress small in/small out buffer");
|
|
|
|
|
|
|
|
// -------------------------------------------------------------------------------------------------------------------------
|
|
|
|
TEST_TITLE("error on no compression data");
|
|
|
|
|
|
|
|
TEST_ERROR(testDecompress(decompressFilter(type), bufNew(0), 1, 1), FormatError, "unexpected eof in compressed data");
|
|
|
|
|
|
|
|
// -------------------------------------------------------------------------------------------------------------------------
|
|
|
|
TEST_TITLE("error on truncated compression data");
|
|
|
|
|
|
|
|
Buffer *truncated = bufNew(0);
|
|
|
|
bufCatSub(truncated, compressed, 0, bufUsed(compressed) - 1);
|
|
|
|
|
|
|
|
TEST_RESULT_UINT(bufUsed(truncated), bufUsed(compressed) - 1, "check truncated buffer size");
|
|
|
|
TEST_ERROR(testDecompress(decompressFilter(type), truncated, 512, 512), FormatError, "unexpected eof in compressed data");
|
|
|
|
|
|
|
|
// -------------------------------------------------------------------------------------------------------------------------
|
|
|
|
TEST_TITLE("compress a large zero input buffer into small output buffer");
|
|
|
|
|
|
|
|
decompressed = bufNew(1024 * 1024 - 1);
|
|
|
|
memset(bufPtr(decompressed), 0, bufSize(decompressed));
|
|
|
|
bufUsedSet(decompressed, bufSize(decompressed));
|
|
|
|
|
|
|
|
TEST_ASSIGN(
|
2020-05-04 15:25:27 -04:00
|
|
|
compressed, testCompress(compressFilter(type, 3), decompressed, bufSize(decompressed), 32),
|
2020-03-06 14:41:03 -05:00
|
|
|
"zero data - compress large in/small out buffer");
|
|
|
|
|
|
|
|
TEST_RESULT_BOOL(
|
|
|
|
bufEq(decompressed, testDecompress(decompressFilter(type), compressed, bufSize(compressed), 1024 * 256)), true,
|
|
|
|
"zero data - decompress large in/small out buffer");
|
|
|
|
}
|
|
|
|
|
2018-08-14 14:56:59 -04:00
|
|
|
/***********************************************************************************************************************************
|
|
|
|
Test Run
|
|
|
|
***********************************************************************************************************************************/
|
|
|
|
void
|
|
|
|
testRun(void)
|
|
|
|
{
|
|
|
|
FUNCTION_HARNESS_VOID();
|
|
|
|
|
|
|
|
// *****************************************************************************************************************************
|
2020-03-06 14:41:03 -05:00
|
|
|
if (testBegin("gz"))
|
2018-08-14 14:56:59 -04:00
|
|
|
{
|
2020-03-06 14:41:03 -05:00
|
|
|
// Run standard test suite
|
|
|
|
testSuite(compressTypeGz, "gzip -dc");
|
|
|
|
|
|
|
|
// -------------------------------------------------------------------------------------------------------------------------
|
|
|
|
TEST_TITLE("gzError()");
|
|
|
|
|
2020-02-27 12:09:05 -05:00
|
|
|
TEST_RESULT_INT(gzError(Z_OK), Z_OK, "check ok");
|
|
|
|
TEST_RESULT_INT(gzError(Z_STREAM_END), Z_STREAM_END, "check stream end");
|
|
|
|
TEST_ERROR(gzError(Z_NEED_DICT), AssertError, "zlib threw error: [2] need dictionary");
|
|
|
|
TEST_ERROR(gzError(Z_ERRNO), AssertError, "zlib threw error: [-1] file error");
|
|
|
|
TEST_ERROR(gzError(Z_STREAM_ERROR), FormatError, "zlib threw error: [-2] stream error");
|
|
|
|
TEST_ERROR(gzError(Z_DATA_ERROR), FormatError, "zlib threw error: [-3] data error");
|
|
|
|
TEST_ERROR(gzError(Z_MEM_ERROR), MemoryError, "zlib threw error: [-4] insufficient memory");
|
|
|
|
TEST_ERROR(gzError(Z_BUF_ERROR), AssertError, "zlib threw error: [-5] no space in buffer");
|
|
|
|
TEST_ERROR(gzError(Z_VERSION_ERROR), FormatError, "zlib threw error: [-6] incompatible version");
|
|
|
|
TEST_ERROR(gzError(999), AssertError, "zlib threw error: [999] unknown error");
|
2018-08-14 14:56:59 -04:00
|
|
|
|
2020-03-06 14:41:03 -05:00
|
|
|
// -------------------------------------------------------------------------------------------------------------------------
|
|
|
|
TEST_TITLE("gzDecompressToLog() and gzCompressToLog()");
|
2019-06-24 10:20:47 -04:00
|
|
|
|
2020-03-06 14:41:03 -05:00
|
|
|
GzDecompress *decompress = (GzDecompress *)ioFilterDriver(gzDecompressNew());
|
2018-08-14 14:56:59 -04:00
|
|
|
|
2020-03-06 14:41:03 -05:00
|
|
|
TEST_RESULT_STR_Z(gzDecompressToLog(decompress), "{inputSame: false, done: false, availIn: 0}", "format object");
|
2018-08-14 14:56:59 -04:00
|
|
|
|
2020-03-06 14:41:03 -05:00
|
|
|
decompress->inputSame = true;
|
|
|
|
decompress->done = true;
|
2018-08-14 14:56:59 -04:00
|
|
|
|
2020-03-06 14:41:03 -05:00
|
|
|
TEST_RESULT_STR_Z(gzDecompressToLog(decompress), "{inputSame: true, done: true, availIn: 0}", "format object");
|
|
|
|
}
|
2018-08-14 14:56:59 -04:00
|
|
|
|
2020-03-10 14:45:27 -04:00
|
|
|
// *****************************************************************************************************************************
|
|
|
|
if (testBegin("lz4"))
|
|
|
|
{
|
|
|
|
#ifdef HAVE_LIBLZ4
|
|
|
|
// Run standard test suite
|
|
|
|
testSuite(compressTypeLz4, "lz4 -dc");
|
|
|
|
|
|
|
|
// -------------------------------------------------------------------------------------------------------------------------
|
|
|
|
TEST_TITLE("lz4Error()");
|
|
|
|
|
|
|
|
TEST_RESULT_UINT(lz4Error(0), 0, "check success");
|
|
|
|
TEST_ERROR(lz4Error((size_t)-2), FormatError, "lz4 error: [-2] ERROR_maxBlockSize_invalid");
|
|
|
|
|
|
|
|
// -------------------------------------------------------------------------------------------------------------------------
|
|
|
|
TEST_TITLE("lz4DecompressToLog() and lz4CompressToLog()");
|
|
|
|
|
|
|
|
Lz4Compress *compress = (Lz4Compress *)ioFilterDriver(lz4CompressNew(7));
|
|
|
|
|
|
|
|
compress->inputSame = true;
|
|
|
|
compress->flushing = true;
|
|
|
|
|
|
|
|
TEST_RESULT_STR_Z(
|
|
|
|
lz4CompressToLog(compress), "{level: 7, first: true, inputSame: true, flushing: true}", "format object");
|
|
|
|
|
|
|
|
Lz4Decompress *decompress = (Lz4Decompress *)ioFilterDriver(lz4DecompressNew());
|
|
|
|
|
|
|
|
decompress->inputSame = true;
|
|
|
|
decompress->done = true;
|
|
|
|
decompress->inputOffset = 999;
|
|
|
|
|
|
|
|
TEST_RESULT_STR_Z(
|
|
|
|
lz4DecompressToLog(decompress), "{inputSame: true, inputOffset: 999, frameDone false, done: true}",
|
|
|
|
"format object");
|
|
|
|
#else
|
|
|
|
TEST_ERROR(compressTypePresent(compressTypeLz4), OptionInvalidValueError, "pgBackRest not compiled with lz4 support");
|
|
|
|
#endif // HAVE_LIBLZ4
|
|
|
|
}
|
|
|
|
|
2020-05-04 15:25:27 -04:00
|
|
|
// *****************************************************************************************************************************
|
|
|
|
if (testBegin("zst"))
|
|
|
|
{
|
|
|
|
#ifdef HAVE_LIBZST
|
|
|
|
// Run standard test suite
|
|
|
|
testSuite(compressTypeZst, "zstd -dc");
|
|
|
|
|
|
|
|
// -------------------------------------------------------------------------------------------------------------------------
|
|
|
|
TEST_TITLE("zstError()");
|
|
|
|
|
|
|
|
TEST_RESULT_UINT(zstError(0), 0, "check success");
|
|
|
|
TEST_ERROR(zstError((size_t)-12), FormatError, "zst error: [-12] Version not supported");
|
|
|
|
|
|
|
|
// -------------------------------------------------------------------------------------------------------------------------
|
|
|
|
TEST_TITLE("zstDecompressToLog() and zstCompressToLog()");
|
|
|
|
|
|
|
|
ZstCompress *compress = (ZstCompress *)ioFilterDriver(zstCompressNew(14));
|
|
|
|
|
|
|
|
compress->inputSame = true;
|
|
|
|
compress->inputOffset = 49;
|
|
|
|
compress->flushing = true;
|
|
|
|
|
|
|
|
TEST_RESULT_STR_Z(
|
|
|
|
zstCompressToLog(compress), "{level: 14, inputSame: true, inputOffset: 49, flushing: true}", "format object");
|
|
|
|
|
|
|
|
ZstDecompress *decompress = (ZstDecompress *)ioFilterDriver(zstDecompressNew());
|
|
|
|
|
|
|
|
decompress->inputSame = true;
|
|
|
|
decompress->done = true;
|
|
|
|
decompress->inputOffset = 999;
|
|
|
|
|
|
|
|
TEST_RESULT_STR_Z(
|
|
|
|
zstDecompressToLog(decompress), "{inputSame: true, inputOffset: 999, frameDone false, done: true}",
|
|
|
|
"format object");
|
|
|
|
#else
|
|
|
|
TEST_ERROR(compressTypePresent(compressTypeZst), OptionInvalidValueError, "pgBackRest not compiled with zst support");
|
|
|
|
#endif // HAVE_LIBZST
|
|
|
|
}
|
|
|
|
|
2020-03-06 14:41:03 -05:00
|
|
|
// Test everything in the helper that is not tested in the individual compression type tests
|
|
|
|
// *****************************************************************************************************************************
|
|
|
|
if (testBegin("helper"))
|
|
|
|
{
|
|
|
|
TEST_TITLE("compressTypeEnum()");
|
2018-08-14 14:56:59 -04:00
|
|
|
|
2020-03-06 14:41:03 -05:00
|
|
|
TEST_RESULT_UINT(compressTypeEnum(STRDEF("none")), compressTypeNone, "none enum");
|
|
|
|
TEST_RESULT_UINT(compressTypeEnum(STRDEF("gz")), compressTypeGz, "gz enum");
|
|
|
|
TEST_ERROR(compressTypeEnum(strNew(BOGUS_STR)), AssertError, "invalid compression type 'BOGUS'");
|
2018-08-14 14:56:59 -04:00
|
|
|
|
2020-03-06 14:41:03 -05:00
|
|
|
// -------------------------------------------------------------------------------------------------------------------------
|
|
|
|
TEST_TITLE("compressTypePresent()");
|
2018-08-14 14:56:59 -04:00
|
|
|
|
2020-03-06 14:41:03 -05:00
|
|
|
TEST_RESULT_VOID(compressTypePresent(compressTypeNone), "type none always present");
|
2020-05-04 15:25:27 -04:00
|
|
|
TEST_ERROR(compressTypePresent(compressTypeXz), OptionInvalidValueError, "pgBackRest not compiled with xz support");
|
2018-08-14 14:56:59 -04:00
|
|
|
|
2019-12-11 08:48:46 -05:00
|
|
|
// -------------------------------------------------------------------------------------------------------------------------
|
2020-03-06 14:41:03 -05:00
|
|
|
TEST_TITLE("compressTypeFromName()");
|
2019-12-11 08:48:46 -05:00
|
|
|
|
2020-03-06 14:41:03 -05:00
|
|
|
TEST_RESULT_UINT(compressTypeFromName(STRDEF("file")), compressTypeNone, "type from name");
|
|
|
|
TEST_RESULT_UINT(compressTypeFromName(STRDEF("file.gz")), compressTypeGz, "type from name");
|
2019-12-11 08:48:46 -05:00
|
|
|
|
|
|
|
// -------------------------------------------------------------------------------------------------------------------------
|
2020-03-06 14:41:03 -05:00
|
|
|
TEST_TITLE("compressFilterVar()");
|
2019-12-11 08:48:46 -05:00
|
|
|
|
2020-03-06 14:41:03 -05:00
|
|
|
TEST_RESULT_PTR(compressFilterVar(STRDEF("BOGUS"), 0), NULL, "no filter match");
|
2019-12-11 08:48:46 -05:00
|
|
|
|
2020-03-06 14:41:03 -05:00
|
|
|
// -------------------------------------------------------------------------------------------------------------------------
|
|
|
|
TEST_TITLE("compressExtStr()");
|
|
|
|
|
|
|
|
TEST_RESULT_STR_Z(compressExtStr(compressTypeNone), "", "one ext");
|
|
|
|
TEST_RESULT_STR_Z(compressExtStr(compressTypeGz), ".gz", "gz ext");
|
2019-12-11 08:48:46 -05:00
|
|
|
|
2018-08-14 14:56:59 -04:00
|
|
|
// -------------------------------------------------------------------------------------------------------------------------
|
2020-03-06 14:41:03 -05:00
|
|
|
TEST_TITLE("compressExtCat()");
|
2018-08-14 14:56:59 -04:00
|
|
|
|
2020-03-06 14:41:03 -05:00
|
|
|
String *file = strNew("file");
|
|
|
|
TEST_RESULT_VOID(compressExtCat(file, compressTypeGz), "cat gz ext");
|
|
|
|
TEST_RESULT_STR_Z(file, "file.gz", " check gz ext");
|
2018-08-14 14:56:59 -04:00
|
|
|
|
2020-03-06 14:41:03 -05:00
|
|
|
// -------------------------------------------------------------------------------------------------------------------------
|
|
|
|
TEST_TITLE("compressExtStrip()");
|
2018-08-14 14:56:59 -04:00
|
|
|
|
2020-03-06 14:41:03 -05:00
|
|
|
TEST_ERROR(compressExtStrip(STRDEF("file"), compressTypeGz), FormatError, "'file' must have '.gz' extension");
|
|
|
|
TEST_RESULT_STR_Z(compressExtStrip(STRDEF("file"), compressTypeNone), "file", "nothing to strip");
|
|
|
|
TEST_RESULT_STR_Z(compressExtStrip(STRDEF("file.gz"), compressTypeGz), "file", "strip gz");
|
2018-08-14 14:56:59 -04:00
|
|
|
|
2020-03-06 14:41:03 -05:00
|
|
|
// -------------------------------------------------------------------------------------------------------------------------
|
|
|
|
TEST_TITLE("compressLevelDefault()");
|
2018-08-14 14:56:59 -04:00
|
|
|
|
2020-03-06 14:41:03 -05:00
|
|
|
TEST_RESULT_INT(compressLevelDefault(compressTypeNone), 0, "none level=0");
|
|
|
|
TEST_RESULT_INT(compressLevelDefault(compressTypeGz), 6, "gz level=6");
|
2018-08-14 14:56:59 -04:00
|
|
|
}
|
|
|
|
|
|
|
|
FUNCTION_HARNESS_RESULT_VOID();
|
|
|
|
}
|