db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2025-03-03 14:52:21 +02:00
Code Issues Releases Activity

Add gzip compression/decompression filters for C.

Browse Source
This commit is contained in:
David Steele 2018-08-14 14:56:59 -04:00
parent e3ff6b209d
commit 6643afe9a8
13 changed files with 861 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
doc/xml/release.xml
View File

@ -57,6 +57,10 @@
<p>Validate configuration options in a single pass. By pre-calculating and storing the option dependencies in <file>parse.auto.c</file> validation can be completed in a single pass, which is both simpler and faster.</p>
</release-item>
<release-item>
<p>Add gzip compression/decompression filters for C.</p>
</release-item>
<release-item>
<release-item-contributor-list>
<release-item-ideator id="stephen.frost"/>

3
libc/Makefile.PL
View File

@ -73,6 +73,9 @@ my @stryCFile =
'common/type/variant.c',
'common/type/variantList.c',
'common/wait.c',
'compress/gzip.c',
'compress/gzipCompress.c',
'compress/gzipDecompress.c',
'config/config.c',
'config/define.c',
'config/load.c',

14
src/Makefile
View File

@ -42,7 +42,7 @@ LDPERL = `perl -MExtUtils::Embed -e ldopts`
LDEXTRA =
# Concatenate options for easy usage
LDFLAGS = -lcrypto $(LDPERL) $(LDEXTRA)
LDFLAGS = -lcrypto -lz $(LDPERL) $(LDEXTRA)
####################################################################################################################################
# Install options
@ -91,6 +91,9 @@ SRCS = \
common/type/variant.c \
common/type/variantList.c \
common/wait.c \
compress/gzip.c \
compress/gzipCompress.c \
compress/gzipDecompress.c \
config/config.c \
config/define.c \
config/load.c \
@ -246,6 +249,15 @@ common/type/variantList.o: common/type/variantList.c common/debug.h common/error
common/wait.o: common/wait.c common/debug.h common/error.auto.h common/error.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/convert.h common/wait.h
$(CC) $(CFLAGS) -c common/wait.c -o common/wait.o
compress/gzip.o: compress/gzip.c common/debug.h common/error.auto.h common/error.h common/logLevel.h common/memContext.h common/stackTrace.h common/type/convert.h compress/gzip.h
$(CC) $(CFLAGS) -c compress/gzip.c -o compress/gzip.o
compress/gzipCompress.o: compress/gzipCompress.c common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/variant.h common/type/variantList.h compress/gzip.h compress/gzipCompress.h
$(CC) $(CFLAGS) -c compress/gzipCompress.c -o compress/gzipCompress.o
compress/gzipDecompress.o: compress/gzipDecompress.c common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/variant.h common/type/variantList.h compress/gzip.h compress/gzipDecompress.h
$(CC) $(CFLAGS) -c compress/gzipDecompress.c -o compress/gzipDecompress.o
config/config.o: config/config.c common/assert.h common/debug.h common/error.auto.h common/error.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.c config/config.auto.h config/config.h config/define.auto.h config/define.h
$(CC) $(CFLAGS) -c config/config.c -o config/config.o

102
src/compress/gzip.c Normal file
View File

@ -0,0 +1,102 @@
/***********************************************************************************************************************************
Gzip Common
***********************************************************************************************************************************/
#include <zlib.h>
#include "common/debug.h"
#include "common/memContext.h"
#include "compress/gzip.h"
/***********************************************************************************************************************************
Constants
***********************************************************************************************************************************/
#define WINDOW_BITS 15
#define WANT_GZIP 16
/***********************************************************************************************************************************
Process gzip errors
***********************************************************************************************************************************/
int
gzipError(int error)
{
if (error != Z_OK && error != Z_STREAM_END)
{
const char *errorMsg;
const ErrorType *errorType = &FormatError;
switch (error)
{
// Not exactly an error, but since we are not using custom dictionaries it shouldn't be possible to get this result
case Z_NEED_DICT:
{
errorMsg = "need dictionary";
errorType = &AssertError;
break;
}
// We should not get this error -- included for completeness
case Z_ERRNO:
{
errorMsg = "file error";
errorType = &AssertError;
break;
}
case Z_STREAM_ERROR:
{
errorMsg = "stream error";
break;
}
case Z_DATA_ERROR:
{
errorMsg = "data error";
break;
}
case Z_MEM_ERROR:
{
errorMsg = "insufficient memory";
errorType = &MemoryError;
break;
}
// This error indicates an error in the code -- there should always be space in the buffer
case Z_BUF_ERROR:
{
errorMsg = "no space in buffer";
errorType = &AssertError;
break;
}
case Z_VERSION_ERROR:
{
errorMsg = "incompatible version";
break;
}
default:
{
errorMsg = "unknown error";
errorType = &AssertError;
}
}
THROWP_FMT(errorType, "zlib threw error: [%d] %s", error, errorMsg);
}
return error;
}
/***********************************************************************************************************************************
Get gzip window bits
Window bits define how large the compression window is. Larger window sizes generally result in better compression so we'll always
use the largest size. When raw is specified disable the gzip header and produce raw compressed output (this is indicated by passing
negative window bits).
***********************************************************************************************************************************/
int
gzipWindowBits(bool raw)
{
return raw ? -WINDOW_BITS : WANT_GZIP | WINDOW_BITS;
}

13
src/compress/gzip.h Normal file
View File

@ -0,0 +1,13 @@
/***********************************************************************************************************************************
Gzip Common
***********************************************************************************************************************************/
#ifndef COMPRESS_GZIP_H
#define COMPRESS_GZIP_H
/***********************************************************************************************************************************
Functions
***********************************************************************************************************************************/
int gzipError(int error);
int gzipWindowBits(bool raw);
#endif

227
src/compress/gzipCompress.c Normal file
View File

@ -0,0 +1,227 @@
/***********************************************************************************************************************************
Gzip Compress
***********************************************************************************************************************************/
#include <stdio.h>
#include <zlib.h>
#include "common/assert.h"
#include "common/debug.h"
#include "common/log.h"
#include "common/memContext.h"
#include "compress/gzip.h"
#include "compress/gzipCompress.h"
/***********************************************************************************************************************************
Filter type constant
***********************************************************************************************************************************/
#define GZIP_COMPRESS_FILTER_TYPE "gzipCompress"
/***********************************************************************************************************************************
Object type
***********************************************************************************************************************************/
struct GzipCompress
{
MemContext *memContext; // Context to store data
z_stream *stream; // Compression stream state
IoFilter *filter; // Filter interface
bool inputSame; // Is the same input required on the next process call?
bool flush; // Is input complete and flushing in progress?
bool done; // Is compression done?
};
/***********************************************************************************************************************************
Compression constants
***********************************************************************************************************************************/
#define MEM_LEVEL 9
/***********************************************************************************************************************************
New object
***********************************************************************************************************************************/
GzipCompress *
gzipCompressNew(int level, bool raw)
{
FUNCTION_DEBUG_BEGIN(logLevelTrace);
FUNCTION_DEBUG_PARAM(INT, level);
FUNCTION_DEBUG_PARAM(BOOL, raw);
FUNCTION_DEBUG_ASSERT(level >= -1 && level <= 9);
FUNCTION_DEBUG_END();
GzipCompress *this = NULL;
MEM_CONTEXT_NEW_BEGIN("GzipCompress")
{
// Allocate state and set context
this = memNew(sizeof(GzipCompress));
this->memContext = MEM_CONTEXT_NEW();
// Create gzip stream
this->stream = memNew(sizeof(z_stream));
gzipError(deflateInit2(this->stream, level, Z_DEFLATED, gzipWindowBits(raw), MEM_LEVEL, Z_DEFAULT_STRATEGY));
// Set free callback to ensure gzip context is freed
memContextCallback(this->memContext, (MemContextCallback)gzipCompressFree, this);
// Create filter interface
this->filter = ioFilterNew(
strNew(GZIP_COMPRESS_FILTER_TYPE), this, (IoFilterDone)gzipCompressDone, (IoFilterInputSame)gzipCompressInputSame,
NULL, (IoFilterProcessInOut)gzipCompressProcess, NULL);
}
MEM_CONTEXT_NEW_END();
FUNCTION_DEBUG_RESULT(GZIP_COMPRESS, this);
}
/***********************************************************************************************************************************
Compress data
***********************************************************************************************************************************/
void
gzipCompressProcess(GzipCompress *this, const Buffer *uncompressed, Buffer *compressed)
{
FUNCTION_DEBUG_BEGIN(logLevelTrace);
FUNCTION_DEBUG_PARAM(GZIP_COMPRESS, this);
FUNCTION_DEBUG_PARAM(BUFFER, uncompressed);
FUNCTION_DEBUG_PARAM(BUFFER, compressed);
FUNCTION_DEBUG_ASSERT(this != NULL);
FUNCTION_DEBUG_ASSERT(!this->done);
FUNCTION_DEBUG_ASSERT(this->stream != NULL);
FUNCTION_DEBUG_ASSERT(compressed != NULL);
FUNCTION_DEBUG_ASSERT(!this->flush || uncompressed == NULL);
FUNCTION_DEBUG_ASSERT(this->flush || (!this->inputSame || this->stream->avail_in != 0));
FUNCTION_DEBUG_END();
// Flushing
if (uncompressed == NULL)
{
this->stream->avail_in = 0;
this->flush = true;
}
// More input
else
{
// Is new input allowed?
if (!this->inputSame)
{
this->stream->avail_in = (unsigned int)bufUsed(uncompressed);
this->stream->next_in = bufPtr(uncompressed);
}
}
// Initialize compressed output buffer
this->stream->avail_out = (unsigned int)bufRemains(compressed);
this->stream->next_out = bufPtr(compressed) + bufUsed(compressed);
// Perform compression
gzipError(deflate(this->stream, this->flush ? Z_FINISH : Z_NO_FLUSH));
// Set buffer used space
bufUsedSet(compressed, bufSize(compressed) - (size_t)this->stream->avail_out);
// Is compression done?
if (this->flush && this->stream->avail_out > 0)
this->done = true;
// Can more input be provided on the next call?
this->inputSame = this->flush ? !this->done : this->stream->avail_in != 0;
FUNCTION_DEBUG_RESULT_VOID();
}
/***********************************************************************************************************************************
Is compress done?
***********************************************************************************************************************************/
bool
gzipCompressDone(const GzipCompress *this)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(GZIP_COMPRESS, this);
FUNCTION_TEST_ASSERT(this != NULL);
FUNCTION_TEST_END();
FUNCTION_TEST_RESULT(BOOL, this->done);
}
/***********************************************************************************************************************************
Get filter interface
***********************************************************************************************************************************/
IoFilter *
gzipCompressFilter(const GzipCompress *this)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(GZIP_COMPRESS, this);
FUNCTION_TEST_ASSERT(this != NULL);
FUNCTION_TEST_END();
FUNCTION_TEST_RESULT(IO_FILTER, this->filter);
}
/***********************************************************************************************************************************
Is the same input required on the next process call?
***********************************************************************************************************************************/
bool
gzipCompressInputSame(const GzipCompress *this)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(GZIP_COMPRESS, this);
FUNCTION_TEST_ASSERT(this != NULL);
FUNCTION_TEST_END();
FUNCTION_TEST_RESULT(BOOL, this->inputSame);
}
/***********************************************************************************************************************************
Convert to a zero-terminated string for logging
***********************************************************************************************************************************/
size_t
gzipCompressToLog(const GzipCompress *this, char *buffer, size_t bufferSize)
{
size_t result = 0;
MEM_CONTEXT_TEMP_BEGIN()
{
String *string = NULL;
if (this == NULL)
string = strNew("null");
else
{
string = strNewFmt(
"{inputSame: %s, done: %s, flushing: %s, availIn: %u}", cvtBoolToConstZ(this->inputSame),
cvtBoolToConstZ(this->done), cvtBoolToConstZ(this->done), this->stream != NULL ? this->stream->avail_in : 0);
}
result = (size_t)snprintf(buffer, bufferSize, "%s", strPtr(string));
}
MEM_CONTEXT_TEMP_END();
return result;
}
/***********************************************************************************************************************************
Free memory
***********************************************************************************************************************************/
void
gzipCompressFree(GzipCompress *this)
{
FUNCTION_DEBUG_BEGIN(logLevelTrace);
FUNCTION_DEBUG_PARAM(GZIP_COMPRESS, this);
FUNCTION_DEBUG_END();
if (this != NULL)
{
if (this->stream != NULL)
{
deflateEnd(this->stream);
this->stream = NULL;
}
memContextFree(this->memContext);
}
FUNCTION_DEBUG_RESULT_VOID();
}

49
src/compress/gzipCompress.h Normal file
View File

@ -0,0 +1,49 @@
/***********************************************************************************************************************************
Gzip Compress
Compress IO using the gzip format.
***********************************************************************************************************************************/
#ifndef COMPRESS_GZIPCOMPRESS_H
#define COMPRESS_GZIPCOMPRESS_H
/***********************************************************************************************************************************
Object type
***********************************************************************************************************************************/
typedef struct GzipCompress GzipCompress;
#include "common/io/filter/filter.h"
#include "common/type/buffer.h"
/***********************************************************************************************************************************
Constructor
***********************************************************************************************************************************/
GzipCompress *gzipCompressNew(int level, bool raw);
/***********************************************************************************************************************************
Functions
***********************************************************************************************************************************/
void gzipCompressProcess(GzipCompress *this, const Buffer *uncompressed, Buffer *compressed);
/***********************************************************************************************************************************
Getters
***********************************************************************************************************************************/
bool gzipCompressDone(const GzipCompress *this);
IoFilter *gzipCompressFilter(const GzipCompress *this);
bool gzipCompressInputSame(const GzipCompress *this);
/***********************************************************************************************************************************
Destructor
***********************************************************************************************************************************/
void gzipCompressFree(GzipCompress *this);
/***********************************************************************************************************************************
Macros for function logging
***********************************************************************************************************************************/
size_t gzipCompressToLog(const GzipCompress *this, char *buffer, size_t bufferSize);
#define FUNCTION_DEBUG_GZIP_COMPRESS_TYPE \
GzipCompress *
#define FUNCTION_DEBUG_GZIP_COMPRESS_FORMAT(value, buffer, bufferSize) \
gzipCompressToLog(value, buffer, bufferSize)
#endif

202
src/compress/gzipDecompress.c Normal file
View File

@ -0,0 +1,202 @@
/***********************************************************************************************************************************
Gzip Decompress
***********************************************************************************************************************************/
#include <stdio.h>
#include <zlib.h>
#include "common/debug.h"
#include "common/log.h"
#include "common/memContext.h"
#include "compress/gzip.h"
#include "compress/gzipDecompress.h"
/***********************************************************************************************************************************
Filter type constant
***********************************************************************************************************************************/
#define GZIP_DECOMPRESS_FILTER_TYPE "gzipDecompress"
/***********************************************************************************************************************************
Object type
***********************************************************************************************************************************/
struct GzipDecompress
{
MemContext *memContext; // Context to store data
z_stream *stream; // Decompression stream state
IoFilter *filter; // Filter interface
int result; // Result of last operation
bool inputSame; // Is the same input required on the next process call?
bool done; // Is decompression done?
};
/***********************************************************************************************************************************
New object
***********************************************************************************************************************************/
GzipDecompress *
gzipDecompressNew(bool raw)
{
FUNCTION_DEBUG_BEGIN(logLevelTrace);
FUNCTION_DEBUG_PARAM(BOOL, raw);
FUNCTION_DEBUG_END();
GzipDecompress *this = NULL;
MEM_CONTEXT_NEW_BEGIN("GzipDecompress")
{
// Allocate state and set context
this = memNew(sizeof(GzipDecompress));
this->memContext = MEM_CONTEXT_NEW();
// Create gzip stream
this->stream = memNew(sizeof(z_stream));
gzipError(this->result = inflateInit2(this->stream, gzipWindowBits(raw)));
// Set free callback to ensure gzip context is freed
memContextCallback(this->memContext, (MemContextCallback)gzipDecompressFree, this);
// Create filter interface
this->filter = ioFilterNew(
strNew(GZIP_DECOMPRESS_FILTER_TYPE), this, (IoFilterDone)gzipDecompressDone, (IoFilterInputSame)gzipDecompressInputSame,
NULL, (IoFilterProcessInOut)gzipDecompressProcess, NULL);
}
MEM_CONTEXT_NEW_END();
FUNCTION_DEBUG_RESULT(GZIP_DECOMPRESS, this);
}
/***********************************************************************************************************************************
Decompress data
***********************************************************************************************************************************/
void
gzipDecompressProcess(GzipDecompress *this, const Buffer *compressed, Buffer *uncompressed)
{
FUNCTION_DEBUG_BEGIN(logLevelTrace);
FUNCTION_DEBUG_PARAM(GZIP_DECOMPRESS, this);
FUNCTION_DEBUG_PARAM(BUFFER, compressed);
FUNCTION_DEBUG_PARAM(BUFFER, uncompressed);
FUNCTION_DEBUG_ASSERT(this != NULL);
FUNCTION_DEBUG_ASSERT(this->stream != NULL);
FUNCTION_DEBUG_ASSERT(compressed != NULL);
FUNCTION_DEBUG_ASSERT(uncompressed != NULL);
FUNCTION_DEBUG_END();
if (!this->inputSame)
{
this->stream->avail_in = (unsigned int)bufUsed(compressed);
this->stream->next_in = bufPtr(compressed);
}
this->stream->avail_out = (unsigned int)bufRemains(uncompressed);
this->stream->next_out = bufPtr(uncompressed) + bufUsed(uncompressed);
this->result = gzipError(inflate(this->stream, Z_NO_FLUSH));
// Set buffer used space
bufUsedSet(uncompressed, bufSize(uncompressed) - (size_t)this->stream->avail_out);
// Is decompression done?
this->done = this->result == Z_STREAM_END;
// Is the same input expected on the next call?
this->inputSame = this->done ? false : this->stream->avail_in != 0;
FUNCTION_DEBUG_RESULT_VOID();
}
/***********************************************************************************************************************************
Is decompress done?
***********************************************************************************************************************************/
bool
gzipDecompressDone(const GzipDecompress *this)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(GZIP_DECOMPRESS, this);
FUNCTION_TEST_ASSERT(this != NULL);
FUNCTION_TEST_END();
FUNCTION_TEST_RESULT(BOOL, this->done);
}
/***********************************************************************************************************************************
Get filter interface
***********************************************************************************************************************************/
IoFilter *
gzipDecompressFilter(const GzipDecompress *this)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(GZIP_DECOMPRESS, this);
FUNCTION_TEST_ASSERT(this != NULL);
FUNCTION_TEST_END();
FUNCTION_TEST_RESULT(IO_FILTER, this->filter);
}
/***********************************************************************************************************************************
Is the same input required on the next process call?
***********************************************************************************************************************************/
bool
gzipDecompressInputSame(const GzipDecompress *this)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(GZIP_DECOMPRESS, this);
FUNCTION_TEST_ASSERT(this != NULL);
FUNCTION_TEST_END();
FUNCTION_TEST_RESULT(BOOL, this->inputSame);
}
/***********************************************************************************************************************************
Convert to a zero-terminated string for logging
***********************************************************************************************************************************/
size_t
gzipDecompressToLog(const GzipDecompress *this, char *buffer, size_t bufferSize)
{
size_t result = 0;
MEM_CONTEXT_TEMP_BEGIN()
{
String *string = NULL;
if (this == NULL)
string = strNew("null");
else
{
string = strNewFmt(
"{inputSame: %s, done: %s, availIn: %u}", cvtBoolToConstZ(this->inputSame), cvtBoolToConstZ(this->done),
this->stream != NULL ? this->stream->avail_in : 0);
}
result = (size_t)snprintf(buffer, bufferSize, "%s", strPtr(string));
}
MEM_CONTEXT_TEMP_END();
return result;
}
/***********************************************************************************************************************************
Free memory
***********************************************************************************************************************************/
void
gzipDecompressFree(GzipDecompress *this)
{
FUNCTION_DEBUG_BEGIN(logLevelTrace);
FUNCTION_DEBUG_PARAM(GZIP_DECOMPRESS, this);
FUNCTION_DEBUG_END();
if (this != NULL)
{
if (this->stream != NULL)
{
inflateEnd(this->stream);
this->stream = NULL;
}
memContextFree(this->memContext);
}
FUNCTION_DEBUG_RESULT_VOID();
}

49
src/compress/gzipDecompress.h Normal file
View File

@ -0,0 +1,49 @@
/***********************************************************************************************************************************
Gzip Decompress
Decompress IO from the gzip format.
***********************************************************************************************************************************/
#ifndef COMPRESS_GZIPDECOMPRESS_H
#define COMPRESS_GZIPDECOMPRESS_H
/***********************************************************************************************************************************
Object type
***********************************************************************************************************************************/
typedef struct GzipDecompress GzipDecompress;
#include "common/io/filter/filter.h"
#include "common/type/string.h"
/***********************************************************************************************************************************
Constructor
***********************************************************************************************************************************/
GzipDecompress *gzipDecompressNew(bool raw);
/***********************************************************************************************************************************
Functions
***********************************************************************************************************************************/
void gzipDecompressProcess(GzipDecompress *this, const Buffer *compressed, Buffer *uncompressed);
/***********************************************************************************************************************************
Getters
***********************************************************************************************************************************/
bool gzipDecompressDone(const GzipDecompress *this);
IoFilter *gzipDecompressFilter(const GzipDecompress *this);
bool gzipDecompressInputSame(const GzipDecompress *this);
/***********************************************************************************************************************************
Destructor
***********************************************************************************************************************************/
void gzipDecompressFree(GzipDecompress *this);
/***********************************************************************************************************************************
Macros for function logging
***********************************************************************************************************************************/
size_t gzipDecompressToLog(const GzipDecompress *this, char *buffer, size_t bufferSize);
#define FUNCTION_DEBUG_GZIP_DECOMPRESS_TYPE \
GzipDecompress *
#define FUNCTION_DEBUG_GZIP_DECOMPRESS_FORMAT(value, buffer, bufferSize) \
gzipDecompressToLog(value, buffer, bufferSize)
#endif

2
test/Vagrantfile vendored
View File

@ -53,7 +53,7 @@ Vagrant.configure(2) do |config|
#---------------------------------------------------------------------------------------------------------------------------
echo 'Install Build Tools' && date
apt-get install -y devscripts build-essential lintian git txt2man debhelper libssl-dev lcov cloc
apt-get install -y devscripts build-essential lintian git txt2man debhelper libssl-dev zlib1g-dev lcov cloc
#---------------------------------------------------------------------------------------------------------------------------
echo 'Install AWS CLI' && date

13
test/define.yaml
View File

@ -315,6 +315,19 @@ unit:
coverage:
crypto/cipherBlock: full
# ********************************************************************************************************************************
- name: compress
test:
# ----------------------------------------------------------------------------------------------------------------------------
- name: gzip
total: 4
coverage:
compress/gzip: full
compress/gzipCompress: full
compress/gzipDecompress: full
# ********************************************************************************************************************************
- name: postgres

2
test/lib/pgBackRestTest/Common/JobTest.pm
View File

@ -373,7 +373,7 @@ sub run
# " -Wpedantic \\\n" : '') .
" -Wformat=2 -Wformat-nonliteral -Wstrict-prototypes -Wpointer-arith \\\n" .
" `perl -MExtUtils::Embed -e ccopts`\n" .
"LDFLAGS=-lcrypto" . (vmCoverage($self->{oTest}->{&TEST_VM}) && $self->{bCoverageUnit} ? " -lgcov" : '') .
"LDFLAGS=-lcrypto -lz" . (vmCoverage($self->{oTest}->{&TEST_VM}) && $self->{bCoverageUnit} ? " -lgcov" : '') .
(vmWithBackTrace($self->{oTest}->{&TEST_VM}) && $self->{bBackTrace} ? ' -lbacktrace' : '') .
" `perl -MExtUtils::Embed -e ldopts`\n" .
'TESTFLAGS=' . ($self->{oTest}->{&TEST_DEBUG_UNIT_SUPPRESS} ? '' : "-DDEBUG_UNIT") .

184
test/src/module/compress/gzipTest.c Normal file
View File

@ -0,0 +1,184 @@
/***********************************************************************************************************************************
Test Gzip
***********************************************************************************************************************************/
#include "common/io/filter/group.h"
#include "common/io/bufferRead.h"
#include "common/io/bufferWrite.h"
#include "common/io/io.h"
/***********************************************************************************************************************************
Compress data
***********************************************************************************************************************************/
static Buffer *
testCompress(GzipCompress *compress, Buffer *decompressed, size_t inputSize, size_t outputSize)
{
Buffer *compressed = bufNew(1024 * 1024);
size_t inputTotal = 0;
ioBufferSizeSet(outputSize);
IoFilterGroup *filterGroup = ioFilterGroupNew();
ioFilterGroupAdd(filterGroup, gzipCompressFilter(compress));
IoWrite *write = ioBufferWriteIo(ioBufferWriteNew(compressed));
ioWriteFilterGroupSet(write, filterGroup);
ioWriteOpen(write);
// Compress input data
while (inputTotal < bufSize(decompressed))
{
// Generate the input buffer based on input size. This breaks the data up into chunks as it would be in a real scenario.
Buffer *input = bufNewC(
inputSize > bufSize(decompressed) - inputTotal ? bufSize(decompressed) - inputTotal : inputSize,
bufPtr(decompressed) + inputTotal);
ioWrite(write, input);
inputTotal += bufUsed(input);
bufFree(input);
}
ioWriteClose(write);
gzipCompressFree(compress);
return compressed;
}
/***********************************************************************************************************************************
Decompress data
***********************************************************************************************************************************/
static Buffer *
testDecompress(GzipDecompress *decompress, Buffer *compressed, size_t inputSize, size_t outputSize)
{
Buffer *decompressed = bufNew(1024 * 1024);
Buffer *output = bufNew(outputSize);
ioBufferSizeSet(inputSize);
IoFilterGroup *filterGroup = ioFilterGroupNew();
ioFilterGroupAdd(filterGroup, gzipDecompressFilter(decompress));
IoRead *read = ioBufferReadIo(ioBufferReadNew(compressed));
ioReadFilterGroupSet(read, filterGroup);
ioReadOpen(read);
while (!ioReadEof(read))
{
ioRead(read, output);
bufCat(decompressed, output);
bufUsedZero(output);
}
ioReadClose(read);
bufFree(output);
gzipDecompressFree(decompress);
return decompressed;
}
/***********************************************************************************************************************************
Test Run
***********************************************************************************************************************************/
void
testRun(void)
{
FUNCTION_HARNESS_VOID();
// *****************************************************************************************************************************
if (testBegin("gzipError"))
{
TEST_RESULT_INT(gzipError(Z_OK), Z_OK, "check ok");
TEST_RESULT_INT(gzipError(Z_STREAM_END), Z_STREAM_END, "check stream end");
TEST_ERROR(gzipError(Z_NEED_DICT), AssertError, "zlib threw error: [2] need dictionary");
TEST_ERROR(gzipError(Z_ERRNO), AssertError, "zlib threw error: [-1] file error");
TEST_ERROR(gzipError(Z_STREAM_ERROR), FormatError, "zlib threw error: [-2] stream error");
TEST_ERROR(gzipError(Z_DATA_ERROR), FormatError, "zlib threw error: [-3] data error");
TEST_ERROR(gzipError(Z_MEM_ERROR), MemoryError, "zlib threw error: [-4] insufficient memory");
TEST_ERROR(gzipError(Z_BUF_ERROR), AssertError, "zlib threw error: [-5] no space in buffer");
TEST_ERROR(gzipError(Z_VERSION_ERROR), FormatError, "zlib threw error: [-6] incompatible version");
TEST_ERROR(gzipError(999), AssertError, "zlib threw error: [999] unknown error");
}
// *****************************************************************************************************************************
if (testBegin("gzipWindowBits"))
{
TEST_RESULT_INT(gzipWindowBits(true), -15, "raw window bits");
TEST_RESULT_INT(gzipWindowBits(false), 31, "gzip window bits");
}
// *****************************************************************************************************************************
if (testBegin("GzipCompress and GzipDecompress"))
{
const char *simpleData = "A simple string";
Buffer *compressed = NULL;
Buffer *decompressed = bufNewC(strlen(simpleData), simpleData);
TEST_ASSIGN(
compressed, testCompress(gzipCompressNew(3, false), decompressed, 1024, 1024),
"simple data - compress large in/large out buffer");
TEST_RESULT_BOOL(
bufEq(compressed, testCompress(gzipCompressNew(3, false), decompressed, 1024, 1)), true,
"simple data - compress large in/small out buffer");
TEST_RESULT_BOOL(
bufEq(compressed, testCompress(gzipCompressNew(3, false), decompressed, 1, 1024)), true,
"simple data - compress small in/large out buffer");
TEST_RESULT_BOOL(
bufEq(compressed, testCompress(gzipCompressNew(3, false), decompressed, 1, 1)), true,
"simple data - compress small in/small out buffer");
TEST_RESULT_BOOL(
bufEq(decompressed, testDecompress(gzipDecompressNew(false), compressed, 1024, 1024)), true,
"simple data - decompress large in/large out buffer");
TEST_RESULT_BOOL(
bufEq(decompressed, testDecompress(gzipDecompressNew(false), compressed, 1024, 1)), true,
"simple data - decompress large in/small out buffer");
TEST_RESULT_BOOL(
bufEq(decompressed, testDecompress(gzipDecompressNew(false), compressed, 1, 1024)), true,
"simple data - decompress small in/large out buffer");
TEST_RESULT_BOOL(
bufEq(decompressed, testDecompress(gzipDecompressNew(false), compressed, 1, 1)), true,
"simple data - decompress small in/small out buffer");
// Compress a large zero input buffer into small output buffer
// -------------------------------------------------------------------------------------------------------------------------
decompressed = bufNew(1024 * 1024 - 1);
memset(bufPtr(decompressed), 0, bufSize(decompressed));
bufUsedSet(decompressed, bufSize(decompressed));
TEST_ASSIGN(
compressed, testCompress(gzipCompressNew(3, true), decompressed, bufSize(decompressed), 1024),
"zero data - compress large in/small out buffer");
TEST_RESULT_BOOL(
bufEq(decompressed, testDecompress(gzipDecompressNew(true), compressed, bufSize(compressed), 1024 * 256)), true,
"zero data - decompress large in/small out buffer");
// -------------------------------------------------------------------------------------------------------------------------
TEST_RESULT_VOID(gzipCompressFree(NULL), "free null decompress object");
TEST_RESULT_VOID(gzipDecompressFree(NULL), "free null decompress object");
}
// *****************************************************************************************************************************
if (testBegin("gzipDecompressToLog() and gzipCompressToLog()"))
{
char buffer[STACK_TRACE_PARAM_MAX];
GzipDecompress *decompress = gzipDecompressNew(false);
TEST_RESULT_INT(gzipDecompressToLog(NULL, buffer, 4), 4, "format object with too small buffer");
TEST_RESULT_STR(buffer, "nul", " check format");
TEST_RESULT_INT(gzipDecompressToLog(decompress, buffer, STACK_TRACE_PARAM_MAX), 43, "format object");
TEST_RESULT_STR(buffer, "{inputSame: false, done: false, availIn: 0}", " check format");
decompress->inputSame = true;
decompress->done = true;
decompress->stream = NULL;
TEST_RESULT_INT(gzipDecompressToLog(decompress, buffer, STACK_TRACE_PARAM_MAX), 41, "format object");
TEST_RESULT_STR(buffer, "{inputSame: true, done: true, availIn: 0}", " check format");
}
FUNCTION_HARNESS_RESULT_VOID();
}