1
0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2025-07-15 01:04:37 +02:00

Add bzip2 compression support.

bzip2 is a widely available, high-quality data compressor. It typically compresses files to within 10% to 15% of the best available techniques (the PPM family of statistical compressors), while being around twice as fast at compression and six times faster at decompression.

bzip2 is currently available on all supported platforms.
This commit is contained in:
Stephen Frost
2020-05-05 16:49:01 -04:00
committed by GitHub
parent 98f30ef222
commit a021c9fe05
31 changed files with 720 additions and 40 deletions

View File

@ -1234,6 +1234,7 @@ my %hConfigDefine =
&CFGDEF_ALLOW_LIST => &CFGDEF_ALLOW_LIST =>
[ [
'none', 'none',
'bz2',
'gz', 'gz',
'lz4', 'lz4',
'zst', 'zst',

View File

@ -64,7 +64,7 @@
apt-get install rsync git devscripts build-essential valgrind lcov autoconf apt-get install rsync git devscripts build-essential valgrind lcov autoconf
autoconf-archive libssl-dev zlib1g-dev libxml2-dev libpq-dev pkg-config autoconf-archive libssl-dev zlib1g-dev libxml2-dev libpq-dev pkg-config
libxml-checker-perl libyaml-libyaml-perl libdbd-pg-perl liblz4-dev liblz4-tool libxml-checker-perl libyaml-libyaml-perl libdbd-pg-perl liblz4-dev liblz4-tool
zstd libzstd-dev zstd libzstd-dev bzip2 libbz2-dev
</exe-cmd> </exe-cmd>
<exe-cmd-extra>-y 2>&amp;1</exe-cmd-extra> <exe-cmd-extra>-y 2>&amp;1</exe-cmd-extra>
</execute> </execute>

View File

@ -141,6 +141,7 @@
<text>The following compression types are supported: <text>The following compression types are supported:
<ul> <ul>
<li><id>bz2</id> - bzip2 compression format</li>
<li><id>gz</id> - gzip compression format</li> <li><id>gz</id> - gzip compression format</li>
<li><id>lz4</id> - lz4 compression format (not available on all platforms)</li> <li><id>lz4</id> - lz4 compression format (not available on all platforms)</li>
<li><id>zst</id> - Zstandard compression format (not available on all platforms)</li> <li><id>zst</id> - Zstandard compression format (not available on all platforms)</li>

View File

@ -63,6 +63,16 @@
<p>Note that setting <br-option>compress-type=zst</br-option> will make new backups and archive incompatible (unrestorable) with prior versions of <backrest/>.</p> <p>Note that setting <br-option>compress-type=zst</br-option> will make new backups and archive incompatible (unrestorable) with prior versions of <backrest/>.</p>
</release-item> </release-item>
<release-item>
<release-item-contributor-list>
<release-item-contributor id="stephen.frost"/>
</release-item-contributor-list>
<p>Add <proper>bzip2</proper> compression support.</p>
<p>Note that setting <br-option>compress-type=bz2</br-option> will make new backups and archive incompatible (unrestorable) with prior versions of <backrest/>.</p>
</release-item>
<release-item> <release-item>
<release-item-contributor-list> <release-item-contributor-list>
<release-item-contributor id="stefan.fercot"/> <release-item-contributor id="stefan.fercot"/>

View File

@ -767,7 +767,7 @@
<execute if="{[os-type-is-debian]}" user="root" pre="y"> <execute if="{[os-type-is-debian]}" user="root" pre="y">
<exe-cmd> <exe-cmd>
apt-get install make gcc libpq-dev libssl-dev libxml2-dev pkg-config apt-get install make gcc libpq-dev libssl-dev libxml2-dev pkg-config
liblz4-dev libzstd-dev liblz4-dev libzstd-dev libbz2-dev
</exe-cmd> </exe-cmd>
<exe-cmd-extra>-y 2>&amp;1</exe-cmd-extra> <exe-cmd-extra>-y 2>&amp;1</exe-cmd-extra>
</execute> </execute>
@ -775,7 +775,7 @@
<execute if="{[os-type-is-centos]}" user="root" pre="y"> <execute if="{[os-type-is-centos]}" user="root" pre="y">
<exe-cmd> <exe-cmd>
yum install make gcc postgresql-devel openssl-devel libxml2-devel yum install make gcc postgresql-devel openssl-devel libxml2-devel
lz4-devel libzstd-devel lz4-devel libzstd-devel bzip2-devel
</exe-cmd> </exe-cmd>
<exe-cmd-extra>-y 2>&amp;1</exe-cmd-extra> <exe-cmd-extra>-y 2>&amp;1</exe-cmd-extra>
</execute> </execute>

View File

@ -42,6 +42,9 @@ SRCS = \
command/stanza/delete.c \ command/stanza/delete.c \
command/stanza/upgrade.c \ command/stanza/upgrade.c \
common/compress/helper.c \ common/compress/helper.c \
common/compress/bz2/common.c \
common/compress/bz2/compress.c \
common/compress/bz2/decompress.c \
common/compress/gz/common.c \ common/compress/gz/common.c \
common/compress/gz/compress.c \ common/compress/gz/compress.c \
common/compress/gz/decompress.c \ common/compress/gz/decompress.c \

View File

@ -118,6 +118,11 @@ AC_CHECK_HEADER(libxml/parser.h, [], [AC_MSG_ERROR([header file <libxml/parser.h
AC_CHECK_LIB([z], [deflate], [], [AC_MSG_ERROR([library 'z' is required])]) AC_CHECK_LIB([z], [deflate], [], [AC_MSG_ERROR([library 'z' is required])])
AC_CHECK_HEADER(zlib.h, [], [AC_MSG_ERROR([header file <zlib.h> is required])]) AC_CHECK_HEADER(zlib.h, [], [AC_MSG_ERROR([header file <zlib.h> is required])])
# Check required bzip2 library
# ----------------------------------------------------------------------------------------------------------------------------------
AC_CHECK_LIB([bz2], [BZ2_bzCompress], [], [AC_MSG_ERROR([library 'bz2' is required])])
AC_CHECK_HEADER(bzlib.h, [], [AC_MSG_ERROR([header file <bzlib.h> is required])])
# Check optional lz4 library # Check optional lz4 library
# ---------------------------------------------------------------------------------------------------------------------------------- # ----------------------------------------------------------------------------------------------------------------------------------
AC_CHECK_LIB( AC_CHECK_LIB(

View File

@ -0,0 +1,101 @@
/***********************************************************************************************************************************
BZ2 Common
***********************************************************************************************************************************/
#include "build.auto.h"
#include <bzlib.h>
#include "common/compress/bz2/common.h"
#include "common/debug.h"
#include "common/memContext.h"
/**********************************************************************************************************************************/
int
bz2Error(int error)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(INT, error);
FUNCTION_TEST_END();
if (error < 0)
{
const char *errorMsg;
const ErrorType *errorType = &FormatError;
switch (error)
{
case BZ_SEQUENCE_ERROR:
{
errorMsg = "sequence error";
errorType = &AssertError;
break;
}
case BZ_PARAM_ERROR:
{
errorMsg = "parameter error";
errorType = &AssertError;
break;
}
case BZ_MEM_ERROR:
{
errorMsg = "memory error";
errorType = &AssertError;
break;
}
case BZ_DATA_ERROR:
{
errorMsg = "data error";
errorType = &AssertError;
break;
}
case BZ_DATA_ERROR_MAGIC:
{
errorMsg = "data error magic";
errorType = &AssertError;
break;
}
case BZ_IO_ERROR:
{
errorMsg = "io error";
errorType = &AssertError;
break;
}
case BZ_UNEXPECTED_EOF:
{
errorMsg = "unexpected eof";
errorType = &AssertError;
break;
}
case BZ_OUTBUFF_FULL:
{
errorMsg = "outbuff full";
errorType = &AssertError;
break;
}
case BZ_CONFIG_ERROR:
{
errorMsg = "config error";
errorType = &AssertError;
break;
}
default:
{
errorMsg = "unknown error";
errorType = &AssertError;
}
}
THROWP_FMT(errorType, "bz2 error: [%d] %s", error, errorMsg);
}
FUNCTION_TEST_RETURN(error);
}

View File

@ -0,0 +1,21 @@
/***********************************************************************************************************************************
BZ2 Common
Developed using the documentation in https://www.sourceware.org/bzip2/manual/manual.html#libprog
***********************************************************************************************************************************/
#ifndef COMMON_COMPRESS_BZ2_COMMON_H
#define COMMON_COMPRESS_BZ2_COMMON_H
#include <stddef.h>
/***********************************************************************************************************************************
BZ2 extension
***********************************************************************************************************************************/
#define BZ2_EXT "bz2"
/***********************************************************************************************************************************
Functions
***********************************************************************************************************************************/
int bz2Error(int error);
#endif

View File

@ -0,0 +1,196 @@
/***********************************************************************************************************************************
BZ2 Compress
***********************************************************************************************************************************/
#include "build.auto.h"
#include <stdio.h>
#include <bzlib.h>
#include "common/compress/bz2/common.h"
#include "common/compress/bz2/compress.h"
#include "common/debug.h"
#include "common/io/filter/filter.intern.h"
#include "common/log.h"
#include "common/memContext.h"
#include "common/type/object.h"
/***********************************************************************************************************************************
Filter type constant
***********************************************************************************************************************************/
STRING_EXTERN(BZ2_COMPRESS_FILTER_TYPE_STR, BZ2_COMPRESS_FILTER_TYPE);
/***********************************************************************************************************************************
Object type
***********************************************************************************************************************************/
#define BZ2_COMPRESS_TYPE Bz2Compress
#define BZ2_COMPRESS_PREFIX bz2Compress
typedef struct Bz2Compress
{
MemContext *memContext; // Context to store data
bz_stream stream; // Compression stream
bool inputSame; // Is the same input required on the next process call?
bool flushing; // Is input complete and flushing in progress?
bool done; // Is compression done?
} Bz2Compress;
/***********************************************************************************************************************************
Render as string for logging
***********************************************************************************************************************************/
static String *
bz2CompressToLog(const Bz2Compress *this)
{
return strNewFmt(
"{inputSame: %s, done: %s, flushing: %s, avail_in: %u}", cvtBoolToConstZ(this->inputSame), cvtBoolToConstZ(this->done),
cvtBoolToConstZ(this->flushing), this->stream.avail_in);
}
#define FUNCTION_LOG_BZ2_COMPRESS_TYPE \
Bz2Compress *
#define FUNCTION_LOG_BZ2_COMPRESS_FORMAT(value, buffer, bufferSize) \
FUNCTION_LOG_STRING_OBJECT_FORMAT(value, bz2CompressToLog, buffer, bufferSize)
/***********************************************************************************************************************************
Free compression stream
***********************************************************************************************************************************/
OBJECT_DEFINE_FREE_RESOURCE_BEGIN(BZ2_COMPRESS, LOG, logLevelTrace)
{
BZ2_bzCompressEnd(&this->stream);
}
OBJECT_DEFINE_FREE_RESOURCE_END(LOG);
/***********************************************************************************************************************************
Compress data
***********************************************************************************************************************************/
static void
bz2CompressProcess(THIS_VOID, const Buffer *uncompressed, Buffer *compressed)
{
THIS(Bz2Compress);
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(BZ2_COMPRESS, this);
FUNCTION_LOG_PARAM(BUFFER, uncompressed);
FUNCTION_LOG_PARAM(BUFFER, compressed);
FUNCTION_LOG_END();
ASSERT(this != NULL);
ASSERT(!this->done);
ASSERT(compressed != NULL);
ASSERT(!this->flushing || uncompressed == NULL);
ASSERT(this->flushing || (!this->inputSame || this->stream.avail_in != 0));
// If input is NULL then start flushing
if (uncompressed == NULL)
{
this->stream.avail_in = 0;
this->flushing = true;
}
// Else still have input data
else
{
// Is new input allowed?
if (!this->inputSame)
{
this->stream.avail_in = (unsigned int)bufUsed(uncompressed);
// bzip2 does not accept const input buffers
this->stream.next_in = (char *)UNCONSTIFY(unsigned char *, bufPtrConst(uncompressed));
}
}
// Initialize compressed output buffer
this->stream.avail_out = (unsigned int)bufRemains(compressed);
this->stream.next_out = (char *)bufPtr(compressed) + bufUsed(compressed);
// Perform compression, check for error
int result = bz2Error(BZ2_bzCompress(&this->stream, this->flushing ? BZ_FINISH : BZ_RUN));
// Set buffer used space
bufUsedSet(compressed, bufSize(compressed) - (size_t)this->stream.avail_out);
// Is compression done?
if (this->flushing && result == BZ_STREAM_END)
this->done = true;
// Can more input be provided on the next call?
this->inputSame = this->flushing ? !this->done : this->stream.avail_in != 0;
FUNCTION_LOG_RETURN_VOID();
}
/***********************************************************************************************************************************
Is compress done?
***********************************************************************************************************************************/
static bool
bz2CompressDone(const THIS_VOID)
{
THIS(const Bz2Compress);
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(BZ2_COMPRESS, this);
FUNCTION_TEST_END();
ASSERT(this != NULL);
FUNCTION_TEST_RETURN(this->done);
}
/***********************************************************************************************************************************
Is the same input required on the next process call?
***********************************************************************************************************************************/
static bool
bz2CompressInputSame(const THIS_VOID)
{
THIS(const Bz2Compress);
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(BZ2_COMPRESS, this);
FUNCTION_TEST_END();
ASSERT(this != NULL);
FUNCTION_TEST_RETURN(this->inputSame);
}
/**********************************************************************************************************************************/
IoFilter *
bz2CompressNew(int level)
{
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(INT, level);
FUNCTION_LOG_END();
ASSERT(level > 0);
IoFilter *this = NULL;
MEM_CONTEXT_NEW_BEGIN("Bz2Compress")
{
Bz2Compress *driver = memNew(sizeof(Bz2Compress));
*driver = (Bz2Compress)
{
.memContext = MEM_CONTEXT_NEW(),
.stream = {.bzalloc = NULL},
};
// Initialize context
bz2Error(BZ2_bzCompressInit(&driver->stream, level, 0, 0));
// Set callback to ensure bz2 stream is freed
memContextCallbackSet(driver->memContext, bz2CompressFreeResource, driver);
// Create param list
VariantList *paramList = varLstNew();
varLstAdd(paramList, varNewInt(level));
// Create filter interface
this = ioFilterNewP(
BZ2_COMPRESS_FILTER_TYPE_STR, driver, paramList, .done = bz2CompressDone, .inOut = bz2CompressProcess,
.inputSame = bz2CompressInputSame);
}
MEM_CONTEXT_NEW_END();
FUNCTION_LOG_RETURN(IO_FILTER, this);
}

View File

@ -0,0 +1,22 @@
/***********************************************************************************************************************************
BZ2 Compress
Compress IO to the bz2 format.
***********************************************************************************************************************************/
#ifndef COMMON_COMPRESS_BZ2_COMPRESS_H
#define COMMON_COMPRESS_BZ2_COMPRESS_H
#include "common/io/filter/filter.h"
/***********************************************************************************************************************************
Filter type constant
***********************************************************************************************************************************/
#define BZ2_COMPRESS_FILTER_TYPE "bz2Compress"
STRING_DECLARE(BZ2_COMPRESS_FILTER_TYPE_STR);
/***********************************************************************************************************************************
Constructors
***********************************************************************************************************************************/
IoFilter *bz2CompressNew(int level);
#endif

View File

@ -0,0 +1,177 @@
/***********************************************************************************************************************************
BZ2 Decompress
***********************************************************************************************************************************/
#include "build.auto.h"
#include <stdio.h>
#include <bzlib.h>
#include "common/compress/bz2/common.h"
#include "common/compress/bz2/decompress.h"
#include "common/debug.h"
#include "common/io/filter/filter.intern.h"
#include "common/log.h"
#include "common/memContext.h"
#include "common/type/object.h"
/***********************************************************************************************************************************
Filter type constant
***********************************************************************************************************************************/
STRING_EXTERN(BZ2_DECOMPRESS_FILTER_TYPE_STR, BZ2_DECOMPRESS_FILTER_TYPE);
/***********************************************************************************************************************************
Object type
***********************************************************************************************************************************/
#define BZ2_DECOMPRESS_TYPE Bz2Decompress
#define BZ2_DECOMPRESS_PREFIX bz2Decompress
typedef struct Bz2Decompress
{
MemContext *memContext; // Context to store data
bz_stream stream; // Decompression stream state
int result; // Result of last operation
bool inputSame; // Is the same input required on the next process call?
bool done; // Is decompression done?
} Bz2Decompress;
/***********************************************************************************************************************************
Macros for function logging
***********************************************************************************************************************************/
static String *
bz2DecompressToLog(const Bz2Decompress *this)
{
return strNewFmt(
"{inputSame: %s, done: %s, avail_in: %u}", cvtBoolToConstZ(this->inputSame), cvtBoolToConstZ(this->done),
this->stream.avail_in);
}
#define FUNCTION_LOG_BZ2_DECOMPRESS_TYPE \
Bz2Decompress *
#define FUNCTION_LOG_BZ2_DECOMPRESS_FORMAT(value, buffer, bufferSize) \
FUNCTION_LOG_STRING_OBJECT_FORMAT(value, bz2DecompressToLog, buffer, bufferSize)
/***********************************************************************************************************************************
Free inflate stream
***********************************************************************************************************************************/
OBJECT_DEFINE_FREE_RESOURCE_BEGIN(BZ2_DECOMPRESS, LOG, logLevelTrace)
{
BZ2_bzDecompressEnd(&this->stream);
}
OBJECT_DEFINE_FREE_RESOURCE_END(LOG);
/***********************************************************************************************************************************
Decompress data
***********************************************************************************************************************************/
static void
bz2DecompressProcess(THIS_VOID, const Buffer *compressed, Buffer *uncompressed)
{
THIS(Bz2Decompress);
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(BZ2_DECOMPRESS, this);
FUNCTION_LOG_PARAM(BUFFER, compressed);
FUNCTION_LOG_PARAM(BUFFER, uncompressed);
FUNCTION_LOG_END();
ASSERT(this != NULL);
ASSERT(uncompressed != NULL);
// There should never be a flush because in a valid compressed stream the end of data can be determined and done will be set.
// If a flush is received it means the compressed stream terminated early, e.g. a zero-length or truncated file.
if (compressed == NULL)
THROW(FormatError, "unexpected eof in compressed data");
if (!this->inputSame)
{
this->stream.avail_in = (unsigned int)bufUsed(compressed);
// bzip2 does not accept const input buffers
this->stream.next_in = (char *)UNCONSTIFY(unsigned char *, bufPtrConst(compressed));
}
this->stream.avail_out = (unsigned int)bufRemains(uncompressed);
this->stream.next_out = (char *)bufPtr(uncompressed) + bufUsed(uncompressed);
this->result = bz2Error(BZ2_bzDecompress(&this->stream));
// Set buffer used space
bufUsedSet(uncompressed, bufSize(uncompressed) - (size_t)this->stream.avail_out);
// Is decompression done?
this->done = this->result == BZ_STREAM_END;
// Is the same input expected on the next call?
this->inputSame = this->done ? false : this->stream.avail_in != 0;
FUNCTION_LOG_RETURN_VOID();
}
/***********************************************************************************************************************************
Is decompress done?
***********************************************************************************************************************************/
static bool
bz2DecompressDone(const THIS_VOID)
{
THIS(const Bz2Decompress);
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(BZ2_DECOMPRESS, this);
FUNCTION_TEST_END();
ASSERT(this != NULL);
FUNCTION_TEST_RETURN(this->done);
}
/***********************************************************************************************************************************
Is the same input required on the next process call?
***********************************************************************************************************************************/
static bool
bz2DecompressInputSame(const THIS_VOID)
{
THIS(const Bz2Decompress);
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(BZ2_DECOMPRESS, this);
FUNCTION_TEST_END();
ASSERT(this != NULL);
FUNCTION_TEST_RETURN(this->inputSame);
}
/**********************************************************************************************************************************/
IoFilter *
bz2DecompressNew(void)
{
FUNCTION_LOG_VOID(logLevelTrace);
IoFilter *this = NULL;
MEM_CONTEXT_NEW_BEGIN("Bz2Decompress")
{
// Allocate state and set context
Bz2Decompress *driver = memNew(sizeof(Bz2Decompress));
*driver = (Bz2Decompress)
{
.memContext = MEM_CONTEXT_NEW(),
.stream = {.bzalloc = NULL},
};
// Create bz2 stream
bz2Error(driver->result = BZ2_bzDecompressInit(&driver->stream, 0, 0));
// Set free callback to ensure bz2 context is freed
memContextCallbackSet(driver->memContext, bz2DecompressFreeResource, driver);
// Create filter interface
this = ioFilterNewP(
BZ2_DECOMPRESS_FILTER_TYPE_STR, driver, NULL, .done = bz2DecompressDone, .inOut = bz2DecompressProcess,
.inputSame = bz2DecompressInputSame);
}
MEM_CONTEXT_NEW_END();
FUNCTION_LOG_RETURN(IO_FILTER, this);
}

View File

@ -0,0 +1,22 @@
/***********************************************************************************************************************************
BZ2 Decompress
Decompress IO from the bz2 format.
***********************************************************************************************************************************/
#ifndef COMMON_COMPRESS_BZ2_DECOMPRESS_H
#define COMMON_COMPRESS_BZ2_DECOMPRESS_H
#include "common/io/filter/filter.h"
/***********************************************************************************************************************************
Filter type constant
***********************************************************************************************************************************/
#define BZ2_DECOMPRESS_FILTER_TYPE "bz2Decompress"
STRING_DECLARE(BZ2_DECOMPRESS_FILTER_TYPE_STR);
/***********************************************************************************************************************************
Constructors
***********************************************************************************************************************************/
IoFilter *bz2DecompressNew(void);
#endif

View File

@ -6,6 +6,9 @@ Compression Helper
#include <string.h> #include <string.h>
#include "common/compress/helper.h" #include "common/compress/helper.h"
#include "common/compress/bz2/common.h"
#include "common/compress/bz2/compress.h"
#include "common/compress/bz2/decompress.h"
#include "common/compress/gz/common.h" #include "common/compress/gz/common.h"
#include "common/compress/gz/compress.h" #include "common/compress/gz/compress.h"
#include "common/compress/gz/decompress.h" #include "common/compress/gz/decompress.h"
@ -25,7 +28,6 @@ Compression type constants
#define COMPRESS_TYPE_NONE "none" #define COMPRESS_TYPE_NONE "none"
// Constants for currently unsupported compression types // Constants for currently unsupported compression types
#define BZ2_EXT "bz2"
#define XZ_EXT "xz" #define XZ_EXT "xz"
/*********************************************************************************************************************************** /***********************************************************************************************************************************
@ -46,6 +48,15 @@ static const struct CompressHelperLocal
.type = STRDEF(COMPRESS_TYPE_NONE), .type = STRDEF(COMPRESS_TYPE_NONE),
.ext = STRDEF(""), .ext = STRDEF(""),
}, },
{
.type = STRDEF(BZ2_EXT),
.ext = STRDEF("." BZ2_EXT),
.compressType = BZ2_COMPRESS_FILTER_TYPE,
.compressNew = bz2CompressNew,
.decompressType = BZ2_DECOMPRESS_FILTER_TYPE,
.decompressNew = bz2DecompressNew,
.levelDefault = 9,
},
{ {
.type = STRDEF(GZ_EXT), .type = STRDEF(GZ_EXT),
.ext = STRDEF("." GZ_EXT), .ext = STRDEF("." GZ_EXT),

View File

@ -15,6 +15,7 @@ Available compression types
typedef enum typedef enum
{ {
compressTypeNone, // No compression compressTypeNone, // No compression
compressTypeBz2, // bzip2
compressTypeGz, // gzip compressTypeGz, // gzip
compressTypeLz4, // lz4 compressTypeLz4, // lz4
compressTypeZst, // zstandard compressTypeZst, // zstandard
@ -23,7 +24,6 @@ typedef enum
// versions. In that sense this list is speculative, but these seem to be all the types that are likely to be added in the // versions. In that sense this list is speculative, but these seem to be all the types that are likely to be added in the
// foreseeable future. // foreseeable future.
compressTypeXz, // xz/lzma compressTypeXz, // xz/lzma
compressTypeBz2, // bzip2
} CompressType; } CompressType;
#include <common/type/string.h> #include <common/type/string.h>

View File

@ -832,6 +832,7 @@ static ConfigDefineOptionData configDefineOptionData[] = CFGDEFDATA_OPTION_LIST
( (
"The following compression types are supported:\n" "The following compression types are supported:\n"
"\n" "\n"
"* bz2 - bzip2 compression format\n"
"* gz - gzip compression format\n" "* gz - gzip compression format\n"
"* lz4 - lz4 compression format (not available on all platforms)\n" "* lz4 - lz4 compression format (not available on all platforms)\n"
"* zst - Zstandard compression format (not available on all platforms)" "* zst - Zstandard compression format (not available on all platforms)"
@ -848,6 +849,7 @@ static ConfigDefineOptionData configDefineOptionData[] = CFGDEFDATA_OPTION_LIST
CFGDEFDATA_OPTION_OPTIONAL_ALLOW_LIST CFGDEFDATA_OPTION_OPTIONAL_ALLOW_LIST
( (
"none", "none",
"bz2",
"gz", "gz",
"lz4", "lz4",
"zst" "zst"

60
src/configure vendored
View File

@ -4018,6 +4018,64 @@ fi
# Check required bzip2 library
# ----------------------------------------------------------------------------------------------------------------------------------
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for BZ2_bzCompress in -lbz2" >&5
$as_echo_n "checking for BZ2_bzCompress in -lbz2... " >&6; }
if ${ac_cv_lib_bz2_BZ2_bzCompress+:} false; then :
$as_echo_n "(cached) " >&6
else
ac_check_lib_save_LIBS=$LIBS
LIBS="-lbz2 $LIBS"
cat confdefs.h - <<_ACEOF >conftest.$ac_ext
/* end confdefs.h. */
/* Override any GCC internal prototype to avoid an error.
Use char because int might match the return type of a GCC
builtin and then its argument prototype would still apply. */
#ifdef __cplusplus
extern "C"
#endif
char BZ2_bzCompress ();
int
main ()
{
return BZ2_bzCompress ();
;
return 0;
}
_ACEOF
if ac_fn_c_try_link "$LINENO"; then :
ac_cv_lib_bz2_BZ2_bzCompress=yes
else
ac_cv_lib_bz2_BZ2_bzCompress=no
fi
rm -f core conftest.err conftest.$ac_objext \
conftest$ac_exeext conftest.$ac_ext
LIBS=$ac_check_lib_save_LIBS
fi
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_bz2_BZ2_bzCompress" >&5
$as_echo "$ac_cv_lib_bz2_BZ2_bzCompress" >&6; }
if test "x$ac_cv_lib_bz2_BZ2_bzCompress" = xyes; then :
cat >>confdefs.h <<_ACEOF
#define HAVE_LIBBZ2 1
_ACEOF
LIBS="-lbz2 $LIBS"
else
as_fn_error $? "library 'bz2' is required" "$LINENO" 5
fi
ac_fn_c_check_header_mongrel "$LINENO" "bzlib.h" "ac_cv_header_bzlib_h" "$ac_includes_default"
if test "x$ac_cv_header_bzlib_h" = xyes; then :
else
as_fn_error $? "header file <bzlib.h> is required" "$LINENO" 5
fi
# Check optional lz4 library # Check optional lz4 library
# ---------------------------------------------------------------------------------------------------------------------------------- # ----------------------------------------------------------------------------------------------------------------------------------
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for LZ4F_isError in -llz4" >&5 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for LZ4F_isError in -llz4" >&5
@ -5417,4 +5475,4 @@ if test -n "$ac_unrecognized_opts" && test "$enable_option_checking" != no; then
$as_echo "$as_me: WARNING: unrecognized options: $ac_unrecognized_opts" >&2;} $as_echo "$as_me: WARNING: unrecognized options: $ac_unrecognized_opts" >&2;}
fi fi
# Generated from src/build/configure.ac sha1 93fbf56725f3dab1089ddd4d7768f4414bf7941c # Generated from src/build/configure.ac sha1 a4acd8b59dcdff7e986a8454335119e955d63c91

2
test/Vagrantfile vendored
View File

@ -75,7 +75,7 @@ Vagrant.configure(2) do |config|
#----------------------------------------------------------------------------------------------------------------------- #-----------------------------------------------------------------------------------------------------------------------
echo 'Install Build Tools' && date echo 'Install Build Tools' && date
apt-get install -y devscripts build-essential lintian git cloc txt2man debhelper libssl-dev zlib1g-dev libperl-dev \ apt-get install -y devscripts build-essential lintian git cloc txt2man debhelper libssl-dev zlib1g-dev libperl-dev \
libxml2-dev liblz4-dev liblz4-tool libpq-dev valgrind lcov autoconf-archive zstd libzstd-dev libxml2-dev liblz4-dev liblz4-tool libpq-dev valgrind lcov autoconf-archive zstd libzstd-dev bzip2 libbz2-dev
#----------------------------------------------------------------------------------------------------------------------- #-----------------------------------------------------------------------------------------------------------------------
echo 'Install Docker' && date echo 'Install Docker' && date

View File

@ -12,13 +12,9 @@
# - docker login -u pgbackrest # - docker login -u pgbackrest
# - VM=XXX;DATE=YYYYMMDDX;BASE=pgbackrest/test:${VM?}-base;docker tag ${BASE?} ${BASE?}-${DATE?} && docker push ${BASE?}-${DATE?} # - VM=XXX;DATE=YYYYMMDDX;BASE=pgbackrest/test:${VM?}-base;docker tag ${BASE?} ${BASE?}-${DATE?} && docker push ${BASE?}-${DATE?}
# ********************************************************************************************************************************** # **********************************************************************************************************************************
20200504A: 20200505A:
co7: 9c922915380c0beb3d813c94d53f823d7008c69b co6: c07889acc321e461263fada797941cfa947d2550
f30: 9ad36516c2403b835c24efbbcef359322a80fa1f co7: 538dc7fd9cc129e44d4ae6bacfb6f1db10309993
u18: 32426ecf8cd0e98976298f6f6b41a2b255aca692 f30: 3ae2c1749afa24daa1ca31abe9441d5e1e66d892
u12: 8a88ab44aace049d7da5ca1094375ff8b9aeb7ab
20200310A: u18: 0f88948969b70f300a9134482e43cda7b94cb24d
co6: beb7b5a62ebdf209bb54494129baeadbd0c2954f
20200124A:
u12: 0f2fcf1bc79ee35e78121773c9a2155a77cb10d5

View File

@ -262,9 +262,12 @@ unit:
# ---------------------------------------------------------------------------------------------------------------------------- # ----------------------------------------------------------------------------------------------------------------------------
- name: compress - name: compress
total: 4 total: 5
coverage: coverage:
common/compress/bz2/common: full
common/compress/bz2/compress: full
common/compress/bz2/decompress: full
common/compress/gz/common: full common/compress/gz/common: full
common/compress/gz/compress: full common/compress/gz/compress: full
common/compress/gz/decompress: full common/compress/gz/decompress: full

View File

@ -361,7 +361,7 @@ sub containerBuild
" yum -y install openssh-server openssh-clients wget sudo valgrind git \\\n" . " yum -y install openssh-server openssh-clients wget sudo valgrind git \\\n" .
" perl perl-Digest-SHA perl-DBD-Pg perl-YAML-LibYAML openssl \\\n" . " perl perl-Digest-SHA perl-DBD-Pg perl-YAML-LibYAML openssl \\\n" .
" gcc make perl-ExtUtils-MakeMaker perl-Test-Simple openssl-devel perl-ExtUtils-Embed rpm-build \\\n" . " gcc make perl-ExtUtils-MakeMaker perl-Test-Simple openssl-devel perl-ExtUtils-Embed rpm-build \\\n" .
" zlib-devel libxml2-devel lz4-devel lz4"; " zlib-devel libxml2-devel lz4-devel lz4 bzip2-devel bzip2";
if ($strOS eq VM_CO6) if ($strOS eq VM_CO6)
{ {
@ -380,7 +380,8 @@ sub containerBuild
" apt-get -y install openssh-server wget sudo gcc make valgrind git \\\n" . " apt-get -y install openssh-server wget sudo gcc make valgrind git \\\n" .
" libdbd-pg-perl libhtml-parser-perl libssl-dev libperl-dev \\\n" . " libdbd-pg-perl libhtml-parser-perl libssl-dev libperl-dev \\\n" .
" libyaml-libyaml-perl tzdata devscripts lintian libxml-checker-perl txt2man debhelper \\\n" . " libyaml-libyaml-perl tzdata devscripts lintian libxml-checker-perl txt2man debhelper \\\n" .
" libppi-html-perl libtemplate-perl libtest-differences-perl zlib1g-dev libxml2-dev pkg-config"; " libppi-html-perl libtemplate-perl libtest-differences-perl zlib1g-dev libxml2-dev pkg-config \\\n" .
" libbz2-dev bzip2";
if ($strOS eq VM_U12) if ($strOS eq VM_U12)
{ {

View File

@ -518,7 +518,7 @@ sub run
"BUILDFLAGS=${strBuildFlags}\n" . "BUILDFLAGS=${strBuildFlags}\n" .
"HARNESSFLAGS=${strHarnessFlags}\n" . "HARNESSFLAGS=${strHarnessFlags}\n" .
"TESTFLAGS=${strTestFlags}\n" . "TESTFLAGS=${strTestFlags}\n" .
"LDFLAGS=-lcrypto -lssl -lxml2 -lz" . "LDFLAGS=-lcrypto -lssl -lxml2 -lz -lbz2" .
(vmWithLz4($self->{oTest}->{&TEST_VM}) ? ' -llz4' : '') . (vmWithLz4($self->{oTest}->{&TEST_VM}) ? ' -llz4' : '') .
(vmWithZst($self->{oTest}->{&TEST_VM}) ? ' -lzstd' : '') . (vmWithZst($self->{oTest}->{&TEST_VM}) ? ' -lzstd' : '') .
(vmCoverageC($self->{oTest}->{&TEST_VM}) && $self->{bCoverageUnit} ? " -lgcov" : '') . (vmCoverageC($self->{oTest}->{&TEST_VM}) && $self->{bCoverageUnit} ? " -lgcov" : '') .

View File

@ -95,6 +95,8 @@ use constant CFGOPTVAL_RESTORE_TYPE_XID => 'xid';
use constant NONE => 'none'; use constant NONE => 'none';
push @EXPORT, qw(NONE); push @EXPORT, qw(NONE);
use constant BZ2 => 'bz2';
push @EXPORT, qw(BZ2);
use constant GZ => 'gz'; use constant GZ => 'gz';
push @EXPORT, qw(GZ); push @EXPORT, qw(GZ);
use constant LZ4 => 'lz4'; use constant LZ4 => 'lz4';

View File

@ -206,12 +206,12 @@ sub run
foreach my $rhRun foreach my $rhRun
( (
{vm => VM1, remote => false, s3 => true, encrypt => false, delta => true, compress => LZ4}, {vm => VM1, remote => false, s3 => true, encrypt => false, delta => true, compress => LZ4},
{vm => VM1, remote => true, s3 => false, encrypt => true, delta => false, compress => GZ}, {vm => VM1, remote => true, s3 => false, encrypt => true, delta => false, compress => BZ2},
{vm => VM2, remote => false, s3 => false, encrypt => true, delta => true, compress => GZ}, {vm => VM2, remote => false, s3 => false, encrypt => true, delta => true, compress => BZ2},
{vm => VM2, remote => true, s3 => true, encrypt => false, delta => false, compress => GZ}, {vm => VM2, remote => true, s3 => true, encrypt => false, delta => false, compress => GZ},
{vm => VM3, remote => false, s3 => false, encrypt => false, delta => true, compress => ZST}, {vm => VM3, remote => false, s3 => false, encrypt => false, delta => true, compress => ZST},
{vm => VM3, remote => true, s3 => true, encrypt => true, delta => false, compress => LZ4}, {vm => VM3, remote => true, s3 => true, encrypt => true, delta => false, compress => LZ4},
{vm => VM4, remote => false, s3 => false, encrypt => false, delta => false, compress => LZ4}, {vm => VM4, remote => false, s3 => false, encrypt => false, delta => false, compress => GZ},
{vm => VM4, remote => true, s3 => true, encrypt => true, delta => true, compress => ZST}, {vm => VM4, remote => true, s3 => true, encrypt => true, delta => true, compress => ZST},
) )
{ {

View File

@ -46,7 +46,7 @@ sub run
{vm => VM1, remote => false, s3 => false, encrypt => false, compress => LZ4, error => 0}, {vm => VM1, remote => false, s3 => false, encrypt => false, compress => LZ4, error => 0},
{vm => VM1, remote => true, s3 => true, encrypt => true, compress => GZ, error => 1}, {vm => VM1, remote => true, s3 => true, encrypt => true, compress => GZ, error => 1},
{vm => VM2, remote => false, s3 => true, encrypt => false, compress => NONE, error => 0}, {vm => VM2, remote => false, s3 => true, encrypt => false, compress => NONE, error => 0},
{vm => VM2, remote => true, s3 => false, encrypt => true, compress => GZ, error => 0}, {vm => VM2, remote => true, s3 => false, encrypt => true, compress => BZ2, error => 0},
{vm => VM3, remote => false, s3 => false, encrypt => true, compress => NONE, error => 0}, {vm => VM3, remote => false, s3 => false, encrypt => true, compress => NONE, error => 0},
{vm => VM3, remote => true, s3 => true, encrypt => false, compress => LZ4, error => 1}, {vm => VM3, remote => true, s3 => true, encrypt => false, compress => LZ4, error => 1},
{vm => VM4, remote => false, s3 => true, encrypt => true, compress => ZST, error => 0}, {vm => VM4, remote => false, s3 => true, encrypt => true, compress => ZST, error => 0},

View File

@ -83,14 +83,14 @@ sub run
foreach my $rhRun foreach my $rhRun
( (
{vm => VM1, remote => false, s3 => false, encrypt => false, compress => LZ4}, {vm => VM1, remote => false, s3 => false, encrypt => false, compress => GZ},
{vm => VM1, remote => true, s3 => true, encrypt => true, compress => GZ}, {vm => VM1, remote => true, s3 => true, encrypt => true, compress => LZ4},
{vm => VM2, remote => false, s3 => true, encrypt => false, compress => GZ}, {vm => VM2, remote => false, s3 => true, encrypt => false, compress => BZ2},
{vm => VM2, remote => true, s3 => false, encrypt => true, compress => GZ}, {vm => VM2, remote => true, s3 => false, encrypt => true, compress => BZ2},
{vm => VM3, remote => false, s3 => false, encrypt => true, compress => LZ4}, {vm => VM3, remote => false, s3 => false, encrypt => true, compress => LZ4},
{vm => VM3, remote => true, s3 => true, encrypt => false, compress => ZST}, {vm => VM3, remote => true, s3 => true, encrypt => false, compress => ZST},
{vm => VM4, remote => false, s3 => true, encrypt => true, compress => ZST}, {vm => VM4, remote => false, s3 => true, encrypt => true, compress => ZST},
{vm => VM4, remote => true, s3 => false, encrypt => false, compress => LZ4}, {vm => VM4, remote => true, s3 => false, encrypt => false, compress => GZ},
) )
{ {
# Only run tests for this vm # Only run tests for this vm

View File

@ -42,12 +42,12 @@ sub run
foreach my $rhRun foreach my $rhRun
( (
{vm => VM1, remote => false, s3 => false, encrypt => true, compress => LZ4}, {vm => VM1, remote => false, s3 => false, encrypt => true, compress => LZ4},
{vm => VM1, remote => true, s3 => true, encrypt => false, compress => GZ}, {vm => VM1, remote => true, s3 => true, encrypt => false, compress => BZ2},
{vm => VM2, remote => false, s3 => true, encrypt => true, compress => GZ}, {vm => VM2, remote => false, s3 => true, encrypt => true, compress => BZ2},
{vm => VM2, remote => true, s3 => false, encrypt => false, compress => GZ}, {vm => VM2, remote => true, s3 => false, encrypt => false, compress => GZ},
{vm => VM3, remote => false, s3 => false, encrypt => false, compress => ZST}, {vm => VM3, remote => false, s3 => false, encrypt => false, compress => ZST},
{vm => VM3, remote => true, s3 => true, encrypt => true, compress => LZ4}, {vm => VM3, remote => true, s3 => true, encrypt => true, compress => LZ4},
{vm => VM4, remote => false, s3 => true, encrypt => false, compress => LZ4}, {vm => VM4, remote => false, s3 => true, encrypt => false, compress => GZ},
{vm => VM4, remote => true, s3 => false, encrypt => true, compress => ZST}, {vm => VM4, remote => true, s3 => false, encrypt => true, compress => ZST},
) )
{ {

View File

@ -66,7 +66,7 @@ sub run
{ {
my $strCompressType = my $strCompressType =
$bHostBackup && !$bHostStandby ? $bHostBackup && !$bHostStandby ?
(vmWithLz4($self->vm()) && $bLz4Compress ? LZ4 : vmWithZst($self->vm()) ? ZST : GZ) : NONE; (vmWithLz4($self->vm()) && $bLz4Compress ? LZ4 : vmWithZst($self->vm()) ? ZST : ($bS3 ? BZ2 : GZ)) : NONE;
my $bRepoEncrypt = ($strCompressType ne NONE && !$bS3) ? true : false; my $bRepoEncrypt = ($strCompressType ne NONE && !$bS3) ? true : false;
# If compression was used then switch it for the next test that uses compression # If compression was used then switch it for the next test that uses compression

View File

@ -1,12 +1,13 @@
--- control --- control
+++ control +++ control
@@ -13,7 +13,8 @@ @@ -13,7 +13,9 @@
pkg-config, pkg-config,
txt2man, txt2man,
zlib1g-dev, zlib1g-dev,
- liblz4-dev - liblz4-dev
+ liblz4-dev, + liblz4-dev,
+ libzstd-dev, + libzstd-dev,
+ libbz2-dev,
Standards-Version: 4.4.1 Standards-Version: 4.4.1
Homepage: https://www.pgbackrest.org/ Homepage: https://www.pgbackrest.org/
Vcs-Git: https://salsa.debian.org/postgresql/pgbackrest.git Vcs-Git: https://salsa.debian.org/postgresql/pgbackrest.git

View File

@ -147,19 +147,23 @@ testSuite(CompressType type, const char *decompressCmd)
TEST_ERROR(testDecompress(decompressFilter(type), truncated, 512, 512), FormatError, "unexpected eof in compressed data"); TEST_ERROR(testDecompress(decompressFilter(type), truncated, 512, 512), FormatError, "unexpected eof in compressed data");
// ------------------------------------------------------------------------------------------------------------------------- // -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("compress a large zero input buffer into small output buffer"); TEST_TITLE("compress a large non-zero input buffer into small output buffer");
decompressed = bufNew(1024 * 1024 - 1); decompressed = bufNew(1024 * 1024 - 1);
memset(bufPtr(decompressed), 0, bufSize(decompressed)); unsigned char *c = bufPtr(decompressed);
for (size_t i = 0; i < bufSize(decompressed); i++)
c[i] = (unsigned char)(i % 94 + 32);
bufUsedSet(decompressed, bufSize(decompressed)); bufUsedSet(decompressed, bufSize(decompressed));
TEST_ASSIGN( TEST_ASSIGN(
compressed, testCompress(compressFilter(type, 3), decompressed, bufSize(decompressed), 32), compressed, testCompress(compressFilter(type, 3), decompressed, bufSize(decompressed), 32),
"zero data - compress large in/small out buffer"); "non-zero data - compress large in/small out buffer");
TEST_RESULT_BOOL( TEST_RESULT_BOOL(
bufEq(decompressed, testDecompress(decompressFilter(type), compressed, bufSize(compressed), 1024 * 256)), true, bufEq(decompressed, testDecompress(decompressFilter(type), compressed, bufSize(compressed), 1024 * 256)), true,
"zero data - decompress large in/small out buffer"); "non-zero data - decompress large in/small out buffer");
} }
/*********************************************************************************************************************************** /***********************************************************************************************************************************
@ -203,6 +207,49 @@ testRun(void)
TEST_RESULT_STR_Z(gzDecompressToLog(decompress), "{inputSame: true, done: true, availIn: 0}", "format object"); TEST_RESULT_STR_Z(gzDecompressToLog(decompress), "{inputSame: true, done: true, availIn: 0}", "format object");
} }
// *****************************************************************************************************************************
if (testBegin("bz2"))
{
// Run standard test suite
testSuite(compressTypeBz2, "bzip2 -dc");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("bz2Error()");
TEST_RESULT_INT(bz2Error(BZ_OK), BZ_OK, "check ok");
TEST_RESULT_INT(bz2Error(BZ_RUN_OK), BZ_RUN_OK, "check run ok");
TEST_RESULT_INT(bz2Error(BZ_FLUSH_OK), BZ_FLUSH_OK, "check flush ok");
TEST_RESULT_INT(bz2Error(BZ_FINISH_OK), BZ_FINISH_OK, "check finish ok");
TEST_RESULT_INT(bz2Error(BZ_STREAM_END), BZ_STREAM_END, "check stream end");
TEST_ERROR(bz2Error(BZ_SEQUENCE_ERROR), AssertError, "bz2 error: [-1] sequence error");
TEST_ERROR(bz2Error(BZ_PARAM_ERROR), AssertError, "bz2 error: [-2] parameter error");
TEST_ERROR(bz2Error(BZ_MEM_ERROR), AssertError, "bz2 error: [-3] memory error");
TEST_ERROR(bz2Error(BZ_DATA_ERROR), AssertError, "bz2 error: [-4] data error");
TEST_ERROR(bz2Error(BZ_DATA_ERROR_MAGIC), AssertError, "bz2 error: [-5] data error magic");
TEST_ERROR(bz2Error(BZ_IO_ERROR), AssertError, "bz2 error: [-6] io error");
TEST_ERROR(bz2Error(BZ_UNEXPECTED_EOF), AssertError, "bz2 error: [-7] unexpected eof");
TEST_ERROR(bz2Error(BZ_OUTBUFF_FULL), AssertError, "bz2 error: [-8] outbuff full");
TEST_ERROR(bz2Error(BZ_CONFIG_ERROR), AssertError, "bz2 error: [-9] config error");
TEST_ERROR(bz2Error(-999), AssertError, "bz2 error: [-999] unknown error");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("bz2DecompressToLog() and bz2CompressToLog()");
Bz2Compress *compress = (Bz2Compress *)ioFilterDriver(bz2CompressNew(1));
compress->stream.avail_in = 999;
TEST_RESULT_STR_Z(
bz2CompressToLog(compress), "{inputSame: false, done: false, flushing: false, avail_in: 999}", "format object");
Bz2Decompress *decompress = (Bz2Decompress *)ioFilterDriver(bz2DecompressNew());
decompress->inputSame = true;
decompress->done = true;
TEST_RESULT_STR_Z(bz2DecompressToLog(decompress), "{inputSame: true, done: true, avail_in: 0}", "format object");
}
// ***************************************************************************************************************************** // *****************************************************************************************************************************
if (testBegin("lz4")) if (testBegin("lz4"))
{ {

View File

@ -186,7 +186,7 @@ eval
# Extra packages required when testing without containers # Extra packages required when testing without containers
if ($strVm eq VM_NONE) if ($strVm eq VM_NONE)
{ {
$strPackage .= " valgrind liblz4-dev liblz4-tool zstd libzstd-dev"; $strPackage .= " valgrind liblz4-dev liblz4-tool zstd libzstd-dev bzip2 libbz2-dev";
} }
# Else packages needed for integration tests on containers # Else packages needed for integration tests on containers
else else