db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2025-07-15 01:04:37 +02:00
Code Issues Releases Activity

Add bzip2 compression support.

Browse Source 
bzip2 is a widely available, high-quality data compressor. It typically compresses files to within 10% to 15% of the best available techniques (the PPM family of statistical compressors), while being around twice as fast at compression and six times faster at decompression.

bzip2 is currently available on all supported platforms.
This commit is contained in:
Stephen Frost
2020-05-05 16:49:01 -04:00
committed by GitHub
parent 98f30ef222
commit a021c9fe05
31 changed files with 720 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
build/lib/pgBackRestBuild/Config/Data.pm
View File

@ -1234,6 +1234,7 @@ my %hConfigDefine =
&CFGDEF_ALLOW_LIST =>
[
'none',
'bz2',
'gz',
'lz4',
'zst',

2
doc/xml/contributing.xml
View File

@ -64,7 +64,7 @@
apt-get install rsync git devscripts build-essential valgrind lcov autoconf
autoconf-archive libssl-dev zlib1g-dev libxml2-dev libpq-dev pkg-config
libxml-checker-perl libyaml-libyaml-perl libdbd-pg-perl liblz4-dev liblz4-tool
zstd libzstd-dev
zstd libzstd-dev bzip2 libbz2-dev
</exe-cmd>
<exe-cmd-extra>-y 2>&amp;1</exe-cmd-extra>
</execute>

1
doc/xml/reference.xml
View File

@ -141,6 +141,7 @@
<text>The following compression types are supported:
<ul>
<li><id>bz2</id> - bzip2 compression format</li>
<li><id>gz</id> - gzip compression format</li>
<li><id>lz4</id> - lz4 compression format (not available on all platforms)</li>
<li><id>zst</id> - Zstandard compression format (not available on all platforms)</li>

10
doc/xml/release.xml
View File

@ -63,6 +63,16 @@
<p>Note that setting <br-option>compress-type=zst</br-option> will make new backups and archive incompatible (unrestorable) with prior versions of <backrest/>.</p>
</release-item>
<release-item>
<release-item-contributor-list>
<release-item-contributor id="stephen.frost"/>
</release-item-contributor-list>
<p>Add <proper>bzip2</proper> compression support.</p>
<p>Note that setting <br-option>compress-type=bz2</br-option> will make new backups and archive incompatible (unrestorable) with prior versions of <backrest/>.</p>
</release-item>
<release-item>
<release-item-contributor-list>
<release-item-contributor id="stefan.fercot"/>

4
doc/xml/user-guide.xml
View File

@ -767,7 +767,7 @@
<execute if="{[os-type-is-debian]}" user="root" pre="y">
<exe-cmd>
apt-get install make gcc libpq-dev libssl-dev libxml2-dev pkg-config
liblz4-dev libzstd-dev
liblz4-dev libzstd-dev libbz2-dev
</exe-cmd>
<exe-cmd-extra>-y 2>&amp;1</exe-cmd-extra>
</execute>
@ -775,7 +775,7 @@
<execute if="{[os-type-is-centos]}" user="root" pre="y">
<exe-cmd>
yum install make gcc postgresql-devel openssl-devel libxml2-devel
lz4-devel libzstd-devel
lz4-devel libzstd-devel bzip2-devel
</exe-cmd>
<exe-cmd-extra>-y 2>&amp;1</exe-cmd-extra>
</execute>

3
src/Makefile.in
View File

@ -42,6 +42,9 @@ SRCS = \
command/stanza/delete.c \
command/stanza/upgrade.c \
common/compress/helper.c \
common/compress/bz2/common.c \
common/compress/bz2/compress.c \
common/compress/bz2/decompress.c \
common/compress/gz/common.c \
common/compress/gz/compress.c \
common/compress/gz/decompress.c \

5
src/build/configure.ac
View File

@ -118,6 +118,11 @@ AC_CHECK_HEADER(libxml/parser.h, [], [AC_MSG_ERROR([header file <libxml/parser.h
AC_CHECK_LIB([z], [deflate], [], [AC_MSG_ERROR([library 'z' is required])])
AC_CHECK_HEADER(zlib.h, [], [AC_MSG_ERROR([header file <zlib.h> is required])])
# Check required bzip2 library
# ----------------------------------------------------------------------------------------------------------------------------------
AC_CHECK_LIB([bz2], [BZ2_bzCompress], [], [AC_MSG_ERROR([library 'bz2' is required])])
AC_CHECK_HEADER(bzlib.h, [], [AC_MSG_ERROR([header file <bzlib.h> is required])])
# Check optional lz4 library
# ----------------------------------------------------------------------------------------------------------------------------------
AC_CHECK_LIB(

101
src/common/compress/bz2/common.c Normal file
View File

@ -0,0 +1,101 @@
/***********************************************************************************************************************************
BZ2 Common
***********************************************************************************************************************************/
#include "build.auto.h"
#include <bzlib.h>
#include "common/compress/bz2/common.h"
#include "common/debug.h"
#include "common/memContext.h"
/**********************************************************************************************************************************/
int
bz2Error(int error)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(INT, error);
FUNCTION_TEST_END();
if (error < 0)
{
const char *errorMsg;
const ErrorType *errorType = &FormatError;
switch (error)
{
case BZ_SEQUENCE_ERROR:
{
errorMsg = "sequence error";
errorType = &AssertError;
break;
}
case BZ_PARAM_ERROR:
{
errorMsg = "parameter error";
errorType = &AssertError;
break;
}
case BZ_MEM_ERROR:
{
errorMsg = "memory error";
errorType = &AssertError;
break;
}
case BZ_DATA_ERROR:
{
errorMsg = "data error";
errorType = &AssertError;
break;
}
case BZ_DATA_ERROR_MAGIC:
{
errorMsg = "data error magic";
errorType = &AssertError;
break;
}
case BZ_IO_ERROR:
{
errorMsg = "io error";
errorType = &AssertError;
break;
}
case BZ_UNEXPECTED_EOF:
{
errorMsg = "unexpected eof";
errorType = &AssertError;
break;
}
case BZ_OUTBUFF_FULL:
{
errorMsg = "outbuff full";
errorType = &AssertError;
break;
}
case BZ_CONFIG_ERROR:
{
errorMsg = "config error";
errorType = &AssertError;
break;
}
default:
{
errorMsg = "unknown error";
errorType = &AssertError;
}
}
THROWP_FMT(errorType, "bz2 error: [%d] %s", error, errorMsg);
}
FUNCTION_TEST_RETURN(error);
}

21
src/common/compress/bz2/common.h Normal file
View File

@ -0,0 +1,21 @@
/***********************************************************************************************************************************
BZ2 Common
Developed using the documentation in https://www.sourceware.org/bzip2/manual/manual.html#libprog
***********************************************************************************************************************************/
#ifndef COMMON_COMPRESS_BZ2_COMMON_H
#define COMMON_COMPRESS_BZ2_COMMON_H
#include <stddef.h>
/***********************************************************************************************************************************
BZ2 extension
***********************************************************************************************************************************/
#define BZ2_EXT "bz2"
/***********************************************************************************************************************************
Functions
***********************************************************************************************************************************/
int bz2Error(int error);
#endif

196
src/common/compress/bz2/compress.c Normal file
View File

@ -0,0 +1,196 @@
/***********************************************************************************************************************************
BZ2 Compress
***********************************************************************************************************************************/
#include "build.auto.h"
#include <stdio.h>
#include <bzlib.h>
#include "common/compress/bz2/common.h"
#include "common/compress/bz2/compress.h"
#include "common/debug.h"
#include "common/io/filter/filter.intern.h"
#include "common/log.h"
#include "common/memContext.h"
#include "common/type/object.h"
/***********************************************************************************************************************************
Filter type constant
***********************************************************************************************************************************/
STRING_EXTERN(BZ2_COMPRESS_FILTER_TYPE_STR, BZ2_COMPRESS_FILTER_TYPE);
/***********************************************************************************************************************************
Object type
***********************************************************************************************************************************/
#define BZ2_COMPRESS_TYPE Bz2Compress
#define BZ2_COMPRESS_PREFIX bz2Compress
typedef struct Bz2Compress
{
MemContext *memContext; // Context to store data
bz_stream stream; // Compression stream
bool inputSame; // Is the same input required on the next process call?
bool flushing; // Is input complete and flushing in progress?
bool done; // Is compression done?
} Bz2Compress;
/***********************************************************************************************************************************
Render as string for logging
***********************************************************************************************************************************/
static String *
bz2CompressToLog(const Bz2Compress *this)
{
return strNewFmt(
"{inputSame: %s, done: %s, flushing: %s, avail_in: %u}", cvtBoolToConstZ(this->inputSame), cvtBoolToConstZ(this->done),
cvtBoolToConstZ(this->flushing), this->stream.avail_in);
}
#define FUNCTION_LOG_BZ2_COMPRESS_TYPE \
Bz2Compress *
#define FUNCTION_LOG_BZ2_COMPRESS_FORMAT(value, buffer, bufferSize) \
FUNCTION_LOG_STRING_OBJECT_FORMAT(value, bz2CompressToLog, buffer, bufferSize)
/***********************************************************************************************************************************
Free compression stream
***********************************************************************************************************************************/
OBJECT_DEFINE_FREE_RESOURCE_BEGIN(BZ2_COMPRESS, LOG, logLevelTrace)
{
BZ2_bzCompressEnd(&this->stream);
}
OBJECT_DEFINE_FREE_RESOURCE_END(LOG);
/***********************************************************************************************************************************
Compress data
***********************************************************************************************************************************/
static void
bz2CompressProcess(THIS_VOID, const Buffer *uncompressed, Buffer *compressed)
{
THIS(Bz2Compress);
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(BZ2_COMPRESS, this);
FUNCTION_LOG_PARAM(BUFFER, uncompressed);
FUNCTION_LOG_PARAM(BUFFER, compressed);
FUNCTION_LOG_END();
ASSERT(this != NULL);
ASSERT(!this->done);
ASSERT(compressed != NULL);
ASSERT(!this->flushing || uncompressed == NULL);
ASSERT(this->flushing || (!this->inputSame || this->stream.avail_in != 0));
// If input is NULL then start flushing
if (uncompressed == NULL)
{
this->stream.avail_in = 0;
this->flushing = true;
}
// Else still have input data
else
{
// Is new input allowed?
if (!this->inputSame)
{
this->stream.avail_in = (unsigned int)bufUsed(uncompressed);
// bzip2 does not accept const input buffers
this->stream.next_in = (char *)UNCONSTIFY(unsigned char *, bufPtrConst(uncompressed));
}
}
// Initialize compressed output buffer
this->stream.avail_out = (unsigned int)bufRemains(compressed);
this->stream.next_out = (char *)bufPtr(compressed) + bufUsed(compressed);
// Perform compression, check for error
int result = bz2Error(BZ2_bzCompress(&this->stream, this->flushing ? BZ_FINISH : BZ_RUN));
// Set buffer used space
bufUsedSet(compressed, bufSize(compressed) - (size_t)this->stream.avail_out);
// Is compression done?
if (this->flushing && result == BZ_STREAM_END)
this->done = true;
// Can more input be provided on the next call?
this->inputSame = this->flushing ? !this->done : this->stream.avail_in != 0;
FUNCTION_LOG_RETURN_VOID();
}
/***********************************************************************************************************************************
Is compress done?
***********************************************************************************************************************************/
static bool
bz2CompressDone(const THIS_VOID)
{
THIS(const Bz2Compress);
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(BZ2_COMPRESS, this);
FUNCTION_TEST_END();
ASSERT(this != NULL);
FUNCTION_TEST_RETURN(this->done);
}
/***********************************************************************************************************************************
Is the same input required on the next process call?
***********************************************************************************************************************************/
static bool
bz2CompressInputSame(const THIS_VOID)
{
THIS(const Bz2Compress);
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(BZ2_COMPRESS, this);
FUNCTION_TEST_END();
ASSERT(this != NULL);
FUNCTION_TEST_RETURN(this->inputSame);
}
/**********************************************************************************************************************************/
IoFilter *
bz2CompressNew(int level)
{
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(INT, level);
FUNCTION_LOG_END();
ASSERT(level > 0);
IoFilter *this = NULL;
MEM_CONTEXT_NEW_BEGIN("Bz2Compress")
{
Bz2Compress *driver = memNew(sizeof(Bz2Compress));
*driver = (Bz2Compress)
{
.memContext = MEM_CONTEXT_NEW(),
.stream = {.bzalloc = NULL},
};
// Initialize context
bz2Error(BZ2_bzCompressInit(&driver->stream, level, 0, 0));
// Set callback to ensure bz2 stream is freed
memContextCallbackSet(driver->memContext, bz2CompressFreeResource, driver);
// Create param list
VariantList *paramList = varLstNew();
varLstAdd(paramList, varNewInt(level));
// Create filter interface
this = ioFilterNewP(
BZ2_COMPRESS_FILTER_TYPE_STR, driver, paramList, .done = bz2CompressDone, .inOut = bz2CompressProcess,
.inputSame = bz2CompressInputSame);
}
MEM_CONTEXT_NEW_END();
FUNCTION_LOG_RETURN(IO_FILTER, this);
}

22
src/common/compress/bz2/compress.h Normal file
View File

@ -0,0 +1,22 @@
/***********************************************************************************************************************************
BZ2 Compress
Compress IO to the bz2 format.
***********************************************************************************************************************************/
#ifndef COMMON_COMPRESS_BZ2_COMPRESS_H
#define COMMON_COMPRESS_BZ2_COMPRESS_H
#include "common/io/filter/filter.h"
/***********************************************************************************************************************************
Filter type constant
***********************************************************************************************************************************/
#define BZ2_COMPRESS_FILTER_TYPE "bz2Compress"
STRING_DECLARE(BZ2_COMPRESS_FILTER_TYPE_STR);
/***********************************************************************************************************************************
Constructors
***********************************************************************************************************************************/
IoFilter *bz2CompressNew(int level);
#endif

177
src/common/compress/bz2/decompress.c Normal file
View File

@ -0,0 +1,177 @@
/***********************************************************************************************************************************
BZ2 Decompress
***********************************************************************************************************************************/
#include "build.auto.h"
#include <stdio.h>
#include <bzlib.h>
#include "common/compress/bz2/common.h"
#include "common/compress/bz2/decompress.h"
#include "common/debug.h"
#include "common/io/filter/filter.intern.h"
#include "common/log.h"
#include "common/memContext.h"
#include "common/type/object.h"
/***********************************************************************************************************************************
Filter type constant
***********************************************************************************************************************************/
STRING_EXTERN(BZ2_DECOMPRESS_FILTER_TYPE_STR, BZ2_DECOMPRESS_FILTER_TYPE);
/***********************************************************************************************************************************
Object type
***********************************************************************************************************************************/
#define BZ2_DECOMPRESS_TYPE Bz2Decompress
#define BZ2_DECOMPRESS_PREFIX bz2Decompress
typedef struct Bz2Decompress
{
MemContext *memContext; // Context to store data
bz_stream stream; // Decompression stream state
int result; // Result of last operation
bool inputSame; // Is the same input required on the next process call?
bool done; // Is decompression done?
} Bz2Decompress;
/***********************************************************************************************************************************
Macros for function logging
***********************************************************************************************************************************/
static String *
bz2DecompressToLog(const Bz2Decompress *this)
{
return strNewFmt(
"{inputSame: %s, done: %s, avail_in: %u}", cvtBoolToConstZ(this->inputSame), cvtBoolToConstZ(this->done),
this->stream.avail_in);
}
#define FUNCTION_LOG_BZ2_DECOMPRESS_TYPE \
Bz2Decompress *
#define FUNCTION_LOG_BZ2_DECOMPRESS_FORMAT(value, buffer, bufferSize) \
FUNCTION_LOG_STRING_OBJECT_FORMAT(value, bz2DecompressToLog, buffer, bufferSize)
/***********************************************************************************************************************************
Free inflate stream
***********************************************************************************************************************************/
OBJECT_DEFINE_FREE_RESOURCE_BEGIN(BZ2_DECOMPRESS, LOG, logLevelTrace)
{
BZ2_bzDecompressEnd(&this->stream);
}
OBJECT_DEFINE_FREE_RESOURCE_END(LOG);
/***********************************************************************************************************************************
Decompress data
***********************************************************************************************************************************/
static void
bz2DecompressProcess(THIS_VOID, const Buffer *compressed, Buffer *uncompressed)
{
THIS(Bz2Decompress);
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(BZ2_DECOMPRESS, this);
FUNCTION_LOG_PARAM(BUFFER, compressed);
FUNCTION_LOG_PARAM(BUFFER, uncompressed);
FUNCTION_LOG_END();
ASSERT(this != NULL);
ASSERT(uncompressed != NULL);
// There should never be a flush because in a valid compressed stream the end of data can be determined and done will be set.
// If a flush is received it means the compressed stream terminated early, e.g. a zero-length or truncated file.
if (compressed == NULL)
THROW(FormatError, "unexpected eof in compressed data");
if (!this->inputSame)
{
this->stream.avail_in = (unsigned int)bufUsed(compressed);
// bzip2 does not accept const input buffers
this->stream.next_in = (char *)UNCONSTIFY(unsigned char *, bufPtrConst(compressed));
}
this->stream.avail_out = (unsigned int)bufRemains(uncompressed);
this->stream.next_out = (char *)bufPtr(uncompressed) + bufUsed(uncompressed);
this->result = bz2Error(BZ2_bzDecompress(&this->stream));
// Set buffer used space
bufUsedSet(uncompressed, bufSize(uncompressed) - (size_t)this->stream.avail_out);
// Is decompression done?
this->done = this->result == BZ_STREAM_END;
// Is the same input expected on the next call?
this->inputSame = this->done ? false : this->stream.avail_in != 0;
FUNCTION_LOG_RETURN_VOID();
}
/***********************************************************************************************************************************
Is decompress done?
***********************************************************************************************************************************/
static bool
bz2DecompressDone(const THIS_VOID)
{
THIS(const Bz2Decompress);
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(BZ2_DECOMPRESS, this);
FUNCTION_TEST_END();
ASSERT(this != NULL);
FUNCTION_TEST_RETURN(this->done);
}
/***********************************************************************************************************************************
Is the same input required on the next process call?
***********************************************************************************************************************************/
static bool
bz2DecompressInputSame(const THIS_VOID)
{
THIS(const Bz2Decompress);
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(BZ2_DECOMPRESS, this);
FUNCTION_TEST_END();
ASSERT(this != NULL);
FUNCTION_TEST_RETURN(this->inputSame);
}
/**********************************************************************************************************************************/
IoFilter *
bz2DecompressNew(void)
{
FUNCTION_LOG_VOID(logLevelTrace);
IoFilter *this = NULL;
MEM_CONTEXT_NEW_BEGIN("Bz2Decompress")
{
// Allocate state and set context
Bz2Decompress *driver = memNew(sizeof(Bz2Decompress));
*driver = (Bz2Decompress)
{
.memContext = MEM_CONTEXT_NEW(),
.stream = {.bzalloc = NULL},
};
// Create bz2 stream
bz2Error(driver->result = BZ2_bzDecompressInit(&driver->stream, 0, 0));
// Set free callback to ensure bz2 context is freed
memContextCallbackSet(driver->memContext, bz2DecompressFreeResource, driver);
// Create filter interface
this = ioFilterNewP(
BZ2_DECOMPRESS_FILTER_TYPE_STR, driver, NULL, .done = bz2DecompressDone, .inOut = bz2DecompressProcess,
.inputSame = bz2DecompressInputSame);
}
MEM_CONTEXT_NEW_END();
FUNCTION_LOG_RETURN(IO_FILTER, this);
}

22
src/common/compress/bz2/decompress.h Normal file
View File

@ -0,0 +1,22 @@
/***********************************************************************************************************************************
BZ2 Decompress
Decompress IO from the bz2 format.
***********************************************************************************************************************************/
#ifndef COMMON_COMPRESS_BZ2_DECOMPRESS_H
#define COMMON_COMPRESS_BZ2_DECOMPRESS_H
#include "common/io/filter/filter.h"
/***********************************************************************************************************************************
Filter type constant
***********************************************************************************************************************************/
#define BZ2_DECOMPRESS_FILTER_TYPE "bz2Decompress"
STRING_DECLARE(BZ2_DECOMPRESS_FILTER_TYPE_STR);
/***********************************************************************************************************************************
Constructors
***********************************************************************************************************************************/
IoFilter *bz2DecompressNew(void);
#endif

13
src/common/compress/helper.c
View File

@ -6,6 +6,9 @@ Compression Helper
#include <string.h>
#include "common/compress/helper.h"
#include "common/compress/bz2/common.h"
#include "common/compress/bz2/compress.h"
#include "common/compress/bz2/decompress.h"
#include "common/compress/gz/common.h"
#include "common/compress/gz/compress.h"
#include "common/compress/gz/decompress.h"
@ -25,7 +28,6 @@ Compression type constants
#define COMPRESS_TYPE_NONE "none"
// Constants for currently unsupported compression types
#define BZ2_EXT "bz2"
#define XZ_EXT "xz"
/***********************************************************************************************************************************
@ -46,6 +48,15 @@ static const struct CompressHelperLocal
.type = STRDEF(COMPRESS_TYPE_NONE),
.ext = STRDEF(""),
},
{
.type = STRDEF(BZ2_EXT),
.ext = STRDEF("." BZ2_EXT),
.compressType = BZ2_COMPRESS_FILTER_TYPE,
.compressNew = bz2CompressNew,
.decompressType = BZ2_DECOMPRESS_FILTER_TYPE,
.decompressNew = bz2DecompressNew,
.levelDefault = 9,
},
{
.type = STRDEF(GZ_EXT),
.ext = STRDEF("." GZ_EXT),

2
src/common/compress/helper.h
View File

@ -15,6 +15,7 @@ Available compression types
typedef enum
{
compressTypeNone, // No compression
compressTypeBz2, // bzip2
compressTypeGz, // gzip
compressTypeLz4, // lz4
compressTypeZst, // zstandard
@ -23,7 +24,6 @@ typedef enum
// versions. In that sense this list is speculative, but these seem to be all the types that are likely to be added in the
// foreseeable future.
compressTypeXz, // xz/lzma
compressTypeBz2, // bzip2
} CompressType;
#include <common/type/string.h>

2
src/config/define.auto.c
View File

@ -832,6 +832,7 @@ static ConfigDefineOptionData configDefineOptionData[] = CFGDEFDATA_OPTION_LIST
(
"The following compression types are supported:\n"
"\n"
"* bz2 - bzip2 compression format\n"
"* gz - gzip compression format\n"
"* lz4 - lz4 compression format (not available on all platforms)\n"
"* zst - Zstandard compression format (not available on all platforms)"
@ -848,6 +849,7 @@ static ConfigDefineOptionData configDefineOptionData[] = CFGDEFDATA_OPTION_LIST
CFGDEFDATA_OPTION_OPTIONAL_ALLOW_LIST
(
"none",
"bz2",
"gz",
"lz4",
"zst"

60
src/configure vendored
View File

@ -4018,6 +4018,64 @@ fi
# Check required bzip2 library
# ----------------------------------------------------------------------------------------------------------------------------------
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for BZ2_bzCompress in -lbz2" >&5
$as_echo_n "checking for BZ2_bzCompress in -lbz2... " >&6; }
if ${ac_cv_lib_bz2_BZ2_bzCompress+:} false; then :
$as_echo_n "(cached) " >&6
else
ac_check_lib_save_LIBS=$LIBS
LIBS="-lbz2 $LIBS"
cat confdefs.h - <<_ACEOF >conftest.$ac_ext
/* end confdefs.h. */
/* Override any GCC internal prototype to avoid an error.
Use char because int might match the return type of a GCC
builtin and then its argument prototype would still apply. */
#ifdef __cplusplus
extern "C"
#endif
char BZ2_bzCompress ();
int
main ()
{
return BZ2_bzCompress ();
;
return 0;
}
_ACEOF
if ac_fn_c_try_link "$LINENO"; then :
ac_cv_lib_bz2_BZ2_bzCompress=yes
else
ac_cv_lib_bz2_BZ2_bzCompress=no
fi
rm -f core conftest.err conftest.$ac_objext \
conftest$ac_exeext conftest.$ac_ext
LIBS=$ac_check_lib_save_LIBS
fi
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_bz2_BZ2_bzCompress" >&5
$as_echo "$ac_cv_lib_bz2_BZ2_bzCompress" >&6; }
if test "x$ac_cv_lib_bz2_BZ2_bzCompress" = xyes; then :
cat >>confdefs.h <<_ACEOF
#define HAVE_LIBBZ2 1
_ACEOF
LIBS="-lbz2 $LIBS"
else
as_fn_error $? "library 'bz2' is required" "$LINENO" 5
fi
ac_fn_c_check_header_mongrel "$LINENO" "bzlib.h" "ac_cv_header_bzlib_h" "$ac_includes_default"
if test "x$ac_cv_header_bzlib_h" = xyes; then :
else
as_fn_error $? "header file <bzlib.h> is required" "$LINENO" 5
fi
# Check optional lz4 library
# ----------------------------------------------------------------------------------------------------------------------------------
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for LZ4F_isError in -llz4" >&5
@ -5417,4 +5475,4 @@ if test -n "$ac_unrecognized_opts" && test "$enable_option_checking" != no; then
$as_echo "$as_me: WARNING: unrecognized options: $ac_unrecognized_opts" >&2;}
fi
# Generated from src/build/configure.ac sha1 93fbf56725f3dab1089ddd4d7768f4414bf7941c
# Generated from src/build/configure.ac sha1 a4acd8b59dcdff7e986a8454335119e955d63c91

2
test/Vagrantfile vendored
View File

@ -75,7 +75,7 @@ Vagrant.configure(2) do |config|
#-----------------------------------------------------------------------------------------------------------------------
echo 'Install Build Tools' && date
apt-get install -y devscripts build-essential lintian git cloc txt2man debhelper libssl-dev zlib1g-dev libperl-dev \
libxml2-dev liblz4-dev liblz4-tool libpq-dev valgrind lcov autoconf-archive zstd libzstd-dev
libxml2-dev liblz4-dev liblz4-tool libpq-dev valgrind lcov autoconf-archive zstd libzstd-dev bzip2 libbz2-dev
#-----------------------------------------------------------------------------------------------------------------------
echo 'Install Docker' && date

16
test/container.yaml
View File

@ -12,13 +12,9 @@
# - docker login -u pgbackrest
# - VM=XXX;DATE=YYYYMMDDX;BASE=pgbackrest/test:${VM?}-base;docker tag ${BASE?} ${BASE?}-${DATE?} && docker push ${BASE?}-${DATE?}
# **********************************************************************************************************************************
20200504A:
co7: 9c922915380c0beb3d813c94d53f823d7008c69b
f30: 9ad36516c2403b835c24efbbcef359322a80fa1f
u18: 32426ecf8cd0e98976298f6f6b41a2b255aca692
20200310A:
co6: beb7b5a62ebdf209bb54494129baeadbd0c2954f
20200124A:
u12: 0f2fcf1bc79ee35e78121773c9a2155a77cb10d5
20200505A:
co6: c07889acc321e461263fada797941cfa947d2550
co7: 538dc7fd9cc129e44d4ae6bacfb6f1db10309993
f30: 3ae2c1749afa24daa1ca31abe9441d5e1e66d892
u12: 8a88ab44aace049d7da5ca1094375ff8b9aeb7ab
u18: 0f88948969b70f300a9134482e43cda7b94cb24d

5
test/define.yaml
View File

@ -262,9 +262,12 @@ unit:
# ----------------------------------------------------------------------------------------------------------------------------
- name: compress
total: 4
total: 5
coverage:
common/compress/bz2/common: full
common/compress/bz2/compress: full
common/compress/bz2/decompress: full
common/compress/gz/common: full
common/compress/gz/compress: full
common/compress/gz/decompress: full

5
test/lib/pgBackRestTest/Common/ContainerTest.pm
View File

@ -361,7 +361,7 @@ sub containerBuild
" yum -y install openssh-server openssh-clients wget sudo valgrind git \\\n" .
" perl perl-Digest-SHA perl-DBD-Pg perl-YAML-LibYAML openssl \\\n" .
" gcc make perl-ExtUtils-MakeMaker perl-Test-Simple openssl-devel perl-ExtUtils-Embed rpm-build \\\n" .
" zlib-devel libxml2-devel lz4-devel lz4";
" zlib-devel libxml2-devel lz4-devel lz4 bzip2-devel bzip2";
if ($strOS eq VM_CO6)
{
@ -380,7 +380,8 @@ sub containerBuild
" apt-get -y install openssh-server wget sudo gcc make valgrind git \\\n" .
" libdbd-pg-perl libhtml-parser-perl libssl-dev libperl-dev \\\n" .
" libyaml-libyaml-perl tzdata devscripts lintian libxml-checker-perl txt2man debhelper \\\n" .
" libppi-html-perl libtemplate-perl libtest-differences-perl zlib1g-dev libxml2-dev pkg-config";
" libppi-html-perl libtemplate-perl libtest-differences-perl zlib1g-dev libxml2-dev pkg-config \\\n" .
" libbz2-dev bzip2";
if ($strOS eq VM_U12)
{

2
test/lib/pgBackRestTest/Common/JobTest.pm
View File

@ -518,7 +518,7 @@ sub run
"BUILDFLAGS=${strBuildFlags}\n" .
"HARNESSFLAGS=${strHarnessFlags}\n" .
"TESTFLAGS=${strTestFlags}\n" .
"LDFLAGS=-lcrypto -lssl -lxml2 -lz" .
"LDFLAGS=-lcrypto -lssl -lxml2 -lz -lbz2" .
(vmWithLz4($self->{oTest}->{&TEST_VM}) ? ' -llz4' : '') .
(vmWithZst($self->{oTest}->{&TEST_VM}) ? ' -lzstd' : '') .
(vmCoverageC($self->{oTest}->{&TEST_VM}) && $self->{bCoverageUnit} ? " -lgcov" : '') .

2
test/lib/pgBackRestTest/Env/Host/HostBackupTest.pm
View File

@ -95,6 +95,8 @@ use constant CFGOPTVAL_RESTORE_TYPE_XID => 'xid';
use constant NONE => 'none';
push @EXPORT, qw(NONE);
use constant BZ2 => 'bz2';
push @EXPORT, qw(BZ2);
use constant GZ => 'gz';
push @EXPORT, qw(GZ);
use constant LZ4 => 'lz4';

6
test/lib/pgBackRestTest/Module/Mock/MockAllTest.pm
View File

@ -206,12 +206,12 @@ sub run
foreach my $rhRun
(
{vm => VM1, remote => false, s3 => true, encrypt => false, delta => true, compress => LZ4},
{vm => VM1, remote => true, s3 => false, encrypt => true, delta => false, compress => GZ},
{vm => VM2, remote => false, s3 => false, encrypt => true, delta => true, compress => GZ},
{vm => VM1, remote => true, s3 => false, encrypt => true, delta => false, compress => BZ2},
{vm => VM2, remote => false, s3 => false, encrypt => true, delta => true, compress => BZ2},
{vm => VM2, remote => true, s3 => true, encrypt => false, delta => false, compress => GZ},
{vm => VM3, remote => false, s3 => false, encrypt => false, delta => true, compress => ZST},
{vm => VM3, remote => true, s3 => true, encrypt => true, delta => false, compress => LZ4},
{vm => VM4, remote => false, s3 => false, encrypt => false, delta => false, compress => LZ4},
{vm => VM4, remote => false, s3 => false, encrypt => false, delta => false, compress => GZ},
{vm => VM4, remote => true, s3 => true, encrypt => true, delta => true, compress => ZST},
)
{

2
test/lib/pgBackRestTest/Module/Mock/MockArchiveStopTest.pm
View File

@ -46,7 +46,7 @@ sub run
{vm => VM1, remote => false, s3 => false, encrypt => false, compress => LZ4, error => 0},
{vm => VM1, remote => true, s3 => true, encrypt => true, compress => GZ, error => 1},
{vm => VM2, remote => false, s3 => true, encrypt => false, compress => NONE, error => 0},
{vm => VM2, remote => true, s3 => false, encrypt => true, compress => GZ, error => 0},
{vm => VM2, remote => true, s3 => false, encrypt => true, compress => BZ2, error => 0},
{vm => VM3, remote => false, s3 => false, encrypt => true, compress => NONE, error => 0},
{vm => VM3, remote => true, s3 => true, encrypt => false, compress => LZ4, error => 1},
{vm => VM4, remote => false, s3 => true, encrypt => true, compress => ZST, error => 0},

10
test/lib/pgBackRestTest/Module/Mock/MockArchiveTest.pm
View File

@ -83,14 +83,14 @@ sub run
foreach my $rhRun
(
{vm => VM1, remote => false, s3 => false, encrypt => false, compress => LZ4},
{vm => VM1, remote => true, s3 => true, encrypt => true, compress => GZ},
{vm => VM2, remote => false, s3 => true, encrypt => false, compress => GZ},
{vm => VM2, remote => true, s3 => false, encrypt => true, compress => GZ},
{vm => VM1, remote => false, s3 => false, encrypt => false, compress => GZ},
{vm => VM1, remote => true, s3 => true, encrypt => true, compress => LZ4},
{vm => VM2, remote => false, s3 => true, encrypt => false, compress => BZ2},
{vm => VM2, remote => true, s3 => false, encrypt => true, compress => BZ2},
{vm => VM3, remote => false, s3 => false, encrypt => true, compress => LZ4},
{vm => VM3, remote => true, s3 => true, encrypt => false, compress => ZST},
{vm => VM4, remote => false, s3 => true, encrypt => true, compress => ZST},
{vm => VM4, remote => true, s3 => false, encrypt => false, compress => LZ4},
{vm => VM4, remote => true, s3 => false, encrypt => false, compress => GZ},
)
{
# Only run tests for this vm

6
test/lib/pgBackRestTest/Module/Mock/MockStanzaTest.pm
View File

@ -42,12 +42,12 @@ sub run
foreach my $rhRun
(
{vm => VM1, remote => false, s3 => false, encrypt => true, compress => LZ4},
{vm => VM1, remote => true, s3 => true, encrypt => false, compress => GZ},
{vm => VM2, remote => false, s3 => true, encrypt => true, compress => GZ},
{vm => VM1, remote => true, s3 => true, encrypt => false, compress => BZ2},
{vm => VM2, remote => false, s3 => true, encrypt => true, compress => BZ2},
{vm => VM2, remote => true, s3 => false, encrypt => false, compress => GZ},
{vm => VM3, remote => false, s3 => false, encrypt => false, compress => ZST},
{vm => VM3, remote => true, s3 => true, encrypt => true, compress => LZ4},
{vm => VM4, remote => false, s3 => true, encrypt => false, compress => LZ4},
{vm => VM4, remote => false, s3 => true, encrypt => false, compress => GZ},
{vm => VM4, remote => true, s3 => false, encrypt => true, compress => ZST},
)
{

2
test/lib/pgBackRestTest/Module/Real/RealAllTest.pm
View File

@ -66,7 +66,7 @@ sub run
{
my $strCompressType =
$bHostBackup && !$bHostStandby ?
(vmWithLz4($self->vm()) && $bLz4Compress ? LZ4 : vmWithZst($self->vm()) ? ZST : GZ) : NONE;
(vmWithLz4($self->vm()) && $bLz4Compress ? LZ4 : vmWithZst($self->vm()) ? ZST : ($bS3 ? BZ2 : GZ)) : NONE;
my $bRepoEncrypt = ($strCompressType ne NONE && !$bS3) ? true : false;
# If compression was used then switch it for the next test that uses compression

3
test/patch/debian-package.patch
View File

@ -1,12 +1,13 @@
--- control
+++ control
@@ -13,7 +13,8 @@
@@ -13,7 +13,9 @@
pkg-config,
txt2man,
zlib1g-dev,
- liblz4-dev
+ liblz4-dev,
+ libzstd-dev,
+ libbz2-dev,
Standards-Version: 4.4.1
Homepage: https://www.pgbackrest.org/
Vcs-Git: https://salsa.debian.org/postgresql/pgbackrest.git

55
test/src/module/common/compressTest.c
View File

@ -147,19 +147,23 @@ testSuite(CompressType type, const char *decompressCmd)
TEST_ERROR(testDecompress(decompressFilter(type), truncated, 512, 512), FormatError, "unexpected eof in compressed data");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("compress a large zero input buffer into small output buffer");
TEST_TITLE("compress a large non-zero input buffer into small output buffer");
decompressed = bufNew(1024 * 1024 - 1);
memset(bufPtr(decompressed), 0, bufSize(decompressed));
unsigned char *c = bufPtr(decompressed);
for (size_t i = 0; i < bufSize(decompressed); i++)
c[i] = (unsigned char)(i % 94 + 32);
bufUsedSet(decompressed, bufSize(decompressed));
TEST_ASSIGN(
compressed, testCompress(compressFilter(type, 3), decompressed, bufSize(decompressed), 32),
"zero data - compress large in/small out buffer");
"non-zero data - compress large in/small out buffer");
TEST_RESULT_BOOL(
bufEq(decompressed, testDecompress(decompressFilter(type), compressed, bufSize(compressed), 1024 * 256)), true,
"zero data - decompress large in/small out buffer");
"non-zero data - decompress large in/small out buffer");
}
/***********************************************************************************************************************************
@ -203,6 +207,49 @@ testRun(void)
TEST_RESULT_STR_Z(gzDecompressToLog(decompress), "{inputSame: true, done: true, availIn: 0}", "format object");
}
// *****************************************************************************************************************************
if (testBegin("bz2"))
{
// Run standard test suite
testSuite(compressTypeBz2, "bzip2 -dc");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("bz2Error()");
TEST_RESULT_INT(bz2Error(BZ_OK), BZ_OK, "check ok");
TEST_RESULT_INT(bz2Error(BZ_RUN_OK), BZ_RUN_OK, "check run ok");
TEST_RESULT_INT(bz2Error(BZ_FLUSH_OK), BZ_FLUSH_OK, "check flush ok");
TEST_RESULT_INT(bz2Error(BZ_FINISH_OK), BZ_FINISH_OK, "check finish ok");
TEST_RESULT_INT(bz2Error(BZ_STREAM_END), BZ_STREAM_END, "check stream end");
TEST_ERROR(bz2Error(BZ_SEQUENCE_ERROR), AssertError, "bz2 error: [-1] sequence error");
TEST_ERROR(bz2Error(BZ_PARAM_ERROR), AssertError, "bz2 error: [-2] parameter error");
TEST_ERROR(bz2Error(BZ_MEM_ERROR), AssertError, "bz2 error: [-3] memory error");
TEST_ERROR(bz2Error(BZ_DATA_ERROR), AssertError, "bz2 error: [-4] data error");
TEST_ERROR(bz2Error(BZ_DATA_ERROR_MAGIC), AssertError, "bz2 error: [-5] data error magic");
TEST_ERROR(bz2Error(BZ_IO_ERROR), AssertError, "bz2 error: [-6] io error");
TEST_ERROR(bz2Error(BZ_UNEXPECTED_EOF), AssertError, "bz2 error: [-7] unexpected eof");
TEST_ERROR(bz2Error(BZ_OUTBUFF_FULL), AssertError, "bz2 error: [-8] outbuff full");
TEST_ERROR(bz2Error(BZ_CONFIG_ERROR), AssertError, "bz2 error: [-9] config error");
TEST_ERROR(bz2Error(-999), AssertError, "bz2 error: [-999] unknown error");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("bz2DecompressToLog() and bz2CompressToLog()");
Bz2Compress *compress = (Bz2Compress *)ioFilterDriver(bz2CompressNew(1));
compress->stream.avail_in = 999;
TEST_RESULT_STR_Z(
bz2CompressToLog(compress), "{inputSame: false, done: false, flushing: false, avail_in: 999}", "format object");
Bz2Decompress *decompress = (Bz2Decompress *)ioFilterDriver(bz2DecompressNew());
decompress->inputSame = true;
decompress->done = true;
TEST_RESULT_STR_Z(bz2DecompressToLog(decompress), "{inputSame: true, done: true, avail_in: 0}", "format object");
}
// *****************************************************************************************************************************
if (testBegin("lz4"))
{

2
test/travis.pl
View File

@ -186,7 +186,7 @@ eval
# Extra packages required when testing without containers
if ($strVm eq VM_NONE)
{
$strPackage .= " valgrind liblz4-dev liblz4-tool zstd libzstd-dev";
$strPackage .= " valgrind liblz4-dev liblz4-tool zstd libzstd-dev bzip2 libbz2-dev";
}
# Else packages needed for integration tests on containers
else