mirror of
https://github.com/pgbackrest/pgbackrest.git
synced 2025-01-18 04:58:51 +02:00
Add Zstandard compression support.
Zstandard is a fast lossless compression algorithm targeting real-time compression scenarios at zlib-level and better compression ratios. It's backed by a very fast entropy stage, provided by Huff0 and FSE library. Zstandard version >= 1.0 is required, which is generally only available on newer distributions.
This commit is contained in:
David Steele
parent
1aaaa94253
commit
47aa765375
@ -1236,6 +1236,7 @@ my %hConfigDefine =
|
||||
'none',
|
||||
'gz',
|
||||
'lz4',
|
||||
'zst',
|
||||
],
|
||||
&CFGDEF_COMMAND => CFGOPT_COMPRESS,
|
||||
},
|
||||
|
||||
@ -64,6 +64,7 @@
|
||||
apt-get install rsync git devscripts build-essential valgrind lcov autoconf
|
||||
autoconf-archive libssl-dev zlib1g-dev libxml2-dev libpq-dev pkg-config
|
||||
libxml-checker-perl libyaml-libyaml-perl libdbd-pg-perl liblz4-dev liblz4-tool
|
||||
zstd libzstd-dev
|
||||
</exe-cmd>
|
||||
<exe-cmd-extra>-y 2>&1</exe-cmd-extra>
|
||||
</execute>
|
||||
|
||||
@ -143,6 +143,7 @@
|
||||
<ul>
|
||||
<li><id>gz</id> - gzip compression format</li>
|
||||
<li><id>lz4</id> - lz4 compression format (not available on all platforms)</li>
|
||||
<li><id>zst</id> - Zstandard compression format (not available on all platforms)</li>
|
||||
</ul></text>
|
||||
|
||||
<example>n</example>
|
||||
|
||||
@ -53,6 +53,16 @@
|
||||
<p>Allow the user to remove a specified backup regardless of retention settings.</p>
|
||||
</release-item>
|
||||
|
||||
<release-item>
|
||||
<release-item-contributor-list>
|
||||
<release-item-reviewer id="cynthia.shang"/>
|
||||
</release-item-contributor-list>
|
||||
|
||||
<p>Add <proper>Zstandard</proper> compression support.</p>
|
||||
|
||||
<p>Note that setting <br-option>compress-type=zst</br-option> will make new backups and archive incompatible (unrestorable) with prior versions of <backrest/>.</p>
|
||||
</release-item>
|
||||
|
||||
<release-item>
|
||||
<release-item-contributor-list>
|
||||
<release-item-contributor id="stefan.fercot"/>
|
||||
|
||||
@ -274,14 +274,13 @@
|
||||
{[copy-ca-cert]}
|
||||
|
||||
# Fix root tty
|
||||
RUN sed -i 's/^mesg n/tty -s \&\& mesg n/g' /root/.profile && \
|
||||
RUN sed -i 's/^mesg n/tty -s \&\& mesg n/g' /root/.profile
|
||||
|
||||
# Suppress dpkg interactive output
|
||||
rm /etc/apt/apt.conf.d/70debconf
|
||||
|
||||
# Install base packages
|
||||
RUN apt-get update && \
|
||||
apt-get install -y sudo ssh wget vim gnupg lsb-release iputils-ping ca-certificates locales 2>&1
|
||||
# Install base packages (suppress dpkg interactive output)
|
||||
RUN export DEBIAN_FRONTEND=noninteractive && \
|
||||
rm /etc/apt/apt.conf.d/70debconf && \
|
||||
apt-get update && \
|
||||
apt-get install -y sudo ssh wget vim gnupg lsb-release iputils-ping ca-certificates tzdata locales 2>&1
|
||||
|
||||
# Install CA certificate
|
||||
RUN update-ca-certificates
|
||||
@ -767,13 +766,16 @@
|
||||
|
||||
<execute if="{[os-type-is-debian]}" user="root" pre="y">
|
||||
<exe-cmd>
|
||||
apt-get install make gcc libpq-dev libssl-dev libxml2-dev pkg-config liblz4-dev</exe-cmd>
|
||||
apt-get install make gcc libpq-dev libssl-dev libxml2-dev pkg-config
|
||||
liblz4-dev libzstd-dev
|
||||
</exe-cmd>
|
||||
<exe-cmd-extra>-y 2>&1</exe-cmd-extra>
|
||||
</execute>
|
||||
|
||||
<execute if="{[os-type-is-centos]}" user="root" pre="y">
|
||||
<exe-cmd>
|
||||
yum install make gcc postgresql-devel openssl-devel libxml2-devel lz4-devel
|
||||
yum install make gcc postgresql-devel openssl-devel libxml2-devel
|
||||
lz4-devel libzstd-devel
|
||||
</exe-cmd>
|
||||
<exe-cmd-extra>-y 2>&1</exe-cmd-extra>
|
||||
</execute>
|
||||
|
||||
@ -48,6 +48,9 @@ SRCS = \
|
||||
common/compress/lz4/common.c \
|
||||
common/compress/lz4/compress.c \
|
||||
common/compress/lz4/decompress.c \
|
||||
common/compress/zst/common.c \
|
||||
common/compress/zst/compress.c \
|
||||
common/compress/zst/decompress.c \
|
||||
common/crypto/cipherBlock.c \
|
||||
common/crypto/common.c \
|
||||
common/crypto/hash.c \
|
||||
|
||||
@ -12,3 +12,6 @@ Build Flags Generated by Configure
|
||||
|
||||
// Is liblz4 present?
|
||||
#undef HAVE_LIBLZ4
|
||||
|
||||
// Is libzstd present?
|
||||
#undef HAVE_LIBZST
|
||||
|
||||
@ -125,6 +125,20 @@ AC_CHECK_LIB(
|
||||
[AC_CHECK_HEADER(lz4frame.h, [AC_DEFINE(HAVE_LIBLZ4) AC_SUBST(LIBS, "${LIBS} -llz4")],
|
||||
[AC_MSG_ERROR([header file <lz4frame.h> is required])])])
|
||||
|
||||
# Check optional zst library. Ignore any versions below 1.0.
|
||||
# ----------------------------------------------------------------------------------------------------------------------------------
|
||||
AC_CHECK_LIB(
|
||||
[zstd], [ZSTD_isError],
|
||||
[AC_CHECK_HEADER(zstd.h,
|
||||
[AC_LINK_IFELSE(
|
||||
[AC_LANG_PROGRAM(
|
||||
[#include <zstd.h>],
|
||||
[#if ZSTD_VERSION_MAJOR < 1
|
||||
#error "ZSTD_VERSION_MAJOR must be >= 1"
|
||||
#endif])],
|
||||
[AC_DEFINE(HAVE_LIBZST) AC_SUBST(LIBS, "${LIBS} -lzstd")])],
|
||||
[AC_MSG_ERROR([header file <zstd.h> is required])])])
|
||||
|
||||
# Write output
|
||||
# ----------------------------------------------------------------------------------------------------------------------------------
|
||||
AC_CONFIG_HEADERS([build.auto.h])
|
||||
|
||||
@ -12,6 +12,9 @@ Compression Helper
|
||||
#include "common/compress/lz4/common.h"
|
||||
#include "common/compress/lz4/compress.h"
|
||||
#include "common/compress/lz4/decompress.h"
|
||||
#include "common/compress/zst/common.h"
|
||||
#include "common/compress/zst/compress.h"
|
||||
#include "common/compress/zst/decompress.h"
|
||||
#include "common/debug.h"
|
||||
#include "common/log.h"
|
||||
#include "version.h"
|
||||
@ -22,7 +25,6 @@ Compression type constants
|
||||
#define COMPRESS_TYPE_NONE "none"
|
||||
|
||||
// Constants for currently unsupported compression types
|
||||
#define ZST_EXT "zst"
|
||||
#define BZ2_EXT "bz2"
|
||||
#define XZ_EXT "xz"
|
||||
|
||||
@ -67,6 +69,13 @@ static const struct CompressHelperLocal
|
||||
{
|
||||
.type = STRDEF(ZST_EXT),
|
||||
.ext = STRDEF("." ZST_EXT),
|
||||
#ifdef HAVE_LIBZST
|
||||
.compressType = ZST_COMPRESS_FILTER_TYPE,
|
||||
.compressNew = zstCompressNew,
|
||||
.decompressType = ZST_DECOMPRESS_FILTER_TYPE,
|
||||
.decompressNew = zstDecompressNew,
|
||||
.levelDefault = 3,
|
||||
#endif
|
||||
},
|
||||
{
|
||||
.type = STRDEF(XZ_EXT),
|
||||
|
||||
@ -17,11 +17,11 @@ typedef enum
|
||||
compressTypeNone, // No compression
|
||||
compressTypeGz, // gzip
|
||||
compressTypeLz4, // lz4
|
||||
compressTypeZst, // zstandard
|
||||
|
||||
// These types have not been implemented but are included here so older versions can identify compression types added by future
|
||||
// versions. In that sense this list is speculative, but these seem to be all the types that are likely to be added in the
|
||||
// foreseeable future.
|
||||
compressTypeZst, // zstandard
|
||||
compressTypeXz, // xz/lzma
|
||||
compressTypeBz2, // bzip2
|
||||
} CompressType;
|
||||
|
||||
32
src/common/compress/zst/common.c
Normal file
32
src/common/compress/zst/common.c
Normal file
@ -0,0 +1,32 @@
|
||||
/***********************************************************************************************************************************
|
||||
ZST Common
|
||||
***********************************************************************************************************************************/
|
||||
#include "build.auto.h"
|
||||
|
||||
#ifdef HAVE_LIBZST
|
||||
|
||||
#include <zstd.h>
|
||||
|
||||
// Check the version -- this is done in configure but it makes sense to be sure
|
||||
#if ZSTD_VERSION_MAJOR < 1
|
||||
#error "ZSTD_VERSION_MAJOR must be >= 1"
|
||||
#endif
|
||||
|
||||
#include "common/compress/zst/common.h"
|
||||
#include "common/debug.h"
|
||||
|
||||
/**********************************************************************************************************************************/
|
||||
size_t
|
||||
zstError(size_t error)
|
||||
{
|
||||
FUNCTION_TEST_BEGIN();
|
||||
FUNCTION_TEST_PARAM(SIZE, error);
|
||||
FUNCTION_TEST_END();
|
||||
|
||||
if (ZSTD_isError(error))
|
||||
THROW_FMT(FormatError, "zst error: [%zd] %s", (ssize_t)error, ZSTD_getErrorName(error));
|
||||
|
||||
FUNCTION_TEST_RETURN(error);
|
||||
}
|
||||
|
||||
#endif // HAVE_LIBZST
|
||||
25
src/common/compress/zst/common.h
Normal file
25
src/common/compress/zst/common.h
Normal file
@ -0,0 +1,25 @@
|
||||
/***********************************************************************************************************************************
|
||||
ZST Common
|
||||
|
||||
Developed using the documentation in https://github.com/facebook/zstd/blob/v1.0.0/lib/zstd.h.
|
||||
***********************************************************************************************************************************/
|
||||
#ifndef COMMON_COMPRESS_ZST_COMMON_H
|
||||
#define COMMON_COMPRESS_ZST_COMMON_H
|
||||
|
||||
#include <stddef.h>
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
ZST extension
|
||||
***********************************************************************************************************************************/
|
||||
#define ZST_EXT "zst"
|
||||
|
||||
#ifdef HAVE_LIBZST
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Functions
|
||||
***********************************************************************************************************************************/
|
||||
size_t zstError(size_t error);
|
||||
|
||||
#endif // HAVE_LIBZST
|
||||
|
||||
#endif
|
||||
207
src/common/compress/zst/compress.c
Normal file
207
src/common/compress/zst/compress.c
Normal file
@ -0,0 +1,207 @@
|
||||
/***********************************************************************************************************************************
|
||||
ZST Compress
|
||||
***********************************************************************************************************************************/
|
||||
#include "build.auto.h"
|
||||
|
||||
#ifdef HAVE_LIBZST
|
||||
|
||||
#include <zstd.h>
|
||||
|
||||
#include "common/compress/zst/common.h"
|
||||
#include "common/compress/zst/compress.h"
|
||||
#include "common/debug.h"
|
||||
#include "common/io/filter/filter.intern.h"
|
||||
#include "common/log.h"
|
||||
#include "common/memContext.h"
|
||||
#include "common/type/object.h"
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Filter type constant
|
||||
***********************************************************************************************************************************/
|
||||
STRING_EXTERN(ZST_COMPRESS_FILTER_TYPE_STR, ZST_COMPRESS_FILTER_TYPE);
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Object type
|
||||
***********************************************************************************************************************************/
|
||||
#define ZST_COMPRESS_TYPE ZstCompress
|
||||
#define ZST_COMPRESS_PREFIX zstCompress
|
||||
|
||||
typedef struct ZstCompress
|
||||
{
|
||||
MemContext *memContext; // Context to store data
|
||||
ZSTD_CStream *context; // Compression context
|
||||
int level; // Compression level
|
||||
IoFilter *filter; // Filter interface
|
||||
|
||||
bool inputSame; // Is the same input required on the next process call?
|
||||
size_t inputOffset; // Current offset in input buffer
|
||||
bool flushing; // Is input complete and flushing in progress?
|
||||
} ZstCompress;
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Render as string for logging
|
||||
***********************************************************************************************************************************/
|
||||
static String *
|
||||
zstCompressToLog(const ZstCompress *this)
|
||||
{
|
||||
return strNewFmt(
|
||||
"{level: %d, inputSame: %s, inputOffset: %zu, flushing: %s}", this->level, cvtBoolToConstZ(this->inputSame),
|
||||
this->inputOffset, cvtBoolToConstZ(this->flushing));
|
||||
}
|
||||
|
||||
#define FUNCTION_LOG_ZST_COMPRESS_TYPE \
|
||||
ZstCompress *
|
||||
#define FUNCTION_LOG_ZST_COMPRESS_FORMAT(value, buffer, bufferSize) \
|
||||
FUNCTION_LOG_STRING_OBJECT_FORMAT(value, zstCompressToLog, buffer, bufferSize)
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Free compression context
|
||||
***********************************************************************************************************************************/
|
||||
OBJECT_DEFINE_FREE_RESOURCE_BEGIN(ZST_COMPRESS, LOG, logLevelTrace)
|
||||
{
|
||||
ZSTD_freeCStream(this->context);
|
||||
}
|
||||
OBJECT_DEFINE_FREE_RESOURCE_END(LOG);
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Compress data
|
||||
***********************************************************************************************************************************/
|
||||
static void
|
||||
zstCompressProcess(THIS_VOID, const Buffer *uncompressed, Buffer *compressed)
|
||||
{
|
||||
THIS(ZstCompress);
|
||||
|
||||
FUNCTION_LOG_BEGIN(logLevelTrace);
|
||||
FUNCTION_LOG_PARAM(ZST_COMPRESS, this);
|
||||
FUNCTION_LOG_PARAM(BUFFER, uncompressed);
|
||||
FUNCTION_LOG_PARAM(BUFFER, compressed);
|
||||
FUNCTION_LOG_END();
|
||||
|
||||
ASSERT(this != NULL);
|
||||
ASSERT(!(this->flushing && !this->inputSame));
|
||||
ASSERT(this->context != NULL);
|
||||
ASSERT(compressed != NULL);
|
||||
ASSERT(!this->flushing || uncompressed == NULL);
|
||||
|
||||
// Initialize output buffer
|
||||
ZSTD_outBuffer out = {.dst = bufRemainsPtr(compressed), .size = bufRemains(compressed)};
|
||||
|
||||
// If input is NULL then start flushing
|
||||
if (uncompressed == NULL)
|
||||
{
|
||||
this->flushing = true;
|
||||
this->inputSame = zstError(ZSTD_endStream(this->context, &out)) != 0;
|
||||
}
|
||||
// Else still have input data
|
||||
else
|
||||
{
|
||||
// Initialize input buffer
|
||||
ZSTD_inBuffer in =
|
||||
{
|
||||
.src = bufPtrConst(uncompressed) + this->inputOffset,
|
||||
.size = bufUsed(uncompressed) - this->inputOffset,
|
||||
};
|
||||
|
||||
// Perform compression
|
||||
zstError(ZSTD_compressStream(this->context, &out, &in));
|
||||
|
||||
// If the input buffer was not entirely consumed then set inputSame and store the offset where processing will restart
|
||||
if (in.pos < in.size)
|
||||
{
|
||||
// Output buffer should be completely full
|
||||
ASSERT(out.pos == out.size);
|
||||
|
||||
this->inputSame = true;
|
||||
this->inputOffset += in.pos;
|
||||
}
|
||||
// Else ready for more input
|
||||
else
|
||||
{
|
||||
this->inputSame = false;
|
||||
this->inputOffset = 0;
|
||||
}
|
||||
}
|
||||
|
||||
bufUsedInc(compressed, out.pos);
|
||||
|
||||
FUNCTION_LOG_RETURN_VOID();
|
||||
}
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Is compress done?
|
||||
***********************************************************************************************************************************/
|
||||
static bool
|
||||
zstCompressDone(const THIS_VOID)
|
||||
{
|
||||
THIS(const ZstCompress);
|
||||
|
||||
FUNCTION_TEST_BEGIN();
|
||||
FUNCTION_TEST_PARAM(ZST_COMPRESS, this);
|
||||
FUNCTION_TEST_END();
|
||||
|
||||
ASSERT(this != NULL);
|
||||
|
||||
FUNCTION_TEST_RETURN(this->flushing && !this->inputSame);
|
||||
}
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Is the same input required on the next process call?
|
||||
***********************************************************************************************************************************/
|
||||
static bool
|
||||
zstCompressInputSame(const THIS_VOID)
|
||||
{
|
||||
THIS(const ZstCompress);
|
||||
|
||||
FUNCTION_TEST_BEGIN();
|
||||
FUNCTION_TEST_PARAM(ZST_COMPRESS, this);
|
||||
FUNCTION_TEST_END();
|
||||
|
||||
ASSERT(this != NULL);
|
||||
|
||||
FUNCTION_TEST_RETURN(this->inputSame);
|
||||
}
|
||||
|
||||
/**********************************************************************************************************************************/
|
||||
IoFilter *
|
||||
zstCompressNew(int level)
|
||||
{
|
||||
FUNCTION_LOG_BEGIN(logLevelTrace);
|
||||
FUNCTION_LOG_PARAM(INT, level);
|
||||
FUNCTION_LOG_END();
|
||||
|
||||
ASSERT(level >= 0);
|
||||
|
||||
IoFilter *this = NULL;
|
||||
|
||||
MEM_CONTEXT_NEW_BEGIN("ZstCompress")
|
||||
{
|
||||
ZstCompress *driver = memNew(sizeof(ZstCompress));
|
||||
|
||||
*driver = (ZstCompress)
|
||||
{
|
||||
.memContext = MEM_CONTEXT_NEW(),
|
||||
.context = ZSTD_createCStream(),
|
||||
.level = level,
|
||||
};
|
||||
|
||||
// Set callback to ensure zst context is freed
|
||||
memContextCallbackSet(driver->memContext, zstCompressFreeResource, driver);
|
||||
|
||||
// Initialize context
|
||||
zstError(ZSTD_initCStream(driver->context, driver->level));
|
||||
|
||||
// Create param list
|
||||
VariantList *paramList = varLstNew();
|
||||
varLstAdd(paramList, varNewInt(level));
|
||||
|
||||
// Create filter interface
|
||||
this = ioFilterNewP(
|
||||
ZST_COMPRESS_FILTER_TYPE_STR, driver, paramList, .done = zstCompressDone, .inOut = zstCompressProcess,
|
||||
.inputSame = zstCompressInputSame);
|
||||
}
|
||||
MEM_CONTEXT_NEW_END();
|
||||
|
||||
FUNCTION_LOG_RETURN(IO_FILTER, this);
|
||||
}
|
||||
|
||||
#endif // HAVE_LIBZST
|
||||
26
src/common/compress/zst/compress.h
Normal file
26
src/common/compress/zst/compress.h
Normal file
@ -0,0 +1,26 @@
|
||||
/***********************************************************************************************************************************
|
||||
ZST Compress
|
||||
|
||||
Compress IO to the zst format.
|
||||
***********************************************************************************************************************************/
|
||||
#ifdef HAVE_LIBZST
|
||||
|
||||
#ifndef COMMON_COMPRESS_ZST_COMPRESS_H
|
||||
#define COMMON_COMPRESS_ZST_COMPRESS_H
|
||||
|
||||
#include "common/io/filter/filter.h"
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Filter type constant
|
||||
***********************************************************************************************************************************/
|
||||
#define ZST_COMPRESS_FILTER_TYPE "zstCompress"
|
||||
STRING_DECLARE(ZST_COMPRESS_FILTER_TYPE_STR);
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Constructors
|
||||
***********************************************************************************************************************************/
|
||||
IoFilter *zstCompressNew(int level);
|
||||
|
||||
#endif
|
||||
|
||||
#endif // HAVE_LIBZST
|
||||
191
src/common/compress/zst/decompress.c
Normal file
191
src/common/compress/zst/decompress.c
Normal file
@ -0,0 +1,191 @@
|
||||
/***********************************************************************************************************************************
|
||||
ZST Decompress
|
||||
***********************************************************************************************************************************/
|
||||
#include "build.auto.h"
|
||||
|
||||
#ifdef HAVE_LIBZST
|
||||
|
||||
#include <zstd.h>
|
||||
|
||||
#include "common/compress/zst/common.h"
|
||||
#include "common/compress/zst/decompress.h"
|
||||
#include "common/debug.h"
|
||||
#include "common/io/filter/filter.intern.h"
|
||||
#include "common/log.h"
|
||||
#include "common/memContext.h"
|
||||
#include "common/type/object.h"
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Filter type constant
|
||||
***********************************************************************************************************************************/
|
||||
STRING_EXTERN(ZST_DECOMPRESS_FILTER_TYPE_STR, ZST_DECOMPRESS_FILTER_TYPE);
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Object type
|
||||
***********************************************************************************************************************************/
|
||||
#define ZST_DECOMPRESS_TYPE ZstDecompress
|
||||
#define ZST_DECOMPRESS_PREFIX zstDecompress
|
||||
|
||||
typedef struct ZstDecompress
|
||||
{
|
||||
MemContext *memContext; // Context to store data
|
||||
ZSTD_DStream *context; // Decompression context
|
||||
IoFilter *filter; // Filter interface
|
||||
|
||||
bool inputSame; // Is the same input required on the next process call?
|
||||
size_t inputOffset; // Current offset in input buffer
|
||||
bool frameDone; // Has the current frame completed?
|
||||
bool done; // Is decompression done?
|
||||
} ZstDecompress;
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Render as string for logging
|
||||
***********************************************************************************************************************************/
|
||||
static String *
|
||||
zstDecompressToLog(const ZstDecompress *this)
|
||||
{
|
||||
return strNewFmt(
|
||||
"{inputSame: %s, inputOffset: %zu, frameDone %s, done: %s}", cvtBoolToConstZ(this->inputSame), this->inputOffset,
|
||||
cvtBoolToConstZ(this->frameDone), cvtBoolToConstZ(this->done));
|
||||
}
|
||||
|
||||
#define FUNCTION_LOG_ZST_DECOMPRESS_TYPE \
|
||||
ZstDecompress *
|
||||
#define FUNCTION_LOG_ZST_DECOMPRESS_FORMAT(value, buffer, bufferSize) \
|
||||
FUNCTION_LOG_STRING_OBJECT_FORMAT(value, zstDecompressToLog, buffer, bufferSize)
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Free decompression context
|
||||
***********************************************************************************************************************************/
|
||||
OBJECT_DEFINE_FREE_RESOURCE_BEGIN(ZST_DECOMPRESS, LOG, logLevelTrace)
|
||||
{
|
||||
ZSTD_freeDStream(this->context);
|
||||
}
|
||||
OBJECT_DEFINE_FREE_RESOURCE_END(LOG);
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Decompress data
|
||||
***********************************************************************************************************************************/
|
||||
static void
|
||||
zstDecompressProcess(THIS_VOID, const Buffer *compressed, Buffer *decompressed)
|
||||
{
|
||||
THIS(ZstDecompress);
|
||||
|
||||
FUNCTION_LOG_BEGIN(logLevelTrace);
|
||||
FUNCTION_LOG_PARAM(ZST_DECOMPRESS, this);
|
||||
FUNCTION_LOG_PARAM(BUFFER, compressed);
|
||||
FUNCTION_LOG_PARAM(BUFFER, decompressed);
|
||||
FUNCTION_LOG_END();
|
||||
|
||||
ASSERT(this != NULL);
|
||||
ASSERT(this->context != NULL);
|
||||
ASSERT(decompressed != NULL);
|
||||
|
||||
// When there is no more input then decompression is done
|
||||
if (compressed == NULL)
|
||||
{
|
||||
// If the current frame being decompressed was not completed then error
|
||||
if (!this->frameDone)
|
||||
THROW(FormatError, "unexpected eof in compressed data");
|
||||
|
||||
this->done = true;
|
||||
}
|
||||
else
|
||||
{
|
||||
// Initialize input/output buffer
|
||||
ZSTD_inBuffer in = {.src = bufPtrConst(compressed) + this->inputOffset, .size = bufUsed(compressed) - this->inputOffset};
|
||||
ZSTD_outBuffer out = {.dst = bufRemainsPtr(decompressed), .size = bufRemains(decompressed)};
|
||||
|
||||
// Perform decompression. Track frame done so we can detect unexpected EOF.
|
||||
this->frameDone = zstError(ZSTD_decompressStream(this->context, &out, &in)) == 0;
|
||||
bufUsedInc(decompressed, out.pos);
|
||||
|
||||
// If the input buffer was not entirely consumed then set inputSame and store the offset where processing will restart
|
||||
if (in.pos < in.size)
|
||||
{
|
||||
// Output buffer should be completely full
|
||||
ASSERT(out.pos == out.size);
|
||||
|
||||
this->inputSame = true;
|
||||
this->inputOffset += in.pos;
|
||||
}
|
||||
// Else ready for more input
|
||||
else
|
||||
{
|
||||
this->inputOffset = 0;
|
||||
this->inputSame = false;
|
||||
}
|
||||
}
|
||||
|
||||
FUNCTION_LOG_RETURN_VOID();
|
||||
}
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Is decompress done?
|
||||
***********************************************************************************************************************************/
|
||||
static bool
|
||||
zstDecompressDone(const THIS_VOID)
|
||||
{
|
||||
THIS(const ZstDecompress);
|
||||
|
||||
FUNCTION_TEST_BEGIN();
|
||||
FUNCTION_TEST_PARAM(ZST_DECOMPRESS, this);
|
||||
FUNCTION_TEST_END();
|
||||
|
||||
ASSERT(this != NULL);
|
||||
|
||||
FUNCTION_TEST_RETURN(this->done);
|
||||
}
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Is the same input required on the next process call?
|
||||
***********************************************************************************************************************************/
|
||||
static bool
|
||||
zstDecompressInputSame(const THIS_VOID)
|
||||
{
|
||||
THIS(const ZstDecompress);
|
||||
|
||||
FUNCTION_TEST_BEGIN();
|
||||
FUNCTION_TEST_PARAM(ZST_DECOMPRESS, this);
|
||||
FUNCTION_TEST_END();
|
||||
|
||||
ASSERT(this != NULL);
|
||||
|
||||
FUNCTION_TEST_RETURN(this->inputSame);
|
||||
}
|
||||
|
||||
/**********************************************************************************************************************************/
|
||||
IoFilter *
|
||||
zstDecompressNew(void)
|
||||
{
|
||||
FUNCTION_LOG_VOID(logLevelTrace);
|
||||
|
||||
IoFilter *this = NULL;
|
||||
|
||||
MEM_CONTEXT_NEW_BEGIN("ZstDecompress")
|
||||
{
|
||||
ZstDecompress *driver = memNew(sizeof(ZstDecompress));
|
||||
|
||||
*driver = (ZstDecompress)
|
||||
{
|
||||
.memContext = MEM_CONTEXT_NEW(),
|
||||
.context = ZSTD_createDStream(),
|
||||
};
|
||||
|
||||
// Set callback to ensure zst context is freed
|
||||
memContextCallbackSet(driver->memContext, zstDecompressFreeResource, driver);
|
||||
|
||||
// Initialize context
|
||||
zstError(ZSTD_initDStream(driver->context));
|
||||
|
||||
// Create filter interface
|
||||
this = ioFilterNewP(
|
||||
ZST_DECOMPRESS_FILTER_TYPE_STR, driver, NULL, .done = zstDecompressDone, .inOut = zstDecompressProcess,
|
||||
.inputSame = zstDecompressInputSame);
|
||||
}
|
||||
MEM_CONTEXT_NEW_END();
|
||||
|
||||
FUNCTION_LOG_RETURN(IO_FILTER, this);
|
||||
}
|
||||
|
||||
#endif // HAVE_LIBZST
|
||||
26
src/common/compress/zst/decompress.h
Normal file
26
src/common/compress/zst/decompress.h
Normal file
@ -0,0 +1,26 @@
|
||||
/***********************************************************************************************************************************
|
||||
ZST Decompress
|
||||
|
||||
Decompress IO from the zst format.
|
||||
***********************************************************************************************************************************/
|
||||
#ifdef HAVE_LIBZST
|
||||
|
||||
#ifndef COMMON_COMPRESS_ZST_DECOMPRESS_H
|
||||
#define COMMON_COMPRESS_ZST_DECOMPRESS_H
|
||||
|
||||
#include "common/io/filter/filter.h"
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Filter type constant
|
||||
***********************************************************************************************************************************/
|
||||
#define ZST_DECOMPRESS_FILTER_TYPE "zstDecompress"
|
||||
STRING_DECLARE(ZST_DECOMPRESS_FILTER_TYPE_STR);
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Constructors
|
||||
***********************************************************************************************************************************/
|
||||
IoFilter *zstDecompressNew(void);
|
||||
|
||||
#endif
|
||||
|
||||
#endif // HAVE_LIBZST
|
||||
@ -833,7 +833,8 @@ static ConfigDefineOptionData configDefineOptionData[] = CFGDEFDATA_OPTION_LIST
|
||||
"The following compression types are supported:\n"
|
||||
"\n"
|
||||
"* gz - gzip compression format\n"
|
||||
"* lz4 - lz4 compression format (not available on all platforms)"
|
||||
"* lz4 - lz4 compression format (not available on all platforms)\n"
|
||||
"* zst - Zstandard compression format (not available on all platforms)"
|
||||
)
|
||||
|
||||
CFGDEFDATA_OPTION_COMMAND_LIST
|
||||
@ -848,7 +849,8 @@ static ConfigDefineOptionData configDefineOptionData[] = CFGDEFDATA_OPTION_LIST
|
||||
(
|
||||
"none",
|
||||
"gz",
|
||||
"lz4"
|
||||
"lz4",
|
||||
"zst"
|
||||
)
|
||||
|
||||
CFGDEFDATA_OPTION_OPTIONAL_DEFAULT("gz")
|
||||
|
||||
71
src/configure
vendored
71
src/configure
vendored
@ -4070,6 +4070,75 @@ fi
|
||||
fi
|
||||
|
||||
|
||||
# Check optional zst library. Ignore any versions below 1.0.
|
||||
# ----------------------------------------------------------------------------------------------------------------------------------
|
||||
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for ZSTD_isError in -lzstd" >&5
|
||||
$as_echo_n "checking for ZSTD_isError in -lzstd... " >&6; }
|
||||
if ${ac_cv_lib_zstd_ZSTD_isError+:} false; then :
|
||||
$as_echo_n "(cached) " >&6
|
||||
else
|
||||
ac_check_lib_save_LIBS=$LIBS
|
||||
LIBS="-lzstd $LIBS"
|
||||
cat confdefs.h - <<_ACEOF >conftest.$ac_ext
|
||||
/* end confdefs.h. */
|
||||
|
||||
/* Override any GCC internal prototype to avoid an error.
|
||||
Use char because int might match the return type of a GCC
|
||||
builtin and then its argument prototype would still apply. */
|
||||
#ifdef __cplusplus
|
||||
extern "C"
|
||||
#endif
|
||||
char ZSTD_isError ();
|
||||
int
|
||||
main ()
|
||||
{
|
||||
return ZSTD_isError ();
|
||||
;
|
||||
return 0;
|
||||
}
|
||||
_ACEOF
|
||||
if ac_fn_c_try_link "$LINENO"; then :
|
||||
ac_cv_lib_zstd_ZSTD_isError=yes
|
||||
else
|
||||
ac_cv_lib_zstd_ZSTD_isError=no
|
||||
fi
|
||||
rm -f core conftest.err conftest.$ac_objext \
|
||||
conftest$ac_exeext conftest.$ac_ext
|
||||
LIBS=$ac_check_lib_save_LIBS
|
||||
fi
|
||||
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_zstd_ZSTD_isError" >&5
|
||||
$as_echo "$ac_cv_lib_zstd_ZSTD_isError" >&6; }
|
||||
if test "x$ac_cv_lib_zstd_ZSTD_isError" = xyes; then :
|
||||
ac_fn_c_check_header_mongrel "$LINENO" "zstd.h" "ac_cv_header_zstd_h" "$ac_includes_default"
|
||||
if test "x$ac_cv_header_zstd_h" = xyes; then :
|
||||
cat confdefs.h - <<_ACEOF >conftest.$ac_ext
|
||||
/* end confdefs.h. */
|
||||
#include <zstd.h>
|
||||
int
|
||||
main ()
|
||||
{
|
||||
#if ZSTD_VERSION_MAJOR < 1
|
||||
#error "ZSTD_VERSION_MAJOR must be >= 1"
|
||||
#endif
|
||||
;
|
||||
return 0;
|
||||
}
|
||||
_ACEOF
|
||||
if ac_fn_c_try_link "$LINENO"; then :
|
||||
$as_echo "#define HAVE_LIBZST 1" >>confdefs.h
|
||||
LIBS="${LIBS} -lzstd"
|
||||
|
||||
fi
|
||||
rm -f core conftest.err conftest.$ac_objext \
|
||||
conftest$ac_exeext conftest.$ac_ext
|
||||
else
|
||||
as_fn_error $? "header file <zstd.h> is required" "$LINENO" 5
|
||||
fi
|
||||
|
||||
|
||||
fi
|
||||
|
||||
|
||||
# Write output
|
||||
# ----------------------------------------------------------------------------------------------------------------------------------
|
||||
ac_config_headers="$ac_config_headers build.auto.h"
|
||||
@ -5348,4 +5417,4 @@ if test -n "$ac_unrecognized_opts" && test "$enable_option_checking" != no; then
|
||||
$as_echo "$as_me: WARNING: unrecognized options: $ac_unrecognized_opts" >&2;}
|
||||
fi
|
||||
|
||||
# Generated from src/build/configure.ac sha1 8f4c0f8a2389f1f0f30c72c1a50c0c8905474095
|
||||
# Generated from src/build/configure.ac sha1 93fbf56725f3dab1089ddd4d7768f4414bf7941c
|
||||
|
||||
2
test/Vagrantfile
vendored
2
test/Vagrantfile
vendored
@ -75,7 +75,7 @@ Vagrant.configure(2) do |config|
|
||||
#-----------------------------------------------------------------------------------------------------------------------
|
||||
echo 'Install Build Tools' && date
|
||||
apt-get install -y devscripts build-essential lintian git cloc txt2man debhelper libssl-dev zlib1g-dev libperl-dev \
|
||||
libxml2-dev liblz4-dev liblz4-tool libpq-dev valgrind lcov autoconf-archive
|
||||
libxml2-dev liblz4-dev liblz4-tool libpq-dev valgrind lcov autoconf-archive zstd libzstd-dev
|
||||
|
||||
#-----------------------------------------------------------------------------------------------------------------------
|
||||
echo 'Install Docker' && date
|
||||
|
||||
@ -12,11 +12,13 @@
|
||||
# - docker login -u pgbackrest
|
||||
# - VM=XXX;DATE=YYYYMMDDX;BASE=pgbackrest/test:${VM?}-base;docker tag ${BASE?} ${BASE?}-${DATE?} && docker push ${BASE?}-${DATE?}
|
||||
# **********************************************************************************************************************************
|
||||
20200504A:
|
||||
co7: 9c922915380c0beb3d813c94d53f823d7008c69b
|
||||
f30: 9ad36516c2403b835c24efbbcef359322a80fa1f
|
||||
u18: 32426ecf8cd0e98976298f6f6b41a2b255aca692
|
||||
|
||||
20200310A:
|
||||
co6: beb7b5a62ebdf209bb54494129baeadbd0c2954f
|
||||
co7: 833c4ef292b9b3c957d9d7d296ec7c03ec6310c2
|
||||
f30: 9ef023b179bd9e54bb079d60b75516067a4b4b1d
|
||||
u18: 5e295130131d5fe96e2483885c171677b52c8ce3
|
||||
|
||||
20200124A:
|
||||
u12: 0f2fcf1bc79ee35e78121773c9a2155a77cb10d5
|
||||
|
||||
@ -262,7 +262,7 @@ unit:
|
||||
|
||||
# ----------------------------------------------------------------------------------------------------------------------------
|
||||
- name: compress
|
||||
total: 3
|
||||
total: 4
|
||||
|
||||
coverage:
|
||||
common/compress/gz/common: full
|
||||
@ -271,6 +271,9 @@ unit:
|
||||
common/compress/lz4/common: full
|
||||
common/compress/lz4/compress: full
|
||||
common/compress/lz4/decompress: full
|
||||
common/compress/zst/common: full
|
||||
common/compress/zst/compress: full
|
||||
common/compress/zst/decompress: full
|
||||
common/compress/helper: full
|
||||
|
||||
# ----------------------------------------------------------------------------------------------------------------------------
|
||||
|
||||
@ -2176,11 +2176,11 @@ backrest-checksum="[CHECKSUM]"
|
||||
diff backup - remove files (db-master host)
|
||||
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --no-online --process-max=1 --delta --type=diff --stanza=db backup
|
||||
------------------------------------------------------------------------------------------------------------------------------------
|
||||
P00 INFO: backup command begin [BACKREST-VERSION]: --buffer-size=[BUFFER-SIZE] --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --delta --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --no-online --pg1-path=[TEST_PATH]/db-master/db/base-2 --process-max=1 --protocol-timeout=60 --repo1-hardlink --repo1-path=[TEST_PATH]/db-master/repo --stanza=db --start-fast --type=diff
|
||||
P00 INFO: backup command begin [BACKREST-VERSION]: --buffer-size=[BUFFER-SIZE] --compress-level=3 --compress-type=zst --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --delta --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --no-online --pg1-path=[TEST_PATH]/db-master/db/base-2 --process-max=1 --protocol-timeout=60 --repo1-hardlink --repo1-path=[TEST_PATH]/db-master/repo --stanza=db --start-fast --type=diff
|
||||
P00 WARN: option repo1-retention-full is not set, the repository may run out of space
|
||||
HINT: to retain full backups indefinitely (without warning), set option 'repo1-retention-full' to the maximum.
|
||||
P00 INFO: last backup label = [BACKUP-FULL-2], version = [VERSION-1]
|
||||
P00 WARN: diff backup cannot alter compress-type option to 'gz', reset to value in [BACKUP-FULL-2]
|
||||
P00 WARN: diff backup cannot alter compress-type option to 'zst', reset to value in [BACKUP-FULL-2]
|
||||
P00 WARN: diff backup cannot alter hardlink option to 'true', reset to value in [BACKUP-FULL-2]
|
||||
P00 WARN: diff backup cannot alter 'checksum-page' option to 'false', reset to 'true' from [BACKUP-FULL-2]
|
||||
P01 DETAIL: match file from prior backup [TEST_PATH]/db-master/db/base-2/base/32768/33001 (64KB, 36%) checksum 6bf316f11d28c28914ea9be92c00de9bea6d9a6b
|
||||
@ -2237,6 +2237,7 @@ tablespace-map=2=[TEST_PATH]/db-master/db/tablespace/ts2-2
|
||||
[global]
|
||||
buffer-size=[BUFFER-SIZE]
|
||||
compress-level=3
|
||||
compress-type=zst
|
||||
db-timeout=45
|
||||
lock-path=[TEST_PATH]/db-master/lock
|
||||
log-level-console=detail
|
||||
@ -2391,7 +2392,7 @@ backrest-checksum="[CHECKSUM]"
|
||||
full backup - update file (db-master host)
|
||||
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --no-online --type=full --stanza=db backup
|
||||
------------------------------------------------------------------------------------------------------------------------------------
|
||||
P00 INFO: backup command begin [BACKREST-VERSION]: --buffer-size=[BUFFER-SIZE] --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --no-online --pg1-path=[TEST_PATH]/db-master/db/base-2 --protocol-timeout=60 --repo1-hardlink --repo1-path=[TEST_PATH]/db-master/repo --stanza=db --start-fast --type=full
|
||||
P00 INFO: backup command begin [BACKREST-VERSION]: --buffer-size=[BUFFER-SIZE] --compress-level=3 --compress-type=zst --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --no-online --pg1-path=[TEST_PATH]/db-master/db/base-2 --protocol-timeout=60 --repo1-hardlink --repo1-path=[TEST_PATH]/db-master/repo --stanza=db --start-fast --type=full
|
||||
P00 WARN: option repo1-retention-full is not set, the repository may run out of space
|
||||
HINT: to retain full backups indefinitely (without warning), set option 'repo1-retention-full' to the maximum.
|
||||
P01 INFO: backup file [TEST_PATH]/db-master/db/base-2/base/32768/33001 (64KB, 36%) checksum 6bf316f11d28c28914ea9be92c00de9bea6d9a6b
|
||||
@ -2434,6 +2435,7 @@ tablespace-map=2=[TEST_PATH]/db-master/db/tablespace/ts2-2
|
||||
[global]
|
||||
buffer-size=[BUFFER-SIZE]
|
||||
compress-level=3
|
||||
compress-type=zst
|
||||
db-timeout=45
|
||||
lock-path=[TEST_PATH]/db-master/lock
|
||||
log-level-console=detail
|
||||
@ -2480,7 +2482,7 @@ option-checksum-page=false
|
||||
option-compress=true
|
||||
option-compress-level=3
|
||||
option-compress-level-network=3
|
||||
option-compress-type="gz"
|
||||
option-compress-type="zst"
|
||||
option-delta=false
|
||||
option-hardlink=true
|
||||
option-online=false
|
||||
@ -2603,7 +2605,7 @@ P00 INFO: expire command end: completed successfully
|
||||
diff backup - add file (db-master host)
|
||||
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --no-online --checksum-page --type=diff --stanza=db backup
|
||||
------------------------------------------------------------------------------------------------------------------------------------
|
||||
P00 INFO: backup command begin [BACKREST-VERSION]: --buffer-size=[BUFFER-SIZE] --checksum-page --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --no-online --pg1-path=[TEST_PATH]/db-master/db/base-2 --protocol-timeout=60 --repo1-hardlink --repo1-path=[TEST_PATH]/db-master/repo --stanza=db --start-fast --type=diff
|
||||
P00 INFO: backup command begin [BACKREST-VERSION]: --buffer-size=[BUFFER-SIZE] --checksum-page --compress-level=3 --compress-type=zst --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --no-online --pg1-path=[TEST_PATH]/db-master/db/base-2 --protocol-timeout=60 --repo1-hardlink --repo1-path=[TEST_PATH]/db-master/repo --stanza=db --start-fast --type=diff
|
||||
P00 WARN: option repo1-retention-full is not set, the repository may run out of space
|
||||
HINT: to retain full backups indefinitely (without warning), set option 'repo1-retention-full' to the maximum.
|
||||
P00 INFO: last backup label = [BACKUP-FULL-3], version = [VERSION-1]
|
||||
@ -2648,6 +2650,7 @@ tablespace-map=2=[TEST_PATH]/db-master/db/tablespace/ts2-2
|
||||
[global]
|
||||
buffer-size=[BUFFER-SIZE]
|
||||
compress-level=3
|
||||
compress-type=zst
|
||||
db-timeout=45
|
||||
lock-path=[TEST_PATH]/db-master/lock
|
||||
log-level-console=detail
|
||||
@ -2695,7 +2698,7 @@ option-checksum-page=false
|
||||
option-compress=true
|
||||
option-compress-level=3
|
||||
option-compress-level-network=3
|
||||
option-compress-type="gz"
|
||||
option-compress-type="zst"
|
||||
option-delta=false
|
||||
option-hardlink=true
|
||||
option-online=false
|
||||
@ -3034,7 +3037,7 @@ restore_command = '[BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf
|
||||
diff backup - option backup-standby reset - backup performed from master (db-master host)
|
||||
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --no-online --log-level-console=info --backup-standby --type=diff --stanza=db backup
|
||||
------------------------------------------------------------------------------------------------------------------------------------
|
||||
P00 INFO: backup command begin [BACKREST-VERSION]: --backup-standby --buffer-size=[BUFFER-SIZE] --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=info --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --no-online --pg1-path=[TEST_PATH]/db-master/db/base-2/base --protocol-timeout=60 --repo1-hardlink --repo1-path=[TEST_PATH]/db-master/repo --stanza=db --start-fast --type=diff
|
||||
P00 INFO: backup command begin [BACKREST-VERSION]: --backup-standby --buffer-size=[BUFFER-SIZE] --compress-level=3 --compress-type=zst --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=info --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --no-online --pg1-path=[TEST_PATH]/db-master/db/base-2/base --protocol-timeout=60 --repo1-hardlink --repo1-path=[TEST_PATH]/db-master/repo --stanza=db --start-fast --type=diff
|
||||
P00 WARN: option repo1-retention-full is not set, the repository may run out of space
|
||||
HINT: to retain full backups indefinitely (without warning), set option 'repo1-retention-full' to the maximum.
|
||||
P00 WARN: option backup-standby is enabled but backup is offline - backups will be performed from the primary
|
||||
@ -3057,6 +3060,7 @@ pg1-path=[TEST_PATH]/db-master/db/base-2/base
|
||||
[global]
|
||||
buffer-size=[BUFFER-SIZE]
|
||||
compress-level=3
|
||||
compress-type=zst
|
||||
db-timeout=45
|
||||
lock-path=[TEST_PATH]/db-master/lock
|
||||
log-level-console=detail
|
||||
@ -3104,7 +3108,7 @@ option-checksum-page=false
|
||||
option-compress=true
|
||||
option-compress-level=3
|
||||
option-compress-level-network=3
|
||||
option-compress-type="gz"
|
||||
option-compress-type="zst"
|
||||
option-delta=false
|
||||
option-hardlink=true
|
||||
option-online=false
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
run 002 - rmt 1, s3 1, enc 0, cmp gz
|
||||
====================================
|
||||
run 002 - rmt 1, s3 1, enc 0, cmp zst
|
||||
=====================================
|
||||
|
||||
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001
|
||||
------------------------------------------------------------------------------------------------------------------------------------
|
||||
@ -70,9 +70,9 @@ db-version="9.4"
|
||||
[backrest]
|
||||
backrest-checksum="[CHECKSUM]"
|
||||
|
||||
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-push --cmd-ssh=/usr/bin/ssh --compress-type=gz [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001
|
||||
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-push --cmd-ssh=/usr/bin/ssh --compress-type=zst [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001
|
||||
------------------------------------------------------------------------------------------------------------------------------------
|
||||
P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001] --buffer-size=[BUFFER-SIZE] --cmd-ssh=/usr/bin/ssh --compress-level=3 --compress-level-network=1 --compress-type=gz --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-host=backup --repo1-host-cmd=[BACKREST-BIN] --repo1-host-config=[TEST_PATH]/backup/pgbackrest.conf --repo1-host-user=[USER-1] --stanza=db
|
||||
P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001] --buffer-size=[BUFFER-SIZE] --cmd-ssh=/usr/bin/ssh --compress-level=3 --compress-level-network=1 --compress-type=zst --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-host=backup --repo1-host-cmd=[BACKREST-BIN] --repo1-host-config=[TEST_PATH]/backup/pgbackrest.conf --repo1-host-user=[USER-1] --stanza=db
|
||||
P00 INFO: pushed WAL file '000000010000000100000001' to the archive
|
||||
P00 INFO: archive-push command end: completed successfully
|
||||
|
||||
@ -88,9 +88,9 @@ P00 INFO: archive-get command begin [BACKREST-VERSION]: [000000010000000100000
|
||||
P00 INFO: found 000000010000000100000001 in the archive
|
||||
P00 INFO: archive-get command end: completed successfully
|
||||
|
||||
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-push --compress-type=gz --archive-async --process-max=2 [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002
|
||||
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-push --compress-type=zst --archive-async --process-max=2 [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002
|
||||
------------------------------------------------------------------------------------------------------------------------------------
|
||||
P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002] --archive-async --buffer-size=[BUFFER-SIZE] --compress-level=3 --compress-level-network=1 --compress-type=gz --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --process-max=2 --protocol-timeout=60 --repo1-host=backup --repo1-host-cmd=[BACKREST-BIN] --repo1-host-config=[TEST_PATH]/backup/pgbackrest.conf --repo1-host-user=[USER-1] --spool-path=[TEST_PATH]/db-master/spool --stanza=db
|
||||
P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002] --archive-async --buffer-size=[BUFFER-SIZE] --compress-level=3 --compress-level-network=1 --compress-type=zst --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --process-max=2 --protocol-timeout=60 --repo1-host=backup --repo1-host-cmd=[BACKREST-BIN] --repo1-host-config=[TEST_PATH]/backup/pgbackrest.conf --repo1-host-user=[USER-1] --spool-path=[TEST_PATH]/db-master/spool --stanza=db
|
||||
P00 INFO: pushed WAL file '000000010000000100000002' to the archive asynchronously
|
||||
P00 INFO: archive-push command end: completed successfully
|
||||
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
run 001 - remote 0, s3 0, enc 0, cmp gz
|
||||
=======================================
|
||||
run 001 - remote 0, s3 0, enc 0, cmp zst
|
||||
========================================
|
||||
|
||||
stanza-create db - fail on missing control file (db-master host)
|
||||
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db --no-online --log-level-file=[LOG-LEVEL-FILE] stanza-create
|
||||
@ -197,7 +197,7 @@ backrest-checksum="[CHECKSUM]"
|
||||
|
||||
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001
|
||||
------------------------------------------------------------------------------------------------------------------------------------
|
||||
P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001] --buffer-size=[BUFFER-SIZE] --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-path=[TEST_PATH]/db-master/repo --stanza=db
|
||||
P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001] --buffer-size=[BUFFER-SIZE] --compress-level=3 --compress-type=zst --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-path=[TEST_PATH]/db-master/repo --stanza=db
|
||||
P00 INFO: pushed WAL file '000000010000000100000001' to the archive
|
||||
P00 INFO: archive-push command end: completed successfully
|
||||
|
||||
@ -230,7 +230,7 @@ backrest-checksum="[CHECKSUM]"
|
||||
|
||||
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002
|
||||
------------------------------------------------------------------------------------------------------------------------------------
|
||||
P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002] --buffer-size=[BUFFER-SIZE] --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-path=[TEST_PATH]/db-master/repo --stanza=db
|
||||
P00 INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002] --buffer-size=[BUFFER-SIZE] --compress-level=3 --compress-type=zst --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-path=[TEST_PATH]/db-master/repo --stanza=db
|
||||
P00 INFO: pushed WAL file '000000010000000100000002' to the archive
|
||||
P00 INFO: archive-push command end: completed successfully
|
||||
|
||||
@ -295,7 +295,7 @@ P00 INFO: archive-get command end: completed successfully
|
||||
full backup - create first full backup (db-master host)
|
||||
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --repo1-retention-full=2 --no-online --type=full --stanza=db backup
|
||||
------------------------------------------------------------------------------------------------------------------------------------
|
||||
P00 INFO: backup command begin [BACKREST-VERSION]: --buffer-size=[BUFFER-SIZE] --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --no-online --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-path=[TEST_PATH]/db-master/repo --repo1-retention-full=2 --stanza=db --start-fast --type=full
|
||||
P00 INFO: backup command begin [BACKREST-VERSION]: --buffer-size=[BUFFER-SIZE] --compress-level=3 --compress-type=zst --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --no-online --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-path=[TEST_PATH]/db-master/repo --repo1-retention-full=2 --stanza=db --start-fast --type=full
|
||||
P01 INFO: backup file [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG (16MB, 33%) checksum 488ba4b8b98acc510bce86b8f16e3c1ed9886a29
|
||||
P01 INFO: backup file [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002 (16MB, 66%) checksum 488ba4b8b98acc510bce86b8f16e3c1ed9886a29
|
||||
P01 INFO: backup file [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001 (16MB, 99%) checksum e40de8cea99dd469c3efb47f5f33a73c7390fb9c
|
||||
@ -316,6 +316,7 @@ pg1-path=[TEST_PATH]/db-master/db/base
|
||||
[global]
|
||||
buffer-size=[BUFFER-SIZE]
|
||||
compress-level=3
|
||||
compress-type=zst
|
||||
db-timeout=45
|
||||
lock-path=[TEST_PATH]/db-master/lock
|
||||
log-level-console=detail
|
||||
@ -438,7 +439,7 @@ backrest-checksum="[CHECKSUM]"
|
||||
diff backup - diff changed to full backup (db-master host)
|
||||
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --repo1-retention-full=2 --no-online --type=diff --stanza=db backup
|
||||
------------------------------------------------------------------------------------------------------------------------------------
|
||||
P00 INFO: backup command begin [BACKREST-VERSION]: --buffer-size=[BUFFER-SIZE] --compress-level=3 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --no-online --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-path=[TEST_PATH]/db-master/repo --repo1-retention-full=2 --stanza=db --start-fast --type=diff
|
||||
P00 INFO: backup command begin [BACKREST-VERSION]: --buffer-size=[BUFFER-SIZE] --compress-level=3 --compress-type=zst --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --no-online --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-path=[TEST_PATH]/db-master/repo --repo1-retention-full=2 --stanza=db --start-fast --type=diff
|
||||
P00 WARN: no prior backup exists, diff backup has been changed to full
|
||||
P01 INFO: backup file [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG (16MB, 33%) checksum 488ba4b8b98acc510bce86b8f16e3c1ed9886a29
|
||||
P01 INFO: backup file [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002 (16MB, 66%) checksum 488ba4b8b98acc510bce86b8f16e3c1ed9886a29
|
||||
@ -461,6 +462,7 @@ pg1-path=[TEST_PATH]/db-master/db/base
|
||||
[global]
|
||||
buffer-size=[BUFFER-SIZE]
|
||||
compress-level=3
|
||||
compress-type=zst
|
||||
db-timeout=45
|
||||
lock-path=[TEST_PATH]/db-master/lock
|
||||
log-level-console=detail
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
run 006 - bkp 1, sby 0, dst backup, cmp gz, s3 1, enc 0
|
||||
=======================================================
|
||||
run 006 - bkp 1, sby 0, dst backup, cmp zst, s3 1, enc 0
|
||||
========================================================
|
||||
|
||||
stanza-create db - main create stanza info files (backup host)
|
||||
> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db stanza-create
|
||||
@ -108,6 +108,7 @@ pg1-socket-path=[TEST_PATH]/db-master/db
|
||||
buffer-size=[BUFFER-SIZE]
|
||||
compress-level=3
|
||||
compress-level-network=1
|
||||
compress-type=zst
|
||||
db-timeout=45
|
||||
lock-path=[TEST_PATH]/db-master/lock
|
||||
log-level-console=detail
|
||||
@ -138,6 +139,7 @@ pg1-port=6543
|
||||
buffer-size=[BUFFER-SIZE]
|
||||
compress-level=3
|
||||
compress-level-network=1
|
||||
compress-type=zst
|
||||
db-timeout=45
|
||||
lock-path=[TEST_PATH]/backup/lock
|
||||
log-level-console=detail
|
||||
@ -176,6 +178,7 @@ pg1-socket-path=[TEST_PATH]/db-master/db
|
||||
buffer-size=[BUFFER-SIZE]
|
||||
compress-level=3
|
||||
compress-level-network=1
|
||||
compress-type=zst
|
||||
db-timeout=45
|
||||
lock-path=[TEST_PATH]/db-master/lock
|
||||
log-level-console=detail
|
||||
@ -207,6 +210,7 @@ archive-async=y
|
||||
buffer-size=[BUFFER-SIZE]
|
||||
compress-level=3
|
||||
compress-level-network=1
|
||||
compress-type=zst
|
||||
db-timeout=45
|
||||
lock-path=[TEST_PATH]/backup/lock
|
||||
log-level-console=detail
|
||||
|
||||
@ -392,6 +392,19 @@ sub containerBuild
|
||||
}
|
||||
}
|
||||
|
||||
# Add zst command-line tool and development libs when available
|
||||
if (vmWithZst($strOS))
|
||||
{
|
||||
if ($oVm->{$strOS}{&VM_OS_BASE} eq VM_OS_BASE_RHEL)
|
||||
{
|
||||
$strScript .= ' zstd libzstd-devel';
|
||||
}
|
||||
else
|
||||
{
|
||||
$strScript .= ' zstd libzstd-dev';
|
||||
}
|
||||
}
|
||||
|
||||
# If no specific version of lcov is requested then install the default package
|
||||
if (!defined($oVm->{$strOS}{&VMDEF_LCOV_VERSION}))
|
||||
{
|
||||
|
||||
@ -440,7 +440,8 @@ sub run
|
||||
my $strBuildAutoH =
|
||||
($self->{oTest}->{&TEST_VM} ne VM_CO6 ? "#define HAVE_STATIC_ASSERT\n" : '') .
|
||||
"#define HAVE_BUILTIN_TYPES_COMPATIBLE_P\n" .
|
||||
(vmWithLz4($self->{oTest}->{&TEST_VM}) ? '#define HAVE_LIBLZ4' : '') . "\n";
|
||||
(vmWithLz4($self->{oTest}->{&TEST_VM}) ? '#define HAVE_LIBLZ4' : '') . "\n" .
|
||||
(vmWithZst($self->{oTest}->{&TEST_VM}) ? '#define HAVE_LIBZST' : '') . "\n";
|
||||
|
||||
buildPutDiffers($self->{oStorageTest}, "$self->{strGCovPath}/" . BUILD_AUTO_H, $strBuildAutoH);
|
||||
|
||||
@ -519,6 +520,7 @@ sub run
|
||||
"TESTFLAGS=${strTestFlags}\n" .
|
||||
"LDFLAGS=-lcrypto -lssl -lxml2 -lz" .
|
||||
(vmWithLz4($self->{oTest}->{&TEST_VM}) ? ' -llz4' : '') .
|
||||
(vmWithZst($self->{oTest}->{&TEST_VM}) ? ' -lzstd' : '') .
|
||||
(vmCoverageC($self->{oTest}->{&TEST_VM}) && $self->{bCoverageUnit} ? " -lgcov" : '') .
|
||||
(vmWithBackTrace($self->{oTest}->{&TEST_VM}) && $self->{bBackTrace} ? ' -lbacktrace' : '') .
|
||||
"\n" .
|
||||
|
||||
@ -51,6 +51,8 @@ use constant VMDEF_WITH_BACKTRACE => 'with-bac
|
||||
push @EXPORT, qw(VMDEF_WITH_BACKTRACE);
|
||||
use constant VMDEF_WITH_LZ4 => 'with-lz4';
|
||||
push @EXPORT, qw(VMDEF_WITH_LZ4);
|
||||
use constant VMDEF_WITH_ZST => 'with-zst';
|
||||
push @EXPORT, qw(VMDEF_WITH_ZST);
|
||||
|
||||
####################################################################################################################################
|
||||
# Valid OS base List
|
||||
@ -139,6 +141,8 @@ my $oyVm =
|
||||
&VMDEF_COVERAGE_C => true,
|
||||
&VMDEF_PGSQL_BIN => '/usr/lib/postgresql/{[version]}/bin',
|
||||
|
||||
&VMDEF_WITH_ZST => true,
|
||||
|
||||
&VM_DB =>
|
||||
[
|
||||
PG_VERSION_10,
|
||||
@ -191,6 +195,7 @@ my $oyVm =
|
||||
&VMDEF_PGSQL_BIN => '/usr/pgsql-{[version]}/bin',
|
||||
|
||||
&VMDEF_DEBUG_INTEGRATION => false,
|
||||
&VMDEF_WITH_ZST => true,
|
||||
|
||||
&VM_DB =>
|
||||
[
|
||||
@ -220,6 +225,7 @@ my $oyVm =
|
||||
&VMDEF_COVERAGE_C => true,
|
||||
|
||||
&VMDEF_DEBUG_INTEGRATION => false,
|
||||
&VMDEF_WITH_ZST => true,
|
||||
|
||||
&VM_DB =>
|
||||
[
|
||||
@ -399,6 +405,7 @@ my $oyVm =
|
||||
&VMDEF_PGSQL_BIN => '/usr/lib/postgresql/{[version]}/bin',
|
||||
|
||||
&VMDEF_WITH_BACKTRACE => true,
|
||||
&VMDEF_WITH_ZST => true,
|
||||
|
||||
&VM_DB =>
|
||||
[
|
||||
@ -575,6 +582,18 @@ sub vmWithLz4
|
||||
|
||||
push @EXPORT, qw(vmWithLz4);
|
||||
|
||||
####################################################################################################################################
|
||||
# Does the VM support liblzst?
|
||||
####################################################################################################################################
|
||||
sub vmWithZst
|
||||
{
|
||||
my $strVm = shift;
|
||||
|
||||
return (defined($oyVm->{$strVm}{&VMDEF_WITH_ZST}) ? $oyVm->{$strVm}{&VMDEF_WITH_ZST} : false);
|
||||
}
|
||||
|
||||
push @EXPORT, qw(vmWithZst);
|
||||
|
||||
####################################################################################################################################
|
||||
# Will integration tests be run in debug mode?
|
||||
####################################################################################################################################
|
||||
|
||||
@ -99,6 +99,8 @@ use constant GZ => 'gz';
|
||||
push @EXPORT, qw(GZ);
|
||||
use constant LZ4 => 'lz4';
|
||||
push @EXPORT, qw(LZ4);
|
||||
use constant ZST => 'zst';
|
||||
push @EXPORT, qw(ZST);
|
||||
|
||||
####################################################################################################################################
|
||||
# new
|
||||
|
||||
@ -209,10 +209,10 @@ sub run
|
||||
{vm => VM1, remote => true, s3 => false, encrypt => true, delta => false, compress => GZ},
|
||||
{vm => VM2, remote => false, s3 => false, encrypt => true, delta => true, compress => GZ},
|
||||
{vm => VM2, remote => true, s3 => true, encrypt => false, delta => false, compress => GZ},
|
||||
{vm => VM3, remote => false, s3 => false, encrypt => false, delta => true, compress => GZ},
|
||||
{vm => VM3, remote => false, s3 => false, encrypt => false, delta => true, compress => ZST},
|
||||
{vm => VM3, remote => true, s3 => true, encrypt => true, delta => false, compress => LZ4},
|
||||
{vm => VM4, remote => false, s3 => false, encrypt => false, delta => false, compress => LZ4},
|
||||
{vm => VM4, remote => true, s3 => true, encrypt => true, delta => true, compress => GZ},
|
||||
{vm => VM4, remote => true, s3 => true, encrypt => true, delta => true, compress => ZST},
|
||||
)
|
||||
{
|
||||
# Only run tests for this vm
|
||||
|
||||
@ -49,7 +49,7 @@ sub run
|
||||
{vm => VM2, remote => true, s3 => false, encrypt => true, compress => GZ, error => 0},
|
||||
{vm => VM3, remote => false, s3 => false, encrypt => true, compress => NONE, error => 0},
|
||||
{vm => VM3, remote => true, s3 => true, encrypt => false, compress => LZ4, error => 1},
|
||||
{vm => VM4, remote => false, s3 => true, encrypt => true, compress => GZ, error => 0},
|
||||
{vm => VM4, remote => false, s3 => true, encrypt => true, compress => ZST, error => 0},
|
||||
{vm => VM4, remote => true, s3 => false, encrypt => false, compress => NONE, error => 0},
|
||||
)
|
||||
{
|
||||
|
||||
@ -88,8 +88,8 @@ sub run
|
||||
{vm => VM2, remote => false, s3 => true, encrypt => false, compress => GZ},
|
||||
{vm => VM2, remote => true, s3 => false, encrypt => true, compress => GZ},
|
||||
{vm => VM3, remote => false, s3 => false, encrypt => true, compress => LZ4},
|
||||
{vm => VM3, remote => true, s3 => true, encrypt => false, compress => GZ},
|
||||
{vm => VM4, remote => false, s3 => true, encrypt => true, compress => GZ},
|
||||
{vm => VM3, remote => true, s3 => true, encrypt => false, compress => ZST},
|
||||
{vm => VM4, remote => false, s3 => true, encrypt => true, compress => ZST},
|
||||
{vm => VM4, remote => true, s3 => false, encrypt => false, compress => LZ4},
|
||||
)
|
||||
{
|
||||
|
||||
@ -45,10 +45,10 @@ sub run
|
||||
{vm => VM1, remote => true, s3 => true, encrypt => false, compress => GZ},
|
||||
{vm => VM2, remote => false, s3 => true, encrypt => true, compress => GZ},
|
||||
{vm => VM2, remote => true, s3 => false, encrypt => false, compress => GZ},
|
||||
{vm => VM3, remote => false, s3 => false, encrypt => false, compress => GZ},
|
||||
{vm => VM3, remote => false, s3 => false, encrypt => false, compress => ZST},
|
||||
{vm => VM3, remote => true, s3 => true, encrypt => true, compress => LZ4},
|
||||
{vm => VM4, remote => false, s3 => true, encrypt => false, compress => LZ4},
|
||||
{vm => VM4, remote => true, s3 => false, encrypt => true, compress => GZ},
|
||||
{vm => VM4, remote => true, s3 => false, encrypt => true, compress => ZST},
|
||||
)
|
||||
{
|
||||
# Only run tests for this vm
|
||||
|
||||
@ -64,7 +64,9 @@ sub run
|
||||
foreach my $strBackupDestination (
|
||||
$bS3 || $bHostBackup ? (HOST_BACKUP) : $bHostStandby ? (HOST_DB_MASTER, HOST_DB_STANDBY) : (HOST_DB_MASTER))
|
||||
{
|
||||
my $strCompressType = $bHostBackup && !$bHostStandby ? (vmWithLz4($self->vm()) && $bLz4Compress ? LZ4 : GZ) : NONE;
|
||||
my $strCompressType =
|
||||
$bHostBackup && !$bHostStandby ?
|
||||
(vmWithLz4($self->vm()) && $bLz4Compress ? LZ4 : vmWithZst($self->vm()) ? ZST : GZ) : NONE;
|
||||
my $bRepoEncrypt = ($strCompressType ne NONE && !$bS3) ? true : false;
|
||||
|
||||
# If compression was used then switch it for the next test that uses compression
|
||||
|
||||
12
test/patch/debian-package.patch
Normal file
12
test/patch/debian-package.patch
Normal file
@ -0,0 +1,12 @@
|
||||
--- control
|
||||
+++ control
|
||||
@@ -13,7 +13,8 @@
|
||||
pkg-config,
|
||||
txt2man,
|
||||
zlib1g-dev,
|
||||
- liblz4-dev
|
||||
+ liblz4-dev,
|
||||
+ libzstd-dev,
|
||||
Standards-Version: 4.4.1
|
||||
Homepage: https://www.pgbackrest.org/
|
||||
Vcs-Git: https://salsa.debian.org/postgresql/pgbackrest.git
|
||||
@ -154,7 +154,7 @@ testSuite(CompressType type, const char *decompressCmd)
|
||||
bufUsedSet(decompressed, bufSize(decompressed));
|
||||
|
||||
TEST_ASSIGN(
|
||||
compressed, testCompress(compressFilter(type, 3), decompressed, bufSize(decompressed), 1024),
|
||||
compressed, testCompress(compressFilter(type, 3), decompressed, bufSize(decompressed), 32),
|
||||
"zero data - compress large in/small out buffer");
|
||||
|
||||
TEST_RESULT_BOOL(
|
||||
@ -241,6 +241,45 @@ testRun(void)
|
||||
#endif // HAVE_LIBLZ4
|
||||
}
|
||||
|
||||
// *****************************************************************************************************************************
|
||||
if (testBegin("zst"))
|
||||
{
|
||||
#ifdef HAVE_LIBZST
|
||||
// Run standard test suite
|
||||
testSuite(compressTypeZst, "zstd -dc");
|
||||
|
||||
// -------------------------------------------------------------------------------------------------------------------------
|
||||
TEST_TITLE("zstError()");
|
||||
|
||||
TEST_RESULT_UINT(zstError(0), 0, "check success");
|
||||
TEST_ERROR(zstError((size_t)-12), FormatError, "zst error: [-12] Version not supported");
|
||||
|
||||
// -------------------------------------------------------------------------------------------------------------------------
|
||||
TEST_TITLE("zstDecompressToLog() and zstCompressToLog()");
|
||||
|
||||
ZstCompress *compress = (ZstCompress *)ioFilterDriver(zstCompressNew(14));
|
||||
|
||||
compress->inputSame = true;
|
||||
compress->inputOffset = 49;
|
||||
compress->flushing = true;
|
||||
|
||||
TEST_RESULT_STR_Z(
|
||||
zstCompressToLog(compress), "{level: 14, inputSame: true, inputOffset: 49, flushing: true}", "format object");
|
||||
|
||||
ZstDecompress *decompress = (ZstDecompress *)ioFilterDriver(zstDecompressNew());
|
||||
|
||||
decompress->inputSame = true;
|
||||
decompress->done = true;
|
||||
decompress->inputOffset = 999;
|
||||
|
||||
TEST_RESULT_STR_Z(
|
||||
zstDecompressToLog(decompress), "{inputSame: true, inputOffset: 999, frameDone false, done: true}",
|
||||
"format object");
|
||||
#else
|
||||
TEST_ERROR(compressTypePresent(compressTypeZst), OptionInvalidValueError, "pgBackRest not compiled with zst support");
|
||||
#endif // HAVE_LIBZST
|
||||
}
|
||||
|
||||
// Test everything in the helper that is not tested in the individual compression type tests
|
||||
// *****************************************************************************************************************************
|
||||
if (testBegin("helper"))
|
||||
@ -255,7 +294,7 @@ testRun(void)
|
||||
TEST_TITLE("compressTypePresent()");
|
||||
|
||||
TEST_RESULT_VOID(compressTypePresent(compressTypeNone), "type none always present");
|
||||
TEST_ERROR(compressTypePresent(compressTypeZst), OptionInvalidValueError, "pgBackRest not compiled with zst support");
|
||||
TEST_ERROR(compressTypePresent(compressTypeXz), OptionInvalidValueError, "pgBackRest not compiled with xz support");
|
||||
|
||||
// -------------------------------------------------------------------------------------------------------------------------
|
||||
TEST_TITLE("compressTypeFromName()");
|
||||
|
||||
@ -186,7 +186,7 @@ eval
|
||||
# Extra packages required when testing without containers
|
||||
if ($strVm eq VM_NONE)
|
||||
{
|
||||
$strPackage .= " valgrind liblz4-dev liblz4-tool";
|
||||
$strPackage .= " valgrind liblz4-dev liblz4-tool zstd libzstd-dev";
|
||||
}
|
||||
# Else packages needed for integration tests on containers
|
||||
else
|
||||
|
||||
Loading…
Reference in New Issue
Block a user