Add lz4 compression support.

LZ4 compresses data faster than gzip but at a lower ratio.  This can be a good tradeoff in certain scenarios.

Note that setting compress-type=lz4 will make new backups and archive incompatible (unrestorable) with prior versions of pgBackRest.

.travis.yml

@@ -13,7 +13,7 @@ services:
 
 matrix:
   include:
-    - env: PGB_CI="test --vm=u12"
+    - env: PGB_CI="test --vm=u12 --param=no-package"
     - env: PGB_CI="test --vm=f30 --param=no-package --param=c-only"
     - env: PGB_CI="test --vm=co6 --param=module=mock --param=module=real"
     - env: PGB_CI="test --vm=u18 --param=container-only"

build/lib/pgBackRestBuild/Config/Data.pm

@@ -1294,6 +1294,7 @@ my %hConfigDefine =
         [
             'none',
             'gz',
+            'lz4',
         ],
         &CFGDEF_COMMAND => CFGOPT_COMPRESS,
     },

doc/xml/reference.xml

@@ -142,6 +142,7 @@
                         <text>The following compression types are supported:
                         <ul>
                             <li><id>gz</id> - gzip compression format</li>
+                            <li><id>lz4</id> - lz4 compression format (not available on all platforms)</li>
                         </ul></text>
 
                         <example>n</example>

doc/xml/release.xml

@@ -14,6 +14,18 @@
     <release-list>
         <release date="XXXX-XX-XX" version="2.25dev" title="UNDER DEVELOPMENT">
             <release-core-list>
+                <release-feature-list>
+                    <release-item>
+                        <release-item-contributor-list>
+                             <release-item-reviewer id="cynthia.shang"/>
+                        </release-item-contributor-list>
+
+                        <p>Add <proper>lz4</proper> compression support.</p>
+
+                        <p>Note that setting <br-option>compress-type=lz4</br-option> will make new backups and archive incompatible (unrestorable) with prior versions of <backrest/>.</p>
+                    </release-item>
+                </release-feature-list>
+
                 <release-improvement-list>
                     <release-item>
                         <release-item-contributor-list>

doc/xml/user-guide.xml

@@ -747,13 +747,13 @@
 
             <execute if="{[os-type-is-debian]}" user="root" pre="y">
                 <exe-cmd>
-                    apt-get install make gcc libpq-dev libssl-dev libxml2-dev pkg-config</exe-cmd>
+                    apt-get install make gcc libpq-dev libssl-dev libxml2-dev pkg-config liblz4-dev</exe-cmd>
                 <exe-cmd-extra>-y 2>&amp;1</exe-cmd-extra>
             </execute>
 
             <execute if="{[os-type-is-centos]}" user="root" pre="y">
                 <exe-cmd>
-                    yum install make gcc postgresql-devel openssl-devel libxml2-devel
+                    yum install make gcc postgresql-devel openssl-devel libxml2-devel lz4-devel
                 </exe-cmd>
                 <exe-cmd-extra>-y 2>&amp;1</exe-cmd-extra>
             </execute>

src/Makefile.in

@@ -83,6 +83,9 @@ SRCS = \
 	common/compress/gz/common.c \
 	common/compress/gz/compress.c \
 	common/compress/gz/decompress.c \
+	common/compress/lz4/common.c \
+	common/compress/lz4/compress.c \
+	common/compress/lz4/decompress.c \
 	common/crypto/cipherBlock.c \
 	common/crypto/common.c \
 	common/crypto/hash.c \
@@ -331,9 +334,18 @@ common/compress/gz/compress.o: common/compress/gz/compress.c build.auto.h common
 common/compress/gz/decompress.o: common/compress/gz/decompress.c build.auto.h common/assert.h common/compress/gz/common.h common/compress/gz/decompress.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/filter.intern.h common/log.h common/logLevel.h common/macro.h common/memContext.h common/object.h common/stackTrace.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h
 	$(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c common/compress/gz/decompress.c -o common/compress/gz/decompress.o
 
-common/compress/helper.o: common/compress/helper.c build.auto.h common/assert.h common/compress/gz/common.h common/compress/gz/compress.h common/compress/gz/decompress.h common/compress/helper.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringz.h common/type/variant.h common/type/variantList.h version.h
+common/compress/helper.o: common/compress/helper.c build.auto.h common/assert.h common/compress/gz/common.h common/compress/gz/compress.h common/compress/gz/decompress.h common/compress/helper.h common/compress/lz4/common.h common/compress/lz4/compress.h common/compress/lz4/decompress.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringz.h common/type/variant.h common/type/variantList.h version.h
 	$(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c common/compress/helper.c -o common/compress/helper.o
 
+common/compress/lz4/common.o: common/compress/lz4/common.c build.auto.h common/assert.h common/compress/lz4/common.h common/debug.h common/error.auto.h common/error.h common/logLevel.h common/stackTrace.h common/type/convert.h common/type/stringz.h
+	$(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c common/compress/lz4/common.c -o common/compress/lz4/common.o
+
+common/compress/lz4/compress.o: common/compress/lz4/compress.c build.auto.h common/assert.h common/compress/lz4/common.h common/compress/lz4/compress.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/filter.intern.h common/log.h common/logLevel.h common/macro.h common/memContext.h common/object.h common/stackTrace.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h
+	$(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c common/compress/lz4/compress.c -o common/compress/lz4/compress.o
+
+common/compress/lz4/decompress.o: common/compress/lz4/decompress.c build.auto.h common/assert.h common/compress/lz4/common.h common/compress/lz4/decompress.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/filter.intern.h common/log.h common/logLevel.h common/macro.h common/memContext.h common/object.h common/stackTrace.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h
+	$(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c common/compress/lz4/decompress.c -o common/compress/lz4/decompress.o
+
 common/crypto/cipherBlock.o: common/crypto/cipherBlock.c build.auto.h common/assert.h common/crypto/cipherBlock.h common/crypto/common.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/filter.intern.h common/io/filter/group.h common/log.h common/logLevel.h common/macro.h common/memContext.h common/object.h common/stackTrace.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/stringz.h common/type/variant.h common/type/variantList.h
 	$(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c common/crypto/cipherBlock.c -o common/crypto/cipherBlock.o
 

src/build.auto.h.in

@@ -3,3 +3,6 @@ Build Flags Generated by Configure
 ***********************************************************************************************************************************/
 // Are test code and asserts disabled?
 #undef NDEBUG
+
+// Is liblz4 present?
+#undef HAVE_LIBLZ4

src/common/compress/helper.c

@@ -9,6 +9,9 @@ Compression Helper
 #include "common/compress/gz/common.h"
 #include "common/compress/gz/compress.h"
 #include "common/compress/gz/decompress.h"
+#include "common/compress/lz4/common.h"
+#include "common/compress/lz4/compress.h"
+#include "common/compress/lz4/decompress.h"
 #include "common/debug.h"
 #include "common/log.h"
 #include "version.h"
@@ -19,7 +22,6 @@ Compression type constants
 #define COMPRESS_TYPE_NONE                                          "none"
 
 // Constants for currently unsupported compression types
-#define LZ4_EXT                                                     "lz4"
 #define ZST_EXT                                                     "zst"
 #define BZ2_EXT                                                     "bz2"
 #define XZ_EXT                                                      "xz"
@@ -54,6 +56,13 @@ static const struct CompressHelperLocal
     {
         .type = STRDEF(LZ4_EXT),
         .ext = STRDEF("." LZ4_EXT),
+#ifdef HAVE_LIBLZ4
+        .compressType = LZ4_COMPRESS_FILTER_TYPE,
+        .compressNew = lz4CompressNew,
+        .decompressType = LZ4_DECOMPRESS_FILTER_TYPE,
+        .decompressNew = lz4DecompressNew,
+        .levelDefault = 1,
+#endif
     },
     {
         .type = STRDEF(ZST_EXT),

src/common/compress/lz4/common.c

@@ -0,0 +1,29 @@
+/***********************************************************************************************************************************
+LZ4 Common
+***********************************************************************************************************************************/
+#include "build.auto.h"
+
+#ifdef HAVE_LIBLZ4
+
+#include <lz4frame.h>
+
+#include "common/compress/lz4/common.h"
+#include "common/debug.h"
+
+/***********************************************************************************************************************************
+Process lz4 errors
+***********************************************************************************************************************************/
+LZ4F_errorCode_t
+lz4Error(LZ4F_errorCode_t error)
+{
+    FUNCTION_TEST_BEGIN();
+        FUNCTION_TEST_PARAM(SIZE, error);
+    FUNCTION_TEST_END();
+
+    if (LZ4F_isError(error))
+        THROW_FMT(FormatError, "lz4 error: [%zd] %s", (ssize_t)error, LZ4F_getErrorName(error));
+
+    FUNCTION_TEST_RETURN(error);
+}
+
+#endif // HAVE_LIBLZ4

src/common/compress/lz4/common.h

@@ -0,0 +1,21 @@
+/***********************************************************************************************************************************
+LZ4 Common
+***********************************************************************************************************************************/
+#ifndef COMMON_COMPRESS_LZ4_COMMON_H
+#define COMMON_COMPRESS_LZ4_COMMON_H
+
+/***********************************************************************************************************************************
+LZ4 extension
+***********************************************************************************************************************************/
+#define LZ4_EXT                                                     "lz4"
+
+#ifdef HAVE_LIBLZ4
+
+/***********************************************************************************************************************************
+Functions
+***********************************************************************************************************************************/
+size_t lz4Error(size_t error);
+
+#endif // HAVE_LIBLZ4
+
+#endif

src/common/compress/lz4/compress.c

@@ -0,0 +1,286 @@
+/***********************************************************************************************************************************
+LZ4 Compress
+
+Developed against version r131 using the documentation in https://github.com/lz4/lz4/blob/r131/lib/lz4frame.h.
+***********************************************************************************************************************************/
+#include "build.auto.h"
+
+#ifdef HAVE_LIBLZ4
+
+#include <stdio.h>
+#include <lz4frame.h>
+#include <string.h>
+
+#include "common/compress/lz4/common.h"
+#include "common/compress/lz4/compress.h"
+#include "common/debug.h"
+#include "common/io/filter/filter.intern.h"
+#include "common/log.h"
+#include "common/memContext.h"
+#include "common/object.h"
+
+/***********************************************************************************************************************************
+Older versions of lz4 do not define the max header size.  This seems to be the max for any version.
+***********************************************************************************************************************************/
+#ifndef LZ4F_HEADER_SIZE_MAX
+    #define LZ4F_HEADER_SIZE_MAX                                    19
+#endif
+
+/***********************************************************************************************************************************
+Filter type constant
+***********************************************************************************************************************************/
+STRING_EXTERN(LZ4_COMPRESS_FILTER_TYPE_STR,                         LZ4_COMPRESS_FILTER_TYPE);
+
+/***********************************************************************************************************************************
+Object type
+***********************************************************************************************************************************/
+#define LZ4_COMPRESS_TYPE                                           Lz4Compress
+#define LZ4_COMPRESS_PREFIX                                         lz4Compress
+
+typedef struct Lz4Compress
+{
+    MemContext *memContext;                                         // Context to store data
+    LZ4F_compressionContext_t context;                              // LZ4 compression context
+    LZ4F_preferences_t prefs;                                       // Preferences -- just compress level set
+    IoFilter *filter;                                               // Filter interface
+
+    Buffer *buffer;                                                 // For when the output buffer can't accept all compressed data
+    bool first;                                                     // Is this the first call to process?
+    bool inputSame;                                                 // Is the same input required on the next process call?
+    bool flushing;                                                  // Is input complete and flushing in progress?
+} Lz4Compress;
+
+/***********************************************************************************************************************************
+Render as string for logging
+***********************************************************************************************************************************/
+static String *
+lz4CompressToLog(const Lz4Compress *this)
+{
+    return strNewFmt(
+        "{level: %d, first: %s, inputSame: %s, flushing: %s}", this->prefs.compressionLevel,
+        cvtBoolToConstZ(this->first), cvtBoolToConstZ(this->inputSame), cvtBoolToConstZ(this->flushing));
+}
+
+#define FUNCTION_LOG_LZ4_COMPRESS_TYPE                                                                                             \
+    Lz4Compress *
+#define FUNCTION_LOG_LZ4_COMPRESS_FORMAT(value, buffer, bufferSize)                                                                \
+    FUNCTION_LOG_STRING_OBJECT_FORMAT(value, lz4CompressToLog, buffer, bufferSize)
+
+/***********************************************************************************************************************************
+Free compression context
+***********************************************************************************************************************************/
+OBJECT_DEFINE_FREE_RESOURCE_BEGIN(LZ4_COMPRESS, LOG, logLevelTrace)
+{
+    LZ4F_freeCompressionContext(this->context);
+}
+OBJECT_DEFINE_FREE_RESOURCE_END(LOG);
+
+/***********************************************************************************************************************************
+Compress data
+***********************************************************************************************************************************/
+// Helper to return a buffer where output will be written.  If there is enough space in the provided output buffer then use it,
+// otherwise allocate an internal buffer to hold the compressed data.  Once we start using the internal buffer we'll need to
+// continue using it until it is completely flushed.
+static Buffer *
+lz4CompressBuffer(Lz4Compress *this, size_t required, Buffer *output)
+{
+    FUNCTION_TEST_BEGIN();
+        FUNCTION_TEST_PARAM(LZ4_COMPRESS, this);
+        FUNCTION_TEST_PARAM(SIZE, required);
+        FUNCTION_TEST_PARAM(BUFFER, output);
+    FUNCTION_TEST_END();
+
+    Buffer *result = output;
+
+    // Is an internal buffer required to hold the compressed data?
+    if (bufUsed(this->buffer) > 0 || required >= bufRemains(output))
+    {
+        // Reallocate buffer if it is not large enough
+        if (required >= bufRemains(this->buffer))
+            bufResize(this->buffer, bufUsed(this->buffer) + required);
+
+        result = this->buffer;
+    }
+
+    FUNCTION_TEST_RETURN(result);
+}
+
+// Helper to flush output data to compressed buffer
+static void
+lz4CompressFlush(Lz4Compress *this, Buffer *output, Buffer *compressed)
+{
+    FUNCTION_TEST_BEGIN();
+        FUNCTION_TEST_PARAM(LZ4_COMPRESS, this);
+        FUNCTION_TEST_PARAM(BUFFER, output);
+        FUNCTION_TEST_PARAM(BUFFER, compressed);
+    FUNCTION_TEST_END();
+
+    // If the compressed buffer can hold all the output
+    if (bufRemains(compressed) >= bufUsed(output))
+    {
+        bufCat(compressed, output);
+        bufUsedZero(output);
+
+        this->inputSame = false;
+    }
+    // Else flush as much as possible and set inputSame to flush more once the compressed buffer has been emptied
+    else
+    {
+        size_t catSize = bufRemains(compressed);
+        bufCatSub(compressed, output, 0, catSize);
+
+        memmove(bufPtr(output), bufPtr(output) + catSize, bufUsed(output) - catSize);
+        bufUsedSet(output, bufUsed(output) - catSize);
+
+        this->inputSame = true;
+    }
+
+    FUNCTION_TEST_RETURN_VOID();
+}
+
+static void
+lz4CompressProcess(THIS_VOID, const Buffer *uncompressed, Buffer *compressed)
+{
+    THIS(Lz4Compress);
+
+    FUNCTION_LOG_BEGIN(logLevelTrace);
+        FUNCTION_LOG_PARAM(LZ4_COMPRESS, this);
+        FUNCTION_LOG_PARAM(BUFFER, uncompressed);
+        FUNCTION_LOG_PARAM(BUFFER, compressed);
+    FUNCTION_LOG_END();
+
+    ASSERT(this != NULL);
+    ASSERT(!(this->flushing && !this->inputSame));
+    ASSERT(this->context != NULL);
+    ASSERT(compressed != NULL);
+    ASSERT(!this->flushing || uncompressed == NULL);
+
+    // Flush overflow output to the compressed buffer
+    if (this->inputSame)
+    {
+        lz4CompressFlush(this, this->buffer, compressed);
+    }
+    else
+    {
+        Buffer *output = NULL;
+
+        // If first call to process then begin compression
+        if (this->first)
+        {
+            output = lz4CompressBuffer(this, LZ4F_HEADER_SIZE_MAX, compressed);
+            bufUsedInc(
+                output, lz4Error(LZ4F_compressBegin(this->context, bufRemainsPtr(output), bufRemains(output), &this->prefs)));
+
+            this->first = false;
+        }
+
+        // Normal processing call
+        if (uncompressed != NULL)
+        {
+            output = lz4CompressBuffer(this, lz4Error(LZ4F_compressBound(bufUsed(uncompressed), &this->prefs)), compressed);
+
+            bufUsedInc(
+                output,
+                lz4Error(
+                    LZ4F_compressUpdate(
+                        this->context, bufRemainsPtr(output), bufRemains(output), bufPtr(uncompressed), bufUsed(uncompressed),
+                        NULL)));
+        }
+        // Else flush remaining output
+        else
+        {
+            output = lz4CompressBuffer(this, lz4Error(LZ4F_compressBound(0, &this->prefs)), compressed);
+            bufUsedInc(output, lz4Error(LZ4F_compressEnd(this->context, bufRemainsPtr(output), bufRemains(output), NULL)));
+
+            this->flushing = true;
+        }
+
+        // If the output buffer was allocated locally it will need to be flushed to the compressed buffer
+        if (output != compressed)
+            lz4CompressFlush(this, output, compressed);
+    }
+
+    FUNCTION_LOG_RETURN_VOID();
+}
+
+/***********************************************************************************************************************************
+Is compress done?
+***********************************************************************************************************************************/
+static bool
+lz4CompressDone(const THIS_VOID)
+{
+    THIS(const Lz4Compress);
+
+    FUNCTION_TEST_BEGIN();
+        FUNCTION_TEST_PARAM(LZ4_COMPRESS, this);
+    FUNCTION_TEST_END();
+
+    ASSERT(this != NULL);
+
+    FUNCTION_TEST_RETURN(this->flushing && !this->inputSame);
+}
+
+/***********************************************************************************************************************************
+Is the same input required on the next process call?
+***********************************************************************************************************************************/
+static bool
+lz4CompressInputSame(const THIS_VOID)
+{
+    THIS(const Lz4Compress);
+
+    FUNCTION_TEST_BEGIN();
+        FUNCTION_TEST_PARAM(LZ4_COMPRESS, this);
+    FUNCTION_TEST_END();
+
+    ASSERT(this != NULL);
+
+    FUNCTION_TEST_RETURN(this->inputSame);
+}
+
+/***********************************************************************************************************************************
+New object
+***********************************************************************************************************************************/
+IoFilter *
+lz4CompressNew(int level)
+{
+    FUNCTION_LOG_BEGIN(logLevelTrace);
+        FUNCTION_LOG_PARAM(INT, level);
+    FUNCTION_LOG_END();
+
+    ASSERT(level >= 0);
+
+    IoFilter *this = NULL;
+
+    MEM_CONTEXT_NEW_BEGIN("Lz4Compress")
+    {
+        Lz4Compress *driver = memNew(sizeof(Lz4Compress));
+
+        *driver = (Lz4Compress)
+        {
+            .memContext = MEM_CONTEXT_NEW(),
+            .prefs = {.compressionLevel = level, .frameInfo = {.contentChecksumFlag = LZ4F_contentChecksumEnabled}},
+            .first = true,
+            .buffer = bufNew(0),
+        };
+
+        // Create lz4 context
+        lz4Error(LZ4F_createCompressionContext(&driver->context, LZ4F_VERSION));
+
+        // Set callback to ensure lz4 context is freed
+        memContextCallbackSet(driver->memContext, lz4CompressFreeResource, driver);
+
+        // Create param list
+        VariantList *paramList = varLstNew();
+        varLstAdd(paramList, varNewInt(level));
+
+        // Create filter interface
+        this = ioFilterNewP(
+            LZ4_COMPRESS_FILTER_TYPE_STR, driver, paramList, .done = lz4CompressDone, .inOut = lz4CompressProcess,
+            .inputSame = lz4CompressInputSame);
+    }
+    MEM_CONTEXT_NEW_END();
+
+    FUNCTION_LOG_RETURN(IO_FILTER, this);
+}
+
+#endif // HAVE_LIBLZ4

src/common/compress/lz4/compress.h

@@ -0,0 +1,26 @@
+/***********************************************************************************************************************************
+LZ4 Compress
+
+Compress IO to the lz4 format.
+***********************************************************************************************************************************/
+#ifdef HAVE_LIBLZ4
+
+#ifndef COMMON_COMPRESS_LZ4_COMPRESS_H
+#define COMMON_COMPRESS_LZ4_COMPRESS_H
+
+#include "common/io/filter/filter.h"
+
+/***********************************************************************************************************************************
+Filter type constant
+***********************************************************************************************************************************/
+#define LZ4_COMPRESS_FILTER_TYPE                                    "lz4Compress"
+    STRING_DECLARE(LZ4_COMPRESS_FILTER_TYPE_STR);
+
+/***********************************************************************************************************************************
+Constructors
+***********************************************************************************************************************************/
+IoFilter *lz4CompressNew(int level);
+
+#endif
+
+#endif // HAVE_LIBLZ4

src/common/compress/lz4/decompress.c

@@ -0,0 +1,192 @@
+/***********************************************************************************************************************************
+LZ4 Decompress
+***********************************************************************************************************************************/
+#include "build.auto.h"
+
+#ifdef HAVE_LIBLZ4
+
+#include <stdio.h>
+#include <lz4frame.h>
+
+#include "common/compress/lz4/common.h"
+#include "common/compress/lz4/decompress.h"
+#include "common/debug.h"
+#include "common/io/filter/filter.intern.h"
+#include "common/log.h"
+#include "common/memContext.h"
+#include "common/object.h"
+
+/***********************************************************************************************************************************
+Filter type constant
+***********************************************************************************************************************************/
+STRING_EXTERN(LZ4_DECOMPRESS_FILTER_TYPE_STR,                       LZ4_DECOMPRESS_FILTER_TYPE);
+
+/***********************************************************************************************************************************
+Object type
+***********************************************************************************************************************************/
+#define LZ4_DECOMPRESS_TYPE                                         Lz4Decompress
+#define LZ4_DECOMPRESS_PREFIX                                       lz4Decompress
+
+typedef struct Lz4Decompress
+{
+    MemContext *memContext;                                         // Context to store data
+    LZ4F_decompressionContext_t context;                            // LZ4 decompression context
+    IoFilter *filter;                                               // Filter interface
+
+    bool inputSame;                                                 // Is the same input required on the next process call?
+    size_t inputOffset;                                             // Current offset from the start of the buffer
+    bool frameDone;                                                 // Has the current frame completed?
+    bool done;                                                      // Is decompression done?
+} Lz4Decompress;
+
+/***********************************************************************************************************************************
+Render as string for logging
+***********************************************************************************************************************************/
+static String *
+lz4DecompressToLog(const Lz4Decompress *this)
+{
+    return strNewFmt(
+        "{inputSame: %s, inputOffset: %zu, frameDone %s, done: %s}", cvtBoolToConstZ(this->inputSame), this->inputOffset,
+        cvtBoolToConstZ(this->frameDone), cvtBoolToConstZ(this->done));
+}
+
+#define FUNCTION_LOG_LZ4_DECOMPRESS_TYPE                                                                                           \
+    Lz4Decompress *
+#define FUNCTION_LOG_LZ4_DECOMPRESS_FORMAT(value, buffer, bufferSize)                                                              \
+    FUNCTION_LOG_STRING_OBJECT_FORMAT(value, lz4DecompressToLog, buffer, bufferSize)
+
+/***********************************************************************************************************************************
+Free decompression context
+***********************************************************************************************************************************/
+OBJECT_DEFINE_FREE_RESOURCE_BEGIN(LZ4_DECOMPRESS, LOG, logLevelTrace)
+{
+    LZ4F_freeDecompressionContext(this->context);
+}
+OBJECT_DEFINE_FREE_RESOURCE_END(LOG);
+
+/***********************************************************************************************************************************
+Decompress data
+***********************************************************************************************************************************/
+static void
+lz4DecompressProcess(THIS_VOID, const Buffer *compressed, Buffer *decompressed)
+{
+    THIS(Lz4Decompress);
+
+    FUNCTION_LOG_BEGIN(logLevelTrace);
+        FUNCTION_LOG_PARAM(LZ4_DECOMPRESS, this);
+        FUNCTION_LOG_PARAM(BUFFER, compressed);
+        FUNCTION_LOG_PARAM(BUFFER, decompressed);
+    FUNCTION_LOG_END();
+
+    ASSERT(this != NULL);
+    ASSERT(this->context != NULL);
+    ASSERT(decompressed != NULL);
+
+    // When there is no more input then decompression is done
+    if (compressed == NULL)
+    {
+        // If the current frame being decompressed was not completed then error
+        if (!this->frameDone)
+            THROW(FormatError, "unexpected eof in compressed data");
+
+        this->done = true;
+    }
+    else
+    {
+        // Decompress as much data as possible
+        size_t srcSize = bufUsed(compressed) - this->inputOffset;
+        size_t dstSize = bufRemains(decompressed);
+
+        this->frameDone = lz4Error(
+            LZ4F_decompress(
+                this->context, bufRemainsPtr(decompressed), &dstSize, bufPtr(compressed) + this->inputOffset, &srcSize, NULL)) == 0;
+
+        bufUsedInc(decompressed, dstSize);
+
+        // If the compressed data was not fully processed then update the offset and set inputSame
+        if (srcSize < bufUsed(compressed) - this->inputOffset)
+        {
+            this->inputOffset += srcSize;
+            this->inputSame = true;
+        }
+        // Else all compressed data was processed
+        else
+        {
+            this->inputOffset = 0;
+            this->inputSame = false;
+        }
+    }
+
+    FUNCTION_LOG_RETURN_VOID();
+}
+
+/***********************************************************************************************************************************
+Is decompress done?
+***********************************************************************************************************************************/
+static bool
+lz4DecompressDone(const THIS_VOID)
+{
+    THIS(const Lz4Decompress);
+
+    FUNCTION_TEST_BEGIN();
+        FUNCTION_TEST_PARAM(LZ4_DECOMPRESS, this);
+    FUNCTION_TEST_END();
+
+    ASSERT(this != NULL);
+
+    FUNCTION_TEST_RETURN(this->done);
+}
+
+/***********************************************************************************************************************************
+Is the same input required on the next process call?
+***********************************************************************************************************************************/
+static bool
+lz4DecompressInputSame(const THIS_VOID)
+{
+    THIS(const Lz4Decompress);
+
+    FUNCTION_TEST_BEGIN();
+        FUNCTION_TEST_PARAM(LZ4_DECOMPRESS, this);
+    FUNCTION_TEST_END();
+
+    ASSERT(this != NULL);
+
+    FUNCTION_TEST_RETURN(this->inputSame);
+}
+
+/***********************************************************************************************************************************
+New object
+***********************************************************************************************************************************/
+IoFilter *
+lz4DecompressNew(void)
+{
+    FUNCTION_LOG_VOID(logLevelTrace);
+
+    IoFilter *this = NULL;
+
+    MEM_CONTEXT_NEW_BEGIN("Lz4Decompress")
+    {
+        Lz4Decompress *driver = memNew(sizeof(Lz4Decompress));
+
+        *driver = (Lz4Decompress)
+        {
+            .memContext = MEM_CONTEXT_NEW(),
+        };
+
+        // Create lz4 context
+        lz4Error(LZ4F_createDecompressionContext(&driver->context, LZ4F_VERSION));
+
+        // Set callback to ensure lz4 context is freed
+        memContextCallbackSet(driver->memContext, lz4DecompressFreeResource, driver);
+
+        // Create filter interface
+        this = ioFilterNewP(
+            LZ4_DECOMPRESS_FILTER_TYPE_STR, driver, NULL, .done = lz4DecompressDone, .inOut = lz4DecompressProcess,
+            .inputSame = lz4DecompressInputSame);
+    }
+    MEM_CONTEXT_NEW_END();
+
+    FUNCTION_LOG_RETURN(IO_FILTER, this);
+}
+
+#endif // HAVE_LIBLZ4

src/common/compress/lz4/decompress.h

@@ -0,0 +1,26 @@
+/***********************************************************************************************************************************
+LZ4 Decompress
+
+Decompress IO from the lz4 format.
+***********************************************************************************************************************************/
+#ifdef HAVE_LIBLZ4
+
+#ifndef COMMON_COMPRESS_LZ4_DECOMPRESS_H
+#define COMMON_COMPRESS_LZ4_DECOMPRESS_H
+
+#include "common/io/filter/filter.h"
+
+/***********************************************************************************************************************************
+Filter type constant
+***********************************************************************************************************************************/
+#define LZ4_DECOMPRESS_FILTER_TYPE                                  "lz4Decompress"
+    STRING_DECLARE(LZ4_DECOMPRESS_FILTER_TYPE_STR);
+
+/***********************************************************************************************************************************
+Constructor
+***********************************************************************************************************************************/
+IoFilter *lz4DecompressNew(void);
+
+#endif
+
+#endif // HAVE_LIBLZ4

src/config/define.auto.c

@@ -832,7 +832,8 @@ static ConfigDefineOptionData configDefineOptionData[] = CFGDEFDATA_OPTION_LIST
         (
             "The following compression types are supported:\n"
             "\n"
-            "* gz - gzip compression format"
+            "* gz - gzip compression format\n"
+            "* lz4 - lz4 compression format (not available on all platforms)"
         )
 
         CFGDEFDATA_OPTION_COMMAND_LIST
@@ -846,7 +847,8 @@ static ConfigDefineOptionData configDefineOptionData[] = CFGDEFDATA_OPTION_LIST
             CFGDEFDATA_OPTION_OPTIONAL_ALLOW_LIST
             (
                 "none",
-                "gz"
+                "gz",
+                "lz4"
             )
 
             CFGDEFDATA_OPTION_OPTIONAL_DEFAULT("gz")

src/configure

@@ -3709,6 +3709,53 @@ else
 fi
 
 
+# Check optional lz4 library
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for LZ4F_isError in -llz4" >&5
+$as_echo_n "checking for LZ4F_isError in -llz4... " >&6; }
+if ${ac_cv_lib_lz4_LZ4F_isError+:} false; then :
+  $as_echo_n "(cached) " >&6
+else
+  ac_check_lib_save_LIBS=$LIBS
+LIBS="-llz4  $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+
+/* Override any GCC internal prototype to avoid an error.
+   Use char because int might match the return type of a GCC
+   builtin and then its argument prototype would still apply.  */
+#ifdef __cplusplus
+extern "C"
+#endif
+char LZ4F_isError ();
+int
+main ()
+{
+return LZ4F_isError ();
+  ;
+  return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+  ac_cv_lib_lz4_LZ4F_isError=yes
+else
+  ac_cv_lib_lz4_LZ4F_isError=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+    conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_lz4_LZ4F_isError" >&5
+$as_echo "$ac_cv_lib_lz4_LZ4F_isError" >&6; }
+if test "x$ac_cv_lib_lz4_LZ4F_isError" = xyes; then :
+  cat >>confdefs.h <<_ACEOF
+#define HAVE_LIBLZ4 1
+_ACEOF
+
+  LIBS="-llz4 $LIBS"
+
+fi
+
+
 # Write output
 ac_config_headers="$ac_config_headers build.auto.h"
 

src/configure.ac

@@ -57,6 +57,9 @@ AC_SUBST(CLIBRARY, "$CLIBRARY `pkg-config libxml-2.0 --cflags`")
 # Check required gz library
 AC_CHECK_LIB([z], [deflate], [], [AC_MSG_ERROR([library 'z' is required])])
 
+# Check optional lz4 library
+AC_CHECK_LIB([lz4], [LZ4F_isError])
+
 # Write output
 AC_CONFIG_HEADERS([build.auto.h])
 AC_CONFIG_FILES([Makefile])

test/Vagrantfile

@@ -71,7 +71,7 @@ Vagrant.configure(2) do |config|
         #---------------------------------------------------------------------------------------------------------------------------
         echo 'Install Build Tools' && date
         apt-get install -y devscripts build-essential lintian git cloc txt2man debhelper libssl-dev zlib1g-dev libperl-dev \
-             libxml2-dev liblz4-dev libpq-dev valgrind lcov
+             libxml2-dev liblz4-dev liblz4-tool libpq-dev valgrind lcov
 
         #---------------------------------------------------------------------------------------------------------------------------
         echo 'Install Docker' && date

test/container.yaml

@@ -12,12 +12,12 @@
 # - docker login -u pgbackrest
 # - VM=XXX;DATE=YYYYMMDDX;BASE=pgbackrest/test:${VM?}-base;docker tag ${BASE?} ${BASE?}-${DATE?} && docker push ${BASE?}-${DATE?}
 # **********************************************************************************************************************************
+20200310A:
+  co6: beb7b5a62ebdf209bb54494129baeadbd0c2954f
+  co7: 833c4ef292b9b3c957d9d7d296ec7c03ec6310c2
+  f30: 9ef023b179bd9e54bb079d60b75516067a4b4b1d
+  u18: 5e295130131d5fe96e2483885c171677b52c8ce3
+  u19: 9eb97f565e47a76e98743c98a862f91e8df5e2b1
+
 20200124A:
   u12: 0f2fcf1bc79ee35e78121773c9a2155a77cb10d5
-  u18: e0d670be1f7167152748e36832081b5b09aabc70
-  u19: 9d057b1081efedfce0f7ba38092b867a125fde37
-
-20191122A:
-  f30: 3a60318c275cdbfa6daa0f2b502a5551562e6db6
-  co6: 63527115fd269dfee74bda70da8a5b0569f94886
-  co7: 0b4c4b953709dc086f95fcc5c8ca45dee1a5eee9

test/define.yaml

@@ -258,12 +258,15 @@ unit:
 
       # ----------------------------------------------------------------------------------------------------------------------------
       - name: compress
-        total: 2
+        total: 3
 
         coverage:
           common/compress/gz/common: full
           common/compress/gz/compress: full
           common/compress/gz/decompress: full
+          common/compress/lz4/common: full
+          common/compress/lz4/compress: full
+          common/compress/lz4/decompress: full
           common/compress/helper: full
 
       # ----------------------------------------------------------------------------------------------------------------------------

test/expect/mock-all-002.log

@@ -1755,7 +1755,7 @@ diff backup - remove files (backup host)
 ------------------------------------------------------------------------------------------------------------------------------------
 P00   WARN: option repo1-retention-full is not set, the repository may run out of space
             HINT: to retain full backups indefinitely (without warning), set option 'repo1-retention-full' to the maximum.
-P00   WARN: diff backup cannot alter compress-type option to 'gz', reset to value in [BACKUP-FULL-2]
+P00   WARN: diff backup cannot alter compress-type option to 'lz4', reset to value in [BACKUP-FULL-2]
 P00   WARN: diff backup cannot alter 'checksum-page' option to 'false', reset to 'true' from [BACKUP-FULL-2]
 P00   WARN: file 'changetime.txt' has timestamp earlier than prior backup, enabling delta checksum
 P00   WARN: page misalignment in file db-master:[TEST_PATH]/db-master/db/base-2/pg_tblspc/2/[TS_PATH-1]/32768/tablespace2c.txt: file size 12 is not divisible by page size 8192
@@ -1804,6 +1804,7 @@ pg1-path=[TEST_PATH]/db-master/db/base-2
 buffer-size=[BUFFER-SIZE]
 compress-level=3
 compress-level-network=1
+compress-type=lz4
 db-timeout=45
 lock-path=[TEST_PATH]/backup/lock
 log-level-console=warn
@@ -2011,6 +2012,7 @@ pg1-path=[TEST_PATH]/db-master/db/base-2
 buffer-size=[BUFFER-SIZE]
 compress-level=3
 compress-level-network=1
+compress-type=lz4
 db-timeout=45
 lock-path=[TEST_PATH]/backup/lock
 log-level-console=warn
@@ -2065,7 +2067,7 @@ option-checksum-page=false
 option-compress=true
 option-compress-level=3
 option-compress-level-network=1
-option-compress-type="gz"
+option-compress-type="lz4"
 option-delta=false
 option-hardlink=false
 option-online=false
@@ -2223,6 +2225,7 @@ pg1-path=[TEST_PATH]/db-master/db/base-2
 buffer-size=[BUFFER-SIZE]
 compress-level=3
 compress-level-network=1
+compress-type=lz4
 db-timeout=45
 lock-path=[TEST_PATH]/backup/lock
 log-level-console=warn
@@ -2278,7 +2281,7 @@ option-checksum-page=false
 option-compress=true
 option-compress-level=3
 option-compress-level-network=1
-option-compress-type="gz"
+option-compress-type="lz4"
 option-delta=false
 option-hardlink=false
 option-online=false
@@ -2418,7 +2421,7 @@ restore_command = '[BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf
 diff backup - option backup-standby reset - backup performed from master (backup host)
 > [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --no-online --log-level-console=info --backup-standby --type=diff --stanza=db backup
 ------------------------------------------------------------------------------------------------------------------------------------
-P00   INFO: backup command begin [BACKREST-VERSION]: --backup-standby --buffer-size=[BUFFER-SIZE] --compress-level=3 --compress-level-network=1 --config=[TEST_PATH]/backup/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/backup/lock --log-level-console=info --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/backup/log[] --no-log-timestamp --no-online --pg1-host=db-master --pg1-host-cmd=[BACKREST-BIN] --pg1-host-config=[TEST_PATH]/db-master/pgbackrest.conf --pg1-host-user=[USER-1] --pg1-path=[TEST_PATH]/db-master/db/base-2/base --process-max=2 --protocol-timeout=60 --repo1-cipher-pass=<redacted> --repo1-cipher-type=aes-256-cbc --repo1-path=/ --repo1-s3-bucket=pgbackrest-dev --repo1-s3-endpoint=s3.amazonaws.com --repo1-s3-key=<redacted> --repo1-s3-key-secret=<redacted> --repo1-s3-region=us-east-1 --no-repo1-s3-verify-tls --repo1-type=s3 --stanza=db --start-fast --type=diff
+P00   INFO: backup command begin [BACKREST-VERSION]: --backup-standby --buffer-size=[BUFFER-SIZE] --compress-level=3 --compress-level-network=1 --compress-type=lz4 --config=[TEST_PATH]/backup/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/backup/lock --log-level-console=info --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/backup/log[] --no-log-timestamp --no-online --pg1-host=db-master --pg1-host-cmd=[BACKREST-BIN] --pg1-host-config=[TEST_PATH]/db-master/pgbackrest.conf --pg1-host-user=[USER-1] --pg1-path=[TEST_PATH]/db-master/db/base-2/base --process-max=2 --protocol-timeout=60 --repo1-cipher-pass=<redacted> --repo1-cipher-type=aes-256-cbc --repo1-path=/ --repo1-s3-bucket=pgbackrest-dev --repo1-s3-endpoint=s3.amazonaws.com --repo1-s3-key=<redacted> --repo1-s3-key-secret=<redacted> --repo1-s3-region=us-east-1 --no-repo1-s3-verify-tls --repo1-type=s3 --stanza=db --start-fast --type=diff
 P00   WARN: option repo1-retention-full is not set, the repository may run out of space
             HINT: to retain full backups indefinitely (without warning), set option 'repo1-retention-full' to the maximum.
 P00   WARN: option backup-standby is enabled but backup is offline - backups will be performed from the primary
@@ -2474,6 +2477,7 @@ pg1-path=[TEST_PATH]/db-master/db/base-2/base
 buffer-size=[BUFFER-SIZE]
 compress-level=3
 compress-level-network=1
+compress-type=lz4
 db-timeout=45
 lock-path=[TEST_PATH]/backup/lock
 log-level-console=warn
@@ -2529,7 +2533,7 @@ option-checksum-page=false
 option-compress=true
 option-compress-level=3
 option-compress-level-network=1
-option-compress-type="gz"
+option-compress-type="lz4"
 option-delta=false
 option-hardlink=false
 option-online=false

test/expect/mock-archive-001.log

@@ -1,5 +1,5 @@
-run 001 - rmt 0, s3 0, enc 1, cmp gz
+run 001 - rmt 0, s3 0, enc 1, cmp lz4
-====================================
+=====================================
 
 > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001
 ------------------------------------------------------------------------------------------------------------------------------------
@@ -73,9 +73,9 @@ db-version="9.4"
 [backrest]
 backrest-checksum="[CHECKSUM]"
 
-> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-push --compress-type=gz [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001
+> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-push --compress-type=lz4 [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001
 ------------------------------------------------------------------------------------------------------------------------------------
-P00   INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001] --buffer-size=[BUFFER-SIZE] --compress-level=3 --compress-type=gz --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-cipher-pass=<redacted> --repo1-cipher-type=aes-256-cbc --repo1-path=[TEST_PATH]/db-master/repo --stanza=db
+P00   INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001] --buffer-size=[BUFFER-SIZE] --compress-level=3 --compress-type=lz4 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-cipher-pass=<redacted> --repo1-cipher-type=aes-256-cbc --repo1-path=[TEST_PATH]/db-master/repo --stanza=db
 P00   INFO: pushed WAL file '000000010000000100000001' to the archive
 P00   INFO: archive-push command end: completed successfully
 
@@ -91,9 +91,9 @@ P00   INFO: archive-get command begin [BACKREST-VERSION]: [000000010000000100000
 P00   INFO: found 000000010000000100000001 in the archive
 P00   INFO: archive-get command end: completed successfully
 
-> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-push --compress-type=gz --archive-async --process-max=2 [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002
+> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-push --compress-type=lz4 --archive-async --process-max=2 [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002
 ------------------------------------------------------------------------------------------------------------------------------------
-P00   INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002] --archive-async --buffer-size=[BUFFER-SIZE] --compress-level=3 --compress-type=gz --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --process-max=2 --protocol-timeout=60 --repo1-cipher-pass=<redacted> --repo1-cipher-type=aes-256-cbc --repo1-path=[TEST_PATH]/db-master/repo --spool-path=[TEST_PATH]/db-master/spool --stanza=db
+P00   INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002] --archive-async --buffer-size=[BUFFER-SIZE] --compress-level=3 --compress-type=lz4 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --process-max=2 --protocol-timeout=60 --repo1-cipher-pass=<redacted> --repo1-cipher-type=aes-256-cbc --repo1-path=[TEST_PATH]/db-master/repo --spool-path=[TEST_PATH]/db-master/spool --stanza=db
 P00   INFO: pushed WAL file '000000010000000100000002' to the archive asynchronously
 P00   INFO: archive-push command end: completed successfully
 

test/expect/mock-archive-stop-002.log

@@ -1,5 +1,5 @@
-run 002 - rmt 1, cmp gz, error connect, s3 1, enc 0
+run 002 - rmt 1, cmp lz4, error connect, s3 1, enc 0
-===================================================
+====================================================
 
 stanza-create db - create required data for stanza (backup host)
 > [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db --no-online stanza-create

test/expect/mock-stanza-002.log

@@ -1,5 +1,5 @@
-run 002 - remote 1, s3 1, enc 1, cmp gz
+run 002 - remote 1, s3 1, enc 1, cmp lz4
-=======================================
+========================================
 
 stanza-create db - fail on missing control file (backup host)
 > [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db --no-online --log-level-file=[LOG-LEVEL-FILE] stanza-create
@@ -231,13 +231,13 @@ backrest-checksum="[CHECKSUM]"
 
 > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001
 ------------------------------------------------------------------------------------------------------------------------------------
-P00   INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001] --buffer-size=[BUFFER-SIZE] --compress-level=3 --compress-level-network=1 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-host=backup --repo1-host-cmd=[BACKREST-BIN] --repo1-host-config=[TEST_PATH]/backup/pgbackrest.conf --repo1-host-user=[USER-1] --stanza=db
+P00   INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001] --buffer-size=[BUFFER-SIZE] --compress-level=3 --compress-level-network=1 --compress-type=lz4 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-host=backup --repo1-host-cmd=[BACKREST-BIN] --repo1-host-config=[TEST_PATH]/backup/pgbackrest.conf --repo1-host-user=[USER-1] --stanza=db
 P00   INFO: pushed WAL file '000000010000000100000001' to the archive
 P00   INFO: archive-push command end: completed successfully
 
 > [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-push [TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002
 ------------------------------------------------------------------------------------------------------------------------------------
-P00   INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002] --buffer-size=[BUFFER-SIZE] --compress-level=3 --compress-level-network=1 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-host=backup --repo1-host-cmd=[BACKREST-BIN] --repo1-host-config=[TEST_PATH]/backup/pgbackrest.conf --repo1-host-user=[USER-1] --stanza=db
+P00   INFO: archive-push command begin [BACKREST-VERSION]: [[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002] --buffer-size=[BUFFER-SIZE] --compress-level=3 --compress-level-network=1 --compress-type=lz4 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log[] --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-host=backup --repo1-host-cmd=[BACKREST-BIN] --repo1-host-config=[TEST_PATH]/backup/pgbackrest.conf --repo1-host-user=[USER-1] --stanza=db
 P00   INFO: pushed WAL file '000000010000000100000002' to the archive
 P00   INFO: archive-push command end: completed successfully
 
@@ -310,7 +310,7 @@ P00   INFO: archive-get command end: completed successfully
 full backup - create first full backup  (backup host)
 > [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --repo1-retention-full=2 --no-online --type=full --stanza=db backup
 ------------------------------------------------------------------------------------------------------------------------------------
-P00   INFO: backup command begin [BACKREST-VERSION]: --buffer-size=[BUFFER-SIZE] --compress-level=3 --compress-level-network=1 --config=[TEST_PATH]/backup/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/backup/log[] --no-log-timestamp --no-online --pg1-host=db-master --pg1-host-cmd=[BACKREST-BIN] --pg1-host-config=[TEST_PATH]/db-master/pgbackrest.conf --pg1-host-user=[USER-1] --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-cipher-pass=<redacted> --repo1-cipher-type=aes-256-cbc --repo1-path=/ --repo1-retention-full=2 --repo1-s3-bucket=pgbackrest-dev --repo1-s3-endpoint=s3.amazonaws.com --repo1-s3-key=<redacted> --repo1-s3-key-secret=<redacted> --repo1-s3-region=us-east-1 --no-repo1-s3-verify-tls --repo1-type=s3 --stanza=db --start-fast --type=full
+P00   INFO: backup command begin [BACKREST-VERSION]: --buffer-size=[BUFFER-SIZE] --compress-level=3 --compress-level-network=1 --compress-type=lz4 --config=[TEST_PATH]/backup/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/backup/log[] --no-log-timestamp --no-online --pg1-host=db-master --pg1-host-cmd=[BACKREST-BIN] --pg1-host-config=[TEST_PATH]/db-master/pgbackrest.conf --pg1-host-user=[USER-1] --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-cipher-pass=<redacted> --repo1-cipher-type=aes-256-cbc --repo1-path=/ --repo1-retention-full=2 --repo1-s3-bucket=pgbackrest-dev --repo1-s3-endpoint=s3.amazonaws.com --repo1-s3-key=<redacted> --repo1-s3-key-secret=<redacted> --repo1-s3-region=us-east-1 --no-repo1-s3-verify-tls --repo1-type=s3 --stanza=db --start-fast --type=full
 P01   INFO: backup file db-master:[TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG (16MB, 33%) checksum 488ba4b8b98acc510bce86b8f16e3c1ed9886a29
 P01   INFO: backup file db-master:[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002 (16MB, 66%) checksum 488ba4b8b98acc510bce86b8f16e3c1ed9886a29
 P01   INFO: backup file db-master:[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000001 (16MB, 99%) checksum e40de8cea99dd469c3efb47f5f33a73c7390fb9c
@@ -338,6 +338,7 @@ pg1-path=[TEST_PATH]/db-master/db/base
 buffer-size=[BUFFER-SIZE]
 compress-level=3
 compress-level-network=1
+compress-type=lz4
 db-timeout=45
 lock-path=[TEST_PATH]/db-master/lock
 log-level-console=detail
@@ -366,6 +367,7 @@ pg1-path=[TEST_PATH]/db-master/db/base
 buffer-size=[BUFFER-SIZE]
 compress-level=3
 compress-level-network=1
+compress-type=lz4
 db-timeout=45
 lock-path=[TEST_PATH]/backup/lock
 log-level-console=detail
@@ -512,7 +514,7 @@ backrest-checksum="[CHECKSUM]"
 diff backup - diff changed to full backup (backup host)
 > [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --repo1-retention-full=2 --no-online --type=diff --stanza=db backup
 ------------------------------------------------------------------------------------------------------------------------------------
-P00   INFO: backup command begin [BACKREST-VERSION]: --buffer-size=[BUFFER-SIZE] --compress-level=3 --compress-level-network=1 --config=[TEST_PATH]/backup/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/backup/log[] --no-log-timestamp --no-online --pg1-host=db-master --pg1-host-cmd=[BACKREST-BIN] --pg1-host-config=[TEST_PATH]/db-master/pgbackrest.conf --pg1-host-user=[USER-1] --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-cipher-pass=<redacted> --repo1-cipher-type=aes-256-cbc --repo1-path=/ --repo1-retention-full=2 --repo1-s3-bucket=pgbackrest-dev --repo1-s3-endpoint=s3.amazonaws.com --repo1-s3-key=<redacted> --repo1-s3-key-secret=<redacted> --repo1-s3-region=us-east-1 --no-repo1-s3-verify-tls --repo1-type=s3 --stanza=db --start-fast --type=diff
+P00   INFO: backup command begin [BACKREST-VERSION]: --buffer-size=[BUFFER-SIZE] --compress-level=3 --compress-level-network=1 --compress-type=lz4 --config=[TEST_PATH]/backup/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/backup/log[] --no-log-timestamp --no-online --pg1-host=db-master --pg1-host-cmd=[BACKREST-BIN] --pg1-host-config=[TEST_PATH]/db-master/pgbackrest.conf --pg1-host-user=[USER-1] --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-cipher-pass=<redacted> --repo1-cipher-type=aes-256-cbc --repo1-path=/ --repo1-retention-full=2 --repo1-s3-bucket=pgbackrest-dev --repo1-s3-endpoint=s3.amazonaws.com --repo1-s3-key=<redacted> --repo1-s3-key-secret=<redacted> --repo1-s3-region=us-east-1 --no-repo1-s3-verify-tls --repo1-type=s3 --stanza=db --start-fast --type=diff
 P00   WARN: no prior backup exists, diff backup has been changed to full
 P01   INFO: backup file db-master:[TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG (16MB, 33%) checksum 488ba4b8b98acc510bce86b8f16e3c1ed9886a29
 P01   INFO: backup file db-master:[TEST_PATH]/db-master/db/base/pg_xlog/000000010000000100000002 (16MB, 66%) checksum 488ba4b8b98acc510bce86b8f16e3c1ed9886a29
@@ -539,6 +541,7 @@ pg1-path=[TEST_PATH]/db-master/db/base
 buffer-size=[BUFFER-SIZE]
 compress-level=3
 compress-level-network=1
+compress-type=lz4
 db-timeout=45
 lock-path=[TEST_PATH]/db-master/lock
 log-level-console=detail
@@ -567,6 +570,7 @@ pg1-path=[TEST_PATH]/db-master/db/base
 buffer-size=[BUFFER-SIZE]
 compress-level=3
 compress-level-network=1
+compress-type=lz4
 db-timeout=45
 lock-path=[TEST_PATH]/backup/lock
 log-level-console=detail

test/expect/real-all-004.log

@@ -1,5 +1,5 @@
-run 004 - bkp 1, sby 0, dst backup, cmp gz, s3 0, enc 1
+run 004 - bkp 1, sby 0, dst backup, cmp lz4, s3 0, enc 1
-=======================================================
+========================================================
 
 stanza-create db - main create stanza info files (backup host)
 > [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db stanza-create
@@ -108,6 +108,7 @@ pg1-socket-path=[TEST_PATH]/db-master/db
 buffer-size=[BUFFER-SIZE]
 compress-level=3
 compress-level-network=1
+compress-type=lz4
 db-timeout=45
 lock-path=[TEST_PATH]/db-master/lock
 log-level-console=detail
@@ -137,6 +138,7 @@ pg1-port=6543
 buffer-size=[BUFFER-SIZE]
 compress-level=3
 compress-level-network=1
+compress-type=lz4
 db-timeout=45
 lock-path=[TEST_PATH]/backup/lock
 log-level-console=detail
@@ -181,6 +183,7 @@ pg1-socket-path=[TEST_PATH]/db-master/db
 buffer-size=[BUFFER-SIZE]
 compress-level=3
 compress-level-network=1
+compress-type=lz4
 db-timeout=45
 lock-path=[TEST_PATH]/db-master/lock
 log-level-console=detail
@@ -211,6 +214,7 @@ archive-async=y
 buffer-size=[BUFFER-SIZE]
 compress-level=3
 compress-level-network=1
+compress-type=lz4
 db-timeout=45
 lock-path=[TEST_PATH]/backup/lock
 log-level-console=detail
@@ -271,6 +275,7 @@ pg1-socket-path=[TEST_PATH]/db-master/db
 buffer-size=[BUFFER-SIZE]
 compress-level=3
 compress-level-network=1
+compress-type=lz4
 db-timeout=45
 lock-path=[TEST_PATH]/db-master/lock
 log-level-console=detail
@@ -301,6 +306,7 @@ archive-async=y
 buffer-size=[BUFFER-SIZE]
 compress-level=3
 compress-level-network=1
+compress-type=lz4
 db-timeout=45
 lock-path=[TEST_PATH]/backup/lock
 log-level-console=detail

test/lib/pgBackRestTest/Common/ContainerTest.pm

@@ -361,7 +361,7 @@ sub containerBuild
                 "    yum -y install openssh-server openssh-clients wget sudo valgrind git \\\n" .
                 "        perl perl-Digest-SHA perl-DBD-Pg perl-YAML-LibYAML openssl \\\n" .
                 "        gcc make perl-ExtUtils-MakeMaker perl-Test-Simple openssl-devel perl-ExtUtils-Embed rpm-build \\\n" .
-                "        zlib-devel libxml2-devel lz4-devel";
+                "        zlib-devel libxml2-devel lz4-devel lz4";
 
             if ($strOS eq VM_CO6)
             {
@@ -388,7 +388,7 @@ sub containerBuild
             }
             else
             {
-                $strScript .= ' libjson-pp-perl liblz4-dev';
+                $strScript .= ' libjson-pp-perl liblz4-dev liblz4-tool';
             }
         }
 

test/lib/pgBackRestTest/Common/JobTest.pm

@@ -434,7 +434,8 @@ sub run
                 buildPutDiffers($self->{oStorageTest}, "$self->{strGCovPath}/test.c", $strTestC);
 
                 # Create build.auto.h
-                my $strBuildAutoH = "";
+                my $strBuildAutoH =
+                    (vmWithLz4($self->{oTest}->{&TEST_VM}) ? '#define HAVE_LIBLZ4' : '') . "\n";
 
                 buildPutDiffers($self->{oStorageTest}, "$self->{strGCovPath}/" . BUILD_AUTO_H, $strBuildAutoH);
 
@@ -502,6 +503,7 @@ sub run
                     "HARNESSFLAGS=${strHarnessFlags}\n" .
                     "TESTFLAGS=${strTestFlags}\n" .
                     "LDFLAGS=-lcrypto -lssl -lxml2 -lz" .
+                        (vmWithLz4($self->{oTest}->{&TEST_VM}) ? ' -llz4' : '') .
                         (vmCoverageC($self->{oTest}->{&TEST_VM}) && $self->{bCoverageUnit} ? " -lgcov" : '') .
                         (vmWithBackTrace($self->{oTest}->{&TEST_VM}) && $self->{bBackTrace} ? ' -lbacktrace' : '') .
                     "\n" .

test/lib/pgBackRestTest/Common/VmTest.pm

@@ -48,6 +48,8 @@ use constant VMDEF_LCOV_VERSION                                     => 'lcov-ver
     push @EXPORT, qw(VMDEF_LCOV_VERSION);
 use constant VMDEF_WITH_BACKTRACE                                   => 'with-backtrace';
     push @EXPORT, qw(VMDEF_WITH_BACKTRACE);
+use constant VMDEF_WITH_LZ4                                         => 'with-lz4';
+    push @EXPORT, qw(VMDEF_WITH_LZ4);
 
 ####################################################################################################################################
 # Valid OS base List
@@ -302,6 +304,7 @@ my $oyVm =
         &VM_IMAGE => 'i386/ubuntu:12.04',
         &VM_ARCH => VM_ARCH_I386,
         &VMDEF_PGSQL_BIN => '/usr/lib/postgresql/{[version]}/bin',
+        &VMDEF_WITH_LZ4 => false,
 
         &VM_DB =>
         [
@@ -591,6 +594,18 @@ sub vmWithBackTrace
 
 push @EXPORT, qw(vmWithBackTrace);
 
+####################################################################################################################################
+# Does the VM support liblz4?
+####################################################################################################################################
+sub vmWithLz4
+{
+    my $strVm = shift;
+
+    return (defined($oyVm->{$strVm}{&VMDEF_WITH_LZ4}) ? $oyVm->{$strVm}{&VMDEF_WITH_LZ4} : true);
+}
+
+push @EXPORT, qw(vmWithLz4);
+
 ####################################################################################################################################
 # Will integration tests be run in debug mode?
 ####################################################################################################################################

test/lib/pgBackRestTest/Env/Host/HostBackupTest.pm

@@ -91,6 +91,8 @@ use constant NONE                                                   => 'none';
     push @EXPORT, qw(NONE);
 use constant GZ                                                     => 'gz';
     push @EXPORT, qw(GZ);
+use constant LZ4                                                    => 'lz4';
+    push @EXPORT, qw(LZ4);
 
 ####################################################################################################################################
 # new

test/lib/pgBackRestTest/Module/Mock/MockAllTest.pm

@@ -60,13 +60,13 @@ sub run
 
     foreach my $rhRun
     (
-        {vm => VM1, remote => false, s3 =>  true, encrypt => false, delta =>  true, compress =>  GZ},
+        {vm => VM1, remote => false, s3 =>  true, encrypt => false, delta =>  true, compress => LZ4},
         {vm => VM1, remote =>  true, s3 => false, encrypt =>  true, delta => false, compress =>  GZ},
         {vm => VM2, remote => false, s3 => false, encrypt =>  true, delta =>  true, compress =>  GZ},
         {vm => VM2, remote =>  true, s3 =>  true, encrypt => false, delta => false, compress =>  GZ},
         {vm => VM3, remote => false, s3 => false, encrypt => false, delta =>  true, compress =>  GZ},
-        {vm => VM3, remote =>  true, s3 =>  true, encrypt =>  true, delta => false, compress =>  GZ},
+        {vm => VM3, remote =>  true, s3 =>  true, encrypt =>  true, delta => false, compress => LZ4},
-        {vm => VM4, remote => false, s3 => false, encrypt => false, delta => false, compress =>  GZ},
+        {vm => VM4, remote => false, s3 => false, encrypt => false, delta => false, compress => LZ4},
         {vm => VM4, remote =>  true, s3 =>  true, encrypt =>  true, delta =>  true, compress =>  GZ},
     )
     {

test/lib/pgBackRestTest/Module/Mock/MockArchiveStopTest.pm

@@ -43,12 +43,12 @@ sub run
 
     foreach my $rhRun
     (
-        {vm => VM1, remote => false, s3 => false, encrypt => false, compress =>    GZ, error => 0},
+        {vm => VM1, remote => false, s3 => false, encrypt => false, compress =>   LZ4, error => 0},
         {vm => VM1, remote =>  true, s3 =>  true, encrypt =>  true, compress =>    GZ, error => 1},
         {vm => VM2, remote => false, s3 =>  true, encrypt => false, compress =>  NONE, error => 0},
         {vm => VM2, remote =>  true, s3 => false, encrypt =>  true, compress =>    GZ, error => 0},
         {vm => VM3, remote => false, s3 => false, encrypt =>  true, compress =>  NONE, error => 0},
-        {vm => VM3, remote =>  true, s3 =>  true, encrypt => false, compress =>    GZ, error => 1},
+        {vm => VM3, remote =>  true, s3 =>  true, encrypt => false, compress =>   LZ4, error => 1},
         {vm => VM4, remote => false, s3 =>  true, encrypt =>  true, compress =>    GZ, error => 0},
         {vm => VM4, remote =>  true, s3 => false, encrypt => false, compress =>  NONE, error => 0},
     )

test/lib/pgBackRestTest/Module/Mock/MockArchiveTest.pm

@@ -83,14 +83,14 @@ sub run
 
     foreach my $rhRun
     (
-        {vm => VM1, remote => false, s3 => false, encrypt => false, compress =>  GZ},
+        {vm => VM1, remote => false, s3 => false, encrypt => false, compress => LZ4},
         {vm => VM1, remote =>  true, s3 =>  true, encrypt =>  true, compress =>  GZ},
         {vm => VM2, remote => false, s3 =>  true, encrypt => false, compress =>  GZ},
         {vm => VM2, remote =>  true, s3 => false, encrypt =>  true, compress =>  GZ},
-        {vm => VM3, remote => false, s3 => false, encrypt =>  true, compress =>  GZ},
+        {vm => VM3, remote => false, s3 => false, encrypt =>  true, compress => LZ4},
         {vm => VM3, remote =>  true, s3 =>  true, encrypt => false, compress =>  GZ},
         {vm => VM4, remote => false, s3 =>  true, encrypt =>  true, compress =>  GZ},
-        {vm => VM4, remote =>  true, s3 => false, encrypt => false, compress =>  GZ},
+        {vm => VM4, remote =>  true, s3 => false, encrypt => false, compress => LZ4},
     )
     {
         # Only run tests for this vm

test/lib/pgBackRestTest/Module/Mock/MockStanzaTest.pm

@@ -41,13 +41,13 @@ sub run
 
     foreach my $rhRun
     (
-        {vm => VM1, remote => false, s3 => false, encrypt =>  true, compress =>  GZ},
+        {vm => VM1, remote => false, s3 => false, encrypt =>  true, compress => LZ4},
         {vm => VM1, remote =>  true, s3 =>  true, encrypt => false, compress =>  GZ},
         {vm => VM2, remote => false, s3 =>  true, encrypt =>  true, compress =>  GZ},
         {vm => VM2, remote =>  true, s3 => false, encrypt => false, compress =>  GZ},
         {vm => VM3, remote => false, s3 => false, encrypt => false, compress =>  GZ},
-        {vm => VM3, remote =>  true, s3 =>  true, encrypt =>  true, compress =>  GZ},
+        {vm => VM3, remote =>  true, s3 =>  true, encrypt =>  true, compress => LZ4},
-        {vm => VM4, remote => false, s3 =>  true, encrypt => false, compress =>  GZ},
+        {vm => VM4, remote => false, s3 =>  true, encrypt => false, compress => LZ4},
         {vm => VM4, remote =>  true, s3 => false, encrypt =>  true, compress =>  GZ},
     )
     {

test/lib/pgBackRestTest/Module/Real/RealAllTest.pm

@@ -49,6 +49,9 @@ sub run
 {
     my $self = shift;
 
+    # Should the test use lz4 compression?
+    my $bLz4Compress = true;
+
     foreach my $bS3 (false, true)
     {
     foreach my $bHostBackup ($bS3 ? (true) : (false, true))
@@ -60,9 +63,15 @@ sub run
     foreach my $strBackupDestination (
         $bS3 || $bHostBackup ? (HOST_BACKUP) : $bHostStandby ? (HOST_DB_MASTER, HOST_DB_STANDBY) : (HOST_DB_MASTER))
     {
-        my $strCompressType = $bHostBackup && !$bHostStandby ? GZ : NONE;
+        my $strCompressType = $bHostBackup && !$bHostStandby ? (vmWithLz4($self->vm()) && $bLz4Compress ? LZ4 : GZ) : NONE;
         my $bRepoEncrypt = ($strCompressType ne NONE && !$bS3) ? true : false;
 
+        # If compression was used then switch it for the next test that uses compression
+        if ($strCompressType ne NONE)
+        {
+            $bLz4Compress = !$bLz4Compress;
+        }
+
         # Increment the run, log, and decide whether this unit test should be run
         my $hyVm = vmGet();
         my $strDbVersionMostRecent = ${$hyVm->{$self->vm()}{&VM_DB_TEST}}[-1];

test/patch/debian-package.patch

@@ -0,0 +1,12 @@
+--- control
++++ control
+@@ -12,7 +12,8 @@
+                libxml2-dev,
+                pkg-config,
+                txt2man,
+-               zlib1g-dev
++               zlib1g-dev,
++               liblz4-dev
+ Standards-Version: 4.4.1
+ Homepage: https://www.pgbackrest.org/
+ Vcs-Git: https://salsa.debian.org/postgresql/pgbackrest.git

test/patch/rhel-package.patch

@@ -0,0 +1,11 @@
+--- pgbackrest.spec
++++ pgbackrest.spec
+@@ -9,7 +9,7 @@
+ Source0:	https://github.com/pgbackrest/pgbackrest/archive/release/%{version}.tar.gz
+ Source1:	pgbackrest-conf.patch
+ Patch0:		pgbackrest-libxmlinclude.patch
+-BuildRequires:	openssl-devel zlib-devel postgresql%{pgmajorversion}-devel
++BuildRequires:	openssl-devel zlib-devel postgresql%{pgmajorversion}-devel lz4-devel
+ Requires:	postgresql-libs
+ 
+ 

test/src/module/common/compressTest.c

@@ -206,6 +206,44 @@ testRun(void)
         TEST_RESULT_STR_Z(gzDecompressToLog(decompress), "{inputSame: true, done: true, availIn: 0}", "format object");
     }
 
+    // *****************************************************************************************************************************
+    if (testBegin("lz4"))
+    {
+#ifdef HAVE_LIBLZ4
+        // Run standard test suite
+        testSuite(compressTypeLz4, "lz4 -dc");
+
+        // -------------------------------------------------------------------------------------------------------------------------
+        TEST_TITLE("lz4Error()");
+
+        TEST_RESULT_UINT(lz4Error(0), 0, "check success");
+        TEST_ERROR(lz4Error((size_t)-2), FormatError, "lz4 error: [-2] ERROR_maxBlockSize_invalid");
+
+        // -------------------------------------------------------------------------------------------------------------------------
+        TEST_TITLE("lz4DecompressToLog() and lz4CompressToLog()");
+
+        Lz4Compress *compress = (Lz4Compress *)ioFilterDriver(lz4CompressNew(7));
+
+        compress->inputSame = true;
+        compress->flushing = true;
+
+        TEST_RESULT_STR_Z(
+            lz4CompressToLog(compress), "{level: 7, first: true, inputSame: true, flushing: true}", "format object");
+
+        Lz4Decompress *decompress = (Lz4Decompress *)ioFilterDriver(lz4DecompressNew());
+
+        decompress->inputSame = true;
+        decompress->done = true;
+        decompress->inputOffset = 999;
+
+        TEST_RESULT_STR_Z(
+            lz4DecompressToLog(decompress), "{inputSame: true, inputOffset: 999, frameDone false, done: true}",
+            "format object");
+#else
+        TEST_ERROR(compressTypePresent(compressTypeLz4), OptionInvalidValueError, "pgBackRest not compiled with lz4 support");
+#endif // HAVE_LIBLZ4
+    }
+
     // Test everything in the helper that is not tested in the individual compression type tests
     // *****************************************************************************************************************************
     if (testBegin("helper"))

test/travis.pl

@@ -181,7 +181,7 @@ eval
 
         if ($strVm eq VM_NONE)
         {
-            $strPackage .= " valgrind";
+            $strPackage .= " valgrind liblz4-dev liblz4-tool";
         }
         else
         {