Add optional raw format for compression types.

Raw format saves 12 bytes of header for gzip and 4 bytes of checksum for lz4 (plus CPU overhead). This may not seem like much, but over millions of small files or incremental blocks can really add up. Even though it may be a relatively small percentage of the overall backup size it is still objectively a large amount of data.

Use raw format for protocol compression to exercise the feature.

Raw compression format will be added to bundling and block incremental in a followup commit.

doc/xml/release.xml

@@ -25,6 +25,7 @@
                         <commit subject="Rename DeltaMap to BlockHash."/>
                         <commit subject="Add repo-block-age-map and repo-block-size-map options."/>
                         <commit subject="Rename block incremental manifest keys."/>
+                        <commit subject="Add optional raw format for compression types."/>
 
                         <release-item-contributor-list>
                             <release-item-contributor id="david.steele"/>

src/build/help/render.c

@@ -270,7 +270,7 @@ bldHlpRenderHelpAutoCCmp(const BldCfg bldCfg, const BldHlp bldHlp)
     IoRead *const source = ioBufferReadNewOpen(packBuf);
     IoWrite *const destination = ioBufferWriteNew(result);
 
-    ioFilterGroupAdd(ioWriteFilterGroup(destination), bz2CompressNew(9));
+    ioFilterGroupAdd(ioWriteFilterGroup(destination), bz2CompressNew(9, false));
     ioWriteOpen(destination);
 
     // Copy data from source to destination

src/command/archive/get/file.c

@@ -70,7 +70,7 @@ archiveGetFile(
 
                 if (compressType != compressTypeNone)
                 {
-                    ioFilterGroupAdd(ioWriteFilterGroup(storageWriteIo(destination)), decompressFilter(compressType));
+                    ioFilterGroupAdd(ioWriteFilterGroup(storageWriteIo(destination)), decompressFilterP(compressType));
                     compressible = false;
                 }
 

src/command/archive/push/file.c

@@ -241,7 +241,7 @@ archivePushFile(
             if (isSegment && compressType != compressTypeNone)
             {
                 compressExtCat(archiveDestination, compressType);
-                ioFilterGroupAdd(ioReadFilterGroup(storageReadIo(source)), compressFilter(compressType, compressLevel));
+                ioFilterGroupAdd(ioReadFilterGroup(storageReadIo(source)), compressFilterP(compressType, compressLevel));
                 compressible = false;
             }
 

src/command/backup/backup.c

@@ -1085,7 +1085,7 @@ backupFilePut(BackupData *backupData, Manifest *manifest, const String *name, ti
             if (compressType != compressTypeNone)
             {
                 ioFilterGroupAdd(
-                    ioWriteFilterGroup(storageWriteIo(write)), compressFilter(compressType, cfgOptionInt(cfgOptCompressLevel)));
+                    ioWriteFilterGroup(storageWriteIo(write)), compressFilterP(compressType, cfgOptionInt(cfgOptCompressLevel)));
 
                 repoChecksum = true;
             }
@@ -2263,12 +2263,12 @@ backupArchiveCheckCopy(const BackupData *const backupData, Manifest *const manif
                         if (archiveCompressType != backupCompressType)
                         {
                             if (archiveCompressType != compressTypeNone)
-                                ioFilterGroupAdd(filterGroup, decompressFilter(archiveCompressType));
+                                ioFilterGroupAdd(filterGroup, decompressFilterP(archiveCompressType));
 
                             if (backupCompressType != compressTypeNone)
                             {
                                 ioFilterGroupAdd(
-                                    filterGroup, compressFilter(backupCompressType, cfgOptionInt(cfgOptCompressLevel)));
+                                    filterGroup, compressFilterP(backupCompressType, cfgOptionInt(cfgOptCompressLevel)));
                             }
                         }
 
@@ -2365,7 +2365,7 @@ backupComplete(InfoBackup *const infoBackup, Manifest *const manifest)
                 STORAGE_REPO_BACKUP "/" BACKUP_PATH_HISTORY "/%s/%s.manifest%s", strZ(strSubN(backupLabel, 0, 4)),
                 strZ(backupLabel), strZ(compressExtStr(compressTypeGz))));
 
-        ioFilterGroupAdd(ioWriteFilterGroup(storageWriteIo(manifestWrite)), compressFilter(compressTypeGz, 9));
+        ioFilterGroupAdd(ioWriteFilterGroup(storageWriteIo(manifestWrite)), compressFilterP(compressTypeGz, 9));
 
         cipherBlockFilterGroupAdd(
             ioWriteFilterGroup(storageWriteIo(manifestWrite)), cfgOptionStrId(cfgOptRepoCipherType), cipherModeEncrypt,

src/command/backup/file.c

@@ -215,7 +215,7 @@ backupFile(
                     // Compress filter
                     IoFilter *const compress =
                         repoFileCompressType != compressTypeNone ?
-                            compressFilter(repoFileCompressType, repoFileCompressLevel) : NULL;
+                            compressFilterP(repoFileCompressType, repoFileCompressLevel) : NULL;
 
                     // Encrypt filter
                     IoFilter *const encrypt =

src/command/help/help.c

@@ -255,7 +255,7 @@ helpRender(const Buffer *const helpData)
 
         // Read pack from compressed buffer
         IoRead *const helpRead = ioBufferReadNew(helpData);
-        ioFilterGroupAdd(ioReadFilterGroup(helpRead), bz2DecompressNew());
+        ioFilterGroupAdd(ioReadFilterGroup(helpRead), bz2DecompressNew(false));
         ioReadOpen(helpRead);
 
         PackRead *pckHelp = pckReadNewIo(helpRead);

src/command/restore/file.c

@@ -403,7 +403,7 @@ restoreFile(
                                             if (repoFileCompressType != compressTypeNone)
                                             {
                                                 ioFilterGroupAdd(
-                                                    ioReadFilterGroup(chunkedRead), decompressFilter(repoFileCompressType));
+                                                    ioReadFilterGroup(chunkedRead), decompressFilterP(repoFileCompressType));
                                             }
 
                                             // Open chunked read
@@ -454,7 +454,7 @@ restoreFile(
 
                         // Add decompression filter
                         if (repoFileCompressType != compressTypeNone)
-                            ioFilterGroupAdd(filterGroup, decompressFilter(repoFileCompressType));
+                            ioFilterGroupAdd(filterGroup, decompressFilterP(repoFileCompressType));
 
                         // Add sha1 filter
                         ioFilterGroupAdd(filterGroup, cryptoHashNew(hashTypeSha1));

src/command/verify/file.c

@@ -50,7 +50,7 @@ verifyFile(
 
         // Add decompression filter
         if (compressType != compressTypeNone)
-            ioFilterGroupAdd(filterGroup, decompressFilter(compressType));
+            ioFilterGroupAdd(filterGroup, decompressFilterP(compressType));
 
         // Add sha1 filter
         ioFilterGroupAdd(filterGroup, cryptoHashNew(hashTypeSha1));

src/command/verify/verify.c

@@ -184,7 +184,7 @@ verifyFileLoad(const String *pathFileName, const String *cipherPass)
 
     // If the file is compressed, add a decompression filter
     if (compressTypeFromName(pathFileName) != compressTypeNone)
-        ioFilterGroupAdd(ioReadFilterGroup(read), decompressFilter(compressTypeFromName(pathFileName)));
+        ioFilterGroupAdd(ioReadFilterGroup(read), decompressFilterP(compressTypeFromName(pathFileName)));
 
     FUNCTION_TEST_RETURN(STORAGE_READ, result);
 }

src/common/compress/bz2/compress.c

@@ -157,10 +157,11 @@ bz2CompressInputSame(const THIS_VOID)
 
 /**********************************************************************************************************************************/
 FN_EXTERN IoFilter *
-bz2CompressNew(int level)
+bz2CompressNew(const int level, const bool raw)
 {
     FUNCTION_LOG_BEGIN(logLevelTrace);
         FUNCTION_LOG_PARAM(INT, level);
+        (void)raw;                                                  // Raw unsupported
     FUNCTION_LOG_END();
 
     ASSERT(level >= BZ2_COMPRESS_LEVEL_MIN && level <= BZ2_COMPRESS_LEVEL_MAX);

src/common/compress/bz2/compress.h

@@ -23,6 +23,6 @@ Level constants
 /***********************************************************************************************************************************
 Constructors
 ***********************************************************************************************************************************/
-FN_EXTERN IoFilter *bz2CompressNew(int level);
+FN_EXTERN IoFilter *bz2CompressNew(int level, bool raw);
 
 #endif

src/common/compress/bz2/decompress.c

@@ -144,9 +144,11 @@ bz2DecompressInputSame(const THIS_VOID)
 
 /**********************************************************************************************************************************/
 FN_EXTERN IoFilter *
-bz2DecompressNew(void)
+bz2DecompressNew(const bool raw)
 {
-    FUNCTION_LOG_VOID(logLevelTrace);
+    FUNCTION_LOG_BEGIN(logLevelTrace);
+        (void)raw;                                                  // Raw unsupported
+    FUNCTION_LOG_END();
 
     IoFilter *this = NULL;
 

src/common/compress/bz2/decompress.h

@@ -16,6 +16,6 @@ Filter type constant
 /***********************************************************************************************************************************
 Constructors
 ***********************************************************************************************************************************/
-FN_EXTERN IoFilter *bz2DecompressNew(void);
+FN_EXTERN IoFilter *bz2DecompressNew(bool raw);
 
 #endif

src/common/compress/gz/compress.c

@@ -1,5 +1,7 @@
 /***********************************************************************************************************************************
 Gz Compress
+
+Based on the documentation at https://github.com/madler/zlib/blob/master/zlib.h
 ***********************************************************************************************************************************/
 #include "build.auto.h"
 
@@ -162,10 +164,11 @@ gzCompressInputSame(const THIS_VOID)
 
 /**********************************************************************************************************************************/
 FN_EXTERN IoFilter *
-gzCompressNew(int level)
+gzCompressNew(const int level, const bool raw)
 {
     FUNCTION_LOG_BEGIN(logLevelTrace);
         FUNCTION_LOG_PARAM(INT, level);
+        FUNCTION_LOG_PARAM(BOOL, raw);
     FUNCTION_LOG_END();
 
     ASSERT(level >= GZ_COMPRESS_LEVEL_MIN && level <= GZ_COMPRESS_LEVEL_MAX);
@@ -182,7 +185,7 @@ gzCompressNew(int level)
         };
 
         // Create gz stream
-        gzError(deflateInit2(&driver->stream, level, Z_DEFLATED, WANT_GZ | WINDOW_BITS, MEM_LEVEL, Z_DEFAULT_STRATEGY));
+        gzError(deflateInit2(&driver->stream, level, Z_DEFLATED, (raw ? 0 : WANT_GZ) | WINDOW_BITS, MEM_LEVEL, Z_DEFAULT_STRATEGY));
 
         // Set free callback to ensure gz context is freed
         memContextCallbackSet(objMemContext(driver), gzCompressFreeResource, driver);
@@ -195,6 +198,7 @@ gzCompressNew(int level)
             PackWrite *const packWrite = pckWriteNewP();
 
             pckWriteI32P(packWrite, level);
+            pckWriteBoolP(packWrite, raw);
             pckWriteEndP(packWrite);
 
             paramList = pckMove(pckWriteResult(packWrite), memContextPrior());

src/common/compress/gz/compress.h

@@ -23,6 +23,6 @@ Level constants
 /***********************************************************************************************************************************
 Constructors
 ***********************************************************************************************************************************/
-FN_EXTERN IoFilter *gzCompressNew(int level);
+FN_EXTERN IoFilter *gzCompressNew(int level, bool raw);
 
 #endif

src/common/compress/gz/decompress.c

@@ -1,5 +1,7 @@
 /***********************************************************************************************************************************
 Gz Decompress
+
+Based on the documentation at https://github.com/madler/zlib/blob/master/zlib.h
 ***********************************************************************************************************************************/
 #include "build.auto.h"
 
@@ -144,9 +146,11 @@ gzDecompressInputSame(const THIS_VOID)
 
 /**********************************************************************************************************************************/
 FN_EXTERN IoFilter *
-gzDecompressNew(void)
+gzDecompressNew(const bool raw)
 {
-    FUNCTION_LOG_VOID(logLevelTrace);
+    FUNCTION_LOG_BEGIN(logLevelTrace);
+        FUNCTION_LOG_PARAM(BOOL, raw);
+    FUNCTION_LOG_END();
 
     IoFilter *this = NULL;
 
@@ -161,14 +165,28 @@ gzDecompressNew(void)
         };
 
         // Create gz stream
-        gzError(driver->result = inflateInit2(&driver->stream, WANT_GZ | WINDOW_BITS));
+        gzError(driver->result = inflateInit2(&driver->stream, (raw ? 0 : WANT_GZ) | WINDOW_BITS));
 
         // Set free callback to ensure gz context is freed
         memContextCallbackSet(objMemContext(driver), gzDecompressFreeResource, driver);
 
+        // Create param list
+        Pack *paramList = NULL;
+
+        MEM_CONTEXT_TEMP_BEGIN()
+        {
+            PackWrite *const packWrite = pckWriteNewP();
+
+            pckWriteBoolP(packWrite, raw);
+            pckWriteEndP(packWrite);
+
+            paramList = pckMove(pckWriteResult(packWrite), memContextPrior());
+        }
+        MEM_CONTEXT_TEMP_END();
+
         // Create filter interface
         this = ioFilterNewP(
-            GZ_DECOMPRESS_FILTER_TYPE, driver, NULL, .done = gzDecompressDone, .inOut = gzDecompressProcess,
+            GZ_DECOMPRESS_FILTER_TYPE, driver, paramList, .done = gzDecompressDone, .inOut = gzDecompressProcess,
             .inputSame = gzDecompressInputSame);
     }
     OBJ_NEW_END();

src/common/compress/gz/decompress.h

@@ -16,6 +16,6 @@ Filter type constant
 /***********************************************************************************************************************************
 Constructors
 ***********************************************************************************************************************************/
-FN_EXTERN IoFilter *gzDecompressNew(void);
+FN_EXTERN IoFilter *gzDecompressNew(bool raw);
 
 #endif

src/common/compress/helper.c

@@ -39,9 +39,9 @@ static const struct CompressHelperLocal
     const String *const type;                                       // Compress type -- must be extension without period prefixed
     const String *const ext;                                        // File extension with period prefixed
     StringId compressType;                                          // Type of the compression filter
-    IoFilter *(*compressNew)(int);                                  // Function to create new compression filter
+    IoFilter *(*compressNew)(int, bool);                            // Function to create new compression filter
     StringId decompressType;                                        // Type of the decompression filter
-    IoFilter *(*decompressNew)(void);                               // Function to create new decompression filter
+    IoFilter *(*decompressNew)(bool);                               // Function to create new decompression filter
     int levelDefault : 8;                                           // Default compression level
     int levelMin : 8;                                               // Minimum compression level
     int levelMax : 8;                                               // Maximum compression level
@@ -230,18 +230,19 @@ compressLevelMax(CompressType type)
 
 /**********************************************************************************************************************************/
 FN_EXTERN IoFilter *
-compressFilter(CompressType type, int level)
+compressFilter(const CompressType type, const int level, const CompressFilterParam param)
 {
     FUNCTION_TEST_BEGIN();
         FUNCTION_TEST_PARAM(ENUM, type);
         FUNCTION_TEST_PARAM(INT, level);
+        FUNCTION_TEST_PARAM(BOOL, param.raw);
     FUNCTION_TEST_END();
 
     ASSERT(type < LENGTH_OF(compressHelperLocal));
     ASSERT(type != compressTypeNone);
     compressTypePresent(type);
 
-    FUNCTION_TEST_RETURN(IO_FILTER, compressHelperLocal[type].compressNew(level));
+    FUNCTION_TEST_RETURN(IO_FILTER, compressHelperLocal[type].compressNew(level, param.raw));
 }
 
 /**********************************************************************************************************************************/
@@ -267,12 +268,16 @@ compressFilterPack(const StringId filterType, const Pack *const filterParam)
             {
                 ASSERT(filterParam != NULL);
 
-                result = ioFilterMove(compress->compressNew(pckReadI32P(pckReadNew(filterParam))), memContextPrior());
+                PackRead *const paramRead = pckReadNew(filterParam);
+                const int level = pckReadI32P(paramRead);
+                const bool raw = pckReadBoolP(paramRead);
+
+                result = ioFilterMove(compress->compressNew(level, raw), memContextPrior());
                 break;
             }
             else if (filterType == compress->decompressType)
             {
-                result = ioFilterMove(compress->decompressNew(), memContextPrior());
+                result = ioFilterMove(compress->decompressNew(pckReadBoolP(pckReadNew(filterParam))), memContextPrior());
                 break;
             }
         }
@@ -284,17 +289,18 @@ compressFilterPack(const StringId filterType, const Pack *const filterParam)
 
 /**********************************************************************************************************************************/
 FN_EXTERN IoFilter *
-decompressFilter(CompressType type)
+decompressFilter(const CompressType type, const DecompressFilterParam param)
 {
     FUNCTION_TEST_BEGIN();
         FUNCTION_TEST_PARAM(ENUM, type);
+        FUNCTION_TEST_PARAM(BOOL, param.raw);
     FUNCTION_TEST_END();
 
     ASSERT(type < LENGTH_OF(compressHelperLocal));
     ASSERT(type != compressTypeNone);
     compressTypePresent(type);
 
-    FUNCTION_TEST_RETURN(IO_FILTER, compressHelperLocal[type].decompressNew());
+    FUNCTION_TEST_RETURN(IO_FILTER, compressHelperLocal[type].decompressNew(param.raw));
 }
 
 /**********************************************************************************************************************************/

src/common/compress/helper.h

@@ -53,14 +53,32 @@ FN_EXTERN const String *compressTypeStr(CompressType type);
 FN_EXTERN CompressType compressTypeFromName(const String *name);
 
 // Compression filter for the specified type.  Error when compress type is none or invalid.
-FN_EXTERN IoFilter *compressFilter(CompressType type, int level);
+typedef struct CompressFilterParam
+{
+    VAR_PARAM_HEADER;
+    bool raw;                                                       // Omit headers, checksum, etc. when possible
+} CompressFilterParam;
+
+#define compressFilterP(type, level, ...)                                                                                          \
+    compressFilter(type, level, (CompressFilterParam){VAR_PARAM_INIT, __VA_ARGS__})
+
+FN_EXTERN IoFilter *compressFilter(CompressType type, int level, CompressFilterParam param);
 
 // Compression/decompression filter based on string type and a parameter list.  This is useful when a filter must be created on a
 // remote system since the filter type and parameters can be passed through a protocol.
 FN_EXTERN IoFilter *compressFilterPack(StringId filterType, const Pack *filterParam);
 
 // Decompression filter for the specified type.  Error when compress type is none or invalid.
-FN_EXTERN IoFilter *decompressFilter(CompressType type);
+typedef struct DecompressFilterParam
+{
+    VAR_PARAM_HEADER;
+    bool raw;                                                       // Omit headers, checksum, etc. when possible
+} DecompressFilterParam;
+
+#define decompressFilterP(type, ...)                                                                                               \
+    decompressFilter(type, (DecompressFilterParam){VAR_PARAM_INIT, __VA_ARGS__})
+
+FN_EXTERN IoFilter *decompressFilter(CompressType type, DecompressFilterParam param);
 
 // Get extension for the current compression type
 FN_EXTERN const String *compressExtStr(CompressType type);

src/common/compress/lz4/compress.c

@@ -243,10 +243,11 @@ lz4CompressInputSame(const THIS_VOID)
 
 /**********************************************************************************************************************************/
 FN_EXTERN IoFilter *
-lz4CompressNew(int level)
+lz4CompressNew(const int level, const bool raw)
 {
     FUNCTION_LOG_BEGIN(logLevelTrace);
         FUNCTION_LOG_PARAM(INT, level);
+        FUNCTION_LOG_PARAM(BOOL, raw);
     FUNCTION_LOG_END();
 
     ASSERT(level >= LZ4_COMPRESS_LEVEL_MIN && level <= LZ4_COMPRESS_LEVEL_MAX);
@@ -259,7 +260,11 @@ lz4CompressNew(int level)
 
         *driver = (Lz4Compress)
         {
-            .prefs = {.compressionLevel = level, .frameInfo = {.contentChecksumFlag = LZ4F_contentChecksumEnabled}},
+            .prefs =
+            {
+                .compressionLevel = level,
+                .frameInfo = {.contentChecksumFlag = raw ? LZ4F_noContentChecksum : LZ4F_contentChecksumEnabled},
+            },
             .first = true,
             .buffer = bufNew(0),
         };
@@ -278,6 +283,7 @@ lz4CompressNew(int level)
             PackWrite *const packWrite = pckWriteNewP();
 
             pckWriteI32P(packWrite, level);
+            pckWriteBoolP(packWrite, raw);
             pckWriteEndP(packWrite);
 
             paramList = pckMove(pckWriteResult(packWrite), memContextPrior());

src/common/compress/lz4/compress.h

@@ -25,7 +25,7 @@ Level constants
 /***********************************************************************************************************************************
 Constructors
 ***********************************************************************************************************************************/
-FN_EXTERN IoFilter *lz4CompressNew(int level);
+FN_EXTERN IoFilter *lz4CompressNew(int level, bool raw);
 
 #endif
 

src/common/compress/lz4/decompress.c

@@ -157,9 +157,11 @@ lz4DecompressInputSame(const THIS_VOID)
 
 /**********************************************************************************************************************************/
 FN_EXTERN IoFilter *
-lz4DecompressNew(void)
+lz4DecompressNew(const bool raw)
 {
-    FUNCTION_LOG_VOID(logLevelTrace);
+    FUNCTION_LOG_BEGIN(logLevelTrace);
+        (void)raw;                                                  // Not required for decompress
+    FUNCTION_LOG_END();
 
     IoFilter *this = NULL;
 

src/common/compress/lz4/decompress.h

@@ -18,7 +18,7 @@ Filter type constant
 /***********************************************************************************************************************************
 Constructors
 ***********************************************************************************************************************************/
-FN_EXTERN IoFilter *lz4DecompressNew(void);
+FN_EXTERN IoFilter *lz4DecompressNew(bool raw);
 
 #endif
 

src/common/compress/zst/compress.c

@@ -164,10 +164,11 @@ zstCompressInputSame(const THIS_VOID)
 
 /**********************************************************************************************************************************/
 FN_EXTERN IoFilter *
-zstCompressNew(int level)
+zstCompressNew(const int level, const bool raw)
 {
     FUNCTION_LOG_BEGIN(logLevelTrace);
         FUNCTION_LOG_PARAM(INT, level);
+        (void)raw;                                                  // Raw unsupported
     FUNCTION_LOG_END();
 
     ASSERT(level >= ZST_COMPRESS_LEVEL_MIN && level <= ZST_COMPRESS_LEVEL_MAX);

src/common/compress/zst/compress.h

@@ -25,7 +25,7 @@ Level constants
 /***********************************************************************************************************************************
 Constructors
 ***********************************************************************************************************************************/
-FN_EXTERN IoFilter *zstCompressNew(int level);
+FN_EXTERN IoFilter *zstCompressNew(int level, bool raw);
 
 #endif
 

src/common/compress/zst/decompress.c

@@ -156,9 +156,11 @@ zstDecompressInputSame(const THIS_VOID)
 
 /**********************************************************************************************************************************/
 FN_EXTERN IoFilter *
-zstDecompressNew(void)
+zstDecompressNew(const bool raw)
 {
-    FUNCTION_LOG_VOID(logLevelTrace);
+    FUNCTION_LOG_BEGIN(logLevelTrace);
+        (void)raw;                                                  // Raw unsupported
+    FUNCTION_LOG_END();
 
     IoFilter *this = NULL;
 

src/common/compress/zst/decompress.h

@@ -18,7 +18,7 @@ Filter type constant
 /***********************************************************************************************************************************
 Constructors
 ***********************************************************************************************************************************/
-FN_EXTERN IoFilter *zstDecompressNew(void);
+FN_EXTERN IoFilter *zstDecompressNew(bool raw);
 
 #endif
 

src/storage/remote/read.c

@@ -217,7 +217,8 @@ storageReadRemoteOpen(THIS_VOID)
         if (this->interface.compressible)
         {
             ioFilterGroupAdd(
-                ioReadFilterGroup(storageReadIo(this->read)), compressFilter(compressTypeGz, (int)this->interface.compressLevel));
+                ioReadFilterGroup(storageReadIo(this->read)),
+                compressFilterP(compressTypeGz, (int)this->interface.compressLevel, .raw = true));
         }
 
         ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_STORAGE_OPEN_READ);
@@ -246,7 +247,7 @@ storageReadRemoteOpen(THIS_VOID)
 
             // If the file is compressible add decompression filter locally
             if (this->interface.compressible)
-                ioFilterGroupAdd(ioReadFilterGroup(storageReadIo(this->read)), decompressFilter(compressTypeGz));
+                ioFilterGroupAdd(ioReadFilterGroup(storageReadIo(this->read)), decompressFilterP(compressTypeGz, .raw = true));
 
             // Set free callback to ensure the protocol is cleared on a short read
             memContextCallbackSet(objMemContext(this), storageReadRemoteFreeResource, this);

src/storage/remote/write.c

@@ -78,7 +78,7 @@ storageWriteRemoteOpen(THIS_VOID)
     {
         // If the file is compressible add decompression filter on the remote
         if (this->interface.compressible)
-            ioFilterGroupInsert(ioWriteFilterGroup(storageWriteIo(this->write)), 0, decompressFilter(compressTypeGz));
+            ioFilterGroupInsert(ioWriteFilterGroup(storageWriteIo(this->write)), 0, decompressFilterP(compressTypeGz));
 
         ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_STORAGE_OPEN_WRITE);
         PackWrite *const param = protocolCommandParam(command);
@@ -106,7 +106,7 @@ storageWriteRemoteOpen(THIS_VOID)
         {
             ioFilterGroupAdd(
                 ioWriteFilterGroup(storageWriteIo(this->write)),
-                compressFilter(compressTypeGz, (int)this->interface.compressLevel));
+                compressFilterP(compressTypeGz, (int)this->interface.compressLevel));
         }
 
         // Set free callback to ensure remote file is freed

test/src/common/harnessStorage.c

@@ -131,7 +131,7 @@ testStorageGet(const Storage *const storage, const char *const file, const char
     if (param.compressType != compressTypeNone)
     {
         ASSERT(param.compressType == compressTypeGz || param.compressType == compressTypeBz2);
-        ioFilterGroupAdd(filterGroup, decompressFilter(param.compressType));
+        ioFilterGroupAdd(filterGroup, decompressFilterP(param.compressType));
     }
 
     printf("test content of %s'%s'", strEmpty(filter) ? "" : strZ(filter), strZ(fileFull));
@@ -429,7 +429,7 @@ hrnStoragePut(
     if (param.compressType != compressTypeNone)
     {
         ASSERT(param.compressType == compressTypeGz || param.compressType == compressTypeBz2);
-        ioFilterGroupAdd(filterGroup, compressFilter(param.compressType, 1));
+        ioFilterGroupAdd(filterGroup, compressFilterP(param.compressType, 1));
 
         strCatFmt(filter, "%scmp[%s]", strEmpty(filter) ? "" : "/", strZ(compressTypeStr(param.compressType)));
     }

test/src/module/command/backupTest.c

@@ -191,7 +191,7 @@ testBackupValidateList(
                             if (manifestData->backupOptionCompressType != compressTypeNone)
                             {
                                 ioFilterGroupAdd(
-                                    ioReadFilterGroup(chunkRead), decompressFilter(manifestData->backupOptionCompressType));
+                                    ioReadFilterGroup(chunkRead), decompressFilterP(manifestData->backupOptionCompressType));
                             }
 
                             ioReadOpen(chunkRead);
@@ -243,7 +243,7 @@ testBackupValidateList(
                         if (manifestData->backupOptionCompressType != compressTypeNone)
                         {
                             ioFilterGroupAdd(
-                                ioReadFilterGroup(storageReadIo(read)), decompressFilter(manifestData->backupOptionCompressType));
+                                ioReadFilterGroup(storageReadIo(read)), decompressFilterP(manifestData->backupOptionCompressType));
                         }
 
                         ioFilterGroupAdd(ioReadFilterGroup(storageReadIo(read)), cryptoHashNew(hashTypeSha1));
@@ -530,7 +530,7 @@ testBackupPqScript(unsigned int pgVersion, time_t backupTimeStart, TestBackupPqS
                     strZ(walChecksum), strZ(compressExtStr(param.walCompressType))));
 
             if (param.walCompressType != compressTypeNone)
-                ioFilterGroupAdd(ioWriteFilterGroup(storageWriteIo(write)), compressFilter(param.walCompressType, 1));
+                ioFilterGroupAdd(ioWriteFilterGroup(storageWriteIo(write)), compressFilterP(param.walCompressType, 1));
 
             storagePutP(write, walBuffer);
         }
@@ -1252,7 +1252,7 @@ testRun(void)
             blockIncrNewPack(
                 ioFilterParamList(
                     blockIncrNew(
-                        3, 2, 4, 5, NULL, compressFilter(compressTypeGz, 1),
+                        3, 2, 4, 5, NULL, compressFilterP(compressTypeGz, 1),
                         cipherBlockNewP(cipherModeEncrypt, cipherTypeAes256Cbc, BUFSTRDEF(TEST_CIPHER_PASS), .raw = true)))),
             "block incr pack");
     }

test/src/module/common/compressTest.c

@@ -73,14 +73,16 @@ testDecompress(IoFilter *decompress, Buffer *compressed, size_t inputSize, size_
 Standard test suite to be applied to all compression types
 ***********************************************************************************************************************************/
 static void
-testSuite(CompressType type, const char *decompressCmd)
+testSuite(CompressType type, const char *decompressCmd, size_t rawDelta)
 {
     const char *simpleData = "A simple string";
     Buffer *compressed = NULL;
+    Buffer *compressedRaw = NULL;
     Buffer *decompressed = bufNewC(simpleData, strlen(simpleData));
 
     PackWrite *packWrite = pckWriteNewP();
     pckWriteI32P(packWrite, 1);
+    pckWriteBoolP(packWrite, false);
     pckWriteEndP(packWrite);
 
     // Create default storage object for testing
@@ -95,6 +97,20 @@ testSuite(CompressType type, const char *decompressCmd)
             256 * 1024 * 1024),
         "simple data - compress large in/large out buffer");
 
+    packWrite = pckWriteNewP();
+    pckWriteI32P(packWrite, 1);
+    pckWriteBoolP(packWrite, true);
+    pckWriteEndP(packWrite);
+
+    TEST_ASSIGN(
+        compressedRaw,
+        testCompress(
+            compressFilterPack(compressHelperLocal[type].compressType, pckWriteResult(packWrite)), decompressed, 1024,
+            1024),
+        "simple data - compress large in/large out buffer (raw)");
+
+    TEST_RESULT_UINT(bufUsed(compressed) - rawDelta, bufUsed(compressedRaw), "compare to raw");
+
     // -------------------------------------------------------------------------------------------------------------------------
     TEST_TITLE("compressed output can be decompressed with command-line tool");
 
@@ -103,39 +119,60 @@ testSuite(CompressType type, const char *decompressCmd)
     TEST_RESULT_BOOL(bufEq(decompressed, storageGetP(storageNewReadP(storageTest, STRDEF("test.out")))), true, "check output");
 
     TEST_RESULT_BOOL(
-        bufEq(compressed, testCompress(compressFilter(type, 1), decompressed, 1024, 1)), true,
+        bufEq(compressed, testCompress(compressFilterP(type, 1), decompressed, 1024, 1)), true,
         "simple data - compress large in/small out buffer");
 
     TEST_RESULT_BOOL(
-        bufEq(compressed, testCompress(compressFilter(type, 1), decompressed, 1, 1024)), true,
+        bufEq(compressed, testCompress(compressFilterP(type, 1), decompressed, 1, 1024)), true,
         "simple data - compress small in/large out buffer");
 
     TEST_RESULT_BOOL(
-        bufEq(compressed, testCompress(compressFilter(type, 1), decompressed, 1, 1)), true,
+        bufEq(compressed, testCompress(compressFilterP(type, 1), decompressed, 1, 1)), true,
         "simple data - compress small in/small out buffer");
 
+    TEST_RESULT_BOOL(
+        bufEq(compressedRaw, testCompress(compressFilterP(type, 1, .raw = true), decompressed, 1, 1)), true,
+        "simple data - compress small in/small out buffer (raw)");
+
+    packWrite = pckWriteNewP();
+    pckWriteBoolP(packWrite, false);
+    pckWriteEndP(packWrite);
+
     TEST_RESULT_BOOL(
         bufEq(
             decompressed,
-            testDecompress(compressFilterPack(compressHelperLocal[type].decompressType, NULL), compressed, 1024, 1024)),
+            testDecompress(
-        true, "simple data - decompress large in/large out buffer");
+                compressFilterPack(compressHelperLocal[type].decompressType, pckWriteResult(packWrite)), compressed, 1024, 1024)),
+        true, "simple data - decompress large in/small out buffer");
+
+    packWrite = pckWriteNewP();
+    pckWriteBoolP(packWrite, true);
+    pckWriteEndP(packWrite);
 
     TEST_RESULT_BOOL(
-        bufEq(decompressed, testDecompress(decompressFilter(type), compressed, 1024, 1)), true,
+        bufEq(
+            decompressed,
+            testDecompress(
+                compressFilterPack(
+                    compressHelperLocal[type].decompressType, pckWriteResult(packWrite)), compressedRaw, 1024, 1024)),
+        true, "simple data - decompress large in/large out buffer (raw)");
+
+    TEST_RESULT_BOOL(
+        bufEq(decompressed, testDecompress(decompressFilterP(type), compressed, 1024, 1)), true,
         "simple data - decompress large in/small out buffer");
 
     TEST_RESULT_BOOL(
-        bufEq(decompressed, testDecompress(decompressFilter(type), compressed, 1, 1024)), true,
+        bufEq(decompressed, testDecompress(decompressFilterP(type), compressed, 1, 1024)), true,
         "simple data - decompress small in/large out buffer");
 
     TEST_RESULT_BOOL(
-        bufEq(decompressed, testDecompress(decompressFilter(type), compressed, 1, 1)), true,
+        bufEq(decompressed, testDecompress(decompressFilterP(type), compressed, 1, 1)), true,
         "simple data - decompress small in/small out buffer");
 
     // -------------------------------------------------------------------------------------------------------------------------
     TEST_TITLE("error on no compression data");
 
-    TEST_ERROR(testDecompress(decompressFilter(type), bufNew(0), 1, 1), FormatError, "unexpected eof in compressed data");
+    TEST_ERROR(testDecompress(decompressFilterP(type), bufNew(0), 1, 1), FormatError, "unexpected eof in compressed data");
 
     // -------------------------------------------------------------------------------------------------------------------------
     TEST_TITLE("error on truncated compression data");
@@ -144,7 +181,7 @@ testSuite(CompressType type, const char *decompressCmd)
     bufCatSub(truncated, compressed, 0, bufUsed(compressed) - 1);
 
     TEST_RESULT_UINT(bufUsed(truncated), bufUsed(compressed) - 1, "check truncated buffer size");
-    TEST_ERROR(testDecompress(decompressFilter(type), truncated, 512, 512), FormatError, "unexpected eof in compressed data");
+    TEST_ERROR(testDecompress(decompressFilterP(type), truncated, 512, 512), FormatError, "unexpected eof in compressed data");
 
     // -------------------------------------------------------------------------------------------------------------------------
     TEST_TITLE("compress a large non-zero input buffer into small output buffer");
@@ -160,11 +197,11 @@ testSuite(CompressType type, const char *decompressCmd)
     bufUsedSet(decompressed, bufSize(decompressed));
 
     TEST_ASSIGN(
-        compressed, testCompress(compressFilter(type, 3), decompressed, bufSize(decompressed), 32),
+        compressed, testCompress(compressFilterP(type, 3), decompressed, bufSize(decompressed), 32),
         "non-zero data - compress large in/small out buffer");
 
     TEST_RESULT_BOOL(
-        bufEq(decompressed, testDecompress(decompressFilter(type), compressed, bufSize(compressed), 1024 * 256)), true,
+        bufEq(decompressed, testDecompress(decompressFilterP(type), compressed, bufSize(compressed), 1024 * 256)), true,
         "non-zero data - decompress large in/small out buffer");
 }
 
@@ -180,7 +217,7 @@ testRun(void)
     if (testBegin("gz"))
     {
         // Run standard test suite
-        testSuite(compressTypeGz, "gzip -dc");
+        testSuite(compressTypeGz, "gzip -dc", 12);
 
         // -------------------------------------------------------------------------------------------------------------------------
         TEST_TITLE("gzError()");
@@ -208,7 +245,7 @@ testRun(void)
 
         char buffer[STACK_TRACE_PARAM_MAX];
 
-        GzDecompress *decompress = (GzDecompress *)ioFilterDriver(gzDecompressNew());
+        GzDecompress *decompress = (GzDecompress *)ioFilterDriver(gzDecompressNew(false));
 
         TEST_RESULT_VOID(FUNCTION_LOG_OBJECT_FORMAT(decompress, gzDecompressToLog, buffer, sizeof(buffer)), "gzDecompressToLog");
         TEST_RESULT_Z(buffer, "{inputSame: false, done: false, availIn: 0}", "check log");
@@ -224,7 +261,7 @@ testRun(void)
     if (testBegin("bz2"))
     {
         // Run standard test suite
-        testSuite(compressTypeBz2, "bzip2 -dc");
+        testSuite(compressTypeBz2, "bzip2 -dc", 0);
 
         // -------------------------------------------------------------------------------------------------------------------------
         TEST_TITLE("bz2Error()");
@@ -257,14 +294,14 @@ testRun(void)
 
         char buffer[STACK_TRACE_PARAM_MAX];
 
-        Bz2Compress *compress = (Bz2Compress *)ioFilterDriver(bz2CompressNew(1));
+        Bz2Compress *compress = (Bz2Compress *)ioFilterDriver(bz2CompressNew(1, false));
 
         compress->stream.avail_in = 999;
 
         TEST_RESULT_VOID(FUNCTION_LOG_OBJECT_FORMAT(compress, bz2CompressToLog, buffer, sizeof(buffer)), "bz2CompressToLog");
         TEST_RESULT_Z(buffer, "{inputSame: false, done: false, flushing: false, avail_in: 999}", "check log");
 
-        Bz2Decompress *decompress = (Bz2Decompress *)ioFilterDriver(bz2DecompressNew());
+        Bz2Decompress *decompress = (Bz2Decompress *)ioFilterDriver(bz2DecompressNew(false));
 
         decompress->inputSame = true;
         decompress->done = true;
@@ -278,7 +315,7 @@ testRun(void)
     {
 #ifdef HAVE_LIBLZ4
         // Run standard test suite
-        testSuite(compressTypeLz4, "lz4 -dc");
+        testSuite(compressTypeLz4, "lz4 -dc", 4);
 
         // -------------------------------------------------------------------------------------------------------------------------
         TEST_TITLE("lz4Error()");
@@ -298,7 +335,7 @@ testRun(void)
 
         char buffer[STACK_TRACE_PARAM_MAX];
 
-        Lz4Compress *compress = (Lz4Compress *)ioFilterDriver(lz4CompressNew(7));
+        Lz4Compress *compress = (Lz4Compress *)ioFilterDriver(lz4CompressNew(7, false));
 
         compress->inputSame = true;
         compress->flushing = true;
@@ -306,7 +343,7 @@ testRun(void)
         TEST_RESULT_VOID(FUNCTION_LOG_OBJECT_FORMAT(compress, lz4CompressToLog, buffer, sizeof(buffer)), "lz4CompressToLog");
         TEST_RESULT_Z(buffer, "{level: 7, first: true, inputSame: true, flushing: true}", "check log");
 
-        Lz4Decompress *decompress = (Lz4Decompress *)ioFilterDriver(lz4DecompressNew());
+        Lz4Decompress *decompress = (Lz4Decompress *)ioFilterDriver(lz4DecompressNew(false));
 
         decompress->inputSame = true;
         decompress->done = true;
@@ -324,7 +361,7 @@ testRun(void)
     {
 #ifdef HAVE_LIBZST
         // Run standard test suite
-        testSuite(compressTypeZst, "zstd -dc");
+        testSuite(compressTypeZst, "zstd -dc", 0);
 
         // -------------------------------------------------------------------------------------------------------------------------
         TEST_TITLE("zstError()");
@@ -344,7 +381,7 @@ testRun(void)
 
         char buffer[STACK_TRACE_PARAM_MAX];
 
-        ZstCompress *compress = (ZstCompress *)ioFilterDriver(zstCompressNew(14));
+        ZstCompress *compress = (ZstCompress *)ioFilterDriver(zstCompressNew(14, false));
 
         compress->inputSame = true;
         compress->inputOffset = 49;
@@ -353,7 +390,7 @@ testRun(void)
         TEST_RESULT_VOID(FUNCTION_LOG_OBJECT_FORMAT(compress, zstCompressToLog, buffer, sizeof(buffer)), "zstCompressToLog");
         TEST_RESULT_Z(buffer, "{level: 14, inputSame: true, inputOffset: 49, flushing: true}", "check log");
 
-        ZstDecompress *decompress = (ZstDecompress *)ioFilterDriver(zstDecompressNew());
+        ZstDecompress *decompress = (ZstDecompress *)ioFilterDriver(zstDecompressNew(false));
 
         decompress->inputSame = true;
         decompress->done = true;

test/src/module/performance/storageTest.c

@@ -334,7 +334,7 @@ testRun(void)
             MEM_CONTEXT_TEMP_BEGIN()
             {
                 BENCHMARK_BEGIN();
-                BENCHMARK_FILTER_ADD(gzCompressNew(6));
+                BENCHMARK_FILTER_ADD(gzCompressNew(6, false));
                 BENCHMARK_END(gzip6Total);
             }
             MEM_CONTEXT_TEMP_END();
@@ -346,7 +346,7 @@ testRun(void)
             MEM_CONTEXT_TEMP_BEGIN()
             {
                 BENCHMARK_BEGIN();
-                BENCHMARK_FILTER_ADD(lz4CompressNew(1));
+                BENCHMARK_FILTER_ADD(lz4CompressNew(1, false));
                 BENCHMARK_END(lz41Total);
             }
             MEM_CONTEXT_TEMP_END();

test/src/module/storage/remoteTest.c

@@ -346,8 +346,8 @@ testRun(void)
         ioFilterGroupAdd(filterGroup, cryptoHashNew(hashTypeSha1));
         ioFilterGroupAdd(filterGroup, cipherBlockNewP(cipherModeEncrypt, cipherTypeAes256Cbc, BUFSTRZ("x")));
         ioFilterGroupAdd(filterGroup, cipherBlockNewP(cipherModeDecrypt, cipherTypeAes256Cbc, BUFSTRZ("x")));
-        ioFilterGroupAdd(filterGroup, compressFilter(compressTypeGz, 3));
+        ioFilterGroupAdd(filterGroup, compressFilterP(compressTypeGz, 3));
-        ioFilterGroupAdd(filterGroup, decompressFilter(compressTypeGz));
+        ioFilterGroupAdd(filterGroup, decompressFilterP(compressTypeGz));
 
         TEST_RESULT_STR_Z(strNewBuf(storageGetP(fileRead)), "TESTDATA", "check contents");