Add archive-header-check option.

Enabled by default, this option checks the WAL header against the PostgreSQL version and system identifier to ensure that the WAL is being copied to the correct stanza. This is in addition to checking pg_control against the stanza and verifying that WAL is being copied from the same PostgreSQL data directory where pg_control is located.

Therefore, disabling this check is fairly safe but should only be done when required, e.g. if the WAL is encrypted.

doc/xml/reference.xml

@@ -748,6 +748,17 @@
                         <example>y</example>
                     </config-key>
 
+                    <!-- ======================================================================================================= -->
+                    <config-key id="archive-header-check" name="Check WAL Headers">
+                        <summary>Check PostgreSQL version/id in WAL headers.</summary>
+
+                        <text>Enabled by default, this option checks the WAL header against the <postgres/> version and system identifier to ensure that the WAL is being copied to the correct stanza. This is in addition to checking <file>pg_control</file> against the stanza and verifying that WAL is being copied from the same <postgres/> data directory where <file>pg_control</file> is located.
+
+                        Therefore, disabling this check is fairly safe but should only be done when needed, e.g. if the WAL is encrypted.</text>
+
+                        <example>n</example>
+                    </config-key>
+
                     <!-- ======================================================================================================= -->
                     <config-key id="archive-mode-check" name="Check Archive Mode">
                         <summary>Check the <postgres/> <setting>archive_mode</setting> setting.</summary>

doc/xml/release.xml

@@ -90,6 +90,17 @@
 
                         <p>GCS support for repository storage.</p>
                     </release-item>
+
+                    <release-item>
+                        <release-item-contributor-list>
+                            <release-item-ideator id="hans.jurgen.schonig"/>
+                            <release-item-contributor id="david.steele"/>
+                            <release-item-reviewer id="stephen.frost"/>
+                            <release-item-reviewer id="cynthia.shang"/>
+                        </release-item-contributor-list>
+
+                        <p>Add <br-option>archive-header-check</br-option> option.</p>
+                    </release-item>
                 </release-feature-list>
 
                 <release-improvement-list>

src/build/config/config.yaml

@@ -1016,6 +1016,16 @@ option:
     command-role:
       default: {}
 
+  archive-header-check:
+    section: global
+    type: boolean
+    default: true
+    command:
+      archive-push: {}
+    command-role:
+      default: {}
+      async: {}
+
   archive-mode-check:
     section: global
     type: boolean

src/command/archive/push/file.c

@@ -95,11 +95,12 @@ archivePushFileIo(ArchivePushFileIoType type, IoWrite *write, const Buffer *buff
 /**********************************************************************************************************************************/
 ArchivePushFileResult
 archivePushFile(
-    const String *walSource, unsigned int pgVersion, uint64_t pgSystemId, const String *archiveFile, CompressType compressType,
+    const String *walSource, bool headerCheck, unsigned int pgVersion, uint64_t pgSystemId, const String *archiveFile,
-    int compressLevel, const List *const repoList, const StringList *const priorErrorList)
+    CompressType compressType, int compressLevel, const List *const repoList, const StringList *const priorErrorList)
 {
     FUNCTION_LOG_BEGIN(logLevelDebug);
         FUNCTION_LOG_PARAM(STRING, walSource);
+        FUNCTION_LOG_PARAM(BOOL, headerCheck);
         FUNCTION_LOG_PARAM(UINT, pgVersion);
         FUNCTION_LOG_PARAM(UINT64, pgSystemId);
         FUNCTION_LOG_PARAM(STRING, archiveFile);
@@ -124,7 +125,7 @@ archivePushFile(
         bool isSegment = walIsSegment(archiveFile);
 
         // If this is a segment compare archive version and systemId to the WAL header
-        if (isSegment)
+        if (headerCheck && isSegment)
         {
             PgWal walInfo = pgWalFromFile(walSource, storageLocal());
 

src/command/archive/push/file.h

@@ -31,7 +31,7 @@ typedef struct ArchivePushFileResult
 
 // Copy a file from the source to the archive
 ArchivePushFileResult archivePushFile(
-    const String *walSource, unsigned int pgVersion, uint64_t pgSystemId, const String *archiveFile, CompressType compressType,
+    const String *walSource, bool headerCheck, unsigned int pgVersion, uint64_t pgSystemId, const String *archiveFile,
-    int compressLevel, const List *const repoList, const StringList *const priorErrorList);
+    CompressType compressType, int compressLevel, const List *const repoList, const StringList *const priorErrorList);
 
 #endif

src/command/archive/push/protocol.c

@@ -33,8 +33,8 @@ archivePushFileProtocol(const VariantList *paramList, ProtocolServer *server)
     {
         // Build the repo data list
         List *repoList = lstNewP(sizeof(ArchivePushFileRepoData));
-        unsigned int repoListSize = varUIntForce(varLstGet(paramList, 7));
+        unsigned int repoListSize = varUIntForce(varLstGet(paramList, 8));
-        unsigned int paramIdx = 8;
+        unsigned int paramIdx = 9;
 
         for (unsigned int repoListIdx = 0; repoListIdx < repoListSize; repoListIdx++)
         {
@@ -53,9 +53,10 @@ archivePushFileProtocol(const VariantList *paramList, ProtocolServer *server)
 
         // Push the file
         ArchivePushFileResult fileResult = archivePushFile(
-            varStr(varLstGet(paramList, 0)), varUIntForce(varLstGet(paramList, 1)), varUInt64(varLstGet(paramList, 2)),
+            varStr(varLstGet(paramList, 0)), varBool(varLstGet(paramList, 1)), varUIntForce(varLstGet(paramList, 2)),
-            varStr(varLstGet(paramList, 3)), (CompressType)varUIntForce(varLstGet(paramList, 4)),
+            varUInt64(varLstGet(paramList, 3)), varStr(varLstGet(paramList, 4)),
-            varIntForce(varLstGet(paramList, 5)), repoList, strLstNewVarLst(varVarLst(varLstGet(paramList, 6))));
+            (CompressType)varUIntForce(varLstGet(paramList, 5)), varIntForce(varLstGet(paramList, 6)), repoList,
+            strLstNewVarLst(varVarLst(varLstGet(paramList, 7))));
 
         // Return result
         VariantList *result = varLstNew();

src/command/archive/push/push.c

@@ -412,7 +412,7 @@ cmdArchivePush(void)
 
                 // Push the file to the archive
                 ArchivePushFileResult fileResult = archivePushFile(
-                    walFile, archiveInfo.pgVersion, archiveInfo.pgSystemId, archiveFile,
+                    walFile, cfgOptionBool(cfgOptArchiveHeaderCheck), archiveInfo.pgVersion, archiveInfo.pgSystemId, archiveFile,
                     compressTypeEnum(cfgOptionStr(cfgOptCompressType)), cfgOptionInt(cfgOptCompressLevel), archiveInfo.repoList,
                     archiveInfo.errorList);
 
@@ -462,6 +462,7 @@ archivePushAsyncCallback(void *data, unsigned int clientIdx)
 
         ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_ARCHIVE_PUSH_FILE_STR);
         protocolCommandParamAdd(command, VARSTR(strNewFmt("%s/%s", strZ(jobData->walPath), strZ(walFile))));
+        protocolCommandParamAdd(command, VARBOOL(cfgOptionBool(cfgOptArchiveHeaderCheck)));
         protocolCommandParamAdd(command, VARUINT(jobData->archiveInfo.pgVersion));
         protocolCommandParamAdd(command, VARUINT64(jobData->archiveInfo.pgSystemId));
         protocolCommandParamAdd(command, VARSTR(walFile));

src/command/help/help.auto.c

@@ -840,6 +840,39 @@ static const unsigned char helpDataPack[] =
             0x6C, 0x69, 0x65, 0x72, 0x20, 0x69, 0x73, 0x20, 0x61, 0x20, 0x70, 0x6F, 0x77, 0x65, 0x72, 0x20, 0x6F, 0x66, 0x20, 0x31,
             0x30, 0x32, 0x34, 0x2E,
 
+        // archive-header-check option
+        // -------------------------------------------------------------------------------------------------------------------------
+        pckTypeStr << 4 | 0x0B, 0x06, // Section
+            0x62, 0x61, 0x63, 0x6B, 0x75, 0x70,
+        pckTypeStr << 4 | 0x08, 0x2B, // Summary
+            0x43, 0x68, 0x65, 0x63, 0x6B, 0x20, 0x50, 0x6F, 0x73, 0x74, 0x67, 0x72, 0x65, 0x53, 0x51, 0x4C, 0x20, 0x76, 0x65, 0x72,
+            0x73, 0x69, 0x6F, 0x6E, 0x2F, 0x69, 0x64, 0x20, 0x69, 0x6E, 0x20, 0x57, 0x41, 0x4C, 0x20, 0x68, 0x65, 0x61, 0x64, 0x65,
+            0x72, 0x73, 0x2E,
+        pckTypeStr << 4 | 0x08, 0xC5, 0x03, // Description
+            0x45, 0x6E, 0x61, 0x62, 0x6C, 0x65, 0x64, 0x20, 0x62, 0x79, 0x20, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6C, 0x74, 0x2C, 0x20,
+            0x74, 0x68, 0x69, 0x73, 0x20, 0x6F, 0x70, 0x74, 0x69, 0x6F, 0x6E, 0x20, 0x63, 0x68, 0x65, 0x63, 0x6B, 0x73, 0x20, 0x74,
+            0x68, 0x65, 0x20, 0x57, 0x41, 0x4C, 0x20, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x20, 0x61, 0x67, 0x61, 0x69, 0x6E, 0x73,
+            0x74, 0x20, 0x74, 0x68, 0x65, 0x20, 0x50, 0x6F, 0x73, 0x74, 0x67, 0x72, 0x65, 0x53, 0x51, 0x4C, 0x20, 0x76, 0x65, 0x72,
+            0x73, 0x69, 0x6F, 0x6E, 0x20, 0x61, 0x6E, 0x64, 0x20, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6D, 0x20, 0x69, 0x64, 0x65, 0x6E,
+            0x74, 0x69, 0x66, 0x69, 0x65, 0x72, 0x20, 0x74, 0x6F, 0x20, 0x65, 0x6E, 0x73, 0x75, 0x72, 0x65, 0x20, 0x74, 0x68, 0x61,
+            0x74, 0x20, 0x74, 0x68, 0x65, 0x20, 0x57, 0x41, 0x4C, 0x20, 0x69, 0x73, 0x20, 0x62, 0x65, 0x69, 0x6E, 0x67, 0x20, 0x63,
+            0x6F, 0x70, 0x69, 0x65, 0x64, 0x20, 0x74, 0x6F, 0x20, 0x74, 0x68, 0x65, 0x20, 0x63, 0x6F, 0x72, 0x72, 0x65, 0x63, 0x74,
+            0x20, 0x73, 0x74, 0x61, 0x6E, 0x7A, 0x61, 0x2E, 0x20, 0x54, 0x68, 0x69, 0x73, 0x20, 0x69, 0x73, 0x20, 0x69, 0x6E, 0x20,
+            0x61, 0x64, 0x64, 0x69, 0x74, 0x69, 0x6F, 0x6E, 0x20, 0x74, 0x6F, 0x20, 0x63, 0x68, 0x65, 0x63, 0x6B, 0x69, 0x6E, 0x67,
+            0x20, 0x70, 0x67, 0x5F, 0x63, 0x6F, 0x6E, 0x74, 0x72, 0x6F, 0x6C, 0x20, 0x61, 0x67, 0x61, 0x69, 0x6E, 0x73, 0x74, 0x20,
+            0x74, 0x68, 0x65, 0x20, 0x73, 0x74, 0x61, 0x6E, 0x7A, 0x61, 0x20, 0x61, 0x6E, 0x64, 0x20, 0x76, 0x65, 0x72, 0x69, 0x66,
+            0x79, 0x69, 0x6E, 0x67, 0x20, 0x74, 0x68, 0x61, 0x74, 0x20, 0x57, 0x41, 0x4C, 0x20, 0x69, 0x73, 0x20, 0x62, 0x65, 0x69,
+            0x6E, 0x67, 0x20, 0x63, 0x6F, 0x70, 0x69, 0x65, 0x64, 0x20, 0x66, 0x72, 0x6F, 0x6D, 0x20, 0x74, 0x68, 0x65, 0x20, 0x73,
+            0x61, 0x6D, 0x65, 0x20, 0x50, 0x6F, 0x73, 0x74, 0x67, 0x72, 0x65, 0x53, 0x51, 0x4C, 0x20, 0x64, 0x61, 0x74, 0x61, 0x20,
+            0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6F, 0x72, 0x79, 0x20, 0x77, 0x68, 0x65, 0x72, 0x65, 0x20, 0x70, 0x67, 0x5F, 0x63,
+            0x6F, 0x6E, 0x74, 0x72, 0x6F, 0x6C, 0x20, 0x69, 0x73, 0x20, 0x6C, 0x6F, 0x63, 0x61, 0x74, 0x65, 0x64, 0x2E, 0x0A, 0x0A,
+            0x54, 0x68, 0x65, 0x72, 0x65, 0x66, 0x6F, 0x72, 0x65, 0x2C, 0x20, 0x64, 0x69, 0x73, 0x61, 0x62, 0x6C, 0x69, 0x6E, 0x67,
+            0x20, 0x74, 0x68, 0x69, 0x73, 0x20, 0x63, 0x68, 0x65, 0x63, 0x6B, 0x20, 0x69, 0x73, 0x20, 0x66, 0x61, 0x69, 0x72, 0x6C,
+            0x79, 0x20, 0x73, 0x61, 0x66, 0x65, 0x20, 0x62, 0x75, 0x74, 0x20, 0x73, 0x68, 0x6F, 0x75, 0x6C, 0x64, 0x20, 0x6F, 0x6E,
+            0x6C, 0x79, 0x20, 0x62, 0x65, 0x20, 0x64, 0x6F, 0x6E, 0x65, 0x20, 0x77, 0x68, 0x65, 0x6E, 0x20, 0x6E, 0x65, 0x65, 0x64,
+            0x65, 0x64, 0x2C, 0x20, 0x65, 0x2E, 0x67, 0x2E, 0x20, 0x69, 0x66, 0x20, 0x74, 0x68, 0x65, 0x20, 0x57, 0x41, 0x4C, 0x20,
+            0x69, 0x73, 0x20, 0x65, 0x6E, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x2E,
+
         // archive-mode option
         // -------------------------------------------------------------------------------------------------------------------------
         pckTypeStr << 4 | 0x0B, 0x07, // Section

src/config/config.auto.c

@@ -261,6 +261,7 @@ STRING_EXTERN(CFGOPT_ARCHIVE_ASYNC_STR,                             CFGOPT_ARCHI
 STRING_EXTERN(CFGOPT_ARCHIVE_CHECK_STR,                             CFGOPT_ARCHIVE_CHECK);
 STRING_EXTERN(CFGOPT_ARCHIVE_COPY_STR,                              CFGOPT_ARCHIVE_COPY);
 STRING_EXTERN(CFGOPT_ARCHIVE_GET_QUEUE_MAX_STR,                     CFGOPT_ARCHIVE_GET_QUEUE_MAX);
+STRING_EXTERN(CFGOPT_ARCHIVE_HEADER_CHECK_STR,                      CFGOPT_ARCHIVE_HEADER_CHECK);
 STRING_EXTERN(CFGOPT_ARCHIVE_MODE_STR,                              CFGOPT_ARCHIVE_MODE);
 STRING_EXTERN(CFGOPT_ARCHIVE_MODE_CHECK_STR,                        CFGOPT_ARCHIVE_MODE_CHECK);
 STRING_EXTERN(CFGOPT_ARCHIVE_PUSH_QUEUE_MAX_STR,                    CFGOPT_ARCHIVE_PUSH_QUEUE_MAX);

src/config/config.auto.h

@@ -68,6 +68,8 @@ Option constants
     STRING_DECLARE(CFGOPT_ARCHIVE_COPY_STR);
 #define CFGOPT_ARCHIVE_GET_QUEUE_MAX                                "archive-get-queue-max"
     STRING_DECLARE(CFGOPT_ARCHIVE_GET_QUEUE_MAX_STR);
+#define CFGOPT_ARCHIVE_HEADER_CHECK                                 "archive-header-check"
+    STRING_DECLARE(CFGOPT_ARCHIVE_HEADER_CHECK_STR);
 #define CFGOPT_ARCHIVE_MODE                                         "archive-mode"
     STRING_DECLARE(CFGOPT_ARCHIVE_MODE_STR);
 #define CFGOPT_ARCHIVE_MODE_CHECK                                   "archive-mode-check"
@@ -209,7 +211,7 @@ Option constants
 #define CFGOPT_TYPE                                                 "type"
     STRING_DECLARE(CFGOPT_TYPE_STR);
 
-#define CFG_OPTION_TOTAL                                            128
+#define CFG_OPTION_TOTAL                                            129
 
 /***********************************************************************************************************************************
 Command enum
@@ -257,6 +259,7 @@ typedef enum
     cfgOptArchiveCheck,
     cfgOptArchiveCopy,
     cfgOptArchiveGetQueueMax,
+    cfgOptArchiveHeaderCheck,
     cfgOptArchiveMode,
     cfgOptArchiveModeCheck,
     cfgOptArchivePushQueueMax,

src/config/parse.auto.c

@@ -406,6 +406,30 @@ static const ParseRuleOption parseRuleOption[CFG_OPTION_TOTAL] =
         ),
     ),
 
+    // -----------------------------------------------------------------------------------------------------------------------------
+    PARSE_RULE_OPTION
+    (
+        PARSE_RULE_OPTION_NAME("archive-header-check"),
+        PARSE_RULE_OPTION_TYPE(cfgOptTypeBoolean),
+        PARSE_RULE_OPTION_REQUIRED(true),
+        PARSE_RULE_OPTION_SECTION(cfgSectionGlobal),
+
+        PARSE_RULE_OPTION_COMMAND_ROLE_DEFAULT_VALID_LIST
+        (
+            PARSE_RULE_OPTION_COMMAND(cfgCmdArchivePush)
+        ),
+
+        PARSE_RULE_OPTION_COMMAND_ROLE_ASYNC_VALID_LIST
+        (
+            PARSE_RULE_OPTION_COMMAND(cfgCmdArchivePush)
+        ),
+
+        PARSE_RULE_OPTION_OPTIONAL_LIST
+        (
+            PARSE_RULE_OPTION_OPTIONAL_DEFAULT("1"),
+        ),
+    ),
+
     // -----------------------------------------------------------------------------------------------------------------------------
     PARSE_RULE_OPTION
     (
@@ -6528,6 +6552,21 @@ static const struct option optionList[] =
         .val = PARSE_OPTION_FLAG | PARSE_RESET_FLAG | cfgOptArchiveGetQueueMax,
     },
 
+    // archive-header-check option
+    // -----------------------------------------------------------------------------------------------------------------------------
+    {
+        .name = "archive-header-check",
+        .val = PARSE_OPTION_FLAG | cfgOptArchiveHeaderCheck,
+    },
+    {
+        .name = "no-archive-header-check",
+        .val = PARSE_OPTION_FLAG | PARSE_NEGATE_FLAG | cfgOptArchiveHeaderCheck,
+    },
+    {
+        .name = "reset-archive-header-check",
+        .val = PARSE_OPTION_FLAG | PARSE_RESET_FLAG | cfgOptArchiveHeaderCheck,
+    },
+
     // archive-mode option
     // -----------------------------------------------------------------------------------------------------------------------------
     {
@@ -10555,6 +10594,7 @@ static const ConfigOption optionResolveOrder[] =
     cfgOptStanza,
     cfgOptArchiveAsync,
     cfgOptArchiveGetQueueMax,
+    cfgOptArchiveHeaderCheck,
     cfgOptArchiveMode,
     cfgOptArchivePushQueueMax,
     cfgOptArchiveTimeout,

test/src/module/command/archivePushTest.c

@@ -327,6 +327,7 @@ testRun(void)
 
         memset(bufPtr(walBuffer1), 0, bufSize(walBuffer1));
         pgWalTestToBuffer((PgWal){.version = PG_VERSION_11, .systemId = 0xECAFECAFECAFECAF}, walBuffer1);
+        const char *walBuffer1Sha1 = strZ(bufHex(cryptoHashOne(HASH_TYPE_SHA1_STR, walBuffer1)));
 
         storagePutP(storageNewWriteP(storagePgWrite(), strNew("pg_wal/000000010000000100000001")), walBuffer1);
 
@@ -335,14 +336,37 @@ testRun(void)
             "WAL file '{[path]}/pg/pg_wal/000000010000000100000001' version 11, system-id 17055110554209741999 do not match"
                 " stanza version 11, system-id 18072658121562454734");
 
+        // -------------------------------------------------------------------------------------------------------------------------
+        TEST_TITLE("push by ignoring the invalid header");
+
+        argListTemp = strLstDup(argList);
+        hrnCfgArgRawNegate(argListTemp, cfgOptArchiveHeaderCheck);
+        strLstAddZ(argListTemp, "pg_wal/000000010000000100000001");
+        harnessCfgLoad(cfgCmdArchivePush, argListTemp);
+
+        TEST_RESULT_VOID(cmdArchivePush(), "push the WAL segment");
+        harnessLogResult("P00   INFO: pushed WAL file '000000010000000100000001' to the archive");
+
+        TEST_RESULT_BOOL(
+            storageExistsP(
+                storageRepoIdx(0),
+                strNewFmt(STORAGE_REPO_ARCHIVE "/11-1/000000010000000100000001-%s.gz", walBuffer1Sha1)),
+            true, "check repo for WAL file");
+        TEST_STORAGE_REMOVE(
+            storageRepoIdxWrite(0), strZ(strNewFmt(STORAGE_REPO_ARCHIVE "/11-1/000000010000000100000001-%s.gz", walBuffer1Sha1)));
+
         // Generate valid WAL and push them
         // -------------------------------------------------------------------------------------------------------------------------
+        argListTemp = strLstDup(argList);
+        strLstAddZ(argListTemp, "pg_wal/000000010000000100000001");
+        harnessCfgLoad(cfgCmdArchivePush, argListTemp);
+
         memset(bufPtr(walBuffer1), 0, bufSize(walBuffer1));
         pgWalTestToBuffer((PgWal){.version = PG_VERSION_11, .systemId = 0xFACEFACEFACEFACE}, walBuffer1);
 
         // Check sha1 checksum against fixed values once to make sure they are not getting munged. After this we'll calculate them
         // directly from the buffers to reduce the cost of maintaining checksums.
-        const char *walBuffer1Sha1 = TEST_64BIT() ?
+        walBuffer1Sha1 = TEST_64BIT() ?
             (TEST_BIG_ENDIAN() ? "1c5f963d720bb199d7935dbd315447ea2ec3feb2" : "aae7591a1dbc58f21d0d004886075094f622e6dd") :
             "28a13fd8cf6fcd9f9a8108aed4c8bcc58040863a";
 
@@ -452,6 +476,7 @@ testRun(void)
         // -------------------------------------------------------------------------------------------------------------------------
         VariantList *paramList = varLstNew();
         varLstAdd(paramList, varNewStr(strNewFmt("%s/pg/pg_wal/000000010000000100000002", testPath())));
+        varLstAdd(paramList, varNewBool(true));
         varLstAdd(paramList, varNewUInt64(PG_VERSION_11));
         varLstAdd(paramList, varNewUInt64(0xFACEFACEFACEFACE));
         varLstAdd(paramList, varNewStrZ("000000010000000100000002"));