Read segment size from WAL headers.

This allows validation of the WAL segment size for PostgreSQL versions <= 10.

src/postgres/interface.c

@@ -386,6 +386,27 @@ pgCatalogVersion(unsigned int pgVersion)
     FUNCTION_TEST_RETURN(pgInterfaceVersion(pgVersion)->catalogVersion());
 }
 
+/***********************************************************************************************************************************
+Check expected WAL segment size for older PostgreSQL versions
+***********************************************************************************************************************************/
+static void
+pgWalSegmentSizeCheck(unsigned int pgVersion, unsigned int walSegmentSize)
+{
+    FUNCTION_TEST_BEGIN();
+        FUNCTION_TEST_PARAM(UINT, pgVersion);
+        FUNCTION_TEST_PARAM(UINT, walSegmentSize);
+    FUNCTION_TEST_END();
+
+    if (pgVersion < PG_VERSION_11 && walSegmentSize != PG_WAL_SEGMENT_SIZE_DEFAULT)
+    {
+        THROW_FMT(
+            FormatError, "wal segment size is %u but must be %u for " PG_NAME " <= " PG_VERSION_10_STR, walSegmentSize,
+            PG_WAL_SEGMENT_SIZE_DEFAULT);
+    }
+
+    FUNCTION_TEST_RETURN_VOID();
+}
+
 /**********************************************************************************************************************************/
 PgControl
 pgControlFromBuffer(const Buffer *controlFile)
@@ -425,12 +446,7 @@ pgControlFromBuffer(const Buffer *controlFile)
     result.version = interface->version;
 
     // Check the segment size
-    if (result.version < PG_VERSION_11 && result.walSegmentSize != PG_WAL_SEGMENT_SIZE_DEFAULT)
+    pgWalSegmentSizeCheck(result.version, result.walSegmentSize);
-    {
-        THROW_FMT(
-            FormatError, "wal segment size is %u but must be %u for " PG_NAME " <= " PG_VERSION_10_STR, result.walSegmentSize,
-            PG_WAL_SEGMENT_SIZE_DEFAULT);
-    }
 
     // Check the page size
     if (result.pageSize != PG_PAGE_SIZE_DEFAULT)
@@ -526,6 +542,9 @@ pgWalFromBuffer(const Buffer *walBuffer)
     PgWal result = interface->wal(bufPtrConst(walBuffer));
     result.version = interface->version;
 
+    // Check the segment size
+    pgWalSegmentSizeCheck(result.version, result.size);
+
     FUNCTION_LOG_RETURN(PG_WAL, result);
 }
 
@@ -790,6 +809,10 @@ pgWalTestToBuffer(PgWal pgWal, Buffer *walBuffer)
 
     ASSERT(walBuffer != NULL);
 
+    // Set default WAL segment size if not specified
+    if (pgWal.size == 0)
+        pgWal.size = PG_WAL_SEGMENT_SIZE_DEFAULT;
+
     // Generate WAL
     pgInterfaceVersion(pgWal.version)->walTest(pgWal, bufPtr(walBuffer));
 

src/postgres/interface.h

@@ -106,6 +106,7 @@ PostgreSQL WAL Info
 typedef struct PgWal
 {
     unsigned int version;
+    unsigned int size;
     uint64_t systemId;
 } PgWal;
 

src/postgres/interface/version.intern.h

@@ -191,6 +191,7 @@ Read the version specific WAL header into a general data structure
         return (PgWal)                                                                                                             \
         {                                                                                                                          \
             .systemId = ((XLogLongPageHeaderData *)walFile)->xlp_sysid,                                                            \
+            .size = ((XLogLongPageHeaderData *)walFile)->xlp_seg_size,                                                             \
         };                                                                                                                         \
     }
 
@@ -210,6 +211,7 @@ Create a WAL file file for testing
         ((XLogLongPageHeaderData *)buffer)->std.xlp_magic = XLOG_PAGE_MAGIC;                                                       \
         ((XLogLongPageHeaderData *)buffer)->std.xlp_info = XLP_LONG_HEADER;                                                        \
         ((XLogLongPageHeaderData *)buffer)->xlp_sysid = pgWal.systemId;                                                            \
+        ((XLogLongPageHeaderData *)buffer)->xlp_seg_size = pgWal.size;                                                             \
     }
 
 #endif

test/expect/mock-stanza-001.log

@@ -296,9 +296,9 @@ full backup - create first full backup  (db-primary host)
 > [CONTAINER-EXEC] db-primary [BACKREST-BIN] --config=[TEST_PATH]/db-primary/pgbackrest.conf --repo1-retention-full=2 --no-online --type=full --stanza=db backup
 ------------------------------------------------------------------------------------------------------------------------------------
 P00   INFO: backup command begin [BACKREST-VERSION]: --buffer-size=[BUFFER-SIZE] --compress-level=3 --compress-type=zst --config=[TEST_PATH]/db-primary/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-primary/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-primary/log[] --no-log-timestamp --no-online --pg1-path=[TEST_PATH]/db-primary/db/base --protocol-timeout=60 --repo1-path=[TEST_PATH]/db-primary/repo --repo1-retention-full=2 --stanza=db --start-fast --type=full
-P01   INFO: backup file [TEST_PATH]/db-primary/db/base/pg_xlog/RECOVERYXLOG (16MB, 33%) checksum 488ba4b8b98acc510bce86b8f16e3c1ed9886a29
+P01   INFO: backup file [TEST_PATH]/db-primary/db/base/pg_xlog/RECOVERYXLOG (16MB, 33%) checksum 51a8525d254c01f5edddda30b7fe697c7e44705c
-P01   INFO: backup file [TEST_PATH]/db-primary/db/base/pg_xlog/000000010000000100000002 (16MB, 66%) checksum 488ba4b8b98acc510bce86b8f16e3c1ed9886a29
+P01   INFO: backup file [TEST_PATH]/db-primary/db/base/pg_xlog/000000010000000100000002 (16MB, 66%) checksum 51a8525d254c01f5edddda30b7fe697c7e44705c
-P01   INFO: backup file [TEST_PATH]/db-primary/db/base/pg_xlog/000000010000000100000001 (16MB, 99%) checksum e40de8cea99dd469c3efb47f5f33a73c7390fb9c
+P01   INFO: backup file [TEST_PATH]/db-primary/db/base/pg_xlog/000000010000000100000001 (16MB, 99%) checksum f92539dea1f9482e2946c1138eeeecdea29d7f19
 P01   INFO: backup file [TEST_PATH]/db-primary/db/base/global/pg_control (8KB, 100%) checksum 4c77c900f7af0d9ab13fa9982051a42e0b637f6c
 P01   INFO: backup file [TEST_PATH]/db-primary/db/base/pg_xlog/archive_status/000000010000000100000002.ready (0B, 100%)
 P01   INFO: backup file [TEST_PATH]/db-primary/db/base/pg_xlog/archive_status/000000010000000100000001.ready (0B, 100%)
@@ -441,9 +441,9 @@ diff backup - diff changed to full backup (db-primary host)
 ------------------------------------------------------------------------------------------------------------------------------------
 P00   INFO: backup command begin [BACKREST-VERSION]: --buffer-size=[BUFFER-SIZE] --compress-level=3 --compress-type=zst --config=[TEST_PATH]/db-primary/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-primary/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-primary/log[] --no-log-timestamp --no-online --pg1-path=[TEST_PATH]/db-primary/db/base --protocol-timeout=60 --repo1-path=[TEST_PATH]/db-primary/repo --repo1-retention-full=2 --stanza=db --start-fast --type=diff
 P00   WARN: no prior backup exists, diff backup has been changed to full
-P01   INFO: backup file [TEST_PATH]/db-primary/db/base/pg_xlog/RECOVERYXLOG (16MB, 33%) checksum 488ba4b8b98acc510bce86b8f16e3c1ed9886a29
+P01   INFO: backup file [TEST_PATH]/db-primary/db/base/pg_xlog/RECOVERYXLOG (16MB, 33%) checksum 51a8525d254c01f5edddda30b7fe697c7e44705c
-P01   INFO: backup file [TEST_PATH]/db-primary/db/base/pg_xlog/000000010000000100000002 (16MB, 66%) checksum 488ba4b8b98acc510bce86b8f16e3c1ed9886a29
+P01   INFO: backup file [TEST_PATH]/db-primary/db/base/pg_xlog/000000010000000100000002 (16MB, 66%) checksum 51a8525d254c01f5edddda30b7fe697c7e44705c
-P01   INFO: backup file [TEST_PATH]/db-primary/db/base/pg_xlog/000000010000000100000001 (16MB, 99%) checksum 65b64687e7bfd1a918c6144f84a133f03e50ac57
+P01   INFO: backup file [TEST_PATH]/db-primary/db/base/pg_xlog/000000010000000100000001 (16MB, 99%) checksum 762dae884fdccb805c5f3283662ea0f8da55f228
 P01   INFO: backup file [TEST_PATH]/db-primary/db/base/global/pg_control (8KB, 100%) checksum 4969435f3b36bfaa0f5a486bef97f1988a135520
 P01   INFO: backup file [TEST_PATH]/db-primary/db/base/pg_xlog/archive_status/000000010000000100000002.ready (0B, 100%)
 P01   INFO: backup file [TEST_PATH]/db-primary/db/base/pg_xlog/archive_status/000000010000000100000001.ready (0B, 100%)

test/expect/mock-stanza-002.log

@@ -320,9 +320,9 @@ full backup - create first full backup  (backup host)
 > [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --repo1-retention-full=2 --no-online --type=full --stanza=db backup
 ------------------------------------------------------------------------------------------------------------------------------------
 P00   INFO: backup command begin [BACKREST-VERSION]: --buffer-size=[BUFFER-SIZE] --compress-level=3 --compress-level-network=1 --compress-type=lz4 --config=[TEST_PATH]/backup/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/backup/log[] --no-log-timestamp --no-online --pg1-host=db-primary --pg1-host-cmd=[BACKREST-BIN] --pg1-host-config=[TEST_PATH]/db-primary/pgbackrest.conf --pg1-host-user=[USER-1] --pg1-path=[TEST_PATH]/db-primary/db/base --protocol-timeout=60 --repo1-azure-account=<redacted> --repo1-azure-container=azContainer --repo1-azure-host=azure --repo1-azure-key=<redacted> --no-repo1-azure-verify-tls --repo1-cipher-pass=<redacted> --repo1-cipher-type=aes-256-cbc --repo1-path=/ --repo1-retention-full=2 --repo1-type=azure --stanza=db --start-fast --type=full
-P01   INFO: backup file db-primary:[TEST_PATH]/db-primary/db/base/pg_xlog/RECOVERYXLOG (16MB, 33%) checksum 488ba4b8b98acc510bce86b8f16e3c1ed9886a29
+P01   INFO: backup file db-primary:[TEST_PATH]/db-primary/db/base/pg_xlog/RECOVERYXLOG (16MB, 33%) checksum 51a8525d254c01f5edddda30b7fe697c7e44705c
-P01   INFO: backup file db-primary:[TEST_PATH]/db-primary/db/base/pg_xlog/000000010000000100000002 (16MB, 66%) checksum 488ba4b8b98acc510bce86b8f16e3c1ed9886a29
+P01   INFO: backup file db-primary:[TEST_PATH]/db-primary/db/base/pg_xlog/000000010000000100000002 (16MB, 66%) checksum 51a8525d254c01f5edddda30b7fe697c7e44705c
-P01   INFO: backup file db-primary:[TEST_PATH]/db-primary/db/base/pg_xlog/000000010000000100000001 (16MB, 99%) checksum e40de8cea99dd469c3efb47f5f33a73c7390fb9c
+P01   INFO: backup file db-primary:[TEST_PATH]/db-primary/db/base/pg_xlog/000000010000000100000001 (16MB, 99%) checksum f92539dea1f9482e2946c1138eeeecdea29d7f19
 P01   INFO: backup file db-primary:[TEST_PATH]/db-primary/db/base/global/pg_control (8KB, 100%) checksum 4c77c900f7af0d9ab13fa9982051a42e0b637f6c
 P01   INFO: backup file db-primary:[TEST_PATH]/db-primary/db/base/pg_xlog/archive_status/000000010000000100000002.ready (0B, 100%)
 P01   INFO: backup file db-primary:[TEST_PATH]/db-primary/db/base/pg_xlog/archive_status/000000010000000100000001.ready (0B, 100%)
@@ -526,9 +526,9 @@ diff backup - diff changed to full backup (backup host)
 ------------------------------------------------------------------------------------------------------------------------------------
 P00   INFO: backup command begin [BACKREST-VERSION]: --buffer-size=[BUFFER-SIZE] --compress-level=3 --compress-level-network=1 --compress-type=lz4 --config=[TEST_PATH]/backup/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/backup/log[] --no-log-timestamp --no-online --pg1-host=db-primary --pg1-host-cmd=[BACKREST-BIN] --pg1-host-config=[TEST_PATH]/db-primary/pgbackrest.conf --pg1-host-user=[USER-1] --pg1-path=[TEST_PATH]/db-primary/db/base --protocol-timeout=60 --repo1-azure-account=<redacted> --repo1-azure-container=azContainer --repo1-azure-host=azure --repo1-azure-key=<redacted> --no-repo1-azure-verify-tls --repo1-cipher-pass=<redacted> --repo1-cipher-type=aes-256-cbc --repo1-path=/ --repo1-retention-full=2 --repo1-type=azure --stanza=db --start-fast --type=diff
 P00   WARN: no prior backup exists, diff backup has been changed to full
-P01   INFO: backup file db-primary:[TEST_PATH]/db-primary/db/base/pg_xlog/RECOVERYXLOG (16MB, 33%) checksum 488ba4b8b98acc510bce86b8f16e3c1ed9886a29
+P01   INFO: backup file db-primary:[TEST_PATH]/db-primary/db/base/pg_xlog/RECOVERYXLOG (16MB, 33%) checksum 51a8525d254c01f5edddda30b7fe697c7e44705c
-P01   INFO: backup file db-primary:[TEST_PATH]/db-primary/db/base/pg_xlog/000000010000000100000002 (16MB, 66%) checksum 488ba4b8b98acc510bce86b8f16e3c1ed9886a29
+P01   INFO: backup file db-primary:[TEST_PATH]/db-primary/db/base/pg_xlog/000000010000000100000002 (16MB, 66%) checksum 51a8525d254c01f5edddda30b7fe697c7e44705c
-P01   INFO: backup file db-primary:[TEST_PATH]/db-primary/db/base/pg_xlog/000000010000000100000001 (16MB, 99%) checksum 65b64687e7bfd1a918c6144f84a133f03e50ac57
+P01   INFO: backup file db-primary:[TEST_PATH]/db-primary/db/base/pg_xlog/000000010000000100000001 (16MB, 99%) checksum 762dae884fdccb805c5f3283662ea0f8da55f228
 P01   INFO: backup file db-primary:[TEST_PATH]/db-primary/db/base/global/pg_control (8KB, 100%) checksum 4969435f3b36bfaa0f5a486bef97f1988a135520
 P01   INFO: backup file db-primary:[TEST_PATH]/db-primary/db/base/pg_xlog/archive_status/000000010000000100000002.ready (0B, 100%)
 P01   INFO: backup file db-primary:[TEST_PATH]/db-primary/db/base/pg_xlog/archive_status/000000010000000100000001.ready (0B, 100%)

test/lib/pgBackRestTest/Env/HostEnvTest.pm

@@ -458,6 +458,9 @@ sub walGenerateContent
     # Add the system identifier
     $tWalContent .= pack('Q', $self->dbSysId($strPgVersion));
 
+    # Add segment size
+    $tWalContent .= pack('L', PG_WAL_SEGMENT_SIZE);
+
     # Add the source number to produce WAL segments with different checksums
     $tWalContent .= pack('S', $iSourceNo);
 

test/src/module/command/archivePushTest.c

@@ -271,7 +271,7 @@ testRun(void)
                 storageTest,
                 strNewFmt(
                     "repo/archive/test/11-1/0000000100000001/000000010000000100000001-%s.gz",
-                    TEST_64BIT() ? "3e5ecd22712f319b2420d5b901fd29f4f6be2336" : "6903dce7e3cd64ba9a6134056405eaeb8dedcd37")),
+                    TEST_64BIT() ? "aae7591a1dbc58f21d0d004886075094f622e6dd" : "28a13fd8cf6fcd9f9a8108aed4c8bcc58040863a")),
             true, "check repo for WAL file");
 
         TEST_RESULT_VOID(cmdArchivePush(), "push the WAL segment again");
@@ -309,7 +309,7 @@ testRun(void)
                 storageTest,
                 strNewFmt(
                     "repo/archive/test/11-1/0000000100000001/000000010000000100000002-%s.gz",
-                    TEST_64BIT() ? "edad2f5a9d8a03ee3c09e8ce92c771e0d20232f5" : "e7c81f5513e0c6e3f19b9dbfc450019165994dda")),
+                    TEST_64BIT() ? "755defa48a0a0872767b6dea49bdd3b64902f147" : "9c2a6ec4491a2118bcdc9b653366581d8821c982")),
             true, "check repo for WAL file");
 
         // Push a history file
@@ -416,7 +416,7 @@ testRun(void)
                 storageTest,
                 strNewFmt(
                     "repo/archive/test/11-1/0000000100000001/000000010000000100000002-%s",
-                    TEST_64BIT() ? "edad2f5a9d8a03ee3c09e8ce92c771e0d20232f5" : "e7c81f5513e0c6e3f19b9dbfc450019165994dda")),
+                    TEST_64BIT() ? "755defa48a0a0872767b6dea49bdd3b64902f147" : "9c2a6ec4491a2118bcdc9b653366581d8821c982")),
             true, "check repo for WAL file");
     }
 
@@ -585,7 +585,7 @@ testRun(void)
                 storageTest,
                 strNewFmt(
                     "repo/archive/test/9.4-1/0000000100000001/000000010000000100000001-%s",
-                    TEST_64BIT() ? "f81d63dd5e258cd607534f3531bbd71442797e37" : "02d228126281e8e102b35a2737e45a0527946296")),
+                    TEST_64BIT() ? "ce58ec898a080e59d71dce49051843b03a497d99" : "765ee94e4f65d081f2dc3b8e77556da95bc31cbf")),
             true, "check repo for WAL file");
 
         // Direct tests of the async function
@@ -650,7 +650,7 @@ testRun(void)
                 storageTest,
                 strNewFmt(
                     "repo/archive/test/9.4-1/0000000100000001/000000010000000100000001-%s",
-                    TEST_64BIT() ? "f81d63dd5e258cd607534f3531bbd71442797e37" : "02d228126281e8e102b35a2737e45a0527946296")),
+                    TEST_64BIT() ? "ce58ec898a080e59d71dce49051843b03a497d99" : "765ee94e4f65d081f2dc3b8e77556da95bc31cbf")),
             true, "check repo for WAL 1 file");
 
         TEST_RESULT_STR_Z(
@@ -679,7 +679,7 @@ testRun(void)
                 storageTest,
                 strNewFmt(
                     "repo/archive/test/9.4-1/0000000100000001/000000010000000100000002-%s",
-                    TEST_64BIT() ? "0aea6fa5d53500ce548b84a86bc3a29ae77fa048" : "408822a89ef44ef6740e785743bf1b870d8024a2")),
+                    TEST_64BIT() ? "251820f85b31554bbf33061fce7dabf18c16afe0" : "ae8cb1fe45d40bc364df6bc6e8803774f0e3eaeb")),
             true, "check repo for WAL 2 file");
 
         TEST_RESULT_STR_Z(

test/src/module/postgres/interfaceTest.c

@@ -224,22 +224,32 @@ testRun(void)
 
         //--------------------------------------------------------------------------------------------------------------------------
         memset(bufPtr(result), 0, bufSize(result));
-        pgWalTestToBuffer((PgWal){.version = PG_VERSION_11, .systemId = 0xECAFECAF}, result);
+        pgWalTestToBuffer(
+            (PgWal){.version = PG_VERSION_11, .systemId = 0xECAFECAF, .size = PG_WAL_SEGMENT_SIZE_DEFAULT * 2}, result);
         storagePutP(storageNewWriteP(storageTest, walFile), result);
 
         PgWal info = {0};
         TEST_ASSIGN(info, pgWalFromFile(walFile), "get wal info v11");
         TEST_RESULT_UINT(info.systemId, 0xECAFECAF, "   check system id");
         TEST_RESULT_UINT(info.version, PG_VERSION_11, "   check version");
+        TEST_RESULT_UINT(info.size, PG_WAL_SEGMENT_SIZE_DEFAULT * 2, "   check size");
 
         //--------------------------------------------------------------------------------------------------------------------------
         memset(bufPtr(result), 0, bufSize(result));
-        pgWalTestToBuffer((PgWal){.version = PG_VERSION_83, .systemId = 0xEAEAEAEA}, result);
+        pgWalTestToBuffer(
+            (PgWal){.version = PG_VERSION_83, .systemId = 0xEAEAEAEA, .size = PG_WAL_SEGMENT_SIZE_DEFAULT * 2}, result);
+
+        TEST_ERROR(pgWalFromBuffer(result), FormatError, "wal segment size is 33554432 but must be 16777216 for PostgreSQL <= 10");
+
+        //--------------------------------------------------------------------------------------------------------------------------
+        memset(bufPtr(result), 0, bufSize(result));
+        pgWalTestToBuffer((PgWal){.version = PG_VERSION_83, .systemId = 0xEAEAEAEA, .size = PG_WAL_SEGMENT_SIZE_DEFAULT}, result);
         storagePutP(storageNewWriteP(storageTest, walFile), result);
 
         TEST_ASSIGN(info, pgWalFromFile(walFile), "get wal info v8.3");
         TEST_RESULT_UINT(info.systemId, 0xEAEAEAEA, "   check system id");
         TEST_RESULT_UINT(info.version, PG_VERSION_83, "   check version");
+        TEST_RESULT_UINT(info.size, PG_WAL_SEGMENT_SIZE_DEFAULT, "   check size");
     }
 
     // *****************************************************************************************************************************