diff --git a/src/postgres/interface.c b/src/postgres/interface.c index 6699dfe0c..3ea9e80e8 100644 --- a/src/postgres/interface.c +++ b/src/postgres/interface.c @@ -386,6 +386,27 @@ pgCatalogVersion(unsigned int pgVersion) FUNCTION_TEST_RETURN(pgInterfaceVersion(pgVersion)->catalogVersion()); } +/*********************************************************************************************************************************** +Check expected WAL segment size for older PostgreSQL versions +***********************************************************************************************************************************/ +static void +pgWalSegmentSizeCheck(unsigned int pgVersion, unsigned int walSegmentSize) +{ + FUNCTION_TEST_BEGIN(); + FUNCTION_TEST_PARAM(UINT, pgVersion); + FUNCTION_TEST_PARAM(UINT, walSegmentSize); + FUNCTION_TEST_END(); + + if (pgVersion < PG_VERSION_11 && walSegmentSize != PG_WAL_SEGMENT_SIZE_DEFAULT) + { + THROW_FMT( + FormatError, "wal segment size is %u but must be %u for " PG_NAME " <= " PG_VERSION_10_STR, walSegmentSize, + PG_WAL_SEGMENT_SIZE_DEFAULT); + } + + FUNCTION_TEST_RETURN_VOID(); +} + /**********************************************************************************************************************************/ PgControl pgControlFromBuffer(const Buffer *controlFile) @@ -425,12 +446,7 @@ pgControlFromBuffer(const Buffer *controlFile) result.version = interface->version; // Check the segment size - if (result.version < PG_VERSION_11 && result.walSegmentSize != PG_WAL_SEGMENT_SIZE_DEFAULT) - { - THROW_FMT( - FormatError, "wal segment size is %u but must be %u for " PG_NAME " <= " PG_VERSION_10_STR, result.walSegmentSize, - PG_WAL_SEGMENT_SIZE_DEFAULT); - } + pgWalSegmentSizeCheck(result.version, result.walSegmentSize); // Check the page size if (result.pageSize != PG_PAGE_SIZE_DEFAULT) @@ -526,6 +542,9 @@ pgWalFromBuffer(const Buffer *walBuffer) PgWal result = interface->wal(bufPtrConst(walBuffer)); result.version = interface->version; + // Check the segment size + pgWalSegmentSizeCheck(result.version, result.size); + FUNCTION_LOG_RETURN(PG_WAL, result); } @@ -790,6 +809,10 @@ pgWalTestToBuffer(PgWal pgWal, Buffer *walBuffer) ASSERT(walBuffer != NULL); + // Set default WAL segment size if not specified + if (pgWal.size == 0) + pgWal.size = PG_WAL_SEGMENT_SIZE_DEFAULT; + // Generate WAL pgInterfaceVersion(pgWal.version)->walTest(pgWal, bufPtr(walBuffer)); diff --git a/src/postgres/interface.h b/src/postgres/interface.h index 05d8d0b32..8e9cea0cd 100644 --- a/src/postgres/interface.h +++ b/src/postgres/interface.h @@ -106,6 +106,7 @@ PostgreSQL WAL Info typedef struct PgWal { unsigned int version; + unsigned int size; uint64_t systemId; } PgWal; diff --git a/src/postgres/interface/version.intern.h b/src/postgres/interface/version.intern.h index 2afd8de46..e42ac6b2e 100644 --- a/src/postgres/interface/version.intern.h +++ b/src/postgres/interface/version.intern.h @@ -191,6 +191,7 @@ Read the version specific WAL header into a general data structure return (PgWal) \ { \ .systemId = ((XLogLongPageHeaderData *)walFile)->xlp_sysid, \ + .size = ((XLogLongPageHeaderData *)walFile)->xlp_seg_size, \ }; \ } @@ -210,6 +211,7 @@ Create a WAL file file for testing ((XLogLongPageHeaderData *)buffer)->std.xlp_magic = XLOG_PAGE_MAGIC; \ ((XLogLongPageHeaderData *)buffer)->std.xlp_info = XLP_LONG_HEADER; \ ((XLogLongPageHeaderData *)buffer)->xlp_sysid = pgWal.systemId; \ + ((XLogLongPageHeaderData *)buffer)->xlp_seg_size = pgWal.size; \ } #endif diff --git a/test/expect/mock-stanza-001.log b/test/expect/mock-stanza-001.log index 60887d830..cf04c64aa 100644 --- a/test/expect/mock-stanza-001.log +++ b/test/expect/mock-stanza-001.log @@ -296,9 +296,9 @@ full backup - create first full backup (db-primary host) > [CONTAINER-EXEC] db-primary [BACKREST-BIN] --config=[TEST_PATH]/db-primary/pgbackrest.conf --repo1-retention-full=2 --no-online --type=full --stanza=db backup ------------------------------------------------------------------------------------------------------------------------------------ P00 INFO: backup command begin [BACKREST-VERSION]: --buffer-size=[BUFFER-SIZE] --compress-level=3 --compress-type=zst --config=[TEST_PATH]/db-primary/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-primary/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-primary/log[] --no-log-timestamp --no-online --pg1-path=[TEST_PATH]/db-primary/db/base --protocol-timeout=60 --repo1-path=[TEST_PATH]/db-primary/repo --repo1-retention-full=2 --stanza=db --start-fast --type=full -P01 INFO: backup file [TEST_PATH]/db-primary/db/base/pg_xlog/RECOVERYXLOG (16MB, 33%) checksum 488ba4b8b98acc510bce86b8f16e3c1ed9886a29 -P01 INFO: backup file [TEST_PATH]/db-primary/db/base/pg_xlog/000000010000000100000002 (16MB, 66%) checksum 488ba4b8b98acc510bce86b8f16e3c1ed9886a29 -P01 INFO: backup file [TEST_PATH]/db-primary/db/base/pg_xlog/000000010000000100000001 (16MB, 99%) checksum e40de8cea99dd469c3efb47f5f33a73c7390fb9c +P01 INFO: backup file [TEST_PATH]/db-primary/db/base/pg_xlog/RECOVERYXLOG (16MB, 33%) checksum 51a8525d254c01f5edddda30b7fe697c7e44705c +P01 INFO: backup file [TEST_PATH]/db-primary/db/base/pg_xlog/000000010000000100000002 (16MB, 66%) checksum 51a8525d254c01f5edddda30b7fe697c7e44705c +P01 INFO: backup file [TEST_PATH]/db-primary/db/base/pg_xlog/000000010000000100000001 (16MB, 99%) checksum f92539dea1f9482e2946c1138eeeecdea29d7f19 P01 INFO: backup file [TEST_PATH]/db-primary/db/base/global/pg_control (8KB, 100%) checksum 4c77c900f7af0d9ab13fa9982051a42e0b637f6c P01 INFO: backup file [TEST_PATH]/db-primary/db/base/pg_xlog/archive_status/000000010000000100000002.ready (0B, 100%) P01 INFO: backup file [TEST_PATH]/db-primary/db/base/pg_xlog/archive_status/000000010000000100000001.ready (0B, 100%) @@ -441,9 +441,9 @@ diff backup - diff changed to full backup (db-primary host) ------------------------------------------------------------------------------------------------------------------------------------ P00 INFO: backup command begin [BACKREST-VERSION]: --buffer-size=[BUFFER-SIZE] --compress-level=3 --compress-type=zst --config=[TEST_PATH]/db-primary/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-primary/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-primary/log[] --no-log-timestamp --no-online --pg1-path=[TEST_PATH]/db-primary/db/base --protocol-timeout=60 --repo1-path=[TEST_PATH]/db-primary/repo --repo1-retention-full=2 --stanza=db --start-fast --type=diff P00 WARN: no prior backup exists, diff backup has been changed to full -P01 INFO: backup file [TEST_PATH]/db-primary/db/base/pg_xlog/RECOVERYXLOG (16MB, 33%) checksum 488ba4b8b98acc510bce86b8f16e3c1ed9886a29 -P01 INFO: backup file [TEST_PATH]/db-primary/db/base/pg_xlog/000000010000000100000002 (16MB, 66%) checksum 488ba4b8b98acc510bce86b8f16e3c1ed9886a29 -P01 INFO: backup file [TEST_PATH]/db-primary/db/base/pg_xlog/000000010000000100000001 (16MB, 99%) checksum 65b64687e7bfd1a918c6144f84a133f03e50ac57 +P01 INFO: backup file [TEST_PATH]/db-primary/db/base/pg_xlog/RECOVERYXLOG (16MB, 33%) checksum 51a8525d254c01f5edddda30b7fe697c7e44705c +P01 INFO: backup file [TEST_PATH]/db-primary/db/base/pg_xlog/000000010000000100000002 (16MB, 66%) checksum 51a8525d254c01f5edddda30b7fe697c7e44705c +P01 INFO: backup file [TEST_PATH]/db-primary/db/base/pg_xlog/000000010000000100000001 (16MB, 99%) checksum 762dae884fdccb805c5f3283662ea0f8da55f228 P01 INFO: backup file [TEST_PATH]/db-primary/db/base/global/pg_control (8KB, 100%) checksum 4969435f3b36bfaa0f5a486bef97f1988a135520 P01 INFO: backup file [TEST_PATH]/db-primary/db/base/pg_xlog/archive_status/000000010000000100000002.ready (0B, 100%) P01 INFO: backup file [TEST_PATH]/db-primary/db/base/pg_xlog/archive_status/000000010000000100000001.ready (0B, 100%) diff --git a/test/expect/mock-stanza-002.log b/test/expect/mock-stanza-002.log index d83cd58f9..3ba6294b1 100644 --- a/test/expect/mock-stanza-002.log +++ b/test/expect/mock-stanza-002.log @@ -320,9 +320,9 @@ full backup - create first full backup (backup host) > [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --repo1-retention-full=2 --no-online --type=full --stanza=db backup ------------------------------------------------------------------------------------------------------------------------------------ P00 INFO: backup command begin [BACKREST-VERSION]: --buffer-size=[BUFFER-SIZE] --compress-level=3 --compress-level-network=1 --compress-type=lz4 --config=[TEST_PATH]/backup/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/backup/log[] --no-log-timestamp --no-online --pg1-host=db-primary --pg1-host-cmd=[BACKREST-BIN] --pg1-host-config=[TEST_PATH]/db-primary/pgbackrest.conf --pg1-host-user=[USER-1] --pg1-path=[TEST_PATH]/db-primary/db/base --protocol-timeout=60 --repo1-azure-account= --repo1-azure-container=azContainer --repo1-azure-host=azure --repo1-azure-key= --no-repo1-azure-verify-tls --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=/ --repo1-retention-full=2 --repo1-type=azure --stanza=db --start-fast --type=full -P01 INFO: backup file db-primary:[TEST_PATH]/db-primary/db/base/pg_xlog/RECOVERYXLOG (16MB, 33%) checksum 488ba4b8b98acc510bce86b8f16e3c1ed9886a29 -P01 INFO: backup file db-primary:[TEST_PATH]/db-primary/db/base/pg_xlog/000000010000000100000002 (16MB, 66%) checksum 488ba4b8b98acc510bce86b8f16e3c1ed9886a29 -P01 INFO: backup file db-primary:[TEST_PATH]/db-primary/db/base/pg_xlog/000000010000000100000001 (16MB, 99%) checksum e40de8cea99dd469c3efb47f5f33a73c7390fb9c +P01 INFO: backup file db-primary:[TEST_PATH]/db-primary/db/base/pg_xlog/RECOVERYXLOG (16MB, 33%) checksum 51a8525d254c01f5edddda30b7fe697c7e44705c +P01 INFO: backup file db-primary:[TEST_PATH]/db-primary/db/base/pg_xlog/000000010000000100000002 (16MB, 66%) checksum 51a8525d254c01f5edddda30b7fe697c7e44705c +P01 INFO: backup file db-primary:[TEST_PATH]/db-primary/db/base/pg_xlog/000000010000000100000001 (16MB, 99%) checksum f92539dea1f9482e2946c1138eeeecdea29d7f19 P01 INFO: backup file db-primary:[TEST_PATH]/db-primary/db/base/global/pg_control (8KB, 100%) checksum 4c77c900f7af0d9ab13fa9982051a42e0b637f6c P01 INFO: backup file db-primary:[TEST_PATH]/db-primary/db/base/pg_xlog/archive_status/000000010000000100000002.ready (0B, 100%) P01 INFO: backup file db-primary:[TEST_PATH]/db-primary/db/base/pg_xlog/archive_status/000000010000000100000001.ready (0B, 100%) @@ -526,9 +526,9 @@ diff backup - diff changed to full backup (backup host) ------------------------------------------------------------------------------------------------------------------------------------ P00 INFO: backup command begin [BACKREST-VERSION]: --buffer-size=[BUFFER-SIZE] --compress-level=3 --compress-level-network=1 --compress-type=lz4 --config=[TEST_PATH]/backup/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/backup/log[] --no-log-timestamp --no-online --pg1-host=db-primary --pg1-host-cmd=[BACKREST-BIN] --pg1-host-config=[TEST_PATH]/db-primary/pgbackrest.conf --pg1-host-user=[USER-1] --pg1-path=[TEST_PATH]/db-primary/db/base --protocol-timeout=60 --repo1-azure-account= --repo1-azure-container=azContainer --repo1-azure-host=azure --repo1-azure-key= --no-repo1-azure-verify-tls --repo1-cipher-pass= --repo1-cipher-type=aes-256-cbc --repo1-path=/ --repo1-retention-full=2 --repo1-type=azure --stanza=db --start-fast --type=diff P00 WARN: no prior backup exists, diff backup has been changed to full -P01 INFO: backup file db-primary:[TEST_PATH]/db-primary/db/base/pg_xlog/RECOVERYXLOG (16MB, 33%) checksum 488ba4b8b98acc510bce86b8f16e3c1ed9886a29 -P01 INFO: backup file db-primary:[TEST_PATH]/db-primary/db/base/pg_xlog/000000010000000100000002 (16MB, 66%) checksum 488ba4b8b98acc510bce86b8f16e3c1ed9886a29 -P01 INFO: backup file db-primary:[TEST_PATH]/db-primary/db/base/pg_xlog/000000010000000100000001 (16MB, 99%) checksum 65b64687e7bfd1a918c6144f84a133f03e50ac57 +P01 INFO: backup file db-primary:[TEST_PATH]/db-primary/db/base/pg_xlog/RECOVERYXLOG (16MB, 33%) checksum 51a8525d254c01f5edddda30b7fe697c7e44705c +P01 INFO: backup file db-primary:[TEST_PATH]/db-primary/db/base/pg_xlog/000000010000000100000002 (16MB, 66%) checksum 51a8525d254c01f5edddda30b7fe697c7e44705c +P01 INFO: backup file db-primary:[TEST_PATH]/db-primary/db/base/pg_xlog/000000010000000100000001 (16MB, 99%) checksum 762dae884fdccb805c5f3283662ea0f8da55f228 P01 INFO: backup file db-primary:[TEST_PATH]/db-primary/db/base/global/pg_control (8KB, 100%) checksum 4969435f3b36bfaa0f5a486bef97f1988a135520 P01 INFO: backup file db-primary:[TEST_PATH]/db-primary/db/base/pg_xlog/archive_status/000000010000000100000002.ready (0B, 100%) P01 INFO: backup file db-primary:[TEST_PATH]/db-primary/db/base/pg_xlog/archive_status/000000010000000100000001.ready (0B, 100%) diff --git a/test/lib/pgBackRestTest/Env/HostEnvTest.pm b/test/lib/pgBackRestTest/Env/HostEnvTest.pm index 8f960c585..60f3f7584 100644 --- a/test/lib/pgBackRestTest/Env/HostEnvTest.pm +++ b/test/lib/pgBackRestTest/Env/HostEnvTest.pm @@ -458,6 +458,9 @@ sub walGenerateContent # Add the system identifier $tWalContent .= pack('Q', $self->dbSysId($strPgVersion)); + # Add segment size + $tWalContent .= pack('L', PG_WAL_SEGMENT_SIZE); + # Add the source number to produce WAL segments with different checksums $tWalContent .= pack('S', $iSourceNo); diff --git a/test/src/module/command/archivePushTest.c b/test/src/module/command/archivePushTest.c index 7e391b887..2ea173647 100644 --- a/test/src/module/command/archivePushTest.c +++ b/test/src/module/command/archivePushTest.c @@ -271,7 +271,7 @@ testRun(void) storageTest, strNewFmt( "repo/archive/test/11-1/0000000100000001/000000010000000100000001-%s.gz", - TEST_64BIT() ? "3e5ecd22712f319b2420d5b901fd29f4f6be2336" : "6903dce7e3cd64ba9a6134056405eaeb8dedcd37")), + TEST_64BIT() ? "aae7591a1dbc58f21d0d004886075094f622e6dd" : "28a13fd8cf6fcd9f9a8108aed4c8bcc58040863a")), true, "check repo for WAL file"); TEST_RESULT_VOID(cmdArchivePush(), "push the WAL segment again"); @@ -309,7 +309,7 @@ testRun(void) storageTest, strNewFmt( "repo/archive/test/11-1/0000000100000001/000000010000000100000002-%s.gz", - TEST_64BIT() ? "edad2f5a9d8a03ee3c09e8ce92c771e0d20232f5" : "e7c81f5513e0c6e3f19b9dbfc450019165994dda")), + TEST_64BIT() ? "755defa48a0a0872767b6dea49bdd3b64902f147" : "9c2a6ec4491a2118bcdc9b653366581d8821c982")), true, "check repo for WAL file"); // Push a history file @@ -416,7 +416,7 @@ testRun(void) storageTest, strNewFmt( "repo/archive/test/11-1/0000000100000001/000000010000000100000002-%s", - TEST_64BIT() ? "edad2f5a9d8a03ee3c09e8ce92c771e0d20232f5" : "e7c81f5513e0c6e3f19b9dbfc450019165994dda")), + TEST_64BIT() ? "755defa48a0a0872767b6dea49bdd3b64902f147" : "9c2a6ec4491a2118bcdc9b653366581d8821c982")), true, "check repo for WAL file"); } @@ -585,7 +585,7 @@ testRun(void) storageTest, strNewFmt( "repo/archive/test/9.4-1/0000000100000001/000000010000000100000001-%s", - TEST_64BIT() ? "f81d63dd5e258cd607534f3531bbd71442797e37" : "02d228126281e8e102b35a2737e45a0527946296")), + TEST_64BIT() ? "ce58ec898a080e59d71dce49051843b03a497d99" : "765ee94e4f65d081f2dc3b8e77556da95bc31cbf")), true, "check repo for WAL file"); // Direct tests of the async function @@ -650,7 +650,7 @@ testRun(void) storageTest, strNewFmt( "repo/archive/test/9.4-1/0000000100000001/000000010000000100000001-%s", - TEST_64BIT() ? "f81d63dd5e258cd607534f3531bbd71442797e37" : "02d228126281e8e102b35a2737e45a0527946296")), + TEST_64BIT() ? "ce58ec898a080e59d71dce49051843b03a497d99" : "765ee94e4f65d081f2dc3b8e77556da95bc31cbf")), true, "check repo for WAL 1 file"); TEST_RESULT_STR_Z( @@ -679,7 +679,7 @@ testRun(void) storageTest, strNewFmt( "repo/archive/test/9.4-1/0000000100000001/000000010000000100000002-%s", - TEST_64BIT() ? "0aea6fa5d53500ce548b84a86bc3a29ae77fa048" : "408822a89ef44ef6740e785743bf1b870d8024a2")), + TEST_64BIT() ? "251820f85b31554bbf33061fce7dabf18c16afe0" : "ae8cb1fe45d40bc364df6bc6e8803774f0e3eaeb")), true, "check repo for WAL 2 file"); TEST_RESULT_STR_Z( diff --git a/test/src/module/postgres/interfaceTest.c b/test/src/module/postgres/interfaceTest.c index ee71b964e..3a1f69d02 100644 --- a/test/src/module/postgres/interfaceTest.c +++ b/test/src/module/postgres/interfaceTest.c @@ -224,22 +224,32 @@ testRun(void) //-------------------------------------------------------------------------------------------------------------------------- memset(bufPtr(result), 0, bufSize(result)); - pgWalTestToBuffer((PgWal){.version = PG_VERSION_11, .systemId = 0xECAFECAF}, result); + pgWalTestToBuffer( + (PgWal){.version = PG_VERSION_11, .systemId = 0xECAFECAF, .size = PG_WAL_SEGMENT_SIZE_DEFAULT * 2}, result); storagePutP(storageNewWriteP(storageTest, walFile), result); PgWal info = {0}; TEST_ASSIGN(info, pgWalFromFile(walFile), "get wal info v11"); TEST_RESULT_UINT(info.systemId, 0xECAFECAF, " check system id"); TEST_RESULT_UINT(info.version, PG_VERSION_11, " check version"); + TEST_RESULT_UINT(info.size, PG_WAL_SEGMENT_SIZE_DEFAULT * 2, " check size"); //-------------------------------------------------------------------------------------------------------------------------- memset(bufPtr(result), 0, bufSize(result)); - pgWalTestToBuffer((PgWal){.version = PG_VERSION_83, .systemId = 0xEAEAEAEA}, result); + pgWalTestToBuffer( + (PgWal){.version = PG_VERSION_83, .systemId = 0xEAEAEAEA, .size = PG_WAL_SEGMENT_SIZE_DEFAULT * 2}, result); + + TEST_ERROR(pgWalFromBuffer(result), FormatError, "wal segment size is 33554432 but must be 16777216 for PostgreSQL <= 10"); + + //-------------------------------------------------------------------------------------------------------------------------- + memset(bufPtr(result), 0, bufSize(result)); + pgWalTestToBuffer((PgWal){.version = PG_VERSION_83, .systemId = 0xEAEAEAEA, .size = PG_WAL_SEGMENT_SIZE_DEFAULT}, result); storagePutP(storageNewWriteP(storageTest, walFile), result); TEST_ASSIGN(info, pgWalFromFile(walFile), "get wal info v8.3"); TEST_RESULT_UINT(info.systemId, 0xEAEAEAEA, " check system id"); TEST_RESULT_UINT(info.version, PG_VERSION_83, " check version"); + TEST_RESULT_UINT(info.size, PG_WAL_SEGMENT_SIZE_DEFAULT, " check size"); } // *****************************************************************************************************************************