From 4722ad87a78f1085ef8d823b7460d191477e768e Mon Sep 17 00:00:00 2001 From: David Steele Date: Sun, 2 Oct 2022 17:41:31 -1000 Subject: [PATCH] Add test for differential file bundles to the command/backup unit. This test exposes a small logging issue. The bundle information for the matched delta on PG_VERSION is not correct. This issue will be fixed in the next commit. The information stored in the manifest *is* correct so this bug is essentially cosmetic. --- test/src/module/command/backupTest.c | 112 ++++++++++++++++++++++++++- 1 file changed, 110 insertions(+), 2 deletions(-) diff --git a/test/src/module/command/backupTest.c b/test/src/module/command/backupTest.c index df1e37312..2b6ef0e22 100644 --- a/test/src/module/command/backupTest.c +++ b/test/src/module/command/backupTest.c @@ -72,8 +72,10 @@ testBackupValidateList( for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(manifest); fileIdx++) { ManifestFilePack **const filePack = lstGet(manifest->pub.fileList, fileIdx); + ManifestFile file = manifestFileUnpack(manifest, *filePack); - if (manifestFileUnpack(manifest, *filePack).bundleId == bundleId) + // File bundle is part of this backup + if (file.bundleId == bundleId && file.reference == NULL) lstAdd(fileList, &filePack); } } @@ -200,6 +202,26 @@ testBackupValidateList( } } + // Check all files in manifest. Since the scan above maps from files to the manifest, any referenced files will not be checked. + // ----------------------------------------------------------------------------------------------------------------------------- + for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(manifest); fileIdx++) + { + ManifestFilePack **const filePack = lstGet(manifest->pub.fileList, fileIdx); + ManifestFile file = manifestFileUnpack(manifest, *filePack); + + // If compressed then set the repo-size to size so it will not be in test output. Even the same compression algorithm can + // give slightly different results based on the version so repo-size is not deterministic for compression. + if (manifestData->backupOptionCompressType != compressTypeNone) + file.sizeRepo = file.size; + + // Bundle id/offset are too noisy so remove them. They are verified against size/checksum and listed with the files. + file.bundleId = 0; + file.bundleOffset = 0; + + // Update changes to manifest file + manifestFilePackUpdate(manifest, filePack, &file); + } + return result; } @@ -3174,6 +3196,9 @@ testRun(void) cfgOptionSet(cfgOptRepoBundleSize, cfgSourceParam, VARINT64(PG_PAGE_SIZE_DEFAULT)); cfgOptionSet(cfgOptRepoBundleLimit, cfgSourceParam, VARINT64(PG_PAGE_SIZE_DEFAULT)); + // Zero-length file to be stored + HRN_STORAGE_PUT_EMPTY(storagePgWrite(), "zero", .timeModified = backupTimeStart); + // Zeroed file which passes page checksums Buffer *relation = bufNew(PG_PAGE_SIZE_DEFAULT * 3); memset(bufPtr(relation), 0, bufSize(relation)); @@ -3200,6 +3225,7 @@ testRun(void) "P00 INFO: execute non-exclusive pg_start_backup(): backup begins after the next regular checkpoint completes\n" "P00 INFO: backup start archive = 0000000105DB8EB000000000, lsn = 5db8eb0/0\n" "P00 INFO: check archive for segment 0000000105DB8EB000000000\n" + "P00 DETAIL: store zero-length file " TEST_PATH "/pg1/zero\n" "P00 DETAIL: store zero-length file " TEST_PATH "/pg1/pg_tblspc/32768/PG_11_201809051/1/5\n" "P01 DETAIL: backup file " TEST_PATH "/pg1/base/1/2 (24KB, [PCT]) checksum [SHA1]\n" "P01 DETAIL: backup file " TEST_PATH "/pg1/stuff.conf (bundle 1/0, 12B, [PCT]) checksum [SHA1]\n" @@ -3217,7 +3243,7 @@ testRun(void) "P00 DETAIL: copy segment 0000000105DB8EB000000000 to backup\n" "P00 DETAIL: copy segment 0000000105DB8EB000000001 to backup\n" "P00 INFO: new backup label = 20191030-014640F\n" - "P00 INFO: full backup size = [SIZE], file total = 13"); + "P00 INFO: full backup size = [SIZE], file total = 14"); TEST_RESULT_STR_Z( testBackupValidate(storageRepo(), STRDEF(STORAGE_REPO_BACKUP "/latest")), @@ -3270,6 +3296,7 @@ testRun(void) ",\"timestamp\":1500000000}\n" "pg_data/tablespace_map={\"checksum\":\"87fe624d7976c2144e10afcb7a9a49b071f35e9c\",\"size\":19" ",\"timestamp\":1572400002}\n" + "pg_data/zero={\"size\":0,\"timestamp\":1572400000}\n" "pg_tblspc/32768/PG_11_201809051/1/5={\"checksum-page\":true,\"size\":0,\"timestamp\":1572200000}\n" "\n" "[target:link]\n" @@ -3288,6 +3315,87 @@ testRun(void) "pg_tblspc/32768/PG_11_201809051/1={}\n", "compare file list"); } + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("online 11 diff backup with bundles"); + + backupTimeStart = BACKUP_EPOCH + 2600000; + + { + // Remove old pg data + HRN_STORAGE_PATH_REMOVE(storageTest, "pg1-data", .recurse = true); + HRN_STORAGE_PATH_REMOVE(storageTest, "pg1-tblspc", .recurse = true); + HRN_STORAGE_REMOVE(storageTest, "pg1"); + + // Update pg_control + HRN_PG_CONTROL_PUT(storagePgWrite(), PG_VERSION_11, .pageChecksum = false, .walSegmentSize = 2 * 1024 * 1024); + + // Update version + HRN_STORAGE_PUT_Z(storagePgWrite(), PG_FILE_PGVERSION, PG_VERSION_11_STR, .timeModified = backupTimeStart); + + // Load options + StringList *argList = strLstNew(); + hrnCfgArgRawZ(argList, cfgOptStanza, "test1"); + hrnCfgArgRaw(argList, cfgOptRepoPath, repoPath); + hrnCfgArgRaw(argList, cfgOptPgPath, pg1Path); + hrnCfgArgRawZ(argList, cfgOptRepoRetentionFull, "1"); + hrnCfgArgRawStrId(argList, cfgOptType, backupTypeDiff); + hrnCfgArgRawBool(argList, cfgOptRepoBundle, true); + hrnCfgArgRawBool(argList, cfgOptDelta, true); + HRN_CFG_LOAD(cfgCmdBackup, argList); + + // Zero-length file to be stored + HRN_STORAGE_PUT_EMPTY(storagePgWrite(), "zero", .timeModified = backupTimeStart); + + // Run backup + testBackupPqScriptP(PG_VERSION_11, backupTimeStart, .walCompressType = compressTypeGz, .walTotal = 2); + TEST_RESULT_VOID(testCmdBackup(), "backup"); + + TEST_RESULT_LOG( + "P00 INFO: last backup label = 20191030-014640F, version = 2.42dev\n" + "P00 WARN: diff backup cannot alter 'checksum-page' option to 'false', reset to 'true' from 20191030-014640F\n" + "P00 INFO: execute non-exclusive pg_start_backup(): backup begins after the next regular checkpoint completes\n" + "P00 INFO: backup start archive = 0000000105DBBF8000000000, lsn = 5dbbf80/0\n" + "P00 INFO: check archive for segment 0000000105DBBF8000000000\n" + "P01 DETAIL: backup file " TEST_PATH "/pg1/global/pg_control (bundle 1/0, 8KB, [PCT]) checksum [SHA1]\n" + "P01 DETAIL: match file from prior backup " TEST_PATH "/pg1/PG_VERSION (bundle 1/0, 2B, [PCT]) checksum [SHA1]\n" + "P00 DETAIL: reference pg_data/PG_VERSION to 20191030-014640F\n" + "P00 DETAIL: reference pg_data/zero to 20191030-014640F\n" + "P00 INFO: execute non-exclusive pg_stop_backup() and wait for all WAL segments to archive\n" + "P00 INFO: backup stop archive = 0000000105DBBF8000000001, lsn = 5dbbf80/300000\n" + "P00 DETAIL: wrote 'backup_label' file returned from pg_stop_backup()\n" + "P00 DETAIL: wrote 'tablespace_map' file returned from pg_stop_backup()\n" + "P00 INFO: check archive for segment(s) 0000000105DBBF8000000000:0000000105DBBF8000000001\n" + "P00 INFO: new backup label = 20191030-014640F_20191101-092000D\n" + "P00 INFO: diff backup size = [SIZE], file total = 5"); + + TEST_RESULT_STR_Z( + testBackupValidate(storageRepo(), STRDEF(STORAGE_REPO_BACKUP "/latest")), + ". {link, d=20191030-014640F_20191101-092000D}\n" + "bundle {path}\n" + "bundle/1/pg_data/global/pg_control {file, s=8192}\n" + "pg_data {path}\n" + "pg_data/backup_label.gz {file, s=17}\n" + "pg_data/tablespace_map.gz {file, s=19}\n" + "--------\n" + "[backup:target]\n" + "pg_data={\"path\":\"" TEST_PATH "/pg1\",\"type\":\"path\"}\n" + "\n" + "[target:file]\n" + "pg_data/PG_VERSION={\"checksum\":\"17ba0791499db908433b80f37c5fbc89b870084b\",\"reference\":\"20191030-014640F\"" + ",\"size\":2,\"timestamp\":1572600000}\n" + "pg_data/backup_label={\"checksum\":\"8e6f41ac87a7514be96260d65bacbffb11be77dc\",\"size\":17" + ",\"timestamp\":1572600002}\n" + "pg_data/global/pg_control={\"size\":8192,\"timestamp\":1572600000}\n" + "pg_data/tablespace_map={\"checksum\":\"87fe624d7976c2144e10afcb7a9a49b071f35e9c\",\"size\":19" + ",\"timestamp\":1572600002}\n" + "pg_data/zero={\"reference\":\"20191030-014640F\",\"size\":0,\"timestamp\":1572600000}\n" + "\n" + "[target:path]\n" + "pg_data={}\n" + "pg_data/global={}\n", + "compare file list"); + } } FUNCTION_HARNESS_RETURN_VOID();