db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2025-01-18 04:58:51 +02:00
Code Issues Releases Activity

Add test for differential file bundles to the command/backup unit.

Browse Source 
This test exposes a small logging issue. The bundle information for the matched delta on PG_VERSION is not correct. This issue will be fixed in the next commit.

The information stored in the manifest *is* correct so this bug is essentially cosmetic.
This commit is contained in:
David Steele 2022-10-02 17:41:31 -10:00
parent ac99201c0e
commit 4722ad87a7
1 changed files with 110 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

112
test/src/module/command/backupTest.c
View File

@ -72,8 +72,10 @@ testBackupValidateList(
for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(manifest); fileIdx++)
{
ManifestFilePack **const filePack = lstGet(manifest->pub.fileList, fileIdx);
ManifestFile file = manifestFileUnpack(manifest, *filePack);
if (manifestFileUnpack(manifest, *filePack).bundleId == bundleId)
// File bundle is part of this backup
if (file.bundleId == bundleId && file.reference == NULL)
lstAdd(fileList, &filePack);
}
}
@ -200,6 +202,26 @@ testBackupValidateList(
}
}
// Check all files in manifest. Since the scan above maps from files to the manifest, any referenced files will not be checked.
// -----------------------------------------------------------------------------------------------------------------------------
for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(manifest); fileIdx++)
{
ManifestFilePack **const filePack = lstGet(manifest->pub.fileList, fileIdx);
ManifestFile file = manifestFileUnpack(manifest, *filePack);
// If compressed then set the repo-size to size so it will not be in test output. Even the same compression algorithm can
// give slightly different results based on the version so repo-size is not deterministic for compression.
if (manifestData->backupOptionCompressType != compressTypeNone)
file.sizeRepo = file.size;
// Bundle id/offset are too noisy so remove them. They are verified against size/checksum and listed with the files.
file.bundleId = 0;
file.bundleOffset = 0;
// Update changes to manifest file
manifestFilePackUpdate(manifest, filePack, &file);
}
return result;
}
@ -3174,6 +3196,9 @@ testRun(void)
cfgOptionSet(cfgOptRepoBundleSize, cfgSourceParam, VARINT64(PG_PAGE_SIZE_DEFAULT));
cfgOptionSet(cfgOptRepoBundleLimit, cfgSourceParam, VARINT64(PG_PAGE_SIZE_DEFAULT));
// Zero-length file to be stored
HRN_STORAGE_PUT_EMPTY(storagePgWrite(), "zero", .timeModified = backupTimeStart);
// Zeroed file which passes page checksums
Buffer *relation = bufNew(PG_PAGE_SIZE_DEFAULT * 3);
memset(bufPtr(relation), 0, bufSize(relation));
@ -3200,6 +3225,7 @@ testRun(void)
"P00 INFO: execute non-exclusive pg_start_backup(): backup begins after the next regular checkpoint completes\n"
"P00 INFO: backup start archive = 0000000105DB8EB000000000, lsn = 5db8eb0/0\n"
"P00 INFO: check archive for segment 0000000105DB8EB000000000\n"
"P00 DETAIL: store zero-length file " TEST_PATH "/pg1/zero\n"
"P00 DETAIL: store zero-length file " TEST_PATH "/pg1/pg_tblspc/32768/PG_11_201809051/1/5\n"
"P01 DETAIL: backup file " TEST_PATH "/pg1/base/1/2 (24KB, [PCT]) checksum [SHA1]\n"
"P01 DETAIL: backup file " TEST_PATH "/pg1/stuff.conf (bundle 1/0, 12B, [PCT]) checksum [SHA1]\n"
@ -3217,7 +3243,7 @@ testRun(void)
"P00 DETAIL: copy segment 0000000105DB8EB000000000 to backup\n"
"P00 DETAIL: copy segment 0000000105DB8EB000000001 to backup\n"
"P00 INFO: new backup label = 20191030-014640F\n"
"P00 INFO: full backup size = [SIZE], file total = 13");
"P00 INFO: full backup size = [SIZE], file total = 14");
TEST_RESULT_STR_Z(
testBackupValidate(storageRepo(), STRDEF(STORAGE_REPO_BACKUP "/latest")),
@ -3270,6 +3296,7 @@ testRun(void)
",\"timestamp\":1500000000}\n"
"pg_data/tablespace_map={\"checksum\":\"87fe624d7976c2144e10afcb7a9a49b071f35e9c\",\"size\":19"
",\"timestamp\":1572400002}\n"
"pg_data/zero={\"size\":0,\"timestamp\":1572400000}\n"
"pg_tblspc/32768/PG_11_201809051/1/5={\"checksum-page\":true,\"size\":0,\"timestamp\":1572200000}\n"
"\n"
"[target:link]\n"
@ -3288,6 +3315,87 @@ testRun(void)
"pg_tblspc/32768/PG_11_201809051/1={}\n",
"compare file list");
}
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("online 11 diff backup with bundles");
backupTimeStart = BACKUP_EPOCH + 2600000;
{
// Remove old pg data
HRN_STORAGE_PATH_REMOVE(storageTest, "pg1-data", .recurse = true);
HRN_STORAGE_PATH_REMOVE(storageTest, "pg1-tblspc", .recurse = true);
HRN_STORAGE_REMOVE(storageTest, "pg1");
// Update pg_control
HRN_PG_CONTROL_PUT(storagePgWrite(), PG_VERSION_11, .pageChecksum = false, .walSegmentSize = 2 * 1024 * 1024);
// Update version
HRN_STORAGE_PUT_Z(storagePgWrite(), PG_FILE_PGVERSION, PG_VERSION_11_STR, .timeModified = backupTimeStart);
// Load options
StringList *argList = strLstNew();
hrnCfgArgRawZ(argList, cfgOptStanza, "test1");
hrnCfgArgRaw(argList, cfgOptRepoPath, repoPath);
hrnCfgArgRaw(argList, cfgOptPgPath, pg1Path);
hrnCfgArgRawZ(argList, cfgOptRepoRetentionFull, "1");
hrnCfgArgRawStrId(argList, cfgOptType, backupTypeDiff);
hrnCfgArgRawBool(argList, cfgOptRepoBundle, true);
hrnCfgArgRawBool(argList, cfgOptDelta, true);
HRN_CFG_LOAD(cfgCmdBackup, argList);
// Zero-length file to be stored
HRN_STORAGE_PUT_EMPTY(storagePgWrite(), "zero", .timeModified = backupTimeStart);
// Run backup
testBackupPqScriptP(PG_VERSION_11, backupTimeStart, .walCompressType = compressTypeGz, .walTotal = 2);
TEST_RESULT_VOID(testCmdBackup(), "backup");
TEST_RESULT_LOG(
"P00 INFO: last backup label = 20191030-014640F, version = 2.42dev\n"
"P00 WARN: diff backup cannot alter 'checksum-page' option to 'false', reset to 'true' from 20191030-014640F\n"
"P00 INFO: execute non-exclusive pg_start_backup(): backup begins after the next regular checkpoint completes\n"
"P00 INFO: backup start archive = 0000000105DBBF8000000000, lsn = 5dbbf80/0\n"
"P00 INFO: check archive for segment 0000000105DBBF8000000000\n"
"P01 DETAIL: backup file " TEST_PATH "/pg1/global/pg_control (bundle 1/0, 8KB, [PCT]) checksum [SHA1]\n"
"P01 DETAIL: match file from prior backup " TEST_PATH "/pg1/PG_VERSION (bundle 1/0, 2B, [PCT]) checksum [SHA1]\n"
"P00 DETAIL: reference pg_data/PG_VERSION to 20191030-014640F\n"
"P00 DETAIL: reference pg_data/zero to 20191030-014640F\n"
"P00 INFO: execute non-exclusive pg_stop_backup() and wait for all WAL segments to archive\n"
"P00 INFO: backup stop archive = 0000000105DBBF8000000001, lsn = 5dbbf80/300000\n"
"P00 DETAIL: wrote 'backup_label' file returned from pg_stop_backup()\n"
"P00 DETAIL: wrote 'tablespace_map' file returned from pg_stop_backup()\n"
"P00 INFO: check archive for segment(s) 0000000105DBBF8000000000:0000000105DBBF8000000001\n"
"P00 INFO: new backup label = 20191030-014640F_20191101-092000D\n"
"P00 INFO: diff backup size = [SIZE], file total = 5");
TEST_RESULT_STR_Z(
testBackupValidate(storageRepo(), STRDEF(STORAGE_REPO_BACKUP "/latest")),
". {link, d=20191030-014640F_20191101-092000D}\n"
"bundle {path}\n"
"bundle/1/pg_data/global/pg_control {file, s=8192}\n"
"pg_data {path}\n"
"pg_data/backup_label.gz {file, s=17}\n"
"pg_data/tablespace_map.gz {file, s=19}\n"
"--------\n"
"[backup:target]\n"
"pg_data={\"path\":\"" TEST_PATH "/pg1\",\"type\":\"path\"}\n"
"\n"
"[target:file]\n"
"pg_data/PG_VERSION={\"checksum\":\"17ba0791499db908433b80f37c5fbc89b870084b\",\"reference\":\"20191030-014640F\""
",\"size\":2,\"timestamp\":1572600000}\n"
"pg_data/backup_label={\"checksum\":\"8e6f41ac87a7514be96260d65bacbffb11be77dc\",\"size\":17"
",\"timestamp\":1572600002}\n"
"pg_data/global/pg_control={\"size\":8192,\"timestamp\":1572600000}\n"
"pg_data/tablespace_map={\"checksum\":\"87fe624d7976c2144e10afcb7a9a49b071f35e9c\",\"size\":19"
",\"timestamp\":1572600002}\n"
"pg_data/zero={\"reference\":\"20191030-014640F\",\"size\":0,\"timestamp\":1572600000}\n"
"\n"
"[target:path]\n"
"pg_data={}\n"
"pg_data/global={}\n",
"compare file list");
}
}
FUNCTION_HARNESS_RETURN_VOID();