db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2026-05-22 10:15:16 +02:00
Code Issues Releases Activity

Detect files that have not changed during non-delta incremental backup.

Browse Source 
02eea55 added code to load a buffer of data from a file being backup up to detect files that have been truncated to zero after manifest generation. This mechanism can also be used to detect files that have not changed since the prior backup.

If the result of the file copy fits into a single buffer, then the size and checksum can be compared to the prior file before anything gets stored. If the file matches then it is referenced to the file in to prior backup.

The size that can be compared for normal copies is limited by the buffer size but for block incremental it works with any size file since there is no output from block incremental when the file is identical.
This commit is contained in:
David Steele
2024-03-08 15:07:43 +13:00
committed by GitHub
parent cf17515e40
commit 9d91d1b2f8
10 changed files with 113 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/src/module/command/backupTest.c
+34 -9
View File
@@ -1240,6 +1240,29 @@ testRun(void)
" block {no: 0, offset: 6}\n",
"check delta");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("diff/incr backup with identical data");
ioBufferSizeSet(3);
source = BUFSTRZ("ACCXYZ123@");
destination = bufNew(256);
write = ioBufferWriteNew(destination);
TEST_RESULT_VOID(
ioFilterGroupAdd(ioWriteFilterGroup(write), ioBufferNew()), "buffer to force internal buffer size");
TEST_RESULT_VOID(
ioFilterGroupAdd(
ioWriteFilterGroup(write), blockIncrNewPack(ioFilterParamList(blockIncrNew(3, 3, 8, 3, 0, 0, map, NULL, NULL)))),
"block incr");
TEST_RESULT_VOID(ioWriteOpen(write), "open");
TEST_RESULT_VOID(ioWrite(write, source), "write");
TEST_RESULT_VOID(ioWriteClose(write), "close");
TEST_ASSIGN(mapSize, pckReadU64P(ioFilterGroupResultP(ioWriteFilterGroup(write), BLOCK_INCR_FILTER_TYPE)), "map size");
TEST_RESULT_UINT(mapSize, 0, "map size is zero");
TEST_RESULT_UINT(bufUsed(destination), 0, "repo size is zero");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("full backup with larger super block");
@@ -3382,14 +3405,16 @@ testRun(void)
"P01 DETAIL: backup file " TEST_PATH "/pg1/block-incr-larger (1.4MB, [PCT]) checksum [SHA1]\n"
"P01 DETAIL: backup file " TEST_PATH "/pg1/block-incr-grow (128KB, [PCT]) checksum [SHA1]\n"
"P01 DETAIL: store truncated file " TEST_PATH "/pg1/truncate-to-zero (4B->0B, [PCT])\n"
"P01 DETAIL: backup file " TEST_PATH "/pg1/normal-same (bundle 1/0, 4B, [PCT]) checksum [SHA1]\n"
"P01 DETAIL: backup file " TEST_PATH "/pg1/grow-to-block-incr (bundle 1/4, 16KB, [PCT]) checksum [SHA1]\n"
"P01 DETAIL: backup file " TEST_PATH "/pg1/global/pg_control (bundle 1/16416, 8KB, [PCT]) checksum [SHA1]\n"
"P01 DETAIL: backup file " TEST_PATH "/pg1/block-incr-shrink-block (bundle 1/24608, 8KB, [PCT]) checksum [SHA1]\n"
"P01 DETAIL: backup file " TEST_PATH "/pg1/block-incr-shrink-below (bundle 1/24625, 8B, [PCT]) checksum [SHA1]\n"
"P01 DETAIL: backup file " TEST_PATH "/pg1/block-incr-shrink (bundle 1/24633, 16.0KB, [PCT]) checksum [SHA1]\n"
"P01 DETAIL: backup file " TEST_PATH "/pg1/block-incr-same (bundle 1/32859, 16KB, [PCT]) checksum [SHA1]\n"
"P01 DETAIL: match file from prior backup " TEST_PATH "/pg1/normal-same (4B, [PCT]) checksum [SHA1]\n"
"P01 DETAIL: backup file " TEST_PATH "/pg1/grow-to-block-incr (bundle 1/0, 16KB, [PCT]) checksum [SHA1]\n"
"P01 DETAIL: backup file " TEST_PATH "/pg1/global/pg_control (bundle 1/16411, 8KB, [PCT]) checksum [SHA1]\n"
"P01 DETAIL: backup file " TEST_PATH "/pg1/block-incr-shrink-block (bundle 1/24603, 8KB, [PCT]) checksum [SHA1]\n"
"P01 DETAIL: backup file " TEST_PATH "/pg1/block-incr-shrink-below (bundle 1/24620, 8B, [PCT]) checksum [SHA1]\n"
"P01 DETAIL: backup file " TEST_PATH "/pg1/block-incr-shrink (bundle 1/24628, 16.0KB, [PCT]) checksum [SHA1]\n"
"P01 DETAIL: match file from prior backup " TEST_PATH "/pg1/block-incr-same (16KB, [PCT]) checksum [SHA1]\n"
"P00 DETAIL: reference pg_data/PG_VERSION to 20191103-165320F\n"
"P00 DETAIL: reference pg_data/block-incr-same to 20191103-165320F\n"
"P00 DETAIL: reference pg_data/normal-same to 20191103-165320F\n"
"P00 INFO: execute non-exclusive backup stop and wait for all WAL segments to archive\n"
"P00 INFO: backup stop archive = 0000000105DC213000000001, lsn = 5dc2130/300000\n"
"P00 DETAIL: wrote 'backup_label' file returned from backup stop function\n"
@@ -3400,17 +3425,17 @@ testRun(void)
TEST_RESULT_STR_Z(
testBackupValidateP(storageRepo(), STRDEF(STORAGE_REPO_BACKUP "/latest")),
".> {d=20191103-165320F_20191106-002640D}\n"
"bundle/1/pg_data/block-incr-same {s=16384, m=0:{0,1}}\n"
"bundle/1/pg_data/block-incr-shrink {s=16383, m=0:{0},1:{0}}\n"
"bundle/1/pg_data/block-incr-shrink-below {s=8}\n"
"bundle/1/pg_data/block-incr-shrink-block {s=8192, m=0:{0}}\n"
"bundle/1/pg_data/global/pg_control {s=8192}\n"
"bundle/1/pg_data/grow-to-block-incr {s=16385, m=1:{0,1,2}}\n"
"bundle/1/pg_data/normal-same {s=4}\n"
"pg_data/backup_label {s=17, ts=+2}\n"
"pg_data/block-incr-grow.pgbi {s=131072, m=0:{0},1:{0},0:{2},1:{1,2,3,4,5,6,7,8,9,10,11,12,13}}\n"
"pg_data/block-incr-larger.pgbi {s=1507328, m=1:{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15},1:{0,1,2,3,4,5,6}}\n"
"20191103-165320F/bundle/1/pg_data/PG_VERSION {s=2, ts=-200000}\n"
"20191103-165320F/bundle/1/pg_data/block-incr-same {s=16384, m=0:{0,1}}\n"
"20191103-165320F/bundle/1/pg_data/normal-same {s=4}\n"
"--------\n"
"[backup:target]\n"
"pg_data={\"path\":\"" TEST_PATH "/pg1\",\"type\":\"path\"}\n",
test/src/module/info/manifestTest.c
+10 -1
View File
@@ -1092,7 +1092,8 @@ testRun(void)
"[target:file]\n"
"pg_data/BOGUS={\"size\":6,\"timestamp\":1482182860}\n"
"pg_data/FILE3={\"reference\":\"20190101-010101F\",\"size\":0,\"timestamp\":1482182860}\n"
"pg_data/FILE4={\"size\":55,\"timestamp\":1482182861}\n"
"pg_data/FILE4={\"checksum\":\"ccccccccccaaaaaaaaaabbbbbbbbbbdddddddddd\",\"reference\":\"20190101-010101F\""
",\"size\":55,\"timestamp\":1482182861}\n"
"pg_data/PG_VERSION={\"checksum\":\"aaaaaaaaaabbbbbbbbbbccccccccccdddddddddd\""
",\"reference\":\"20190101-010101F\",\"size\":4,\"timestamp\":1482182860}\n"
TEST_MANIFEST_FILE_DEFAULT
@@ -1369,6 +1370,14 @@ testRun(void)
TEST_RESULT_VOID(
manifestBuildIncr(manifest, manifestPrior, backupTypeIncr, STRDEF("000000030000000300000003")), "incremental manifest");
TEST_RESULT_UINT(
manifestFileFind(manifest, STRDEF(MANIFEST_TARGET_PGDATA "/block-incr-add")).sizePrior, 0, "check prior size");
TEST_RESULT_UINT(
manifestFileFind(manifest, STRDEF(MANIFEST_TARGET_PGDATA "/block-incr-sub")).sizePrior, 0, "check prior size");
TEST_RESULT_UINT(
manifestFileFind(manifest, STRDEF(MANIFEST_TARGET_PGDATA "/block-incr-keep-size")).sizePrior, 16384,
"check prior size");
contentSave = bufNew(0);
TEST_RESULT_VOID(manifestSave(manifest, ioBufferWriteNew(contentSave)), "save manifest");
TEST_RESULT_STR(