diff --git a/doc/xml/release/2024/2.51.xml b/doc/xml/release/2024/2.51.xml index 17c9d220e..50dbff7f2 100644 --- a/doc/xml/release/2024/2.51.xml +++ b/doc/xml/release/2024/2.51.xml @@ -30,6 +30,17 @@ + + + + + + + + +

Detect files that have not changed during non-delta incremental backup.

+ + diff --git a/src/command/backup/backup.c b/src/command/backup/backup.c index f8c25f67d..e34a8be17 100644 --- a/src/command/backup/backup.c +++ b/src/command/backup/backup.c @@ -2006,6 +2006,7 @@ backupJobCallback(void *const data, const unsigned int clientIdx) pckWriteBoolP(param, file.delta); pckWriteBoolP(param, !strEq(file.name, STRDEF(MANIFEST_TARGET_PGDATA "/" PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL))); pckWriteU64P(param, file.size); + pckWriteU64P(param, file.sizePrior); pckWriteBoolP(param, !backupProcessFilePrimary(jobData->standbyExp, file.name)); pckWriteBinP(param, file.checksumSha1 != NULL ? BUF(file.checksumSha1, HASH_TYPE_SHA1_SIZE) : NULL); pckWriteBoolP(param, file.checksumPage); diff --git a/src/command/backup/blockIncr.c b/src/command/backup/blockIncr.c index 6cd23905b..fbdc111fd 100644 --- a/src/command/backup/blockIncr.c +++ b/src/command/backup/blockIncr.c @@ -47,6 +47,7 @@ typedef struct BlockIncr const BlockMap *blockMapPrior; // Prior block map BlockMap *blockMapOut; // Output block map uint64_t blockMapOutSize; // Output block map size (if any) + bool blockMapWrite; // Write block map (at least one new/changed block) size_t inputOffset; // Input offset bool inputSame; // Input the same data @@ -175,7 +176,7 @@ blockIncrProcess(THIS_VOID, const Buffer *const input, Buffer *const output) memcpy(blockMapItem.checksum, bufPtrConst(checksum), bufUsed(checksum)); - unsigned int blockMapItemIdx = blockMapSize(this->blockMapOut); + const unsigned int blockMapItemIdx = blockMapSize(this->blockMapOut); blockMapAdd(this->blockMapOut, &blockMapItem); lstAdd(this->blockOutList, &blockMapItemIdx); @@ -226,10 +227,15 @@ blockIncrProcess(THIS_VOID, const Buffer *const input, Buffer *const output) // Reset block out size and super block no this->blockOutSize = 0; this->superBlockNo = 0; + + // Block map must be written since there are new/changed blocks + this->blockMapWrite = true; } - // Write the block map if done processing and at least one block was written - if (this->done && this->blockOutOffset == 0 && this->blockNo > 0) + // Write the block map if done processing and there are new/changed blocks or block list has been truncated + if (this->done && this->blockOutOffset == 0 && + (this->blockMapWrite || + (this->blockMapPrior != NULL && blockMapSize(this->blockMapOut) < blockMapSize(this->blockMapPrior)))) { MEM_CONTEXT_TEMP_BEGIN() { diff --git a/src/command/backup/file.c b/src/command/backup/file.c index cd1f0c1e9..1297847f2 100644 --- a/src/command/backup/file.c +++ b/src/command/backup/file.c @@ -301,8 +301,8 @@ backupFile( Buffer *const buffer = bufNew(ioBufferSize()); bool readEof = false; - // Read the first buffer to determine if the file was truncated. Detecting truncation matters only when - // bundling is enabled as otherwise the file will be stored anyway. + // Read the first buffer to determine if the file was truncated or was not changed. Detecting truncation + // matters only when bundling is enabled as otherwise the file will be stored anyway. ioRead(readIo, buffer); if (ioReadEof(readIo)) @@ -328,6 +328,26 @@ backupFile( pckReadBinP( ioFilterGroupResultP(ioReadFilterGroup(readIo), CRYPTO_HASH_FILTER_TYPE, .idx = 0)))); } + // Else check if size is equal to prior size + else if (file->manifestFileHasReference && fileResult->copySize == file->pgFileSizePrior) + { + const Buffer *const copyChecksum = pckReadBinP( + ioFilterGroupResultP(ioReadFilterGroup(readIo), CRYPTO_HASH_FILTER_TYPE)); + + // If checksum is also equal then no need to copy the file + if (bufEq(file->pgFileChecksum, copyChecksum)) + { + // If block incremental make sure no map was returned but a prior map was provided + ASSERT( + file->blockIncrSize == 0 || + (pckReadU64P( + ioFilterGroupResultP(ioReadFilterGroup(readIo), BLOCK_INCR_FILTER_TYPE)) == 0 && + file->blockIncrMapPriorFile != NULL)); + + fileResult->backupCopyResult = backupCopyResultNoOp; + fileResult->copyChecksum = file->pgFileChecksum; + } + } } // Copy the file @@ -391,6 +411,9 @@ backupFile( { fileResult->blockIncrMapSize = pckReadU64P( ioFilterGroupResultP(ioReadFilterGroup(readIo), BLOCK_INCR_FILTER_TYPE)); + + // There must be a map because the file should have changed or shrunk + ASSERT(fileResult->blockIncrMapSize > 0); } // Get repo checksum diff --git a/src/command/backup/file.h b/src/command/backup/file.h index 14d794d3b..d3dcfdd3c 100644 --- a/src/command/backup/file.h +++ b/src/command/backup/file.h @@ -31,6 +31,7 @@ typedef struct BackupFile bool pgFileDelta; // Checksum pg file before copying bool pgFileIgnoreMissing; // Ignore missing pg file uint64_t pgFileSize; // Expected pg file size + uint64_t pgFileSizePrior; // Prior pg file size (if manifestFileHasReference) bool pgFileCopyExactSize; // Copy only pg expected size const Buffer *pgFileChecksum; // Expected pg file checksum bool pgFileChecksumPage; // Validate page checksums? diff --git a/src/command/backup/protocol.c b/src/command/backup/protocol.c index 5e9bb9416..15d835482 100644 --- a/src/command/backup/protocol.c +++ b/src/command/backup/protocol.c @@ -49,6 +49,7 @@ backupFileProtocol(PackRead *const param, ProtocolServer *const server) file.pgFileDelta = pckReadBoolP(param); file.pgFileIgnoreMissing = pckReadBoolP(param); file.pgFileSize = pckReadU64P(param); + file.pgFileSizePrior = pckReadU64P(param); file.pgFileCopyExactSize = pckReadBoolP(param); file.pgFileChecksum = pckReadBinP(param); file.pgFileChecksumPage = pckReadBoolP(param); diff --git a/src/info/manifest.c b/src/info/manifest.c index 9a2c6d8dd..a706b0ea4 100644 --- a/src/info/manifest.c +++ b/src/info/manifest.c @@ -104,6 +104,7 @@ typedef enum manifestFilePackFlagBlockIncr, manifestFilePackFlagCopy, manifestFilePackFlagDelta, + manifestFilePackFlagSizePrior, manifestFilePackFlagResume, manifestFilePackFlagChecksumPage, manifestFilePackFlagChecksumPageError, @@ -146,6 +147,9 @@ manifestFilePack(const Manifest *const manifest, const ManifestFile *const file) if (file->delta) flag |= 1 << manifestFilePackFlagDelta; + if (file->reference && file->sizePrior != 0) + flag |= 1 << manifestFilePackFlagSizePrior; + if (file->resume) flag |= 1 << manifestFilePackFlagResume; @@ -192,6 +196,10 @@ manifestFilePack(const Manifest *const manifest, const ManifestFile *const file) if (flag & (1 << manifestFilePackFlagSizeOriginal)) cvtUInt64ToVarInt128(file->sizeOriginal, buffer, &bufferPos, sizeof(buffer)); + // Prior size + if (flag & (1 << manifestFilePackFlagSizePrior)) + cvtUInt64ToVarInt128(cvtInt64ToZigZag((int64_t)file->size - (int64_t)file->sizePrior), buffer, &bufferPos, sizeof(buffer)); + // Use the first timestamp that appears as the base for all other timestamps. Ideally we would like a timestamp as close to the // middle as possible but it doesn't seem worth doing the calculation. if (manifestPackBaseTime == -1) @@ -322,6 +330,14 @@ manifestFileUnpack(const Manifest *const manifest, const ManifestFilePack *const else result.sizeOriginal = result.size; + // Prior size + if (flag & (1 << manifestFilePackFlagSizePrior)) + { + result.sizePrior = + (uint64_t) + ((int64_t)result.size - cvtInt64FromZigZag(cvtUInt64FromVarInt128((const uint8_t *)filePack, &bufferPos, UINT_MAX))); + } + // Timestamp result.timestamp = manifestPackBaseTime - (time_t)cvtInt64FromZigZag(cvtUInt64FromVarInt128((const uint8_t *)filePack, &bufferPos, UINT_MAX)); @@ -1711,9 +1727,11 @@ manifestBuildIncr(Manifest *this, const Manifest *manifestPrior, BackupType type ASSERT(!file.delta || fileSizeEqual); // Preserve values if the file will not be copied, is possibly equal to the prior file, or will be stored with - // block incremental and the prior file is also stored with block incremental - if (!file.copy || file.delta || fileBlockIncrPreserve) + // block incremental and the prior file is also stored with block incremental. If the file will not be copied + // then the file size must be equal to the prior file so there is no need to check that condition separately. + if (fileSizeEqual || fileBlockIncrPreserve) { + file.sizePrior = filePrior.size; file.sizeRepo = filePrior.sizeRepo; file.checksumSha1 = filePrior.checksumSha1; file.checksumRepoSha1 = filePrior.checksumRepoSha1; diff --git a/src/info/manifest.h b/src/info/manifest.h index e76c38034..653c42d85 100644 --- a/src/info/manifest.h +++ b/src/info/manifest.h @@ -159,6 +159,7 @@ typedef struct ManifestFile uint64_t blockIncrMapSize; // Block incremental map size uint64_t size; // Final size (after copy) uint64_t sizeOriginal; // Original size (from manifest build) + uint64_t sizePrior; // Prior size (valid if reference is set, backup only) uint64_t sizeRepo; // Size in repo time_t timestamp; // Original timestamp } ManifestFile; diff --git a/test/src/module/command/backupTest.c b/test/src/module/command/backupTest.c index 3a0753f25..e826c075b 100644 --- a/test/src/module/command/backupTest.c +++ b/test/src/module/command/backupTest.c @@ -1240,6 +1240,29 @@ testRun(void) " block {no: 0, offset: 6}\n", "check delta"); + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("diff/incr backup with identical data"); + + ioBufferSizeSet(3); + + source = BUFSTRZ("ACCXYZ123@"); + destination = bufNew(256); + write = ioBufferWriteNew(destination); + + TEST_RESULT_VOID( + ioFilterGroupAdd(ioWriteFilterGroup(write), ioBufferNew()), "buffer to force internal buffer size"); + TEST_RESULT_VOID( + ioFilterGroupAdd( + ioWriteFilterGroup(write), blockIncrNewPack(ioFilterParamList(blockIncrNew(3, 3, 8, 3, 0, 0, map, NULL, NULL)))), + "block incr"); + TEST_RESULT_VOID(ioWriteOpen(write), "open"); + TEST_RESULT_VOID(ioWrite(write, source), "write"); + TEST_RESULT_VOID(ioWriteClose(write), "close"); + + TEST_ASSIGN(mapSize, pckReadU64P(ioFilterGroupResultP(ioWriteFilterGroup(write), BLOCK_INCR_FILTER_TYPE)), "map size"); + TEST_RESULT_UINT(mapSize, 0, "map size is zero"); + TEST_RESULT_UINT(bufUsed(destination), 0, "repo size is zero"); + // ------------------------------------------------------------------------------------------------------------------------- TEST_TITLE("full backup with larger super block"); @@ -3382,14 +3405,16 @@ testRun(void) "P01 DETAIL: backup file " TEST_PATH "/pg1/block-incr-larger (1.4MB, [PCT]) checksum [SHA1]\n" "P01 DETAIL: backup file " TEST_PATH "/pg1/block-incr-grow (128KB, [PCT]) checksum [SHA1]\n" "P01 DETAIL: store truncated file " TEST_PATH "/pg1/truncate-to-zero (4B->0B, [PCT])\n" - "P01 DETAIL: backup file " TEST_PATH "/pg1/normal-same (bundle 1/0, 4B, [PCT]) checksum [SHA1]\n" - "P01 DETAIL: backup file " TEST_PATH "/pg1/grow-to-block-incr (bundle 1/4, 16KB, [PCT]) checksum [SHA1]\n" - "P01 DETAIL: backup file " TEST_PATH "/pg1/global/pg_control (bundle 1/16416, 8KB, [PCT]) checksum [SHA1]\n" - "P01 DETAIL: backup file " TEST_PATH "/pg1/block-incr-shrink-block (bundle 1/24608, 8KB, [PCT]) checksum [SHA1]\n" - "P01 DETAIL: backup file " TEST_PATH "/pg1/block-incr-shrink-below (bundle 1/24625, 8B, [PCT]) checksum [SHA1]\n" - "P01 DETAIL: backup file " TEST_PATH "/pg1/block-incr-shrink (bundle 1/24633, 16.0KB, [PCT]) checksum [SHA1]\n" - "P01 DETAIL: backup file " TEST_PATH "/pg1/block-incr-same (bundle 1/32859, 16KB, [PCT]) checksum [SHA1]\n" + "P01 DETAIL: match file from prior backup " TEST_PATH "/pg1/normal-same (4B, [PCT]) checksum [SHA1]\n" + "P01 DETAIL: backup file " TEST_PATH "/pg1/grow-to-block-incr (bundle 1/0, 16KB, [PCT]) checksum [SHA1]\n" + "P01 DETAIL: backup file " TEST_PATH "/pg1/global/pg_control (bundle 1/16411, 8KB, [PCT]) checksum [SHA1]\n" + "P01 DETAIL: backup file " TEST_PATH "/pg1/block-incr-shrink-block (bundle 1/24603, 8KB, [PCT]) checksum [SHA1]\n" + "P01 DETAIL: backup file " TEST_PATH "/pg1/block-incr-shrink-below (bundle 1/24620, 8B, [PCT]) checksum [SHA1]\n" + "P01 DETAIL: backup file " TEST_PATH "/pg1/block-incr-shrink (bundle 1/24628, 16.0KB, [PCT]) checksum [SHA1]\n" + "P01 DETAIL: match file from prior backup " TEST_PATH "/pg1/block-incr-same (16KB, [PCT]) checksum [SHA1]\n" "P00 DETAIL: reference pg_data/PG_VERSION to 20191103-165320F\n" + "P00 DETAIL: reference pg_data/block-incr-same to 20191103-165320F\n" + "P00 DETAIL: reference pg_data/normal-same to 20191103-165320F\n" "P00 INFO: execute non-exclusive backup stop and wait for all WAL segments to archive\n" "P00 INFO: backup stop archive = 0000000105DC213000000001, lsn = 5dc2130/300000\n" "P00 DETAIL: wrote 'backup_label' file returned from backup stop function\n" @@ -3400,17 +3425,17 @@ testRun(void) TEST_RESULT_STR_Z( testBackupValidateP(storageRepo(), STRDEF(STORAGE_REPO_BACKUP "/latest")), ".> {d=20191103-165320F_20191106-002640D}\n" - "bundle/1/pg_data/block-incr-same {s=16384, m=0:{0,1}}\n" "bundle/1/pg_data/block-incr-shrink {s=16383, m=0:{0},1:{0}}\n" "bundle/1/pg_data/block-incr-shrink-below {s=8}\n" "bundle/1/pg_data/block-incr-shrink-block {s=8192, m=0:{0}}\n" "bundle/1/pg_data/global/pg_control {s=8192}\n" "bundle/1/pg_data/grow-to-block-incr {s=16385, m=1:{0,1,2}}\n" - "bundle/1/pg_data/normal-same {s=4}\n" "pg_data/backup_label {s=17, ts=+2}\n" "pg_data/block-incr-grow.pgbi {s=131072, m=0:{0},1:{0},0:{2},1:{1,2,3,4,5,6,7,8,9,10,11,12,13}}\n" "pg_data/block-incr-larger.pgbi {s=1507328, m=1:{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15},1:{0,1,2,3,4,5,6}}\n" "20191103-165320F/bundle/1/pg_data/PG_VERSION {s=2, ts=-200000}\n" + "20191103-165320F/bundle/1/pg_data/block-incr-same {s=16384, m=0:{0,1}}\n" + "20191103-165320F/bundle/1/pg_data/normal-same {s=4}\n" "--------\n" "[backup:target]\n" "pg_data={\"path\":\"" TEST_PATH "/pg1\",\"type\":\"path\"}\n", diff --git a/test/src/module/info/manifestTest.c b/test/src/module/info/manifestTest.c index a72146d7e..4b9aec48a 100644 --- a/test/src/module/info/manifestTest.c +++ b/test/src/module/info/manifestTest.c @@ -1092,7 +1092,8 @@ testRun(void) "[target:file]\n" "pg_data/BOGUS={\"size\":6,\"timestamp\":1482182860}\n" "pg_data/FILE3={\"reference\":\"20190101-010101F\",\"size\":0,\"timestamp\":1482182860}\n" - "pg_data/FILE4={\"size\":55,\"timestamp\":1482182861}\n" + "pg_data/FILE4={\"checksum\":\"ccccccccccaaaaaaaaaabbbbbbbbbbdddddddddd\",\"reference\":\"20190101-010101F\"" + ",\"size\":55,\"timestamp\":1482182861}\n" "pg_data/PG_VERSION={\"checksum\":\"aaaaaaaaaabbbbbbbbbbccccccccccdddddddddd\"" ",\"reference\":\"20190101-010101F\",\"size\":4,\"timestamp\":1482182860}\n" TEST_MANIFEST_FILE_DEFAULT @@ -1369,6 +1370,14 @@ testRun(void) TEST_RESULT_VOID( manifestBuildIncr(manifest, manifestPrior, backupTypeIncr, STRDEF("000000030000000300000003")), "incremental manifest"); + TEST_RESULT_UINT( + manifestFileFind(manifest, STRDEF(MANIFEST_TARGET_PGDATA "/block-incr-add")).sizePrior, 0, "check prior size"); + TEST_RESULT_UINT( + manifestFileFind(manifest, STRDEF(MANIFEST_TARGET_PGDATA "/block-incr-sub")).sizePrior, 0, "check prior size"); + TEST_RESULT_UINT( + manifestFileFind(manifest, STRDEF(MANIFEST_TARGET_PGDATA "/block-incr-keep-size")).sizePrior, 16384, + "check prior size"); + contentSave = bufNew(0); TEST_RESULT_VOID(manifestSave(manifest, ioBufferWriteNew(contentSave)), "save manifest"); TEST_RESULT_STR(