db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2025-07-09 00:45:49 +02:00
Code Issues Releases Activity

Limit backup file copy size to size reported at backup start.

Browse Source 
If a file grows during the backup it will be reconstructed by WAL replay during recovery so there is no need to copy the additional data.

This also reduces the likelihood of seeing torn pages during the copy. Torn pages can still occur in the middle of the file, though, so they must be handled.
This commit is contained in:
David Steele
2020-03-19 13:16:05 -04:00
parent e5bcc0c47e
commit dcddf3a58b
3 changed files with 36 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
doc/xml/release.xml
View File

@ -65,6 +65,16 @@
<p>These commands (e.g. <cmd>restore</cmd>, <cmd>archive-get</cmd>) never used the compress options but allowed them to be passed on the command line. Now they will error when these options are passed on the command line. If these errors occur then remove the unused options.</p>
</release-item>
<release-item>
<release-item-contributor-list>
<release-item-reviewer id="cynthia.shang"/>
</release-item-contributor-list>
<p>Limit backup file copy size to size reported at backup start.</p>
<p>If a file grows during the backup it will be reconstructed by WAL replay during recovery so there is no need to copy the additional data.</p>
</release-item>
</release-improvement-list>
<release-development-list>

14
src/command/backup/file.c
View File

@ -84,8 +84,11 @@ backupFile(
// recopy.
if (delta)
{
// Generate checksum/size for the pg file
IoRead *read = storageReadIo(storageNewReadP(storagePg(), pgFile, .ignoreMissing = pgFileIgnoreMissing));
// Generate checksum/size for the pg file. Only read as many bytes as passed in pgFileSize. If the file has grown
// since the manifest was built we don't need to consider the extra bytes since they will be replayed from WAL
// during recovery.
IoRead *read = storageReadIo(
storageNewReadP(storagePg(), pgFile, .ignoreMissing = pgFileIgnoreMissing, .limit = VARUINT64(pgFileSize)));
ioFilterGroupAdd(ioReadFilterGroup(read), cryptoHashNew(HASH_TYPE_SHA1_STR));
ioFilterGroupAdd(ioReadFilterGroup(read), ioSizeNew());
@ -190,9 +193,12 @@ backupFile(
// Is the file compressible during the copy?
bool compressible = repoFileCompressType == compressTypeNone && cipherType == cipherTypeNone;
// Setup pg file for read
// Setup pg file for read. Only read as many bytes as passed in pgFileSize. If the file is growing it does no good to
// copy data past the end of the size recorded in the manifest since those blocks will need to be replayed from WAL
// during recovery.
StorageRead *read = storageNewReadP(
storagePg(), pgFile, .ignoreMissing = pgFileIgnoreMissing, .compressible = compressible);
storagePg(), pgFile, .ignoreMissing = pgFileIgnoreMissing, .compressible = compressible,
.limit = VARUINT64(pgFileSize));
ioFilterGroupAdd(ioReadFilterGroup(storageReadIo(read)), cryptoHashNew(HASH_TYPE_SHA1_STR));
ioFilterGroupAdd(ioReadFilterGroup(storageReadIo(read)), ioSizeNew());

27
test/src/module/command/backupTest.c
View File

@ -562,6 +562,11 @@ testRun(void)
// -------------------------------------------------------------------------------------------------------------------------
// Test pagechecksum
// Increase the file size but most of the following tests will still treat the file as size 9. This tests the common case
// where a file grows while a backup is running.
storagePutP(storageNewWriteP(storagePgWrite(), pgFile), BUFSTRDEF("atestfile!!!"));
TEST_ASSIGN(
result,
backupFile(
@ -665,13 +670,14 @@ testRun(void)
TEST_ASSIGN(
result,
backupFile(
pgFile, false, 8, strNew("9bc8ab2dda60ef4beed07d1e19ce0676d5edde67"), false, 0, pgFile, true, false, 1, backupLabel,
true, cipherTypeNone, NULL),
pgFile, false, 9999999, strNew("9bc8ab2dda60ef4beed07d1e19ce0676d5edde67"), false, 0, pgFile, true, false, 1,
backupLabel, true, cipherTypeNone, NULL),
"db & repo file, pg checksum same, pg size different, no ignoreMissing, no pageChecksum, delta, hasReference");
TEST_RESULT_UINT(result.copySize + result.repoSize, 18, " copy=repo=pgFile size");
TEST_RESULT_UINT(result.copySize + result.repoSize, 24, " copy=repo=pgFile size");
TEST_RESULT_UINT(result.backupCopyResult, backupCopyResultCopy, " copy file");
TEST_RESULT_STR_Z(result.copyChecksum, "719e82b52966b075c1ee276547e924179628fe69", "TEST");
TEST_RESULT_BOOL(
(strEqZ(result.copyChecksum, "9bc8ab2dda60ef4beed07d1e19ce0676d5edde67") &&
(strEqZ(result.copyChecksum, "719e82b52966b075c1ee276547e924179628fe69") &&
storageExistsP(storageRepo(), backupPathFile) && result.pageChecksumResult == NULL),
true, " copy");
@ -853,11 +859,11 @@ testRun(void)
pgFile, false, 8, strNew("9bc8ab2dda60ef4beed07d1e19ce0676d5edde67"), false, 0, pgFile, false, false, 1,
backupLabel, true, cipherTypeAes256Cbc, strNew("12345678")),
"pg and repo file exists, pgFileMatch false, no ignoreMissing, no pageChecksum, delta, no hasReference");
TEST_RESULT_UINT(result.copySize, 9, " copy size set");
TEST_RESULT_UINT(result.copySize, 8, " copy size set");
TEST_RESULT_UINT(result.repoSize, 32, " repo size set");
TEST_RESULT_UINT(result.backupCopyResult, backupCopyResultCopy, " copy file");
TEST_RESULT_BOOL(
(strEqZ(result.copyChecksum, "9bc8ab2dda60ef4beed07d1e19ce0676d5edde67") &&
(strEqZ(result.copyChecksum, "acc972a8319d4903b839c64ec217faa3e77b4fcb") &&
storageExistsP(storageRepo(), backupPathFile) && result.pageChecksumResult == NULL),
true, " copy file (size missmatch) to encrypted repo success");
@ -2126,8 +2132,8 @@ testRun(void)
"pg_data/backup_label {file, s=17}\n"
"pg_data/base {path}\n"
"pg_data/base/1 {path}\n"
"pg_data/base/1/1 {file, s=4}\n"
"pg_data/base/1/2 {file, s=4}\n"
"pg_data/base/1/1 {file, s=0}\n"
"pg_data/base/1/2 {file, s=2}\n"
"pg_data/base/1/3 {file, s=3}\n"
"pg_data/global {path}\n"
"pg_data/global/pg_control {file, s=8192}\n"
@ -2143,9 +2149,8 @@ testRun(void)
",\"timestamp\":1571200000}\n"
"pg_data/backup_label={\"checksum\":\"8e6f41ac87a7514be96260d65bacbffb11be77dc\",\"size\":17"
",\"timestamp\":1571200002}\n"
"pg_data/base/1/1={\"checksum\":\"7110eda4d09e062aa5e4a390b0a572ac0d2c0220\",\"master\":false,\"size\":4"
",\"timestamp\":1571200000}\n"
"pg_data/base/1/2={\"checksum\":\"2abd55e001c524cb2cf6300a89ca6366848a77d5\",\"master\":false,\"size\":4"
"pg_data/base/1/1={\"master\":false,\"size\":0,\"timestamp\":1571200000}\n"
"pg_data/base/1/2={\"checksum\":\"54ceb91256e8190e474aa752a6e0650a2df5ba37\",\"master\":false,\"size\":2"
",\"timestamp\":1571200000}\n"
"pg_data/base/1/3={\"checksum\":\"3c01bdbb26f358bab27f267924aa2c9a03fcfdb8\",\"master\":false,\"size\":3"
",\"timestamp\":1571200000}\n"