Limit backup file copy size to size reported at backup start.

If a file grows during the backup it will be reconstructed by WAL replay during recovery so there is no need to copy the additional data.

This also reduces the likelihood of seeing torn pages during the copy. Torn pages can still occur in the middle of the file, though, so they must be handled.

doc/xml/release.xml

@@ -65,6 +65,16 @@
 
                         <p>These commands (e.g. <cmd>restore</cmd>, <cmd>archive-get</cmd>) never used the compress options but allowed them to be passed on the command line.  Now they will error when these options are passed on the command line.  If these errors occur then remove the unused options.</p>
                     </release-item>
+
+                    <release-item>
+                        <release-item-contributor-list>
+                             <release-item-reviewer id="cynthia.shang"/>
+                        </release-item-contributor-list>
+
+                        <p>Limit backup file copy size to size reported at backup start.</p>
+
+                        <p>If a file grows during the backup it will be reconstructed by WAL replay during recovery so there is no need to copy the additional data.</p>
+                    </release-item>
                 </release-improvement-list>
 
                 <release-development-list>

src/command/backup/file.c

@@ -84,8 +84,11 @@ backupFile(
             // recopy.
             if (delta)
             {
-                // Generate checksum/size for the pg file
+                // Generate checksum/size for the pg file. Only read as many bytes as passed in pgFileSize.  If the file has grown
-                IoRead *read = storageReadIo(storageNewReadP(storagePg(), pgFile, .ignoreMissing = pgFileIgnoreMissing));
+                // since the manifest was built we don't need to consider the extra bytes since they will be replayed from WAL
+                // during recovery.
+                IoRead *read = storageReadIo(
+                    storageNewReadP(storagePg(), pgFile, .ignoreMissing = pgFileIgnoreMissing, .limit = VARUINT64(pgFileSize)));
                 ioFilterGroupAdd(ioReadFilterGroup(read), cryptoHashNew(HASH_TYPE_SHA1_STR));
                 ioFilterGroupAdd(ioReadFilterGroup(read), ioSizeNew());
 
@@ -190,9 +193,12 @@ backupFile(
             // Is the file compressible during the copy?
             bool compressible = repoFileCompressType == compressTypeNone && cipherType == cipherTypeNone;
 
-            // Setup pg file for read
+            // Setup pg file for read. Only read as many bytes as passed in pgFileSize.  If the file is growing it does no good to
+            // copy data past the end of the size recorded in the manifest since those blocks will need to be replayed from WAL
+            // during recovery.
             StorageRead *read = storageNewReadP(
-                storagePg(), pgFile, .ignoreMissing = pgFileIgnoreMissing, .compressible = compressible);
+                storagePg(), pgFile, .ignoreMissing = pgFileIgnoreMissing, .compressible = compressible,
+                .limit = VARUINT64(pgFileSize));
             ioFilterGroupAdd(ioReadFilterGroup(storageReadIo(read)), cryptoHashNew(HASH_TYPE_SHA1_STR));
             ioFilterGroupAdd(ioReadFilterGroup(storageReadIo(read)), ioSizeNew());
 

test/src/module/command/backupTest.c

@@ -562,6 +562,11 @@ testRun(void)
 
         // -------------------------------------------------------------------------------------------------------------------------
         // Test pagechecksum
+
+        // Increase the file size but most of the following tests will still treat the file as size 9.  This tests the common case
+        // where a file grows while a backup is running.
+        storagePutP(storageNewWriteP(storagePgWrite(), pgFile), BUFSTRDEF("atestfile!!!"));
+
         TEST_ASSIGN(
             result,
             backupFile(
@@ -665,13 +670,14 @@ testRun(void)
         TEST_ASSIGN(
             result,
             backupFile(
-                pgFile, false, 8, strNew("9bc8ab2dda60ef4beed07d1e19ce0676d5edde67"), false, 0, pgFile, true, false, 1, backupLabel,
+                pgFile, false, 9999999, strNew("9bc8ab2dda60ef4beed07d1e19ce0676d5edde67"), false, 0, pgFile, true, false, 1,
-                true, cipherTypeNone, NULL),
+                backupLabel, true, cipherTypeNone, NULL),
             "db & repo file, pg checksum same, pg size different, no ignoreMissing, no pageChecksum, delta, hasReference");
-        TEST_RESULT_UINT(result.copySize + result.repoSize, 18, "    copy=repo=pgFile size");
+        TEST_RESULT_UINT(result.copySize + result.repoSize, 24, "    copy=repo=pgFile size");
         TEST_RESULT_UINT(result.backupCopyResult, backupCopyResultCopy, "    copy file");
+        TEST_RESULT_STR_Z(result.copyChecksum, "719e82b52966b075c1ee276547e924179628fe69", "TEST");
         TEST_RESULT_BOOL(
-            (strEqZ(result.copyChecksum, "9bc8ab2dda60ef4beed07d1e19ce0676d5edde67") &&
+            (strEqZ(result.copyChecksum, "719e82b52966b075c1ee276547e924179628fe69") &&
                 storageExistsP(storageRepo(), backupPathFile) && result.pageChecksumResult == NULL),
             true, "    copy");
 
@@ -853,11 +859,11 @@ testRun(void)
                 pgFile, false, 8, strNew("9bc8ab2dda60ef4beed07d1e19ce0676d5edde67"), false, 0, pgFile, false, false, 1,
                 backupLabel, true, cipherTypeAes256Cbc, strNew("12345678")),
             "pg and repo file exists, pgFileMatch false, no ignoreMissing, no pageChecksum, delta, no hasReference");
-        TEST_RESULT_UINT(result.copySize, 9, "    copy size set");
+        TEST_RESULT_UINT(result.copySize, 8, "    copy size set");
         TEST_RESULT_UINT(result.repoSize, 32, "    repo size set");
         TEST_RESULT_UINT(result.backupCopyResult, backupCopyResultCopy, "    copy file");
         TEST_RESULT_BOOL(
-            (strEqZ(result.copyChecksum, "9bc8ab2dda60ef4beed07d1e19ce0676d5edde67") &&
+            (strEqZ(result.copyChecksum, "acc972a8319d4903b839c64ec217faa3e77b4fcb") &&
                 storageExistsP(storageRepo(), backupPathFile) && result.pageChecksumResult == NULL),
             true, "    copy file (size missmatch) to encrypted repo success");
 
@@ -2126,8 +2132,8 @@ testRun(void)
                 "pg_data/backup_label {file, s=17}\n"
                 "pg_data/base {path}\n"
                 "pg_data/base/1 {path}\n"
-                "pg_data/base/1/1 {file, s=4}\n"
+                "pg_data/base/1/1 {file, s=0}\n"
-                "pg_data/base/1/2 {file, s=4}\n"
+                "pg_data/base/1/2 {file, s=2}\n"
                 "pg_data/base/1/3 {file, s=3}\n"
                 "pg_data/global {path}\n"
                 "pg_data/global/pg_control {file, s=8192}\n"
@@ -2143,9 +2149,8 @@ testRun(void)
                     ",\"timestamp\":1571200000}\n"
                 "pg_data/backup_label={\"checksum\":\"8e6f41ac87a7514be96260d65bacbffb11be77dc\",\"size\":17"
                     ",\"timestamp\":1571200002}\n"
-                "pg_data/base/1/1={\"checksum\":\"7110eda4d09e062aa5e4a390b0a572ac0d2c0220\",\"master\":false,\"size\":4"
+                "pg_data/base/1/1={\"master\":false,\"size\":0,\"timestamp\":1571200000}\n"
-                    ",\"timestamp\":1571200000}\n"
+                "pg_data/base/1/2={\"checksum\":\"54ceb91256e8190e474aa752a6e0650a2df5ba37\",\"master\":false,\"size\":2"
-                "pg_data/base/1/2={\"checksum\":\"2abd55e001c524cb2cf6300a89ca6366848a77d5\",\"master\":false,\"size\":4"
                     ",\"timestamp\":1571200000}\n"
                 "pg_data/base/1/3={\"checksum\":\"3c01bdbb26f358bab27f267924aa2c9a03fcfdb8\",\"master\":false,\"size\":3"
                     ",\"timestamp\":1571200000}\n"