db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2025-01-18 04:58:51 +02:00
Code Issues Releases Activity

Fix memory leak in file bundle backup/restore.

Browse Source 
When converting restoreFile() to support file bundling in 34d64957 there were some I/O objects that were only freed at the end of the function that should have been freed at the end of each loop. Wrap the loops in temp mem contexts to fix this.

Do the same to backupFile() since it would have a similar leak when resuming a backup. Since file bundles cannot be resumed the leak would not be as severe, but still seems worth doing to protect against future leaks.
This commit is contained in:
David Steele 2022-09-22 22:42:01 -07:00 committed by GitHub
parent d50a4442e4
commit cd8db7d9e5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 407 additions and 356 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
doc/xml/release.xml
View File

@ -15,6 +15,24 @@
<release-list>
<release date="XXXX-XX-XX" version="2.42dev" title="UNDER DEVELOPMENT">
<release-core-list>
<release-bug-list>
<release-item>
<github-issue id="1877"/>
<github-pull-request id="1878"/>
<release-item-contributor-list>
<release-item-ideator id="MannerMan"/>
<release-item-contributor id="david.steele"/>
<release-item-reviewer id="john.morris"/>
<!-- Actually tester, but we don't have a tag for that yet -->
<release-item-reviewer id="MannerMan"/>
</release-item-contributor-list>
<p>Fix memory leak in file bundle <cmd>backup</cmd>/<cmd>restore</cmd>.</p>
</release-item>
</release-bug-list>
</release-core-list>
</release>
<release date="2022-09-19" version="2.41" title="Backup Annotations">

366
src/command/backup/file.c
View File

@ -65,129 +65,134 @@ backupFile(
// Check files to determine which ones need to be copied
for (unsigned int fileIdx = 0; fileIdx < lstSize(fileList); fileIdx++)
{
const BackupFile *const file = lstGet(fileList, fileIdx);
ASSERT(file->pgFile != NULL);
ASSERT(file->manifestFile != NULL);
BackupFileResult *const fileResult = lstAdd(
result, &(BackupFileResult){.manifestFile = file->manifestFile, .backupCopyResult = backupCopyResultCopy});
// If checksum is defined then the file needs to be checked. If delta option then check the DB and possibly the repo,
// else just check the repo.
if (file->pgFileChecksum != NULL)
// Use a per-file mem context to reduce memory usage
MEM_CONTEXT_TEMP_BEGIN()
{
// Does the file in pg match the checksum and size passed?
bool pgFileMatch = false;
const BackupFile *const file = lstGet(fileList, fileIdx);
ASSERT(file->pgFile != NULL);
ASSERT(file->manifestFile != NULL);
// If delta, then check the DB checksum and possibly the repo. If the checksum does not match in either case then
// recopy.
if (delta)
BackupFileResult *const fileResult = lstAdd(
result, &(BackupFileResult){.manifestFile = file->manifestFile, .backupCopyResult = backupCopyResultCopy});
// If checksum is defined then the file needs to be checked. If delta option then check the DB and possibly the
// repo, else just check the repo.
if (file->pgFileChecksum != NULL)
{
// Generate checksum/size for the pg file. Only read as many bytes as passed in pgFileSize. If the file has
// grown since the manifest was built we don't need to consider the extra bytes since they will be replayed from
// WAL during recovery.
IoRead *read = storageReadIo(
storageNewReadP(
storagePg(), file->pgFile, .ignoreMissing = file->pgFileIgnoreMissing,
.limit = file->pgFileCopyExactSize ? VARUINT64(file->pgFileSize) : NULL));
ioFilterGroupAdd(ioReadFilterGroup(read), cryptoHashNew(hashTypeSha1));
ioFilterGroupAdd(ioReadFilterGroup(read), ioSizeNew());
// Does the file in pg match the checksum and size passed?
bool pgFileMatch = false;
// If the pg file exists check the checksum/size
if (ioReadDrain(read))
// If delta, then check the DB checksum and possibly the repo. If the checksum does not match in either case
// then recopy.
if (delta)
{
const String *pgTestChecksum = pckReadStrP(
ioFilterGroupResultP(ioReadFilterGroup(read), CRYPTO_HASH_FILTER_TYPE));
uint64_t pgTestSize = pckReadU64P(ioFilterGroupResultP(ioReadFilterGroup(read), SIZE_FILTER_TYPE));
// Generate checksum/size for the pg file. Only read as many bytes as passed in pgFileSize. If the file has
// grown since the manifest was built we don't need to consider the extra bytes since they will be replayed
// from WAL during recovery.
IoRead *read = storageReadIo(
storageNewReadP(
storagePg(), file->pgFile, .ignoreMissing = file->pgFileIgnoreMissing,
.limit = file->pgFileCopyExactSize ? VARUINT64(file->pgFileSize) : NULL));
ioFilterGroupAdd(ioReadFilterGroup(read), cryptoHashNew(hashTypeSha1));
ioFilterGroupAdd(ioReadFilterGroup(read), ioSizeNew());
// Does the pg file match?
if (file->pgFileSize == pgTestSize && strEq(file->pgFileChecksum, pgTestChecksum))
// If the pg file exists check the checksum/size
if (ioReadDrain(read))
{
pgFileMatch = true;
// If it matches and is a reference to a previous backup then no need to copy the file
if (file->manifestFileHasReference)
{
MEM_CONTEXT_BEGIN(lstMemContext(result))
{
fileResult->backupCopyResult = backupCopyResultNoOp;
fileResult->copySize = pgTestSize;
fileResult->copyChecksum = strDup(pgTestChecksum);
}
MEM_CONTEXT_END();
}
}
}
// Else the source file is missing from the database so skip this file
else
fileResult->backupCopyResult = backupCopyResultSkip;
}
// If this is not a delta backup or it is and the file exists and the checksum from the DB matches, then also test
// the checksum of the file in the repo (unless it is in a prior backup) and if the checksum doesn't match, then
// there may be corruption in the repo, so recopy
if (!delta || !file->manifestFileHasReference)
{
// If this is a delta backup and the file is missing from the DB, then remove it from the repo
// (backupManifestUpdate will remove it from the manifest)
if (fileResult->backupCopyResult == backupCopyResultSkip)
{
storageRemoveP(storageRepoWrite(), repoFile);
}
else if (!delta || pgFileMatch)
{
// Check the repo file in a try block because on error (e.g. missing or corrupt file that can't be decrypted
// or decompressed) we should recopy rather than ending the backup.
TRY_BEGIN()
{
// Generate checksum/size for the repo file
IoRead *read = storageReadIo(storageNewReadP(storageRepo(), repoFile));
if (cipherType != cipherTypeNone)
{
ioFilterGroupAdd(
ioReadFilterGroup(read),
cipherBlockNew(cipherModeDecrypt, cipherType, BUFSTR(cipherPass), NULL));
}
// Decompress the file if compressed
if (repoFileCompressType != compressTypeNone)
ioFilterGroupAdd(ioReadFilterGroup(read), decompressFilter(repoFileCompressType));
ioFilterGroupAdd(ioReadFilterGroup(read), cryptoHashNew(hashTypeSha1));
ioFilterGroupAdd(ioReadFilterGroup(read), ioSizeNew());
ioReadDrain(read);
// Test checksum/size
const String *pgTestChecksum = pckReadStrP(
ioFilterGroupResultP(ioReadFilterGroup(read), CRYPTO_HASH_FILTER_TYPE));
uint64_t pgTestSize = pckReadU64P(ioFilterGroupResultP(ioReadFilterGroup(read), SIZE_FILTER_TYPE));
// No need to recopy if checksum/size match
// Does the pg file match?
if (file->pgFileSize == pgTestSize && strEq(file->pgFileChecksum, pgTestChecksum))
{
MEM_CONTEXT_BEGIN(lstMemContext(result))
pgFileMatch = true;
// If it matches and is a reference to a previous backup then no need to copy the file
if (file->manifestFileHasReference)
{
fileResult->backupCopyResult = backupCopyResultChecksum;
fileResult->copySize = pgTestSize;
fileResult->copyChecksum = strDup(pgTestChecksum);
MEM_CONTEXT_BEGIN(lstMemContext(result))
{
fileResult->backupCopyResult = backupCopyResultNoOp;
fileResult->copySize = pgTestSize;
fileResult->copyChecksum = strDup(pgTestChecksum);
}
MEM_CONTEXT_END();
}
MEM_CONTEXT_END();
}
// Else recopy when repo file is not as expected
else
fileResult->backupCopyResult = backupCopyResultReCopy;
}
// Recopy on any kind of error
CATCH_ANY()
// Else the source file is missing from the database so skip this file
else
fileResult->backupCopyResult = backupCopyResultSkip;
}
// If this is not a delta backup or it is and the file exists and the checksum from the DB matches, then also
// test the checksum of the file in the repo (unless it is in a prior backup) and if the checksum doesn't match,
// then there may be corruption in the repo, so recopy
if (!delta || !file->manifestFileHasReference)
{
// If this is a delta backup and the file is missing from the DB, then remove it from the repo
// (backupManifestUpdate will remove it from the manifest)
if (fileResult->backupCopyResult == backupCopyResultSkip)
{
fileResult->backupCopyResult = backupCopyResultReCopy;
storageRemoveP(storageRepoWrite(), repoFile);
}
else if (!delta || pgFileMatch)
{
// Check the repo file in a try block because on error (e.g. missing or corrupt file that can't be
// decrypted or decompressed) we should recopy rather than ending the backup.
TRY_BEGIN()
{
// Generate checksum/size for the repo file
IoRead *read = storageReadIo(storageNewReadP(storageRepo(), repoFile));
if (cipherType != cipherTypeNone)
{
ioFilterGroupAdd(
ioReadFilterGroup(read),
cipherBlockNew(cipherModeDecrypt, cipherType, BUFSTR(cipherPass), NULL));
}
// Decompress the file if compressed
if (repoFileCompressType != compressTypeNone)
ioFilterGroupAdd(ioReadFilterGroup(read), decompressFilter(repoFileCompressType));
ioFilterGroupAdd(ioReadFilterGroup(read), cryptoHashNew(hashTypeSha1));
ioFilterGroupAdd(ioReadFilterGroup(read), ioSizeNew());
ioReadDrain(read);
// Test checksum/size
const String *pgTestChecksum = pckReadStrP(
ioFilterGroupResultP(ioReadFilterGroup(read), CRYPTO_HASH_FILTER_TYPE));
uint64_t pgTestSize = pckReadU64P(ioFilterGroupResultP(ioReadFilterGroup(read), SIZE_FILTER_TYPE));
// No need to recopy if checksum/size match
if (file->pgFileSize == pgTestSize && strEq(file->pgFileChecksum, pgTestChecksum))
{
MEM_CONTEXT_BEGIN(lstMemContext(result))
{
fileResult->backupCopyResult = backupCopyResultChecksum;
fileResult->copySize = pgTestSize;
fileResult->copyChecksum = strDup(pgTestChecksum);
}
MEM_CONTEXT_END();
}
// Else recopy when repo file is not as expected
else
fileResult->backupCopyResult = backupCopyResultReCopy;
}
// Recopy on any kind of error
CATCH_ANY()
{
fileResult->backupCopyResult = backupCopyResultReCopy;
}
TRY_END();
}
TRY_END();
}
}
}
MEM_CONTEXT_TEMP_END();
}
// Are the files compressible during the copy?
@ -199,97 +204,104 @@ backupFile(
for (unsigned int fileIdx = 0; fileIdx < lstSize(fileList); fileIdx++)
{
const BackupFile *const file = lstGet(fileList, fileIdx);
BackupFileResult *const fileResult = lstGet(result, fileIdx);
if (fileResult->backupCopyResult == backupCopyResultCopy || fileResult->backupCopyResult == backupCopyResultReCopy)
// Use a per-file mem context to reduce memory usage
MEM_CONTEXT_TEMP_BEGIN()
{
// Setup pg file for read. Only read as many bytes as passed in pgFileSize. If the file is growing it does no good
// to copy data past the end of the size recorded in the manifest since those blocks will need to be replayed from
// WAL during recovery.
StorageRead *read = storageNewReadP(
storagePg(), file->pgFile, .ignoreMissing = file->pgFileIgnoreMissing, .compressible = compressible,
.limit = file->pgFileCopyExactSize ? VARUINT64(file->pgFileSize) : NULL);
ioFilterGroupAdd(ioReadFilterGroup(storageReadIo(read)), cryptoHashNew(hashTypeSha1));
ioFilterGroupAdd(ioReadFilterGroup(storageReadIo(read)), ioSizeNew());
const BackupFile *const file = lstGet(fileList, fileIdx);
BackupFileResult *const fileResult = lstGet(result, fileIdx);
// Add page checksum filter
if (file->pgFileChecksumPage)
if (fileResult->backupCopyResult == backupCopyResultCopy || fileResult->backupCopyResult == backupCopyResultReCopy)
{
ioFilterGroupAdd(
ioReadFilterGroup(storageReadIo(read)),
pageChecksumNew(
segmentNumber(file->pgFile), PG_SEGMENT_PAGE_DEFAULT, storagePathP(storagePg(), file->pgFile)));
}
// Setup pg file for read. Only read as many bytes as passed in pgFileSize. If the file is growing it does no
// good to copy data past the end of the size recorded in the manifest since those blocks will need to be
// replayed from WAL during recovery.
StorageRead *read = storageNewReadP(
storagePg(), file->pgFile, .ignoreMissing = file->pgFileIgnoreMissing, .compressible = compressible,
.limit = file->pgFileCopyExactSize ? VARUINT64(file->pgFileSize) : NULL);
ioFilterGroupAdd(ioReadFilterGroup(storageReadIo(read)), cryptoHashNew(hashTypeSha1));
ioFilterGroupAdd(ioReadFilterGroup(storageReadIo(read)), ioSizeNew());
// Add compression
if (repoFileCompressType != compressTypeNone)
{
ioFilterGroupAdd(
ioReadFilterGroup(storageReadIo(read)), compressFilter(repoFileCompressType, repoFileCompressLevel));
}
// Add page checksum filter
if (file->pgFileChecksumPage)
{
ioFilterGroupAdd(
ioReadFilterGroup(storageReadIo(read)),
pageChecksumNew(
segmentNumber(file->pgFile), PG_SEGMENT_PAGE_DEFAULT, storagePathP(storagePg(), file->pgFile)));
}
// If there is a cipher then add the encrypt filter
if (cipherType != cipherTypeNone)
{
ioFilterGroupAdd(
ioReadFilterGroup(storageReadIo(read)),
cipherBlockNew(cipherModeEncrypt, cipherType, BUFSTR(cipherPass), NULL));
}
// Add compression
if (repoFileCompressType != compressTypeNone)
{
ioFilterGroupAdd(
ioReadFilterGroup(storageReadIo(read)), compressFilter(repoFileCompressType, repoFileCompressLevel));
}
// Add size filter last to calculate repo size
ioFilterGroupAdd(ioReadFilterGroup(storageReadIo(read)), ioSizeNew());
// If there is a cipher then add the encrypt filter
if (cipherType != cipherTypeNone)
{
ioFilterGroupAdd(
ioReadFilterGroup(storageReadIo(read)),
cipherBlockNew(cipherModeEncrypt, cipherType, BUFSTR(cipherPass), NULL));
}
// Open the source and destination and copy the file
if (ioReadOpen(storageReadIo(read)))
{
if (write == NULL)
// Add size filter last to calculate repo size
ioFilterGroupAdd(ioReadFilterGroup(storageReadIo(read)), ioSizeNew());
// Open the source and destination and copy the file
if (ioReadOpen(storageReadIo(read)))
{
// Setup the repo file for write. There is no need to write the file atomically (e.g. via a temp file on
// Posix) because checksums are tested on resume after a failed backup. The path does not need to be synced
// for each file because all paths are synced at the end of the backup.
write = storageNewWriteP(
storageRepoWrite(), repoFile, .compressible = compressible, .noAtomic = true, .noSyncPath = true);
ioWriteOpen(storageWriteIo(write));
}
// Copy data from source to destination
ioCopyP(storageReadIo(read), storageWriteIo(write));
// Close the source
ioReadClose(storageReadIo(read));
MEM_CONTEXT_BEGIN(lstMemContext(result))
{
// Get sizes and checksum
fileResult->copySize = pckReadU64P(
ioFilterGroupResultP(ioReadFilterGroup(storageReadIo(read)), SIZE_FILTER_TYPE, .idx = 0));
fileResult->bundleOffset = bundleOffset;
fileResult->copyChecksum = strDup(
pckReadStrP(ioFilterGroupResultP(ioReadFilterGroup(storageReadIo(read)), CRYPTO_HASH_FILTER_TYPE)));
fileResult->repoSize = pckReadU64P(
ioFilterGroupResultP(ioReadFilterGroup(storageReadIo(read)), SIZE_FILTER_TYPE, .idx = 1));
// Get results of page checksum validation
if (file->pgFileChecksumPage)
// for each file because all paths are synced at the end of the backup. It needs to be created in the prior
// context because it will live longer than a single loop when more than one file is being written.
if (write == NULL)
{
fileResult->pageChecksumResult = pckDup(
ioFilterGroupResultPackP(ioReadFilterGroup(storageReadIo(read)), PAGE_CHECKSUM_FILTER_TYPE));
MEM_CONTEXT_PRIOR_BEGIN()
{
write = storageNewWriteP(
storageRepoWrite(), repoFile, .compressible = compressible, .noAtomic = true,
.noSyncPath = true);
ioWriteOpen(storageWriteIo(write));
}
MEM_CONTEXT_PRIOR_END();
}
// Copy data from source to destination
ioCopyP(storageReadIo(read), storageWriteIo(write));
// Close the source
ioReadClose(storageReadIo(read));
MEM_CONTEXT_BEGIN(lstMemContext(result))
{
// Get sizes and checksum
fileResult->copySize = pckReadU64P(
ioFilterGroupResultP(ioReadFilterGroup(storageReadIo(read)), SIZE_FILTER_TYPE, .idx = 0));
fileResult->bundleOffset = bundleOffset;
fileResult->copyChecksum = strDup(
pckReadStrP(ioFilterGroupResultP(ioReadFilterGroup(storageReadIo(read)), CRYPTO_HASH_FILTER_TYPE)));
fileResult->repoSize = pckReadU64P(
ioFilterGroupResultP(ioReadFilterGroup(storageReadIo(read)), SIZE_FILTER_TYPE, .idx = 1));
// Get results of page checksum validation
if (file->pgFileChecksumPage)
{
fileResult->pageChecksumResult = pckDup(
ioFilterGroupResultPackP(ioReadFilterGroup(storageReadIo(read)), PAGE_CHECKSUM_FILTER_TYPE));
}
}
MEM_CONTEXT_END();
bundleOffset += fileResult->repoSize;
}
MEM_CONTEXT_END();
// Free the read object. This is very important if many files are being read because they can each contain a lot
// of buffers.
storageReadFree(read);
bundleOffset += fileResult->repoSize;
// Else if source file is missing and the read setup indicated ignore a missing file, the database removed it so
// skip it
else
fileResult->backupCopyResult = backupCopyResultSkip;
}
// Else if source file is missing and the read setup indicated ignore a missing file, the database removed it so
// skip it
else
fileResult->backupCopyResult = backupCopyResultSkip;
}
MEM_CONTEXT_TEMP_END();
}
// Close the repository file if it was opened

379
src/command/restore/file.c
View File

@ -47,122 +47,133 @@ List *restoreFile(
// Check files to determine which ones need to be restored
for (unsigned int fileIdx = 0; fileIdx < lstSize(fileList); fileIdx++)
{
const RestoreFile *const file = lstGet(fileList, fileIdx);
ASSERT(file->name != NULL);
ASSERT(file->limit == NULL || varType(file->limit) == varTypeUInt64);
RestoreFileResult *const fileResult = lstAdd(
result, &(RestoreFileResult){.manifestFile = file->manifestFile, .result = restoreResultCopy});
// Perform delta if requested. Delta zero-length files to avoid overwriting the file if the timestamp is correct.
if (delta && !file->zero)
// Use a per-file mem context to reduce memory usage
MEM_CONTEXT_TEMP_BEGIN()
{
// Perform delta if the file exists
StorageInfo info = storageInfoP(storagePg(), file->name, .ignoreMissing = true, .followLink = true);
const RestoreFile *const file = lstGet(fileList, fileIdx);
ASSERT(file->name != NULL);
ASSERT(file->limit == NULL || varType(file->limit) == varTypeUInt64);
if (info.exists)
RestoreFileResult *const fileResult = lstAdd(
result, &(RestoreFileResult){.manifestFile = file->manifestFile, .result = restoreResultCopy});
// Perform delta if requested. Delta zero-length files to avoid overwriting the file if the timestamp is correct.
if (delta && !file->zero)
{
// If force then use size/timestamp delta
if (deltaForce)
// Perform delta if the file exists
StorageInfo info = storageInfoP(storagePg(), file->name, .ignoreMissing = true, .followLink = true);
if (info.exists)
{
// Make sure that timestamp/size are equal and that timestamp is before the copy start time of the backup
if (info.size == file->size && info.timeModified == file->timeModified && info.timeModified < copyTimeBegin)
fileResult->result = restoreResultPreserve;
}
// Else use size and checksum
else
{
// Only continue delta if the file size is as expected or larger
if (info.size >= file->size)
// If force then use size/timestamp delta
if (deltaForce)
{
const char *const fileName = strZ(storagePathP(storagePg(), file->name));
// If the file was extended since the backup, then truncate it to the size it was during the backup as
// it might have only been appended to with the earlier portion being unchanged (we will verify this
// using the checksum below)
if (info.size > file->size)
// Make sure that timestamp/size are equal and the timestamp is before the copy start time of the backup
if (info.size == file->size && info.timeModified == file->timeModified &&
info.timeModified < copyTimeBegin)
{
// Open the file for write
int fd = open(fileName, O_WRONLY, 0);
THROW_ON_SYS_ERROR_FMT(fd == -1, FileReadError, STORAGE_ERROR_WRITE_OPEN, fileName);
TRY_BEGIN()
{
// Truncate to original size
THROW_ON_SYS_ERROR_FMT(
ftruncate(fd, (off_t)file->size) == -1, FileWriteError, "unable to truncate file '%s'",
fileName);
// Sync
THROW_ON_SYS_ERROR_FMT(fsync(fd) == -1, FileSyncError, STORAGE_ERROR_WRITE_SYNC, fileName);
}
FINALLY()
{
THROW_ON_SYS_ERROR_FMT(close(fd) == -1, FileCloseError, STORAGE_ERROR_WRITE_CLOSE, fileName);
}
TRY_END();
// Update info
info = storageInfoP(storagePg(), file->name, .followLink = true);
}
// Generate checksum for the file if size is not zero
IoRead *read = NULL;
if (file->size != 0)
{
read = storageReadIo(storageNewReadP(storagePgWrite(), file->name));
ioFilterGroupAdd(ioReadFilterGroup(read), cryptoHashNew(hashTypeSha1));
ioReadDrain(read);
}
// If the checksum is the same (or file is zero size) then no need to copy the file
if (file->size == 0 ||
strEq(
file->checksum,
pckReadStrP(ioFilterGroupResultP(ioReadFilterGroup(read), CRYPTO_HASH_FILTER_TYPE))))
{
// If the hash/size are now the same but the time is not, then set the time back to the backup time.
// This helps with unit testing, but also presents a pristine version of the database after restore.
if (info.timeModified != file->timeModified)
{
THROW_ON_SYS_ERROR_FMT(
utime(
fileName,
&((struct utimbuf){.actime = file->timeModified, .modtime = file->timeModified})) == -1,
FileInfoError, "unable to set time for '%s'", fileName);
}
fileResult->result = restoreResultPreserve;
}
}
// Else use size and checksum
else
{
// Only continue delta if the file size is as expected or larger
if (info.size >= file->size)
{
const char *const fileName = strZ(storagePathP(storagePg(), file->name));
// If the file was extended since the backup, then truncate it to the size it was during the backup
// as it might have only been appended to with the earlier portion being unchanged (we will verify
// this using the checksum below)
if (info.size > file->size)
{
// Open the file for write
int fd = open(fileName, O_WRONLY, 0);
THROW_ON_SYS_ERROR_FMT(fd == -1, FileReadError, STORAGE_ERROR_WRITE_OPEN, fileName);
TRY_BEGIN()
{
// Truncate to original size
THROW_ON_SYS_ERROR_FMT(
ftruncate(fd, (off_t)file->size) == -1, FileWriteError, "unable to truncate file '%s'",
fileName);
// Sync
THROW_ON_SYS_ERROR_FMT(fsync(fd) == -1, FileSyncError, STORAGE_ERROR_WRITE_SYNC, fileName);
}
FINALLY()
{
THROW_ON_SYS_ERROR_FMT(
close(fd) == -1, FileCloseError, STORAGE_ERROR_WRITE_CLOSE, fileName);
}
TRY_END();
// Update info
info = storageInfoP(storagePg(), file->name, .followLink = true);
}
// Generate checksum for the file if size is not zero
IoRead *read = NULL;
if (file->size != 0)
{
read = storageReadIo(storageNewReadP(storagePgWrite(), file->name));
ioFilterGroupAdd(ioReadFilterGroup(read), cryptoHashNew(hashTypeSha1));
ioReadDrain(read);
}
// If the checksum is the same (or file is zero size) then no need to copy the file
if (file->size == 0 ||
strEq(
file->checksum,
pckReadStrP(ioFilterGroupResultP(ioReadFilterGroup(read), CRYPTO_HASH_FILTER_TYPE))))
{
// If the hash/size are now the same but the time is not, then set the time back to the backup
// time. This helps with unit testing, but also presents a pristine version of the database
// after restore.
if (info.timeModified != file->timeModified)
{
THROW_ON_SYS_ERROR_FMT(
utime(
fileName,
&((struct utimbuf){
.actime = file->timeModified, .modtime = file->timeModified})) == -1,
FileInfoError, "unable to set time for '%s'", fileName);
}
fileResult->result = restoreResultPreserve;
}
}
}
}
}
}
// Create zeroed and zero-length files
if (fileResult->result == restoreResultCopy && (file->size == 0 || file->zero))
{
// Create destination file
StorageWrite *pgFileWrite = storageNewWriteP(
storagePgWrite(), file->name, .modeFile = file->mode, .user = file->user, .group = file->group,
.timeModified = file->timeModified, .noAtomic = true, .noCreatePath = true, .noSyncPath = true);
ioWriteOpen(storageWriteIo(pgFileWrite));
// Truncate the file to specified length (note in this case the file will grow, not shrink)
if (file->zero)
// Create zeroed and zero-length files
if (fileResult->result == restoreResultCopy && (file->size == 0 || file->zero))
{
THROW_ON_SYS_ERROR_FMT(
ftruncate(ioWriteFd(storageWriteIo(pgFileWrite)), (off_t)file->size) == -1, FileWriteError,
"unable to truncate '%s'", strZ(file->name));
// Create destination file
StorageWrite *pgFileWrite = storageNewWriteP(
storagePgWrite(), file->name, .modeFile = file->mode, .user = file->user, .group = file->group,
.timeModified = file->timeModified, .noAtomic = true, .noCreatePath = true, .noSyncPath = true);
ioWriteOpen(storageWriteIo(pgFileWrite));
// Truncate the file to specified length (note in this case the file will grow, not shrink)
if (file->zero)
{
THROW_ON_SYS_ERROR_FMT(
ftruncate(ioWriteFd(storageWriteIo(pgFileWrite)), (off_t)file->size) == -1, FileWriteError,
"unable to truncate '%s'", strZ(file->name));
}
ioWriteClose(storageWriteIo(pgFileWrite));
// Report the file as zeroed or zero-length
fileResult->result = restoreResultZero;
}
ioWriteClose(storageWriteIo(pgFileWrite));
// Report the file as zeroed or zero-length
fileResult->result = restoreResultZero;
}
MEM_CONTEXT_TEMP_END();
}
// Copy files from repository to database
@ -171,97 +182,107 @@ List *restoreFile(
for (unsigned int fileIdx = 0; fileIdx < lstSize(fileList); fileIdx++)
{
const RestoreFile *const file = lstGet(fileList, fileIdx);
const RestoreFileResult *const fileResult = lstGet(result, fileIdx);
// Copy file from repository to database
if (fileResult->result == restoreResultCopy)
// Use a per-file mem context to reduce memory usage
MEM_CONTEXT_TEMP_BEGIN()
{
// If no repo file is currently open
if (repoFileLimit == 0)
const RestoreFile *const file = lstGet(fileList, fileIdx);
const RestoreFileResult *const fileResult = lstGet(result, fileIdx);
// Copy file from repository to database
if (fileResult->result == restoreResultCopy)
{
// If a limit is specified then we need to use it, even if there is only one pg file to copy, because we might
// be reading from the middle of a repo file containing many pg files
if (file->limit != NULL)
// If no repo file is currently open
if (repoFileLimit == 0)
{
ASSERT(varUInt64(file->limit) != 0);
repoFileLimit = varUInt64(file->limit);
// Determine how many files can be copied with one read
for (unsigned int fileNextIdx = fileIdx + 1; fileNextIdx < lstSize(fileList); fileNextIdx++)
// If a limit is specified then we need to use it, even if there is only one pg file to copy, because we
// might be reading from the middle of a repo file containing many pg files
if (file->limit != NULL)
{
// Only files that are being copied are considered
if (((const RestoreFileResult *)lstGet(result, fileNextIdx))->result == restoreResultCopy)
ASSERT(varUInt64(file->limit) != 0);
repoFileLimit = varUInt64(file->limit);
// Determine how many files can be copied with one read
for (unsigned int fileNextIdx = fileIdx + 1; fileNextIdx < lstSize(fileList); fileNextIdx++)
{
const RestoreFile *const fileNext = lstGet(fileList, fileNextIdx);
ASSERT(fileNext->limit != NULL && varUInt64(fileNext->limit) != 0);
// Only files that are being copied are considered
if (((const RestoreFileResult *)lstGet(result, fileNextIdx))->result == restoreResultCopy)
{
const RestoreFile *const fileNext = lstGet(fileList, fileNextIdx);
ASSERT(fileNext->limit != NULL && varUInt64(fileNext->limit) != 0);
// Break if the offset is not the first file's offset + the limit of all additional files so far
if (fileNext->offset != file->offset + repoFileLimit)
// Break if the offset is not the first file's offset + the limit of all additional files so far
if (fileNext->offset != file->offset + repoFileLimit)
break;
repoFileLimit += varUInt64(fileNext->limit);
}
// Else if the file was not copied then there is a gap so break
else
break;
repoFileLimit += varUInt64(fileNext->limit);
}
// Else if the file was not copied then there is a gap so break
else
break;
}
// Create and open the repo file. It needs to be created in the prior context because it will live longer
// than a single loop when more than one file is being read.
MEM_CONTEXT_PRIOR_BEGIN()
{
repoFileRead = storageNewReadP(
storageRepoIdx(repoIdx), repoFile,
.compressible = repoFileCompressType == compressTypeNone && cipherPass == NULL,
.offset = file->offset, .limit = repoFileLimit != 0 ? VARUINT64(repoFileLimit) : NULL);
ioReadOpen(storageReadIo(repoFileRead));
}
MEM_CONTEXT_PRIOR_END();
}
// Create and open the repo file
repoFileRead = storageNewReadP(
storageRepoIdx(repoIdx), repoFile,
.compressible = repoFileCompressType == compressTypeNone && cipherPass == NULL, .offset = file->offset,
.limit = repoFileLimit != 0 ? VARUINT64(repoFileLimit) : NULL);
ioReadOpen(storageReadIo(repoFileRead));
// Create pg file
StorageWrite *pgFileWrite = storageNewWriteP(
storagePgWrite(), file->name, .modeFile = file->mode, .user = file->user, .group = file->group,
.timeModified = file->timeModified, .noAtomic = true, .noCreatePath = true, .noSyncPath = true);
IoFilterGroup *filterGroup = ioWriteFilterGroup(storageWriteIo(pgFileWrite));
// Add decryption filter
if (cipherPass != NULL)
{
ioFilterGroupAdd(
filterGroup, cipherBlockNew(cipherModeDecrypt, cipherTypeAes256Cbc, BUFSTR(cipherPass), NULL));
}
// Add decompression filter
if (repoFileCompressType != compressTypeNone)
ioFilterGroupAdd(filterGroup, decompressFilter(repoFileCompressType));
// Add sha1 filter
ioFilterGroupAdd(filterGroup, cryptoHashNew(hashTypeSha1));
// Add size filter
ioFilterGroupAdd(filterGroup, ioSizeNew());
// Copy file
ioWriteOpen(storageWriteIo(pgFileWrite));
ioCopyP(storageReadIo(repoFileRead), storageWriteIo(pgFileWrite), .limit = file->limit);
ioWriteClose(storageWriteIo(pgFileWrite));
// If more than one file is being copied from a single read then decrement the limit
if (repoFileLimit != 0)
repoFileLimit -= varUInt64(file->limit);
// Free the repo file when there are no more files to copy from it
if (repoFileLimit == 0)
storageReadFree(repoFileRead);
// Validate checksum
if (!strEq(file->checksum, pckReadStrP(ioFilterGroupResultP(filterGroup, CRYPTO_HASH_FILTER_TYPE))))
{
THROW_FMT(
ChecksumError,
"error restoring '%s': actual checksum '%s' does not match expected checksum '%s'", strZ(file->name),
strZ(pckReadStrP(ioFilterGroupResultP(filterGroup, CRYPTO_HASH_FILTER_TYPE))), strZ(file->checksum));
}
}
// Create pg file
StorageWrite *pgFileWrite = storageNewWriteP(
storagePgWrite(), file->name, .modeFile = file->mode, .user = file->user, .group = file->group,
.timeModified = file->timeModified, .noAtomic = true, .noCreatePath = true, .noSyncPath = true);
IoFilterGroup *filterGroup = ioWriteFilterGroup(storageWriteIo(pgFileWrite));
// Add decryption filter
if (cipherPass != NULL)
ioFilterGroupAdd(filterGroup, cipherBlockNew(cipherModeDecrypt, cipherTypeAes256Cbc, BUFSTR(cipherPass), NULL));
// Add decompression filter
if (repoFileCompressType != compressTypeNone)
ioFilterGroupAdd(filterGroup, decompressFilter(repoFileCompressType));
// Add sha1 filter
ioFilterGroupAdd(filterGroup, cryptoHashNew(hashTypeSha1));
// Add size filter
ioFilterGroupAdd(filterGroup, ioSizeNew());
// Copy file
ioWriteOpen(storageWriteIo(pgFileWrite));
ioCopyP(storageReadIo(repoFileRead), storageWriteIo(pgFileWrite), .limit = file->limit);
ioWriteClose(storageWriteIo(pgFileWrite));
// If more than one file is being copied from a single read then decrement the limit
if (repoFileLimit != 0)
repoFileLimit -= varUInt64(file->limit);
// Free the repo file when there are no more files to copy from it
if (repoFileLimit == 0)
storageReadFree(repoFileRead);
// Validate checksum
if (!strEq(file->checksum, pckReadStrP(ioFilterGroupResultP(filterGroup, CRYPTO_HASH_FILTER_TYPE))))
{
THROW_FMT(
ChecksumError,
"error restoring '%s': actual checksum '%s' does not match expected checksum '%s'", strZ(file->name),
strZ(pckReadStrP(ioFilterGroupResultP(filterGroup, CRYPTO_HASH_FILTER_TYPE))), strZ(file->checksum));
}
// Free the pg file
storageWriteFree(pgFileWrite);
}
MEM_CONTEXT_TEMP_END();
}
lstMove(result, memContextPrior());