db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2024-12-12 10:04:14 +02:00
Code Issues Releases Activity

Optimize restore command for file bundling.

Browse Source 
Since files are stored sequentially in a bundle, it is often possible to restore multiple files with a single read. Previously, each restored file required a separate read. Reducing the number of reads is particularly beneficial for object stores, but performance should benefit on any file system.

Currently if there is a gap then a new read is required. In the future we might set a limit for how large a gap we'll skip without starting a new read.
This commit is contained in:
David Steele 2022-03-09 15:03:28 -06:00 committed by GitHub
parent f7ab002aa7
commit dca6da86bf
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 563 additions and 219 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
doc/xml/release.xml
View File

@ -46,9 +46,15 @@
<release-item> <release-item>
<commit subject="Remove redundant restoreFile() test and improve coverage."/> <commit subject="Remove redundant restoreFile() test and improve coverage."/>
<commit subject="Add limit parameter to ioCopyP()."/> <commit subject="Add limit parameter to ioCopyP()."/>
<commit subject="Optimize restore command for file bundling.">
<github-issue id="1149"/>
<github-pull-request id="1683"/>
</commit>
<release-item-contributor-list> <release-item-contributor-list>
<release-item-contributor id="david.steele"/> <release-item-contributor id="david.steele"/>
<release-item-reviewer id="reid.thompson"/>
<release-item-reviewer id="stefan.fercot"/>
</release-item-contributor-list> </release-item-contributor-list>
<p>Improve small file support.</p> <p>Improve small file support.</p>

246
src/command/restore/file.c
View File

@ -20,145 +20,185 @@ Restore File
#include "storage/helper.h" #include "storage/helper.h"
/**********************************************************************************************************************************/ /**********************************************************************************************************************************/
bool List *restoreFile(
restoreFile( const String *const repoFile, const unsigned int repoIdx, const CompressType repoFileCompressType, const time_t copyTimeBegin,
const String *const repoFile, unsigned int repoIdx, const uint64_t offset, const Variant *const limit, const bool delta, const bool deltaForce, const String *const cipherPass, const List *const fileList)
const CompressType repoFileCompressType, const String *const pgFile, const String *const pgFileChecksum, const bool pgFileZero,
const uint64_t pgFileSize, const time_t pgFileModified, const mode_t pgFileMode, const String *const pgFileUser,
const String *const pgFileGroup, const time_t copyTimeBegin, const bool delta, const bool deltaForce,
const String *const cipherPass)
{ {
FUNCTION_LOG_BEGIN(logLevelDebug); FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(STRING, repoFile); FUNCTION_LOG_PARAM(STRING, repoFile);
FUNCTION_LOG_PARAM(UINT, repoIdx); FUNCTION_LOG_PARAM(UINT, repoIdx);
FUNCTION_LOG_PARAM(UINT64, offset);
FUNCTION_LOG_PARAM(VARIANT, limit);
FUNCTION_LOG_PARAM(ENUM, repoFileCompressType); FUNCTION_LOG_PARAM(ENUM, repoFileCompressType);
FUNCTION_LOG_PARAM(STRING, pgFile);
FUNCTION_LOG_PARAM(STRING, pgFileChecksum);
FUNCTION_LOG_PARAM(BOOL, pgFileZero);
FUNCTION_LOG_PARAM(UINT64, pgFileSize);
FUNCTION_LOG_PARAM(TIME, pgFileModified);
FUNCTION_LOG_PARAM(MODE, pgFileMode);
FUNCTION_LOG_PARAM(STRING, pgFileUser);
FUNCTION_LOG_PARAM(STRING, pgFileGroup);
FUNCTION_LOG_PARAM(TIME, copyTimeBegin); FUNCTION_LOG_PARAM(TIME, copyTimeBegin);
FUNCTION_LOG_PARAM(BOOL, delta); FUNCTION_LOG_PARAM(BOOL, delta);
FUNCTION_LOG_PARAM(BOOL, deltaForce); FUNCTION_LOG_PARAM(BOOL, deltaForce);
FUNCTION_TEST_PARAM(STRING, cipherPass); FUNCTION_TEST_PARAM(STRING, cipherPass);
FUNCTION_LOG_PARAM(LIST, fileList); // List of files to restore
FUNCTION_LOG_END(); FUNCTION_LOG_END();
ASSERT(repoFile != NULL); ASSERT(repoFile != NULL);
ASSERT(pgFile != NULL);
ASSERT(limit == NULL || varType(limit) == varTypeUInt64);
// Was the file copied? // Restore file results
bool result = true; List *result = NULL;
// Is the file compressible during the copy?
bool compressible = true;
MEM_CONTEXT_TEMP_BEGIN() MEM_CONTEXT_TEMP_BEGIN()
{ {
// Perform delta if requested. Delta zero-length files to avoid overwriting the file if the timestamp is correct. result = lstNewP(sizeof(RestoreFileResult));
if (delta && !pgFileZero)
// Check files to determine which ones need to be restored
for (unsigned int fileIdx = 0; fileIdx < lstSize(fileList); fileIdx++)
{ {
// Perform delta if the file exists const RestoreFile *const file = lstGet(fileList, fileIdx);
StorageInfo info = storageInfoP(storagePg(), pgFile, .ignoreMissing = true, .followLink = true); ASSERT(file->name != NULL);
ASSERT(file->limit == NULL || varType(file->limit) == varTypeUInt64);
if (info.exists) RestoreFileResult *const fileResult = lstAdd(
result, &(RestoreFileResult){.manifestFile = file->manifestFile, .result = restoreResultCopy});
// Perform delta if requested. Delta zero-length files to avoid overwriting the file if the timestamp is correct.
if (delta && !file->zero)
{ {
// If force then use size/timestamp delta // Perform delta if the file exists
if (deltaForce) StorageInfo info = storageInfoP(storagePg(), file->name, .ignoreMissing = true, .followLink = true);
if (info.exists)
{ {
// Make sure that timestamp/size are equal and that timestamp is before the copy start time of the backup // If force then use size/timestamp delta
if (info.size == pgFileSize && info.timeModified == pgFileModified && info.timeModified < copyTimeBegin) if (deltaForce)
result = false;
}
// Else use size and checksum
else
{
// Only continue delta if the file size is as expected
if (info.size == pgFileSize)
{ {
// Generate checksum for the file if size is not zero // Make sure that timestamp/size are equal and that timestamp is before the copy start time of the backup
IoRead *read = NULL; if (info.size == file->size && info.timeModified == file->timeModified && info.timeModified < copyTimeBegin)
fileResult->result = restoreResultPreserve;
if (info.size != 0) }
// Else use size and checksum
else
{
// Only continue delta if the file size is as expected
if (info.size == file->size)
{ {
read = storageReadIo(storageNewReadP(storagePgWrite(), pgFile)); // Generate checksum for the file if size is not zero
ioFilterGroupAdd(ioReadFilterGroup(read), cryptoHashNew(HASH_TYPE_SHA1_STR)); IoRead *read = NULL;
ioReadDrain(read);
}
// If size and checksum are equal then no need to copy the file if (info.size != 0)
if (pgFileSize == 0 ||
strEq(
pgFileChecksum,
pckReadStrP(ioFilterGroupResultP(ioReadFilterGroup(read), CRYPTO_HASH_FILTER_TYPE))))
{
// Even if hash/size are the same set the time back to backup time. This helps with unit testing, but
// also presents a pristine version of the database after restore.
if (info.timeModified != pgFileModified)
{ {
THROW_ON_SYS_ERROR_FMT( read = storageReadIo(storageNewReadP(storagePgWrite(), file->name));
utime( ioFilterGroupAdd(ioReadFilterGroup(read), cryptoHashNew(HASH_TYPE_SHA1_STR));
strZ(storagePathP(storagePg(), pgFile)), ioReadDrain(read);
&((struct utimbuf){.actime = pgFileModified, .modtime = pgFileModified})) == -1,
FileInfoError, "unable to set time for '%s'", strZ(storagePathP(storagePg(), pgFile)));
} }
result = false; // If the checksum is also equal (or file is zero size) then no need to copy the file
if (file->size == 0 ||
strEq(
file->checksum,
pckReadStrP(ioFilterGroupResultP(ioReadFilterGroup(read), CRYPTO_HASH_FILTER_TYPE))))
{
// Even if hash/size are the same set the time back to backup time. This helps with unit testing,
// but also presents a pristine version of the database after restore.
if (info.timeModified != file->timeModified)
{
THROW_ON_SYS_ERROR_FMT(
utime(
strZ(storagePathP(storagePg(), file->name)),
&((struct utimbuf){.actime = file->timeModified, .modtime = file->timeModified})) == -1,
FileInfoError, "unable to set time for '%s'", strZ(storagePathP(storagePg(), file->name)));
}
fileResult->result = restoreResultPreserve;
}
} }
} }
} }
} }
}
// Copy file from repository to database or create zero-length/sparse file // Create zeroed and zero-length files
if (result) if (fileResult->result == restoreResultCopy && (file->size == 0 || file->zero))
{
// Create destination file
StorageWrite *pgFileWrite = storageNewWriteP(
storagePgWrite(), pgFile, .modeFile = pgFileMode, .user = pgFileUser, .group = pgFileGroup,
.timeModified = pgFileModified, .noAtomic = true, .noCreatePath = true, .noSyncPath = true);
// If size is zero/sparse no need to actually copy
if (pgFileSize == 0 || pgFileZero)
{ {
// Create destination file
StorageWrite *pgFileWrite = storageNewWriteP(
storagePgWrite(), file->name, .modeFile = file->mode, .user = file->user, .group = file->group,
.timeModified = file->timeModified, .noAtomic = true, .noCreatePath = true, .noSyncPath = true);
ioWriteOpen(storageWriteIo(pgFileWrite)); ioWriteOpen(storageWriteIo(pgFileWrite));
// Truncate the file to specified length (note in this case the file with grow, not shrink) // Truncate the file to specified length (note in this case the file will grow, not shrink)
if (pgFileZero) if (file->zero)
{ {
THROW_ON_SYS_ERROR_FMT( THROW_ON_SYS_ERROR_FMT(
ftruncate(ioWriteFd(storageWriteIo(pgFileWrite)), (off_t)pgFileSize) == -1, FileWriteError, ftruncate(ioWriteFd(storageWriteIo(pgFileWrite)), (off_t)file->size) == -1, FileWriteError,
"unable to truncate '%s'", strZ(pgFile)); "unable to truncate '%s'", strZ(file->name));
// Report the file as not copied
result = false;
} }
ioWriteClose(storageWriteIo(pgFileWrite)); ioWriteClose(storageWriteIo(pgFileWrite));
// Report the file as zeroed or zero-length
fileResult->result = restoreResultZero;
} }
// Else perform the copy }
else
// Copy files from repository to database
StorageRead *repoFileRead = NULL;
uint64_t repoFileLimit = 0;
for (unsigned int fileIdx = 0; fileIdx < lstSize(fileList); fileIdx++)
{
const RestoreFile *const file = lstGet(fileList, fileIdx);
const RestoreFileResult *const fileResult = lstGet(result, fileIdx);
// Copy file from repository to database
if (fileResult->result == restoreResultCopy)
{ {
// If no repo file is currently open
if (repoFileLimit == 0)
{
// If a limit is specified then we need to use it, even if there is only one pg file to copy, because we might
// be reading from the middle of a repo file containing many pg files
if (file->limit != NULL)
{
ASSERT(varUInt64(file->limit) != 0);
repoFileLimit = varUInt64(file->limit);
// Determine how many files can be copied with one read
for (unsigned int fileNextIdx = fileIdx + 1; fileNextIdx < lstSize(fileList); fileNextIdx++)
{
// Only files that are being copied are considered
if (((const RestoreFileResult *)lstGet(result, fileNextIdx))->result == restoreResultCopy)
{
const RestoreFile *const fileNext = lstGet(fileList, fileNextIdx);
ASSERT(fileNext->limit != NULL && varUInt64(fileNext->limit) != 0);
// Break if the offset is not the first file's offset + the limit of all additional files so far
if (fileNext->offset != file->offset + repoFileLimit)
break;
repoFileLimit += varUInt64(fileNext->limit);
}
// Else if the file was not copied then there is a gap so break
else
break;
}
}
// Create and open the repo file
repoFileRead = storageNewReadP(
storageRepoIdx(repoIdx), repoFile,
.compressible = repoFileCompressType == compressTypeNone && cipherPass == NULL, .offset = file->offset,
.limit = repoFileLimit != 0 ? VARUINT64(repoFileLimit) : NULL);
ioReadOpen(storageReadIo(repoFileRead));
}
// Create pg file
StorageWrite *pgFileWrite = storageNewWriteP(
storagePgWrite(), file->name, .modeFile = file->mode, .user = file->user, .group = file->group,
.timeModified = file->timeModified, .noAtomic = true, .noCreatePath = true, .noSyncPath = true);
IoFilterGroup *filterGroup = ioWriteFilterGroup(storageWriteIo(pgFileWrite)); IoFilterGroup *filterGroup = ioWriteFilterGroup(storageWriteIo(pgFileWrite));
// Add decryption filter // Add decryption filter
if (cipherPass != NULL) if (cipherPass != NULL)
{
ioFilterGroupAdd(filterGroup, cipherBlockNew(cipherModeDecrypt, cipherTypeAes256Cbc, BUFSTR(cipherPass), NULL)); ioFilterGroupAdd(filterGroup, cipherBlockNew(cipherModeDecrypt, cipherTypeAes256Cbc, BUFSTR(cipherPass), NULL));
compressible = false;
}
// Add decompression filter // Add decompression filter
if (repoFileCompressType != compressTypeNone) if (repoFileCompressType != compressTypeNone)
{
ioFilterGroupAdd(filterGroup, decompressFilter(repoFileCompressType)); ioFilterGroupAdd(filterGroup, decompressFilter(repoFileCompressType));
compressible = false;
}
// Add sha1 filter // Add sha1 filter
ioFilterGroupAdd(filterGroup, cryptoHashNew(HASH_TYPE_SHA1_STR)); ioFilterGroupAdd(filterGroup, cryptoHashNew(HASH_TYPE_SHA1_STR));
@ -167,23 +207,35 @@ restoreFile(
ioFilterGroupAdd(filterGroup, ioSizeNew()); ioFilterGroupAdd(filterGroup, ioSizeNew());
// Copy file // Copy file
storageCopyP( ioWriteOpen(storageWriteIo(pgFileWrite));
storageNewReadP( ioCopyP(storageReadIo(repoFileRead), storageWriteIo(pgFileWrite), .limit = file->limit);
storageRepoIdx(repoIdx), repoFile, .compressible = compressible, .offset = offset, .limit = limit), ioWriteClose(storageWriteIo(pgFileWrite));
pgFileWrite);
// If more than one file is being copied from a single read then decrement the limit
if (repoFileLimit != 0)
repoFileLimit -= varUInt64(file->limit);
// Free the repo file when there are no more files to copy from it
if (repoFileLimit == 0)
storageReadFree(repoFileRead);
// Validate checksum // Validate checksum
if (!strEq(pgFileChecksum, pckReadStrP(ioFilterGroupResultP(filterGroup, CRYPTO_HASH_FILTER_TYPE)))) if (!strEq(file->checksum, pckReadStrP(ioFilterGroupResultP(filterGroup, CRYPTO_HASH_FILTER_TYPE))))
{ {
THROW_FMT( THROW_FMT(
ChecksumError, ChecksumError,
"error restoring '%s': actual checksum '%s' does not match expected checksum '%s'", strZ(pgFile), "error restoring '%s': actual checksum '%s' does not match expected checksum '%s'", strZ(file->name),
strZ(pckReadStrP(ioFilterGroupResultP(filterGroup, CRYPTO_HASH_FILTER_TYPE))), strZ(pgFileChecksum)); strZ(pckReadStrP(ioFilterGroupResultP(filterGroup, CRYPTO_HASH_FILTER_TYPE))), strZ(file->checksum));
} }
// Free the pg file
storageWriteFree(pgFileWrite);
} }
} }
lstMove(result, memContextPrior());
} }
MEM_CONTEXT_TEMP_END(); MEM_CONTEXT_TEMP_END();
FUNCTION_LOG_RETURN(BOOL, result); FUNCTION_LOG_RETURN(LIST, result);
} }

43
src/command/restore/file.h
View File

@ -5,18 +5,45 @@ Restore File
#define COMMAND_RESTORE_FILE_H #define COMMAND_RESTORE_FILE_H
#include "common/compress/helper.h" #include "common/compress/helper.h"
#include "common/crypto/common.h" #include "common/type/variant.h"
#include "common/type/string.h"
#include "storage/storage.h" /***********************************************************************************************************************************
Restore file types
***********************************************************************************************************************************/
typedef enum
{
restoreResultPreserve,
restoreResultZero,
restoreResultCopy,
} RestoreResult;
/*********************************************************************************************************************************** /***********************************************************************************************************************************
Functions Functions
***********************************************************************************************************************************/ ***********************************************************************************************************************************/
// Copy a file from the backup to the specified destination // Copy a file from the backup to the specified destination
bool restoreFile( typedef struct RestoreFile
const String *repoFile, unsigned int repoIdx, uint64_t offset, const Variant *limit, CompressType repoFileCompressType, {
const String *pgFile, const String *pgFileChecksum, bool pgFileZero, uint64_t pgFileSize, time_t pgFileModified, const String *name; // File to restore
mode_t pgFileMode, const String *pgFileUser, const String *pgFileGroup, time_t copyTimeBegin, bool delta, bool deltaForce, const String *checksum; // Expected checksum
const String *cipherPass); uint64_t size; // Expected size
time_t timeModified; // Original modification time
mode_t mode; // Original mode
bool zero; // Should the file be zeroed?
const String *user; // Original user
const String *group; // Original group
uint64_t offset; // Offset into repo file where pg file is located
const Variant *limit; // Limit for read in the repo file
const String *manifestFile; // Manifest file
} RestoreFile;
typedef struct RestoreFileResult
{
const String *manifestFile; // Manifest file
RestoreResult result; // Restore result (e.g. preserve, copy)
} RestoreFileResult;
List *restoreFile(
const String *repoFile, unsigned int repoIdx, CompressType repoFileCompressType, time_t copyTimeBegin, bool delta,
bool deltaForce, const String *cipherPass, const List *fileList);
#endif #endif

61
src/command/restore/protocol.c
View File

@ -28,37 +28,54 @@ restoreFileProtocol(PackRead *const param, ProtocolServer *const server)
{ {
// Restore file // Restore file
const String *const repoFile = pckReadStrP(param); const String *const repoFile = pckReadStrP(param);
uint64_t offset = 0;
const Variant *limit = NULL;
if (pckReadBoolP(param))
{
offset = pckReadU64P(param);
limit = varNewUInt64(pckReadU64P(param));
}
const unsigned int repoIdx = pckReadU32P(param); const unsigned int repoIdx = pckReadU32P(param);
const CompressType repoFileCompressType = (CompressType)pckReadU32P(param); const CompressType repoFileCompressType = (CompressType)pckReadU32P(param);
const String *const pgFile = pckReadStrP(param);
const String *const pgFileChecksum = pckReadStrP(param);
const bool pgFileZero = pckReadBoolP(param);
const uint64_t pgFileSize = pckReadU64P(param);
const time_t pgFileModified = pckReadTimeP(param);
const mode_t pgFileMode = pckReadModeP(param);
const String *const pgFileUser = pckReadStrP(param);
const String *const pgFileGroup = pckReadStrP(param);
const time_t copyTimeBegin = pckReadTimeP(param); const time_t copyTimeBegin = pckReadTimeP(param);
const bool delta = pckReadBoolP(param); const bool delta = pckReadBoolP(param);
const bool deltaForce = pckReadBoolP(param); const bool deltaForce = pckReadBoolP(param);
const String *const cipherPass = pckReadStrP(param); const String *const cipherPass = pckReadStrP(param);
const bool result = restoreFile( // Build the file list
repoFile, repoIdx, offset, limit, repoFileCompressType, pgFile, pgFileChecksum, pgFileZero, pgFileSize, pgFileModified, List *fileList = lstNewP(sizeof(RestoreFile));
pgFileMode, pgFileUser, pgFileGroup, copyTimeBegin, delta, deltaForce, cipherPass);
while (!pckReadNullP(param))
{
RestoreFile file = {.name = pckReadStrP(param)};
file.checksum = pckReadStrP(param);
file.size = pckReadU64P(param);
file.timeModified = pckReadTimeP(param);
file.mode = pckReadModeP(param);
file.zero = pckReadBoolP(param);
file.user = pckReadStrP(param);
file.group = pckReadStrP(param);
if (pckReadBoolP(param))
{
file.offset = pckReadU64P(param);
file.limit = varNewUInt64(pckReadU64P(param));
}
file.manifestFile = pckReadStrP(param);
lstAdd(fileList, &file);
}
// Restore files
const List *const result = restoreFile(
repoFile, repoIdx, repoFileCompressType, copyTimeBegin, delta, deltaForce, cipherPass, fileList);
// Return result // Return result
protocolServerDataPut(server, pckWriteBoolP(protocolPackNew(), result)); PackWrite *const resultPack = protocolPackNew();
for (unsigned int resultIdx = 0; resultIdx < lstSize(result); resultIdx++)
{
const RestoreFileResult *const fileResult = lstGet(result, resultIdx);
pckWriteStrP(resultPack, fileResult->manifestFile);
pckWriteU32P(resultPack, fileResult->result);
}
protocolServerDataPut(server, resultPack);
protocolServerDataEndPut(server); protocolServerDataEndPut(server);
} }
MEM_CONTEXT_TEMP_END(); MEM_CONTEXT_TEMP_END();

254
src/command/restore/restore.c
View File

@ -8,6 +8,7 @@ Restore Command
#include <time.h> #include <time.h>
#include <unistd.h> #include <unistd.h>
#include "command/restore/file.h"
#include "command/restore/protocol.h" #include "command/restore/protocol.h"
#include "command/restore/restore.h" #include "command/restore/restore.h"
#include "common/crypto/cipherBlock.h" #include "common/crypto/cipherBlock.h"
@ -1914,14 +1915,58 @@ restoreProcessQueueComparator(const void *item1, const void *item2)
ManifestFile file1 = manifestFileUnpack(restoreProcessQueueComparatorManifest, *(const ManifestFilePack **)item1); ManifestFile file1 = manifestFileUnpack(restoreProcessQueueComparatorManifest, *(const ManifestFilePack **)item1);
ManifestFile file2 = manifestFileUnpack(restoreProcessQueueComparatorManifest, *(const ManifestFilePack **)item2); ManifestFile file2 = manifestFileUnpack(restoreProcessQueueComparatorManifest, *(const ManifestFilePack **)item2);
// If the size differs then that's enough to determine order // Zero length files should be ordered at the end
if (file1.size < file2.size) if (file1.size == 0)
FUNCTION_TEST_RETURN(-1); {
else if (file1.size > file2.size) if (file2.size != 0)
FUNCTION_TEST_RETURN(-1);
}
else if (file2.size == 0)
FUNCTION_TEST_RETURN(1); FUNCTION_TEST_RETURN(1);
// If size is the same then use name to generate a deterministic ordering (names must be unique) // If the bundle id differs that is enough to determine order
FUNCTION_TEST_RETURN(strCmp(file1.name, file2.name)); if (file1.bundleId < file2.bundleId)
FUNCTION_TEST_RETURN(1);
else if (file1.bundleId > file2.bundleId)
FUNCTION_TEST_RETURN(-1);
// If the bundle ids are 0
if (file1.bundleId == 0)
{
// If the size differs then that's enough to determine order
if (file1.size < file2.size)
FUNCTION_TEST_RETURN(-1);
else if (file1.size > file2.size)
FUNCTION_TEST_RETURN(1);
// If size is the same then use name to generate a deterministic ordering (names must be unique)
ASSERT(!strEq(file1.name, file2.name));
FUNCTION_TEST_RETURN(strCmp(file1.name, file2.name));
}
// If the reference differs that is enough to determine order
if (file1.reference == NULL)
{
if (file2.reference != NULL)
FUNCTION_TEST_RETURN(-1);
}
else if (file2.reference == NULL)
FUNCTION_TEST_RETURN(1);
else
{
const int backupLabelCmp = strCmp(file1.reference, file2.reference) * -1;
if (backupLabelCmp != 0)
FUNCTION_TEST_RETURN(backupLabelCmp);
}
// Finally order by bundle offset
ASSERT(file1.bundleOffset != file2.bundleOffset);
if (file1.bundleOffset < file2.bundleOffset)
FUNCTION_TEST_RETURN(1);
FUNCTION_TEST_RETURN(-1);
} }
static uint64_t static uint64_t
@ -2062,56 +2107,77 @@ restoreJobResult(const Manifest *manifest, ProtocolParallelJob *job, RegExp *zer
{ {
MEM_CONTEXT_TEMP_BEGIN() MEM_CONTEXT_TEMP_BEGIN()
{ {
const ManifestFile file = manifestFileFind(manifest, varStr(protocolParallelJobKey(job))); PackRead *const jobResult = protocolParallelJobResult(job);
bool zeroed = restoreFileZeroed(file.name, zeroExp);
bool copy = pckReadBoolP(protocolParallelJobResult(job));
String *log = strCatZ(strNew(), "restore"); while (!pckReadNullP(jobResult))
// Note if file was zeroed (i.e. selective restore)
if (zeroed)
strCatZ(log, " zeroed");
// Add filename
strCatFmt(log, " file %s", strZ(restoreFilePgPath(manifest, file.name)));
// If not copied and not zeroed add details to explain why it was not copied
if (!copy && !zeroed)
{ {
strCatZ(log, " - "); const ManifestFile file = manifestFileFind(manifest, pckReadStrP(jobResult));
const bool zeroed = restoreFileZeroed(file.name, zeroExp);
const RestoreResult result = (RestoreResult)pckReadU32P(jobResult);
// On force we match on size and modification time String *log = strCatZ(strNew(), "restore");
if (cfgOptionBool(cfgOptForce))
{
strCatFmt(
log, "exists and matches size %" PRIu64 " and modification time %" PRIu64, file.size,
(uint64_t)file.timestamp);
}
// Else a checksum delta or file is zero-length
else
{
strCatZ(log, "exists and ");
// No need to copy zero-length files // Note if file was zeroed (i.e. selective restore)
if (file.size == 0) if (zeroed)
strCatZ(log, " zeroed");
// Add filename
strCatFmt(log, " file %s", strZ(restoreFilePgPath(manifest, file.name)));
// If preserved add details to explain why it was not copied or zeroed
if (result == restoreResultPreserve)
{
strCatZ(log, " - ");
// On force we match on size and modification time
if (cfgOptionBool(cfgOptForce))
{ {
strCatZ(log, "is zero size"); strCatFmt(
log, "exists and matches size %" PRIu64 " and modification time %" PRIu64, file.size,
(uint64_t)file.timestamp);
} }
// The file matched the manifest checksum so did not need to be copied // Else a checksum delta or file is zero-length
else else
strCatZ(log, "matches backup"); {
strCatZ(log, "exists and ");
// No need to copy zero-length files
if (file.size == 0)
{
strCatZ(log, "is zero size");
}
// The file matched the manifest checksum so did not need to be copied
else
strCatZ(log, "matches backup");
}
} }
// Add bundle info
strCatZ(log, " (");
if (file.bundleId != 0)
{
ASSERT(varUInt64(protocolParallelJobKey(job)) == file.bundleId);
strCatZ(log, "bundle ");
if (file.reference != NULL)
strCatFmt(log, "%s/", strZ(file.reference));
strCatFmt(log, "%" PRIu64 "/%" PRIu64 ", ", file.bundleId, file.bundleOffset);
}
// Add size and percent complete
sizeRestored += file.size;
strCatFmt(log, "%s, %.2lf%%)", strZ(strSizeFormat(file.size)), (double)sizeRestored * 100.00 / (double)sizeTotal);
// If not zero-length add the checksum
if (file.size != 0 && !zeroed)
strCatFmt(log, " checksum %s", file.checksumSha1);
LOG_DETAIL_PID(protocolParallelJobProcessId(job), strZ(log));
} }
// Add size and percent complete
sizeRestored += file.size;
strCatFmt(log, " (%s, %.2lf%%)", strZ(strSizeFormat(file.size)), (double)sizeRestored * 100.00 / (double)sizeTotal);
// If not zero-length add the checksum
if (file.size != 0 && !zeroed)
strCatFmt(log, " checksum %s", file.checksumSha1);
LOG_DETAIL_PID(protocolParallelJobProcessId(job), strZ(log));
} }
MEM_CONTEXT_TEMP_END(); MEM_CONTEXT_TEMP_END();
@ -2179,68 +2245,100 @@ static ProtocolParallelJob *restoreJobCallback(void *data, unsigned int clientId
RestoreJobData *jobData = data; RestoreJobData *jobData = data;
// Determine where to begin scanning the queue (we'll stop when we get back here) // Determine where to begin scanning the queue (we'll stop when we get back here)
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_RESTORE_FILE);
PackWrite *param = NULL;
int queueIdx = (int)(clientIdx % lstSize(jobData->queueList)); int queueIdx = (int)(clientIdx % lstSize(jobData->queueList));
int queueEnd = queueIdx; int queueEnd = queueIdx;
// Create restore job
do do
{ {
List *queue = *(List **)lstGet(jobData->queueList, (unsigned int)queueIdx); List *queue = *(List **)lstGet(jobData->queueList, (unsigned int)queueIdx);
bool fileAdded = false;
const String *fileName = NULL;
uint64_t bundleId = 0;
const String *reference = NULL;
if (!lstEmpty(queue)) while (!lstEmpty(queue))
{ {
const ManifestFile file = manifestFileUnpack(jobData->manifest, *(ManifestFilePack **)lstGet(queue, 0)); const ManifestFile file = manifestFileUnpack(jobData->manifest, *(ManifestFilePack **)lstGet(queue, 0));
// Create restore job // Break if bundled files have already been added and 1) the bundleId has changed or 2) the reference has changed
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_RESTORE_FILE); if (fileAdded && (bundleId != file.bundleId || !strEq(reference, file.reference)))
PackWrite *const param = protocolCommandParam(command); break;
const String *const repoPath = strNewFmt( // Add common parameters before first file
STORAGE_REPO_BACKUP "/%s/", if (param == NULL)
strZ(file.reference != NULL ? file.reference : manifestData(jobData->manifest)->backupLabel)); {
param = protocolCommandParam(command);
const String *const repoPath = strNewFmt(
STORAGE_REPO_BACKUP "/%s/",
strZ(file.reference != NULL ? file.reference : manifestData(jobData->manifest)->backupLabel));
if (file.bundleId != 0)
{
pckWriteStrP(param, strNewFmt("%s" MANIFEST_PATH_BUNDLE "/%" PRIu64, strZ(repoPath), file.bundleId));
bundleId = file.bundleId;
reference = file.reference;
}
else
{
pckWriteStrP(
param,
strNewFmt(
"%s%s%s", strZ(repoPath), strZ(file.name),
strZ(compressExtStr(manifestData(jobData->manifest)->backupOptionCompressType))));
fileName = file.name;
}
pckWriteU32P(param, jobData->repoIdx);
pckWriteU32P(param, manifestData(jobData->manifest)->backupOptionCompressType);
pckWriteTimeP(param, manifestData(jobData->manifest)->backupTimestampCopyStart);
pckWriteBoolP(param, cfgOptionBool(cfgOptDelta));
pckWriteBoolP(param, cfgOptionBool(cfgOptDelta) && cfgOptionBool(cfgOptForce));
pckWriteStrP(param, jobData->cipherSubPass);
fileAdded = true;
}
pckWriteStrP(param, restoreFilePgPath(jobData->manifest, file.name));
pckWriteStrP(param, STR(file.checksumSha1));
pckWriteU64P(param, file.size);
pckWriteTimeP(param, file.timestamp);
pckWriteModeP(param, file.mode);
pckWriteBoolP(param, restoreFileZeroed(file.name, jobData->zeroExp));
pckWriteStrP(param, restoreManifestOwnerReplace(file.user, jobData->rootReplaceUser));
pckWriteStrP(param, restoreManifestOwnerReplace(file.group, jobData->rootReplaceGroup));
if (file.bundleId != 0) if (file.bundleId != 0)
{ {
pckWriteStrP(param, strNewFmt("%s" MANIFEST_PATH_BUNDLE "/%" PRIu64, strZ(repoPath), file.bundleId));
pckWriteBoolP(param, true); pckWriteBoolP(param, true);
pckWriteU64P(param, file.bundleOffset); pckWriteU64P(param, file.bundleOffset);
pckWriteU64P(param, file.sizeRepo); pckWriteU64P(param, file.sizeRepo);
} }
else else
{
pckWriteStrP(
param,
strNewFmt(
"%s%s%s", strZ(repoPath), strZ(file.name),
strZ(compressExtStr(manifestData(jobData->manifest)->backupOptionCompressType))));
pckWriteBoolP(param, false); pckWriteBoolP(param, false);
}
pckWriteU32P(param, jobData->repoIdx); pckWriteStrP(param, file.name);
pckWriteU32P(param, manifestData(jobData->manifest)->backupOptionCompressType);
pckWriteStrP(param, restoreFilePgPath(jobData->manifest, file.name));
pckWriteStrP(param, STR(file.checksumSha1));
pckWriteBoolP(param, restoreFileZeroed(file.name, jobData->zeroExp));
pckWriteU64P(param, file.size);
pckWriteTimeP(param, file.timestamp);
pckWriteModeP(param, file.mode);
pckWriteStrP(param, restoreManifestOwnerReplace(file.user, jobData->rootReplaceUser));
pckWriteStrP(param, restoreManifestOwnerReplace(file.group, jobData->rootReplaceGroup));
pckWriteTimeP(param, manifestData(jobData->manifest)->backupTimestampCopyStart);
pckWriteBoolP(param, cfgOptionBool(cfgOptDelta));
pckWriteBoolP(param, cfgOptionBool(cfgOptDelta) && cfgOptionBool(cfgOptForce));
pckWriteStrP(param, jobData->cipherSubPass);
// Remove job from the queue // Remove job from the queue
lstRemoveIdx(queue, 0); lstRemoveIdx(queue, 0);
// Break if the file is not bundled
if (bundleId == 0)
break;
}
if (fileAdded)
{
// Assign job to result // Assign job to result
MEM_CONTEXT_PRIOR_BEGIN() MEM_CONTEXT_PRIOR_BEGIN()
{ {
result = protocolParallelJobNew(VARSTR(file.name), command); result = protocolParallelJobNew(bundleId != 0 ? VARUINT64(bundleId) : VARSTR(fileName), command);
} }
MEM_CONTEXT_PRIOR_END(); MEM_CONTEXT_PRIOR_END();
// Break out of the loop early since we found a job
break; break;
} }

5
src/info/manifest.c
View File

@ -235,6 +235,11 @@ manifestFilePack(const Manifest *const manifest, const ManifestFile *const file)
FUNCTION_TEST_PARAM(MANIFEST_FILE, file); FUNCTION_TEST_PARAM(MANIFEST_FILE, file);
FUNCTION_TEST_END(); FUNCTION_TEST_END();
ASSERT(manifest != NULL);
ASSERT(file != NULL);
CHECK(AssertError, file->size > 0 || file->bundleId == 0, "zero-length files may not be bundled");
uint8_t buffer[512]; uint8_t buffer[512];
size_t bufferPos = 0; size_t bufferPos = 0;

167
test/src/module/command/restoreTest.c
View File

@ -178,11 +178,26 @@ testRun(void)
"acefile", .compressType = compressTypeGz, .cipherType = cipherTypeAes256Cbc, .cipherPass = "badpass", "acefile", .compressType = compressTypeGz, .cipherType = cipherTypeAes256Cbc, .cipherPass = "badpass",
.comment = "create a compressed encrypted repo file"); .comment = "create a compressed encrypted repo file");
List *fileList = lstNewP(sizeof(RestoreFile));
RestoreFile file =
{
.name = STRDEF("normal"),
.checksum = STRDEF("ffffffffffffffffffffffffffffffffffffffff"),
.size = 7,
.timeModified = 1557432154,
.mode = 0600,
.zero = false,
.user = NULL,
.group = NULL,
};
lstAdd(fileList, &file);
TEST_ERROR( TEST_ERROR(
restoreFile( restoreFile(
strNewFmt(STORAGE_REPO_BACKUP "/%s/%s.gz", strZ(repoFileReferenceFull), strZ(repoFile1)), repoIdx, 0, NULL, strNewFmt(STORAGE_REPO_BACKUP "/%s/%s.gz", strZ(repoFileReferenceFull), strZ(repoFile1)), repoIdx, compressTypeGz,
compressTypeGz, STRDEF("normal"), STRDEF("ffffffffffffffffffffffffffffffffffffffff"), false, 7, 1557432154, 0600, 0, false, false, STRDEF("badpass"), fileList),
TEST_USER_STR, TEST_GROUP_STR, 0, false, false, STRDEF("badpass")),
ChecksumError, ChecksumError,
"error restoring 'normal': actual checksum 'd1cd8a7d11daa26814b93eb604e1d49ab4b43770' does not match expected checksum" "error restoring 'normal': actual checksum 'd1cd8a7d11daa26814b93eb604e1d49ab4b43770' does not match expected checksum"
" 'ffffffffffffffffffffffffffffffffffffffff'"); " 'ffffffffffffffffffffffffffffffffffffffff'");
@ -2220,7 +2235,8 @@ testRun(void)
"P01 DETAIL: restore file " TEST_PATH "/pg/postgresql.conf (10B, [PCT]) checksum" "P01 DETAIL: restore file " TEST_PATH "/pg/postgresql.conf (10B, [PCT]) checksum"
" 1a49a3c2240449fee1422e4afcf44d5b96378511\n" " 1a49a3c2240449fee1422e4afcf44d5b96378511\n"
"P01 DETAIL: restore file " TEST_PATH "/pg/PG_VERSION (4B, [PCT]) checksum b74d60e763728399bcd3fb63f7dd1f97b46c6b44\n" "P01 DETAIL: restore file " TEST_PATH "/pg/PG_VERSION (4B, [PCT]) checksum b74d60e763728399bcd3fb63f7dd1f97b46c6b44\n"
"P01 DETAIL: restore file " TEST_PATH "/pg/size-mismatch (1B, [PCT]) checksum c032adc1ff629c9b66f22749ad667e6beadf144b\n" "P01 DETAIL: restore file " TEST_PATH "/pg/size-mismatch (1B, [PCT]) checksum"
" c032adc1ff629c9b66f22749ad667e6beadf144b\n"
"P01 DETAIL: restore file " TEST_PATH "/pg/tablespace_map (0B, [PCT])\n" "P01 DETAIL: restore file " TEST_PATH "/pg/tablespace_map (0B, [PCT])\n"
"P01 DETAIL: restore file " TEST_PATH "/pg/pg_tblspc/1/16384/PG_VERSION (4B, [PCT])" "P01 DETAIL: restore file " TEST_PATH "/pg/pg_tblspc/1/16384/PG_VERSION (4B, [PCT])"
" checksum b74d60e763728399bcd3fb63f7dd1f97b46c6b44\n" " checksum b74d60e763728399bcd3fb63f7dd1f97b46c6b44\n"
@ -2279,6 +2295,9 @@ testRun(void)
hrnCfgArgRawZ(argList, cfgOptLinkMap, "pg_xact=../xact"); hrnCfgArgRawZ(argList, cfgOptLinkMap, "pg_xact=../xact");
HRN_CFG_LOAD(cfgCmdRestore, argList); HRN_CFG_LOAD(cfgCmdRestore, argList);
#define TEST_LABEL_FULL "20161219-212741F"
#define TEST_LABEL_DIFF "20161219-212741F_20161219-212800D"
#define TEST_LABEL_INCR "20161219-212741F_20161219-212900I"
#define TEST_LABEL "20161219-212741F_20161219-212918I" #define TEST_LABEL "20161219-212741F_20161219-212918I"
#define TEST_PGDATA MANIFEST_TARGET_PGDATA "/" #define TEST_PGDATA MANIFEST_TARGET_PGDATA "/"
#define TEST_REPO_PATH STORAGE_REPO_BACKUP "/" TEST_LABEL "/" TEST_PGDATA #define TEST_REPO_PATH STORAGE_REPO_BACKUP "/" TEST_LABEL "/" TEST_PGDATA
@ -2290,7 +2309,7 @@ testRun(void)
manifest->pub.data.backupLabel = STRDEF(TEST_LABEL); manifest->pub.data.backupLabel = STRDEF(TEST_LABEL);
manifest->pub.data.pgVersion = PG_VERSION_10; manifest->pub.data.pgVersion = PG_VERSION_10;
manifest->pub.data.pgCatalogVersion = hrnPgCatalogVersion(PG_VERSION_10); manifest->pub.data.pgCatalogVersion = hrnPgCatalogVersion(PG_VERSION_10);
manifest->pub.data.backupType = backupTypeFull; manifest->pub.data.backupType = backupTypeIncr;
manifest->pub.data.backupTimestampCopyStart = 1482182861; // So file timestamps should be less than this manifest->pub.data.backupTimestampCopyStart = 1482182861; // So file timestamps should be less than this
// Data directory // Data directory
@ -2319,6 +2338,15 @@ testRun(void)
.checksumSha1 = "5e2b96c19c4f5c63a5afa2de504d29fe64a4c908"}); .checksumSha1 = "5e2b96c19c4f5c63a5afa2de504d29fe64a4c908"});
HRN_STORAGE_PUT(storageRepoWrite(), TEST_REPO_PATH PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL, fileBuffer); HRN_STORAGE_PUT(storageRepoWrite(), TEST_REPO_PATH PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL, fileBuffer);
// global/888
manifestFileAdd(
manifest,
&(ManifestFile){
.name = STRDEF(TEST_PGDATA PG_PATH_GLOBAL "/888"), .size = 0, .timestamp = 1482182860,
.mode = 0600, .group = groupName(), .user = userName(),
.checksumSha1 = HASH_TYPE_SHA1_ZERO});
HRN_STORAGE_PUT_EMPTY(storageRepoWrite(), TEST_REPO_PATH PG_PATH_GLOBAL "/888");
// global/999 // global/999
manifestFileAdd( manifestFileAdd(
manifest, manifest,
@ -2334,9 +2362,28 @@ testRun(void)
&(ManifestFile){ &(ManifestFile){
.name = STRDEF(TEST_PGDATA PG_FILE_PGVERSION), .size = 4, .sizeRepo = 4, .timestamp = 1482182860, .name = STRDEF(TEST_PGDATA PG_FILE_PGVERSION), .size = 4, .sizeRepo = 4, .timestamp = 1482182860,
.mode = 0600, .group = groupName(), .user = userName(), .bundleId = 1, .bundleOffset = 0, .mode = 0600, .group = groupName(), .user = userName(), .bundleId = 1, .bundleOffset = 0,
.reference = STRDEF(TEST_LABEL), .checksumSha1 = "8dbabb96e032b8d9f1993c0e4b9141e71ade01a1"}); .reference = NULL, .checksumSha1 = "8dbabb96e032b8d9f1993c0e4b9141e71ade01a1"});
manifestFileAdd(
manifest,
&(ManifestFile){
.name = STRDEF(TEST_PGDATA "yyy"), .size = 3, .sizeRepo = 3, .timestamp = 1482182860,
.mode = 0600, .group = groupName(), .user = userName(), .bundleId = 1, .bundleOffset = 8,
.reference = NULL, .checksumSha1 = "186154712b2d5f6791d85b9a0987b98fa231779c"});
manifestFileAdd(
manifest,
&(ManifestFile){
.name = STRDEF(TEST_PGDATA "xxxxx"), .size = 5, .sizeRepo = 5, .timestamp = 1482182860,
.mode = 0600, .group = groupName(), .user = userName(), .bundleId = 1, .bundleOffset = 11,
.reference = NULL, .checksumSha1 = "9addbf544119efa4a64223b649750a510f0d463f"});
manifestFileAdd(
manifest,
&(ManifestFile){
.name = STRDEF(TEST_PGDATA "zz"), .size = 2, .sizeRepo = 2, .timestamp = 1482182860,
.mode = 0600, .group = groupName(), .user = userName(), .bundleId = 1, .bundleOffset = 17,
.reference = NULL, .checksumSha1 = "d7dacae2c968388960bf8970080a980ed5c5dcb7"});
HRN_STORAGE_PUT_Z( HRN_STORAGE_PUT_Z(
storageRepoWrite(), STORAGE_REPO_BACKUP "/" TEST_LABEL "/bundle/1", PG_VERSION_94_STR "\n" PG_VERSION_94_STR "\n"); storageRepoWrite(), STORAGE_REPO_BACKUP "/" TEST_LABEL "/bundle/1",
PG_VERSION_94_STR "\n" PG_VERSION_94_STR "\nyyyxxxxxAzzA");
// base directory // base directory
manifestPathAdd( manifestPathAdd(
@ -2371,6 +2418,51 @@ testRun(void)
.checksumSha1 = "4d7b2a36c5387decf799352a3751883b7ceb96aa"}); .checksumSha1 = "4d7b2a36c5387decf799352a3751883b7ceb96aa"});
HRN_STORAGE_PUT(storageRepoWrite(), TEST_REPO_PATH "base/1/2", fileBuffer); HRN_STORAGE_PUT(storageRepoWrite(), TEST_REPO_PATH "base/1/2", fileBuffer);
// base/1/10
fileBuffer = bufNew(8194);
memset(bufPtr(fileBuffer), 10, bufSize(fileBuffer));
bufPtr(fileBuffer)[0] = 0xFF;
bufPtr(fileBuffer)[8193] = 0xFF;
bufUsedSet(fileBuffer, bufSize(fileBuffer));
manifestFileAdd(
manifest,
&(ManifestFile){
.name = STRDEF(TEST_PGDATA "base/1/10"), .size = 8192, .sizeRepo = 8192, .timestamp = 1482182860,
.mode = 0600, .group = groupName(), .user = userName(), .bundleId = 1, .bundleOffset = 1,
.reference = STRDEF(TEST_LABEL_FULL), .checksumSha1 = "28757c756c03c37aca13692cb719c18d1510c190"});
HRN_STORAGE_PUT(storageRepoWrite(), STORAGE_REPO_BACKUP "/" TEST_LABEL_FULL "/bundle/1", fileBuffer);
// base/1/20 and base/1/21
manifestFileAdd(
manifest,
&(ManifestFile){
.name = STRDEF(TEST_PGDATA "base/1/20"), .size = 1, .sizeRepo = 1, .timestamp = 1482182860,
.mode = 0600, .group = groupName(), .user = userName(), .bundleId = 2, .bundleOffset = 1,
.reference = STRDEF(TEST_LABEL_DIFF), .checksumSha1 = "c032adc1ff629c9b66f22749ad667e6beadf144b"});
manifestFileAdd(
manifest,
&(ManifestFile){
.name = STRDEF(TEST_PGDATA "base/1/21"), .size = 1, .sizeRepo = 1, .timestamp = 1482182860,
.mode = 0600, .group = groupName(), .user = userName(), .bundleId = 2, .bundleOffset = 2,
.reference = STRDEF(TEST_LABEL_DIFF), .checksumSha1 = "e9d71f5ee7c92d6dc9e92ffdad17b8bd49418f98"});
HRN_STORAGE_PUT_Z(storageRepoWrite(), STORAGE_REPO_BACKUP "/" TEST_LABEL_DIFF "/bundle/2", "aXb");
// base/1/30 and base/1/31
manifestFileAdd(
manifest,
&(ManifestFile){
.name = STRDEF(TEST_PGDATA "base/1/30"), .size = 1, .sizeRepo = 1, .timestamp = 1482182860,
.mode = 0600, .group = groupName(), .user = userName(), .bundleId = 2, .bundleOffset = 1,
.reference = STRDEF(TEST_LABEL_INCR), .checksumSha1 = "c032adc1ff629c9b66f22749ad667e6beadf144b"});
manifestFileAdd(
manifest,
&(ManifestFile){
.name = STRDEF(TEST_PGDATA "base/1/31"), .size = 1, .sizeRepo = 1, .timestamp = 1482182860,
.mode = 0600, .group = groupName(), .user = userName(), .bundleId = 2, .bundleOffset = 2,
.reference = STRDEF(TEST_LABEL_INCR), .checksumSha1 = "e9d71f5ee7c92d6dc9e92ffdad17b8bd49418f98"});
HRN_STORAGE_PUT_Z(storageRepoWrite(), STORAGE_REPO_BACKUP "/" TEST_LABEL_INCR "/bundle/2", "aXb");
// system db name // system db name
manifestDbAdd(manifest, &(ManifestDb){.name = STRDEF("template1"), .id = 1, .lastSystemId = 12168}); manifestDbAdd(manifest, &(ManifestDb){.name = STRDEF("template1"), .id = 1, .lastSystemId = 12168});
@ -2532,6 +2624,9 @@ testRun(void)
HRN_STORAGE_PATH_CREATE(storagePgWrite(), "bogus1/bogus2"); HRN_STORAGE_PATH_CREATE(storagePgWrite(), "bogus1/bogus2");
HRN_STORAGE_PATH_CREATE(storagePgWrite(), PG_PATH_GLOBAL "/bogus3"); HRN_STORAGE_PATH_CREATE(storagePgWrite(), PG_PATH_GLOBAL "/bogus3");
// Create yyy file so it is not copied
HRN_STORAGE_PUT_Z(storagePgWrite(), "yyy", "yyy", .modeFile = 0600);
// Add a few bogus links to be deleted // Add a few bogus links to be deleted
THROW_ON_SYS_ERROR( THROW_ON_SYS_ERROR(
symlink("../wal", strZ(strNewFmt("%s/pg_wal2", strZ(pgPath)))) == -1, FileOpenError, symlink("../wal", strZ(strNewFmt("%s/pg_wal2", strZ(pgPath)))) == -1, FileOpenError,
@ -2578,10 +2673,28 @@ testRun(void)
" checksum 8dbabb96e032b8d9f1993c0e4b9141e71ade01a1\n" " checksum 8dbabb96e032b8d9f1993c0e4b9141e71ade01a1\n"
"P01 DETAIL: restore file " TEST_PATH "/pg/base/16384/PG_VERSION (4B, [PCT])" "P01 DETAIL: restore file " TEST_PATH "/pg/base/16384/PG_VERSION (4B, [PCT])"
" checksum 8dbabb96e032b8d9f1993c0e4b9141e71ade01a1\n" " checksum 8dbabb96e032b8d9f1993c0e4b9141e71ade01a1\n"
"P01 DETAIL: restore file " TEST_PATH "/pg/base/1/PG_VERSION (4B, [PCT]) checksum" "P01 DETAIL: restore file " TEST_PATH "/pg/base/1/10 (bundle 20161219-212741F/1/1, 8KB, [PCT])"
" checksum 28757c756c03c37aca13692cb719c18d1510c190\n"
"P01 DETAIL: restore file " TEST_PATH "/pg/PG_VERSION (bundle 1/0, 4B, [PCT]) checksum"
" 8dbabb96e032b8d9f1993c0e4b9141e71ade01a1\n" " 8dbabb96e032b8d9f1993c0e4b9141e71ade01a1\n"
"P01 DETAIL: restore file " TEST_PATH "/pg/PG_VERSION (4B, [PCT]) checksum 8dbabb96e032b8d9f1993c0e4b9141e71ade01a1\n" "P01 DETAIL: restore file " TEST_PATH "/pg/base/1/PG_VERSION (bundle 1/4, 4B, [PCT]) checksum"
" 8dbabb96e032b8d9f1993c0e4b9141e71ade01a1\n"
"P01 DETAIL: restore file " TEST_PATH "/pg/yyy - exists and matches backup (bundle 1/8, 3B, [PCT]) checksum"
" 186154712b2d5f6791d85b9a0987b98fa231779c\n"
"P01 DETAIL: restore file " TEST_PATH "/pg/xxxxx (bundle 1/11, 5B, [PCT]) checksum"
" 9addbf544119efa4a64223b649750a510f0d463f\n"
"P01 DETAIL: restore file " TEST_PATH "/pg/zz (bundle 1/17, 2B, [PCT]) checksum"
" d7dacae2c968388960bf8970080a980ed5c5dcb7\n"
"P01 DETAIL: restore file " TEST_PATH "/pg/base/1/20 (bundle 20161219-212741F_20161219-212800D/2/1, 1B, [PCT]) checksum"
" c032adc1ff629c9b66f22749ad667e6beadf144b\n"
"P01 DETAIL: restore file " TEST_PATH "/pg/base/1/21 (bundle 20161219-212741F_20161219-212800D/2/2, 1B, [PCT]) checksum"
" e9d71f5ee7c92d6dc9e92ffdad17b8bd49418f98\n"
"P01 DETAIL: restore file " TEST_PATH "/pg/base/1/30 (bundle 20161219-212741F_20161219-212900I/2/1, 1B, [PCT]) checksum"
" c032adc1ff629c9b66f22749ad667e6beadf144b\n"
"P01 DETAIL: restore file " TEST_PATH "/pg/base/1/31 (bundle 20161219-212741F_20161219-212900I/2/2, 1B, [PCT]) checksum"
" e9d71f5ee7c92d6dc9e92ffdad17b8bd49418f98\n"
"P01 DETAIL: restore file " TEST_PATH "/pg/global/999 (0B, [PCT])\n" "P01 DETAIL: restore file " TEST_PATH "/pg/global/999 (0B, [PCT])\n"
"P01 DETAIL: restore file " TEST_PATH "/pg/global/888 (0B, [PCT])\n"
"P00 DETAIL: sync path '" TEST_PATH "/config'\n" "P00 DETAIL: sync path '" TEST_PATH "/config'\n"
"P00 DETAIL: sync path '" TEST_PATH "/pg'\n" "P00 DETAIL: sync path '" TEST_PATH "/pg'\n"
"P00 DETAIL: sync path '" TEST_PATH "/pg/base'\n" "P00 DETAIL: sync path '" TEST_PATH "/pg/base'\n"
@ -2595,7 +2708,7 @@ testRun(void)
"P00 DETAIL: sync path '" TEST_PATH "/pg/pg_tblspc/1/PG_10_201707211'\n" "P00 DETAIL: sync path '" TEST_PATH "/pg/pg_tblspc/1/PG_10_201707211'\n"
"P00 INFO: restore global/pg_control (performed last to ensure aborted restores cannot be started)\n" "P00 INFO: restore global/pg_control (performed last to ensure aborted restores cannot be started)\n"
"P00 DETAIL: sync path '" TEST_PATH "/pg/global'\n" "P00 DETAIL: sync path '" TEST_PATH "/pg/global'\n"
"P00 INFO: restore size = [SIZE], file total = 11"); "P00 INFO: restore size = [SIZE], file total = 20");
testRestoreCompare( testRestoreCompare(
storagePg(), NULL, manifest, storagePg(), NULL, manifest,
@ -2603,7 +2716,12 @@ testRun(void)
"PG_VERSION {file, s=4, t=1482182860}\n" "PG_VERSION {file, s=4, t=1482182860}\n"
"base {path}\n" "base {path}\n"
"base/1 {path}\n" "base/1 {path}\n"
"base/1/10 {file, s=8192, t=1482182860}\n"
"base/1/2 {file, s=8192, t=1482182860}\n" "base/1/2 {file, s=8192, t=1482182860}\n"
"base/1/20 {file, s=1, t=1482182860}\n"
"base/1/21 {file, s=1, t=1482182860}\n"
"base/1/30 {file, s=1, t=1482182860}\n"
"base/1/31 {file, s=1, t=1482182860}\n"
"base/1/PG_VERSION {file, s=4, t=1482182860}\n" "base/1/PG_VERSION {file, s=4, t=1482182860}\n"
"base/16384 {path}\n" "base/16384 {path}\n"
"base/16384/16385 {file, s=16384, t=1482182860}\n" "base/16384/16385 {file, s=16384, t=1482182860}\n"
@ -2612,6 +2730,7 @@ testRun(void)
"base/32768/32769 {file, s=32768, t=1482182860}\n" "base/32768/32769 {file, s=32768, t=1482182860}\n"
"base/32768/PG_VERSION {file, s=4, t=1482182860}\n" "base/32768/PG_VERSION {file, s=4, t=1482182860}\n"
"global {path}\n" "global {path}\n"
"global/888 {file, s=0, t=1482182860}\n"
"global/999 {file, s=0, t=1482182860}\n" "global/999 {file, s=0, t=1482182860}\n"
"global/pg_control {file, s=8192, t=1482182860}\n" "global/pg_control {file, s=8192, t=1482182860}\n"
"pg_hba.conf {link, d=../config/pg_hba.conf}\n" "pg_hba.conf {link, d=../config/pg_hba.conf}\n"
@ -2619,7 +2738,10 @@ testRun(void)
"pg_tblspc/1 {link, d=" TEST_PATH "/ts/1}\n" "pg_tblspc/1 {link, d=" TEST_PATH "/ts/1}\n"
"pg_wal {link, d=../wal}\n" "pg_wal {link, d=../wal}\n"
"pg_xact {link, d=../xact}\n" "pg_xact {link, d=../xact}\n"
"postgresql.conf {link, d=../config/postgresql.conf}\n"); "postgresql.conf {link, d=../config/postgresql.conf}\n"
"xxxxx {file, s=5, t=1482182860}\n"
"yyy {file, s=3, t=1482182860}\n"
"zz {file, s=2, t=1482182860}\n");
testRestoreCompare( testRestoreCompare(
storagePg(), STRDEF("pg_tblspc/1"), manifest, storagePg(), STRDEF("pg_tblspc/1"), manifest,
@ -2726,11 +2848,28 @@ testRun(void)
" checksum 8dbabb96e032b8d9f1993c0e4b9141e71ade01a1\n" " checksum 8dbabb96e032b8d9f1993c0e4b9141e71ade01a1\n"
"P01 DETAIL: restore file " TEST_PATH "/pg/base/16384/PG_VERSION - exists and matches backup (4B, [PCT])" "P01 DETAIL: restore file " TEST_PATH "/pg/base/16384/PG_VERSION - exists and matches backup (4B, [PCT])"
" checksum 8dbabb96e032b8d9f1993c0e4b9141e71ade01a1\n" " checksum 8dbabb96e032b8d9f1993c0e4b9141e71ade01a1\n"
"P01 DETAIL: restore file " TEST_PATH "/pg/base/1/PG_VERSION - exists and matches backup (4B, [PCT])" "P01 DETAIL: restore file " TEST_PATH "/pg/base/1/10 - exists and matches backup (bundle 20161219-212741F/1/1, 8KB,"
" [PCT]) checksum 28757c756c03c37aca13692cb719c18d1510c190\n"
"P01 DETAIL: restore file " TEST_PATH "/pg/PG_VERSION - exists and matches backup (bundle 1/0, 4B, [PCT])"
" checksum 8dbabb96e032b8d9f1993c0e4b9141e71ade01a1\n" " checksum 8dbabb96e032b8d9f1993c0e4b9141e71ade01a1\n"
"P01 DETAIL: restore file " TEST_PATH "/pg/PG_VERSION - exists and matches backup (4B, [PCT])" "P01 DETAIL: restore file " TEST_PATH "/pg/base/1/PG_VERSION - exists and matches backup (bundle 1/4, 4B, [PCT])"
" checksum 8dbabb96e032b8d9f1993c0e4b9141e71ade01a1\n" " checksum 8dbabb96e032b8d9f1993c0e4b9141e71ade01a1\n"
"P01 DETAIL: restore file " TEST_PATH "/pg/yyy - exists and matches backup (bundle 1/8, 3B, [PCT]) checksum"
" 186154712b2d5f6791d85b9a0987b98fa231779c\n"
"P01 DETAIL: restore file " TEST_PATH "/pg/xxxxx - exists and matches backup (bundle 1/11, 5B, [PCT]) checksum"
" 9addbf544119efa4a64223b649750a510f0d463f\n"
"P01 DETAIL: restore file " TEST_PATH "/pg/zz - exists and matches backup (bundle 1/17, 2B, [PCT]) checksum"
" d7dacae2c968388960bf8970080a980ed5c5dcb7\n"
"P01 DETAIL: restore file " TEST_PATH "/pg/base/1/20 - exists and matches backup (bundle"
" 20161219-212741F_20161219-212800D/2/1, 1B, [PCT]) checksum c032adc1ff629c9b66f22749ad667e6beadf144b\n"
"P01 DETAIL: restore file " TEST_PATH "/pg/base/1/21 - exists and matches backup (bundle"
" 20161219-212741F_20161219-212800D/2/2, 1B, [PCT]) checksum e9d71f5ee7c92d6dc9e92ffdad17b8bd49418f98\n"
"P01 DETAIL: restore file " TEST_PATH "/pg/base/1/30 - exists and matches backup (bundle"
" 20161219-212741F_20161219-212900I/2/1, 1B, [PCT]) checksum c032adc1ff629c9b66f22749ad667e6beadf144b\n"
"P01 DETAIL: restore file " TEST_PATH "/pg/base/1/31 - exists and matches backup (bundle"
" 20161219-212741F_20161219-212900I/2/2, 1B, [PCT]) checksum e9d71f5ee7c92d6dc9e92ffdad17b8bd49418f98\n"
"P01 DETAIL: restore file " TEST_PATH "/pg/global/999 - exists and is zero size (0B, [PCT])\n" "P01 DETAIL: restore file " TEST_PATH "/pg/global/999 - exists and is zero size (0B, [PCT])\n"
"P01 DETAIL: restore file " TEST_PATH "/pg/global/888 - exists and is zero size (0B, [PCT])\n"
"P00 DETAIL: sync path '" TEST_PATH "/config'\n" "P00 DETAIL: sync path '" TEST_PATH "/config'\n"
"P00 DETAIL: sync path '" TEST_PATH "/pg'\n" "P00 DETAIL: sync path '" TEST_PATH "/pg'\n"
"P00 DETAIL: sync path '" TEST_PATH "/pg/base'\n" "P00 DETAIL: sync path '" TEST_PATH "/pg/base'\n"
@ -2744,7 +2883,7 @@ testRun(void)
"P00 DETAIL: sync path '" TEST_PATH "/pg/pg_tblspc/1/PG_10_201707211'\n" "P00 DETAIL: sync path '" TEST_PATH "/pg/pg_tblspc/1/PG_10_201707211'\n"
"P00 INFO: restore global/pg_control (performed last to ensure aborted restores cannot be started)\n" "P00 INFO: restore global/pg_control (performed last to ensure aborted restores cannot be started)\n"
"P00 DETAIL: sync path '" TEST_PATH "/pg/global'\n" "P00 DETAIL: sync path '" TEST_PATH "/pg/global'\n"
"P00 INFO: restore size = [SIZE], file total = 11"); "P00 INFO: restore size = [SIZE], file total = 20");
// Check stanza archive spool path was removed // Check stanza archive spool path was removed
TEST_STORAGE_LIST_EMPTY(storageSpool(), STORAGE_PATH_ARCHIVE); TEST_STORAGE_LIST_EMPTY(storageSpool(), STORAGE_PATH_ARCHIVE);