db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2024-12-12 10:04:14 +02:00
Code Issues Releases Activity

Enhance restore command multi-repo support.

Browse Source 
The restore command automatically defaults to selecting the latest backup from a single repository. With multiple repositories configured, the restore command will now default to selecting the latest backup from the first repository where backups exist. The order in which the repositories are checked is dictated by the pgbackrest.conf order.

To select from a specific repository, the --repo option can be passed (e.g. --repo=1). The --set option can be passed if a backup other than the latest is desired.
This commit is contained in:
Cynthia Shang 2021-02-23 16:17:27 -05:00 committed by David Steele
parent bec3e20b2c
commit 118d9e64fe
9 changed files with 406 additions and 139 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
doc/xml/release.xml
View File

@ -41,6 +41,7 @@
<commit subject="Enhance expire command multi-repo support."/>
<commit subject="Expire continues if an error occurs processing a repository."/>
<commit subject="Add archive-get command multi-repo support."/>
<commit subject="Enhance restore command multi-repo support."/>
<release-item-contributor-list>
<release-item-contributor id="cynthia.shang"/>

9
src/command/restore/file.c
View File

@ -21,13 +21,14 @@ Restore File
/**********************************************************************************************************************************/
bool
restoreFile(
const String *repoFile, const String *repoFileReference, CompressType repoFileCompressType, const String *pgFile,
const String *pgFileChecksum, bool pgFileZero, uint64_t pgFileSize, time_t pgFileModified, mode_t pgFileMode,
const String *pgFileUser, const String *pgFileGroup, time_t copyTimeBegin, bool delta, bool deltaForce,
const String *repoFile, unsigned int repoIdx, const String *repoFileReference, CompressType repoFileCompressType,
const String *pgFile, const String *pgFileChecksum, bool pgFileZero, uint64_t pgFileSize, time_t pgFileModified,
mode_t pgFileMode, const String *pgFileUser, const String *pgFileGroup, time_t copyTimeBegin, bool delta, bool deltaForce,
const String *cipherPass)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(STRING, repoFile);
FUNCTION_LOG_PARAM(UINT, repoIdx);
FUNCTION_LOG_PARAM(STRING, repoFileReference);
FUNCTION_LOG_PARAM(ENUM, repoFileCompressType);
FUNCTION_LOG_PARAM(STRING, pgFile);
@ -164,7 +165,7 @@ restoreFile(
// Copy file
storageCopyP(
storageNewReadP(
storageRepo(),
storageRepoIdx(repoIdx),
strNewFmt(
STORAGE_REPO_BACKUP "/%s/%s%s", strZ(repoFileReference), strZ(repoFile),
strZ(compressExtStr(repoFileCompressType))),

6
src/command/restore/file.h
View File

@ -14,9 +14,9 @@ Functions
***********************************************************************************************************************************/
// Copy a file from the backup to the specified destination
bool restoreFile(
const String *repoFile, const String *repoFileReference, CompressType repoFileCompressType, const String *pgFile,
const String *pgFileChecksum, bool pgFileZero, uint64_t pgFileSize, time_t pgFileModified, mode_t pgFileMode,
const String *pgFileUser, const String *pgFileGroup, time_t copyTimeBegin, bool delta, bool deltaForce,
const String *repoFile, unsigned int repoIdx, const String *repoFileReference, CompressType repoFileCompressType,
const String *pgFile, const String *pgFileChecksum, bool pgFileZero, uint64_t pgFileSize, time_t pgFileModified,
mode_t pgFileMode, const String *pgFileUser, const String *pgFileGroup, time_t copyTimeBegin, bool delta, bool deltaForce,
const String *cipherPass);
#endif

16
src/command/restore/protocol.c
View File

@ -40,14 +40,14 @@ restoreProtocol(const String *command, const VariantList *paramList, ProtocolSer
server,
VARBOOL(
restoreFile(
varStr(varLstGet(paramList, 0)), varStr(varLstGet(paramList, 1)),
(CompressType)varUIntForce(varLstGet(paramList, 2)), varStr(varLstGet(paramList, 3)),
varStr(varLstGet(paramList, 4)), varBoolForce(varLstGet(paramList, 5)), varUInt64(varLstGet(paramList, 6)),
(time_t)varInt64Force(varLstGet(paramList, 7)),
(mode_t)cvtZToUIntBase(strZ(varStr(varLstGet(paramList, 8))), 8),
varStr(varLstGet(paramList, 9)), varStr(varLstGet(paramList, 10)),
(time_t)varInt64Force(varLstGet(paramList, 11)), varBoolForce(varLstGet(paramList, 12)),
varBoolForce(varLstGet(paramList, 13)), varStr(varLstGet(paramList, 14)))));
varStr(varLstGet(paramList, 0)), varUIntForce(varLstGet(paramList, 1)), varStr(varLstGet(paramList, 2)),
(CompressType)varUIntForce(varLstGet(paramList, 3)), varStr(varLstGet(paramList, 4)),
varStr(varLstGet(paramList, 5)), varBoolForce(varLstGet(paramList, 6)), varUInt64(varLstGet(paramList, 7)),
(time_t)varInt64Force(varLstGet(paramList, 8)),
(mode_t)cvtZToUIntBase(strZ(varStr(varLstGet(paramList, 9))), 8),
varStr(varLstGet(paramList, 10)), varStr(varLstGet(paramList, 11)),
(time_t)varInt64Force(varLstGet(paramList, 12)), varBoolForce(varLstGet(paramList, 13)),
varBoolForce(varLstGet(paramList, 14)), varStr(varLstGet(paramList, 15)))));
}
else
found = false;

220
src/command/restore/restore.c
View File

@ -207,37 +207,120 @@ getEpoch(const String *targetTime)
/***********************************************************************************************************************************
Get the backup set to restore
***********************************************************************************************************************************/
static String *
restoreBackupSet(InfoBackup *infoBackup)
typedef struct RestoreBackupData
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(INFO_BACKUP, infoBackup);
FUNCTION_LOG_END();
unsigned int repoIdx; // Internal repo index
CipherType repoCipherType; // Repo encryption type (0 = none)
const String *backupCipherPass; // Passphrase of backup files if repo is encrypted (else NULL)
const String *backupSet; // Backup set to restore
} RestoreBackupData;
ASSERT(infoBackup != NULL);
#define FUNCTION_LOG_RESTORE_BACKUP_DATA_TYPE \
RestoreBackupData
#define FUNCTION_LOG_RESTORE_BACKUP_DATA_FORMAT(value, buffer, bufferSize) \
objToLog(&value, "RestoreBackupData", buffer, bufferSize)
String *result = NULL;
// Helper function for restoreBackupSet
static RestoreBackupData
restoreBackupData(const String *backupLabel, unsigned int repoIdx, const String *backupCipherPass)
{
ASSERT(backupLabel != NULL);
RestoreBackupData restoreBackup = {0};
MEM_CONTEXT_PRIOR_BEGIN()
{
restoreBackup.backupSet = strDup(backupLabel);
restoreBackup.repoIdx = repoIdx;
restoreBackup.repoCipherType = cipherType(cfgOptionIdxStr(cfgOptRepoCipherType, repoIdx));
restoreBackup.backupCipherPass = strDup(backupCipherPass);
}
MEM_CONTEXT_PRIOR_END();
return restoreBackup;
}
static RestoreBackupData
restoreBackupSet(void)
{
FUNCTION_LOG_VOID(logLevelDebug);
RestoreBackupData result = {0};
MEM_CONTEXT_TEMP_BEGIN()
{
// If backup set to restore is default (i.e. latest) then get the actual set
const String *backupSet = NULL;
// Initialize the repo index
unsigned int repoIdxMin = 0;
unsigned int repoIdxMax = cfgOptionGroupIdxTotal(cfgOptGrpRepo) - 1;
// If the repo was specified then set index to the array location and max to loop only once
if (cfgOptionTest(cfgOptRepo))
{
repoIdxMin = cfgOptionGroupIdxDefault(cfgOptGrpRepo);
repoIdxMax = repoIdxMin;
}
// Initialize a backup candidate list
List *backupCandidateList = lstNewP(sizeof(RestoreBackupData));
const String *backupSetRequested = NULL;
time_t timeTargetEpoch = 0;
// If the set option was not provided by the user but a time to recover was set, then we will need to search for a backup
// set that satisfies the time condition, else we will use the backup provided
if (cfgOptionSource(cfgOptSet) == cfgSourceDefault)
{
if (infoBackupDataTotal(infoBackup) == 0)
THROW(BackupSetInvalidError, "no backup sets to restore");
time_t timeTargetEpoch = 0;
// If the recovery type is time, attempt to determine the backup set
if (strEq(cfgOptionStr(cfgOptType), RECOVERY_TYPE_TIME_STR))
{
timeTargetEpoch = getEpoch(cfgOptionStr(cfgOptTarget));
}
else
backupSetRequested = cfgOptionStr(cfgOptSet);
// Try to find the newest backup set with a stop time before the target recovery time
// Search through the repo list for a backup set to use for recovery
for (unsigned int repoIdx = repoIdxMin; repoIdx <= repoIdxMax; repoIdx++)
{
// Get the repo storage in case it is remote and encryption settings need to be pulled down
storageRepoIdx(repoIdx);
InfoBackup *infoBackup = NULL;
// Attempt to load backup.info
TRY_BEGIN()
{
infoBackup = infoBackupLoadFile(
storageRepoIdx(repoIdx), INFO_BACKUP_PATH_FILE_STR, cipherType(cfgOptionIdxStr(cfgOptRepoCipherType, repoIdx)),
cfgOptionIdxStrNull(cfgOptRepoCipherPass, repoIdx));
}
CATCH_ANY()
{
LOG_WARN_FMT(
"repo%u: [%s] %s", cfgOptionGroupIdxToKey(cfgOptGrpRepo, repoIdx), errorTypeName(errorType()), errorMessage());
}
TRY_END();
// If unable to load the backup info file, then move on to next repo
if (infoBackup == NULL)
continue;
if (infoBackupDataTotal(infoBackup) == 0)
{
LOG_WARN_FMT(
"repo%u: [%s] no backup sets to restore", cfgOptionGroupIdxToKey(cfgOptGrpRepo, repoIdx),
errorTypeName(&BackupSetInvalidError));
continue;
}
// If a backup set was not specified, then see if a time to recover was requested
if (backupSetRequested == NULL)
{
// Get the latest backup
InfoBackupData latestBackup = infoBackupData(infoBackup, infoBackupDataTotal(infoBackup) - 1);
// If the recovery type is time, attempt to determine the backup set
if (timeTargetEpoch != 0)
{
bool found = false;
// Search current backups from newest to oldest
for (unsigned int keyIdx = infoBackupDataTotal(infoBackup) - 1; (int)keyIdx >= 0; keyIdx--)
{
@ -247,52 +330,76 @@ restoreBackupSet(InfoBackup *infoBackup)
// If the end of the backup is before the target time, then select this backup
if (backupData.backupTimestampStop < timeTargetEpoch)
{
backupSet = backupData.backupLabel;
found = true;
result = restoreBackupData(
backupData.backupLabel, repoIdx, infoPgCipherPass(infoBackupPg(infoBackup)));
break;
}
}
if (backupSet == NULL)
// If a backup was found on this repo matching the criteria for time then exit, else determine if the latest
// backup from this repo might be used
if (found)
break;
else
{
LOG_WARN_FMT(
"unable to find backup set with stop time less than '%s', latest backup set will be used",
strZ(cfgOptionStr(cfgOptTarget)));
// If a backup was not yet found then set the latest from this repo as the backup that might be used
RestoreBackupData candidate = restoreBackupData(
latestBackup.backupLabel, repoIdx, infoPgCipherPass(infoBackupPg(infoBackup)));
lstAdd(backupCandidateList, &candidate);
}
}
}
// If a backup set was not found or the recovery type was not time, then use the latest backup
if (backupSet == NULL)
backupSet = infoBackupData(infoBackup, infoBackupDataTotal(infoBackup) - 1).backupLabel;
}
// Otherwise check to make sure the specified backup set is valid
else
{
bool found = false;
backupSet = cfgOptionStr(cfgOptSet);
for (unsigned int backupIdx = 0; backupIdx < infoBackupDataTotal(infoBackup); backupIdx++)
{
if (strEq(infoBackupData(infoBackup, backupIdx).backupLabel, backupSet))
else
{
found = true;
// If the recovery type was not time (or time provided was not valid), then use the latest backup from this repo
result = restoreBackupData(latestBackup.backupLabel, repoIdx, infoPgCipherPass(infoBackupPg(infoBackup)));
break;
}
}
// Otherwise check to see if the specified backup set is on this repo
else
{
for (unsigned int backupIdx = 0; backupIdx < infoBackupDataTotal(infoBackup); backupIdx++)
{
if (strEq(infoBackupData(infoBackup, backupIdx).backupLabel, backupSetRequested))
{
result = restoreBackupData(backupSetRequested, repoIdx, infoPgCipherPass(infoBackupPg(infoBackup)));
break;
}
}
if (!found)
THROW_FMT(BackupSetInvalidError, "backup set %s is not valid", strZ(backupSet));
// If the backup set is found, then exit, else continue to next repo
if (result.backupSet != NULL)
break;
}
}
MEM_CONTEXT_PRIOR_BEGIN()
// Still no backup set to use after checking all the repos required to be checked?
if (result.backupSet == NULL)
{
result = strDup(backupSet);
if (backupSetRequested != NULL)
THROW_FMT(BackupSetInvalidError, "backup set %s is not valid", strZ(backupSetRequested));
else if (timeTargetEpoch != 0 && lstSize(backupCandidateList) > 0)
{
// Since the repos were scanned in priority order, use the first candidate found
result = restoreBackupData(
((RestoreBackupData *)lstGet(backupCandidateList, 0))->backupSet,
((RestoreBackupData *)lstGet(backupCandidateList, 0))->repoIdx,
((RestoreBackupData *)lstGet(backupCandidateList, 0))->backupCipherPass);
LOG_WARN_FMT(
"unable to find backup set with stop time less than '%s', repo%u: latest backup set will be used",
strZ(cfgOptionStr(cfgOptTarget)), cfgOptionGroupIdxToKey(cfgOptGrpRepo, result.repoIdx));
}
else
THROW(BackupSetInvalidError, "no backup set found to restore");
}
MEM_CONTEXT_PRIOR_END();
}
MEM_CONTEXT_TEMP_END();
FUNCTION_LOG_RETURN(STRING, result);
FUNCTION_LOG_RETURN_STRUCT(result);
}
/***********************************************************************************************************************************
@ -1928,6 +2035,7 @@ Return new restore jobs as requested
***********************************************************************************************************************************/
typedef struct RestoreJobData
{
unsigned int repoIdx; // Internal repo idx
Manifest *manifest; // Backup manifest
List *queueList; // List of processing queues
RegExp *zeroExp; // Identify files that should be sparse zeroed
@ -1987,8 +2095,8 @@ static ProtocolParallelJob *restoreJobCallback(void *data, unsigned int clientId
// Create restore job
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_RESTORE_FILE_STR);
protocolCommandParamAdd(command, VARSTR(file->name));
protocolCommandParamAdd(command, VARUINT(jobData->repoIdx));
protocolCommandParamAdd(
command, file->reference != NULL ?
VARSTR(file->reference) : VARSTR(manifestData(jobData->manifest)->backupLabel));
@ -2042,23 +2150,16 @@ cmdRestore(void)
// Validate restore path
restorePathValidate();
// Get the repo storage in case it is remote and encryption settings need to be pulled down
storageRepo();
// Load backup.info
InfoBackup *infoBackup = infoBackupLoadFile(
storageRepo(), INFO_BACKUP_PATH_FILE_STR, cipherType(cfgOptionStr(cfgOptRepoCipherType)),
cfgOptionStrNull(cfgOptRepoCipherPass));
// Get the backup set
const String *backupSet = restoreBackupSet(infoBackup);
RestoreBackupData backupData = restoreBackupSet();
// Load manifest
RestoreJobData jobData = {0};
RestoreJobData jobData = {.repoIdx = backupData.repoIdx};
jobData.manifest = manifestLoadFile(
storageRepo(), strNewFmt(STORAGE_REPO_BACKUP "/%s/" BACKUP_MANIFEST_FILE, strZ(backupSet)),
cipherType(cfgOptionStr(cfgOptRepoCipherType)), infoPgCipherPass(infoBackupPg(infoBackup)));
storageRepoIdx(backupData.repoIdx),
strNewFmt(STORAGE_REPO_BACKUP "/%s/" BACKUP_MANIFEST_FILE, strZ(backupData.backupSet)), backupData.repoCipherType,
backupData.backupCipherPass);
// Validate manifest. Don't use strict mode because we'd rather ignore problems that won't affect a restore.
manifestValidate(jobData.manifest, false);
@ -2067,10 +2168,11 @@ cmdRestore(void)
jobData.cipherSubPass = manifestCipherSubPass(jobData.manifest);
// Validate the manifest
restoreManifestValidate(jobData.manifest, backupSet);
restoreManifestValidate(jobData.manifest, backupData.backupSet);
// Log the backup set to restore
LOG_INFO_FMT("restore backup set %s", strZ(backupSet));
LOG_INFO_FMT(
"repo%u: restore backup set %s", cfgOptionGroupIdxToKey(cfgOptGrpRepo, backupData.repoIdx), strZ(backupData.backupSet));
// Map manifest
restoreManifestMap(jobData.manifest);

4
src/config/load.c
View File

@ -70,8 +70,8 @@ cfgLoadUpdateOption(void)
// Make sure repo option is set for the default command role when it is not internal and more than one repo is configured or the
// first configured repo is not key 1. Filter out any commands where this does not apply.
if (!cfgCommandHelp() && cfgOptionValid(cfgOptRepo) && !cfgOptionTest(cfgOptRepo) && cfgCommand() != cfgCmdArchiveGet &&
cfgCommand() != cfgCmdInfo && cfgCommand() != cfgCmdExpire &&
if (!cfgCommandHelp() && cfgCommand() != cfgCmdInfo && cfgCommand() != cfgCmdExpire && cfgCommand() != cfgCmdArchiveGet &&
cfgCommand() != cfgCmdRestore && cfgOptionValid(cfgOptRepo) && !cfgOptionTest(cfgOptRepo) &&
(cfgOptionGroupIdxTotal(cfgOptGrpRepo) > 1 || cfgOptionGroupIdxToKey(cfgOptGrpRepo, 0) != 1))
{
THROW_FMT(

18
test/expect/mock-all-001.log
View File

@ -489,7 +489,7 @@ restore delta, backup '[BACKUP-FULL-2]' - add and delete files (db-primary host)
> [CONTAINER-EXEC] db-primary [BACKREST-BIN] --config=[TEST_PATH]/db-primary/pgbackrest.conf --delta --set=[BACKUP-FULL-2] --link-all --repo=1 --stanza=db restore
------------------------------------------------------------------------------------------------------------------------------------
P00 INFO: restore command begin [BACKREST-VERSION]: --buffer-size=[BUFFER-SIZE] --config=[TEST_PATH]/db-primary/pgbackrest.conf --delta --exec-id=[EXEC-ID] --job-retry=0 --link-all --lock-path=[TEST_PATH]/db-primary/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-primary/log[] --no-log-timestamp --pg1-path=[TEST_PATH]/db-primary/db/base --protocol-timeout=60 --repo=1 --repo1-path=[TEST_PATH]/db-primary/repo --set=[BACKUP-FULL-2] --stanza=db
P00 INFO: restore backup set [BACKUP-FULL-2]
P00 INFO: repo1: restore backup set [BACKUP-FULL-2]
P00 WARN: unknown user in backup manifest mapped to '[USER-2]'
P00 WARN: unknown group in backup manifest mapped to '[GROUP-2]'
P00 DETAIL: check '[TEST_PATH]/db-primary/db/base' exists
@ -574,7 +574,7 @@ restore delta, backup '[BACKUP-FULL-2]' - fix permissions (db-primary host)
> [CONTAINER-EXEC] db-primary [BACKREST-BIN] --config=[TEST_PATH]/db-primary/pgbackrest.conf --delta --set=[BACKUP-FULL-2] --link-all --log-level-console=detail --repo=1 --stanza=db restore
------------------------------------------------------------------------------------------------------------------------------------
P00 INFO: restore command begin [BACKREST-VERSION]: --buffer-size=[BUFFER-SIZE] --config=[TEST_PATH]/db-primary/pgbackrest.conf --delta --exec-id=[EXEC-ID] --job-retry=0 --link-all --lock-path=[TEST_PATH]/db-primary/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-primary/log[] --no-log-timestamp --pg1-path=[TEST_PATH]/db-primary/db/base --protocol-timeout=60 --repo=1 --repo1-path=[TEST_PATH]/db-primary/repo --set=[BACKUP-FULL-2] --stanza=db
P00 INFO: restore backup set [BACKUP-FULL-2]
P00 INFO: repo1: restore backup set [BACKUP-FULL-2]
P00 WARN: unknown user in backup manifest mapped to '[USER-1]'
P00 WARN: unknown group in backup manifest mapped to '[GROUP-1]'
P00 DETAIL: check '[TEST_PATH]/db-primary/db/base' exists
@ -635,7 +635,7 @@ restore delta, backup '[BACKUP-FULL-2]' - fix broken symlink (db-primary host)
> [CONTAINER-EXEC] db-primary [BACKREST-BIN] --config=[TEST_PATH]/db-primary/pgbackrest.conf --delta --set=[BACKUP-FULL-2] --link-all --repo=1 --stanza=db restore
------------------------------------------------------------------------------------------------------------------------------------
P00 INFO: restore command begin [BACKREST-VERSION]: --buffer-size=[BUFFER-SIZE] --config=[TEST_PATH]/db-primary/pgbackrest.conf --delta --exec-id=[EXEC-ID] --job-retry=0 --link-all --lock-path=[TEST_PATH]/db-primary/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-primary/log[] --no-log-timestamp --pg1-path=[TEST_PATH]/db-primary/db/base --protocol-timeout=60 --repo=1 --repo1-path=[TEST_PATH]/db-primary/repo --set=[BACKUP-FULL-2] --stanza=db
P00 INFO: restore backup set [BACKUP-FULL-2]
P00 INFO: repo1: restore backup set [BACKUP-FULL-2]
P00 WARN: unknown user in backup manifest mapped to current user
P00 WARN: unknown group in backup manifest mapped to current group
P00 DETAIL: check '[TEST_PATH]/db-primary/db/base' exists
@ -696,7 +696,7 @@ restore delta, force, backup '[BACKUP-FULL-2]' - restore links as directories (d
> [CONTAINER-EXEC] db-primary [BACKREST-BIN] --config=[TEST_PATH]/db-primary/pgbackrest.conf --delta --force --set=[BACKUP-FULL-2] --repo=1 --stanza=db restore
------------------------------------------------------------------------------------------------------------------------------------
P00 INFO: restore command begin [BACKREST-VERSION]: --buffer-size=[BUFFER-SIZE] --config=[TEST_PATH]/db-primary/pgbackrest.conf --delta --exec-id=[EXEC-ID] --force --job-retry=0 --lock-path=[TEST_PATH]/db-primary/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-primary/log[] --no-log-timestamp --pg1-path=[TEST_PATH]/db-primary/db/base --protocol-timeout=60 --repo=1 --repo1-path=[TEST_PATH]/db-primary/repo --set=[BACKUP-FULL-2] --stanza=db
P00 INFO: restore backup set [BACKUP-FULL-2]
P00 INFO: repo1: restore backup set [BACKUP-FULL-2]
P00 WARN: file link 'pg_hba.conf' will be restored as a file at the same location
P00 WARN: contents of directory link 'pg_stat' will be restored in a directory at the same location
P00 WARN: file link 'postgresql.conf' will be restored as a file at the same location
@ -1392,7 +1392,7 @@ restore, backup '[BACKUP-DIFF-1]', remap - remap all paths (db-primary host)
> [CONTAINER-EXEC] db-primary [BACKREST-BIN] --config=[TEST_PATH]/db-primary/pgbackrest.conf --set=[BACKUP-DIFF-1] --repo=1 --stanza=db restore
------------------------------------------------------------------------------------------------------------------------------------
P00 INFO: restore command begin [BACKREST-VERSION]: --buffer-size=[BUFFER-SIZE] --config=[TEST_PATH]/db-primary/pgbackrest.conf --exec-id=[EXEC-ID] --job-retry=0 --lock-path=[TEST_PATH]/db-primary/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-primary/log[] --no-log-timestamp --pg1-path=[TEST_PATH]/db-primary/db/base-2 --protocol-timeout=60 --repo=1 --repo1-path=[TEST_PATH]/db-primary/repo --set=[BACKUP-DIFF-1] --stanza=db --tablespace-map=1=[TEST_PATH]/db-primary/db/tablespace/ts1-2 --tablespace-map=2=[TEST_PATH]/db-primary/db/tablespace/ts2-2
P00 INFO: restore backup set [BACKUP-DIFF-1]
P00 INFO: repo1: restore backup set [BACKUP-DIFF-1]
P00 INFO: remap data directory to '[TEST_PATH]/db-primary/db/base-2'
P00 INFO: map tablespace 'pg_tblspc/1' to '[TEST_PATH]/db-primary/db/tablespace/ts1-2'
P00 INFO: map tablespace 'pg_tblspc/2' to '[TEST_PATH]/db-primary/db/tablespace/ts2-2'
@ -1477,7 +1477,7 @@ restore delta, backup '[BACKUP-DIFF-1]', remap - ensure file in tblspc root rema
> [CONTAINER-EXEC] db-primary [BACKREST-BIN] --config=[TEST_PATH]/db-primary/pgbackrest.conf --delta --set=[BACKUP-DIFF-1] --repo=1 --stanza=db restore
------------------------------------------------------------------------------------------------------------------------------------
P00 INFO: restore command begin [BACKREST-VERSION]: --buffer-size=[BUFFER-SIZE] --config=[TEST_PATH]/db-primary/pgbackrest.conf --delta --exec-id=[EXEC-ID] --job-retry=0 --lock-path=[TEST_PATH]/db-primary/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-primary/log[] --no-log-timestamp --pg1-path=[TEST_PATH]/db-primary/db/base-2 --protocol-timeout=60 --repo=1 --repo1-path=[TEST_PATH]/db-primary/repo --set=[BACKUP-DIFF-1] --stanza=db --tablespace-map=1=[TEST_PATH]/db-primary/db/tablespace/ts1-2 --tablespace-map=2=[TEST_PATH]/db-primary/db/tablespace/ts2-2
P00 INFO: restore backup set [BACKUP-DIFF-1]
P00 INFO: repo1: restore backup set [BACKUP-DIFF-1]
P00 INFO: remap data directory to '[TEST_PATH]/db-primary/db/base-2'
P00 INFO: map tablespace 'pg_tblspc/1' to '[TEST_PATH]/db-primary/db/tablespace/ts1-2'
P00 INFO: map tablespace 'pg_tblspc/2' to '[TEST_PATH]/db-primary/db/tablespace/ts2-2'
@ -2815,7 +2815,7 @@ restore delta, remap - selective restore 16384 (db-primary host)
> [CONTAINER-EXEC] db-primary [BACKREST-BIN] --config=[TEST_PATH]/db-primary/pgbackrest.conf --delta --db-include=16384 --repo=1 --stanza=db restore
------------------------------------------------------------------------------------------------------------------------------------
P00 INFO: restore command begin [BACKREST-VERSION]: --buffer-size=[BUFFER-SIZE] --config=[TEST_PATH]/db-primary/pgbackrest.conf --db-include=16384 --delta --exec-id=[EXEC-ID] --job-retry=0 --lock-path=[TEST_PATH]/db-primary/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-primary/log[] --no-log-timestamp --pg1-path=[TEST_PATH]/db-primary/db/base-2 --protocol-timeout=60 --repo=1 --repo1-path=[TEST_PATH]/db-primary/repo --stanza=db --tablespace-map=2=[TEST_PATH]/db-primary/db/tablespace/ts2-2
P00 INFO: restore backup set [BACKUP-DIFF-4]
P00 INFO: repo1: restore backup set [BACKUP-DIFF-4]
P00 INFO: map tablespace 'pg_tblspc/2' to '[TEST_PATH]/db-primary/db/tablespace/ts2-2'
P00 DETAIL: databases found for selective restore (1, 16384, 32768)
P00 DETAIL: check '[TEST_PATH]/db-primary/db/base-2' exists
@ -2879,7 +2879,7 @@ restore delta, remap - selective restore 32768 (db-primary host)
> [CONTAINER-EXEC] db-primary [BACKREST-BIN] --config=[TEST_PATH]/db-primary/pgbackrest.conf --delta --db-include=32768 --repo=1 --stanza=db restore
------------------------------------------------------------------------------------------------------------------------------------
P00 INFO: restore command begin [BACKREST-VERSION]: --buffer-size=[BUFFER-SIZE] --config=[TEST_PATH]/db-primary/pgbackrest.conf --db-include=32768 --delta --exec-id=[EXEC-ID] --job-retry=0 --lock-path=[TEST_PATH]/db-primary/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-primary/log[] --no-log-timestamp --pg1-path=[TEST_PATH]/db-primary/db/base-2 --protocol-timeout=60 --repo=1 --repo1-path=[TEST_PATH]/db-primary/repo --stanza=db --tablespace-map=2=[TEST_PATH]/db-primary/db/tablespace/ts2-2
P00 INFO: restore backup set [BACKUP-DIFF-4]
P00 INFO: repo1: restore backup set [BACKUP-DIFF-4]
P00 INFO: map tablespace 'pg_tblspc/2' to '[TEST_PATH]/db-primary/db/tablespace/ts2-2'
P00 DETAIL: databases found for selective restore (1, 16384, 32768)
P00 DETAIL: check '[TEST_PATH]/db-primary/db/base-2' exists
@ -2953,7 +2953,7 @@ restore, remap - no tablespace remap (db-primary host)
> [CONTAINER-EXEC] db-primary [BACKREST-BIN] --config=[TEST_PATH]/db-primary/pgbackrest.conf --tablespace-map-all=../../tablespace --repo=1 --stanza=db restore
------------------------------------------------------------------------------------------------------------------------------------
P00 INFO: restore command begin [BACKREST-VERSION]: --buffer-size=[BUFFER-SIZE] --config=[TEST_PATH]/db-primary/pgbackrest.conf --exec-id=[EXEC-ID] --job-retry=0 --lock-path=[TEST_PATH]/db-primary/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/db-primary/log[] --no-log-timestamp --pg1-path=[TEST_PATH]/db-primary/db/base-2/base --protocol-timeout=60 --repo=1 --repo1-path=[TEST_PATH]/db-primary/repo --stanza=db --tablespace-map-all=../../tablespace
P00 INFO: restore backup set [BACKUP-DIFF-4]
P00 INFO: repo1: restore backup set [BACKUP-DIFF-4]
P00 INFO: remap data directory to '[TEST_PATH]/db-primary/db/base-2/base'
P00 INFO: map tablespace 'pg_tblspc/2' to '../../tablespace/ts2'
P00 DETAIL: check '[TEST_PATH]/db-primary/db/base-2/base' exists

250
test/src/module/command/restoreTest.c
View File

@ -155,6 +155,7 @@ testRun(void)
{
const String *repoFileReferenceFull = strNew("20190509F");
const String *repoFile1 = strNew("pg_data/testfile");
unsigned int repoIdx = 0;
// Start a protocol server to test the protocol directly
Buffer *serverWrite = bufNew(8192);
@ -178,7 +179,7 @@ testRun(void)
TEST_RESULT_BOOL(
restoreFile(
repoFile1, repoFileReferenceFull, compressTypeNone, strNew("sparse-zero"),
repoFile1, repoIdx, repoFileReferenceFull, compressTypeNone, strNew("sparse-zero"),
strNew("9bc8ab2dda60ef4beed07d1e19ce0676d5edde67"), true, 0x10000000000UL, 1557432154, 0600, strNew(testUser()),
strNew(testGroup()), 0, true, false, NULL),
false, "zero sparse 1TB file");
@ -186,7 +187,7 @@ testRun(void)
TEST_RESULT_BOOL(
restoreFile(
repoFile1, repoFileReferenceFull, compressTypeNone, strNew("normal-zero"),
repoFile1, repoIdx, repoFileReferenceFull, compressTypeNone, strNew("normal-zero"),
strNew("9bc8ab2dda60ef4beed07d1e19ce0676d5edde67"), false, 0, 1557432154, 0600, strNew(testUser()),
strNew(testGroup()), 0, false, false, NULL),
true, "zero-length file");
@ -204,7 +205,7 @@ testRun(void)
TEST_ERROR(
restoreFile(
repoFile1, repoFileReferenceFull, compressTypeGz, strNew("normal"),
repoFile1, repoIdx, repoFileReferenceFull, compressTypeGz, strNew("normal"),
strNew("ffffffffffffffffffffffffffffffffffffffff"), false, 7, 1557432154, 0600, strNew(testUser()),
strNew(testGroup()), 0, false, false, strNew("badpass")),
ChecksumError,
@ -219,7 +220,7 @@ testRun(void)
TEST_RESULT_BOOL(
restoreFile(
repoFile1, repoFileReferenceFull, compressTypeGz, strNew("normal"),
repoFile1, repoIdx, repoFileReferenceFull, compressTypeGz, strNew("normal"),
strNew("d1cd8a7d11daa26814b93eb604e1d49ab4b43770"), false, 7, 1557432154, 0600, strNew(testUser()),
strNew(testGroup()), 0, false, false, strNew("badpass")),
true, "copy file");
@ -242,7 +243,7 @@ testRun(void)
TEST_RESULT_BOOL(
restoreFile(
repoFile1, repoFileReferenceFull, compressTypeNone, strNew("delta"),
repoFile1, repoIdx, repoFileReferenceFull, compressTypeNone, strNew("delta"),
strNew("9bc8ab2dda60ef4beed07d1e19ce0676d5edde67"), false, 9, 1557432154, 0600, strNew(testUser()),
strNew(testGroup()), 0, true, false, NULL),
true, "sha1 delta missing");
@ -254,7 +255,7 @@ testRun(void)
TEST_RESULT_BOOL(
restoreFile(
repoFile1, repoFileReferenceFull, compressTypeNone, strNew("delta"),
repoFile1, repoIdx, repoFileReferenceFull, compressTypeNone, strNew("delta"),
strNew("9bc8ab2dda60ef4beed07d1e19ce0676d5edde67"), false, 9, 1557432154, 0600, strNew(testUser()),
strNew(testGroup()), 0, true, false, NULL),
false, "sha1 delta existing");
@ -263,7 +264,7 @@ testRun(void)
TEST_RESULT_BOOL(
restoreFile(
repoFile1, repoFileReferenceFull, compressTypeNone, strNew("delta"),
repoFile1, repoIdx, repoFileReferenceFull, compressTypeNone, strNew("delta"),
strNew("9bc8ab2dda60ef4beed07d1e19ce0676d5edde67"), false, 9, 1557432154, 0600, strNew(testUser()),
strNew(testGroup()), 1557432155, true, true, NULL),
false, "sha1 delta force existing");
@ -273,7 +274,7 @@ testRun(void)
TEST_RESULT_BOOL(
restoreFile(
repoFile1, repoFileReferenceFull, compressTypeNone, strNew("delta"),
repoFile1, repoIdx, repoFileReferenceFull, compressTypeNone, strNew("delta"),
strNew("9bc8ab2dda60ef4beed07d1e19ce0676d5edde67"), false, 9, 1557432154, 0600, strNew(testUser()),
strNew(testGroup()), 0, true, false, NULL),
true, "sha1 delta existing, size differs");
@ -284,7 +285,7 @@ testRun(void)
TEST_RESULT_BOOL(
restoreFile(
repoFile1, repoFileReferenceFull, compressTypeNone, strNew("delta"),
repoFile1, repoIdx, repoFileReferenceFull, compressTypeNone, strNew("delta"),
strNew("9bc8ab2dda60ef4beed07d1e19ce0676d5edde67"), false, 9, 1557432154, 0600, strNew(testUser()),
strNew(testGroup()), 1557432155, true, true, NULL),
true, "delta force existing, size differs");
@ -296,7 +297,7 @@ testRun(void)
TEST_RESULT_BOOL(
restoreFile(
repoFile1, repoFileReferenceFull, compressTypeNone, strNew("delta"),
repoFile1, repoIdx, repoFileReferenceFull, compressTypeNone, strNew("delta"),
strNew("9bc8ab2dda60ef4beed07d1e19ce0676d5edde67"), false, 9, 1557432154, 0600, strNew(testUser()),
strNew(testGroup()), 0, true, false, NULL),
true, "sha1 delta existing, content differs");
@ -307,14 +308,14 @@ testRun(void)
TEST_RESULT_BOOL(
restoreFile(
repoFile1, repoFileReferenceFull, compressTypeNone, strNew("delta"),
repoFile1, repoIdx, repoFileReferenceFull, compressTypeNone, strNew("delta"),
strNew("9bc8ab2dda60ef4beed07d1e19ce0676d5edde67"), false, 9, 1557432154, 0600, strNew(testUser()),
strNew(testGroup()), 1557432155, true, true, NULL),
true, "delta force existing, timestamp differs");
TEST_RESULT_BOOL(
restoreFile(
repoFile1, repoFileReferenceFull, compressTypeNone, strNew("delta"),
repoFile1, repoIdx, repoFileReferenceFull, compressTypeNone, strNew("delta"),
strNew("9bc8ab2dda60ef4beed07d1e19ce0676d5edde67"), false, 9, 1557432154, 0600, strNew(testUser()),
strNew(testGroup()), 1557432153, true, true, NULL),
true, "delta force existing, timestamp after copy time");
@ -324,7 +325,7 @@ testRun(void)
TEST_RESULT_BOOL(
restoreFile(
repoFile1, repoFileReferenceFull, compressTypeNone, strNew("delta"),
repoFile1, repoIdx, repoFileReferenceFull, compressTypeNone, strNew("delta"),
strNew("9bc8ab2dda60ef4beed07d1e19ce0676d5edde67"), false, 0, 1557432154, 0600, strNew(testUser()),
strNew(testGroup()), 0, true, false, NULL),
false, "sha1 delta existing, content differs");
@ -333,6 +334,7 @@ testRun(void)
// -------------------------------------------------------------------------------------------------------------------------
VariantList *paramList = varLstNew();
varLstAdd(paramList, varNewStr(repoFile1));
varLstAdd(paramList, varNewUInt(repoIdx));
varLstAdd(paramList, varNewStr(repoFileReferenceFull));
varLstAdd(paramList, varNewUInt(compressTypeNone));
varLstAdd(paramList, varNewStrZ("protocol"));
@ -364,6 +366,7 @@ testRun(void)
paramList = varLstNew();
varLstAdd(paramList, varNewStr(repoFile1));
varLstAdd(paramList, varNewUInt(repoIdx));
varLstAdd(paramList, varNewStr(repoFileReferenceFull));
varLstAdd(paramList, varNewUInt(compressTypeNone));
varLstAdd(paramList, varNewStrZ("protocol"));
@ -517,14 +520,18 @@ testRun(void)
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("error when no backups are present");
InfoBackup *infoBackup = infoBackupNewLoad(ioBufferReadNew(harnessInfoChecksumZ(TEST_RESTORE_BACKUP_INFO_DB)));
TEST_ERROR_FMT(restoreBackupSet(infoBackup), BackupSetInvalidError, "no backup sets to restore");
HRN_INFO_PUT(storageRepoWrite(), INFO_BACKUP_PATH_FILE, TEST_RESTORE_BACKUP_INFO_DB);
TEST_ERROR_FMT(restoreBackupSet(), BackupSetInvalidError, "no backup set found to restore");
TEST_RESULT_LOG("P00 WARN: repo1: [BackupSetInvalidError] no backup sets to restore");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("error on invalid backup set");
infoBackup = infoBackupNewLoad(
ioBufferReadNew(harnessInfoChecksumZ(TEST_RESTORE_BACKUP_INFO "\n" TEST_RESTORE_BACKUP_INFO_DB)));
HRN_INFO_PUT(
storageRepoWrite(), INFO_BACKUP_PATH_FILE,
TEST_RESTORE_BACKUP_INFO
"\n"
TEST_RESTORE_BACKUP_INFO_DB);
argList = strLstNew();
strLstAddZ(argList, "--stanza=test1");
@ -533,55 +540,154 @@ testRun(void)
strLstAddZ(argList, "--set=BOGUS");
harnessCfgLoad(cfgCmdRestore, argList);
TEST_ERROR(restoreBackupSet(infoBackup), BackupSetInvalidError, "backup set BOGUS is not valid");
TEST_ERROR(restoreBackupSet(), BackupSetInvalidError, "backup set BOGUS is not valid");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("target time");
setenv("TZ", "UTC", true);
infoBackup = infoBackupNewLoad(
ioBufferReadNew(harnessInfoChecksumZ(TEST_RESTORE_BACKUP_INFO "\n" TEST_RESTORE_BACKUP_INFO_DB)));
const String *repoPath2 = strNewFmt("%s/repo2", testPath());
argList = strLstNew();
strLstAddZ(argList, "--stanza=test1");
strLstAdd(argList, strNewFmt("--repo1-path=%s", strZ(repoPath)));
hrnCfgArgKeyRaw(argList, cfgOptRepoPath, 1, repoPath2);
hrnCfgArgKeyRaw(argList, cfgOptRepoPath, 2, repoPath);
strLstAdd(argList, strNewFmt("--pg1-path=%s", strZ(pgPath)));
strLstAddZ(argList, "--type=time");
strLstAddZ(argList, "--target=2016-12-19 16:28:04-0500");
harnessCfgLoad(cfgCmdRestore, argList);
TEST_RESULT_STR_Z(restoreBackupSet(infoBackup), "20161219-212741F_20161219-212803D", "backup set found");
// Write out backup.info with no current backups to repo1
HRN_INFO_PUT(storageRepoIdxWrite(0), INFO_BACKUP_PATH_FILE, TEST_RESTORE_BACKUP_INFO_DB);
RestoreBackupData backupData = {0};
TEST_ASSIGN(backupData, restoreBackupSet(), "get backup set");
TEST_RESULT_STR_Z(backupData.backupSet, "20161219-212741F_20161219-212803D", "backup set found");
TEST_RESULT_UINT(backupData.repoIdx, 1, "backup set found, repo2");
TEST_RESULT_LOG("P00 WARN: repo1: [BackupSetInvalidError] no backup sets to restore");
// Switch repo paths and confirm same result but on repo1
argList = strLstNew();
strLstAddZ(argList, "--stanza=test1");
strLstAdd(argList, strNewFmt("--repo1-path=%s", strZ(repoPath)));
hrnCfgArgKeyRaw(argList, cfgOptRepoPath, 1, repoPath);
hrnCfgArgKeyRaw(argList, cfgOptRepoPath, 2, repoPath2);
strLstAdd(argList, strNewFmt("--pg1-path=%s", strZ(pgPath)));
strLstAddZ(argList, "--type=time");
strLstAddZ(argList, "--target=2016-12-19 16:27:30-0500");
strLstAddZ(argList, "--target=2016-12-19 16:28:04-0500");
harnessCfgLoad(cfgCmdRestore, argList);
TEST_RESULT_STR_Z(restoreBackupSet(infoBackup), "20161219-212741F_20161219-212918I", "default to latest backup set");
TEST_ASSIGN(backupData, restoreBackupSet(), "get backup set");
TEST_RESULT_STR_Z(backupData.backupSet, "20161219-212741F_20161219-212803D", "backup set found");
TEST_RESULT_UINT(backupData.repoIdx, 0, "backup set found, repo1");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("target time, multi repo, latest used");
argList = strLstNew();
hrnCfgArgRawZ(argList, cfgOptStanza ,"test1");
hrnCfgArgKeyRaw(argList, cfgOptRepoPath, 1, repoPath);
hrnCfgArgKeyRaw(argList, cfgOptRepoPath, 2, repoPath2);
hrnCfgArgKeyRaw(argList, cfgOptPgPath, 1, pgPath);
hrnCfgArgRawZ(argList, cfgOptType, "time");
hrnCfgArgRawZ(argList, cfgOptTarget, "2016-12-19 16:27:30-0500");
harnessCfgLoad(cfgCmdRestore, argList);
#define TEST_RESTORE_BACKUP_INFO_NEWEST \
"[backup:current]\n" \
"20201212-201243F={\"backrest-format\":5,\"backrest-version\":\"2.04\"," \
"\"backup-archive-start\":\"00000007000000000000001C\",\"backup-archive-stop\":\"00000007000000000000001C\"," \
"\"backup-info-repo-size\":3159776,\"backup-info-repo-size-delta\":3159776,\"backup-info-size\":26897030," \
"\"backup-info-size-delta\":26897030,\"backup-timestamp-start\":1607803000,\"backup-timestamp-stop\":1607803963," \
"\"backup-type\":\"full\",\"db-id\":1,\"option-archive-check\":true,\"option-archive-copy\":false," \
"\"option-backup-standby\":false,\"option-checksum-page\":false,\"option-compress\":true,\"option-hardlink\":false," \
"\"option-online\":true}\n"
// Write out backup.info with current backup newest to repo2 but still does not satisfy time requirement, so repo1 chosen
HRN_INFO_PUT(
storageRepoIdxWrite(1), INFO_BACKUP_PATH_FILE,
TEST_RESTORE_BACKUP_INFO_NEWEST
"\n"
TEST_RESTORE_BACKUP_INFO_DB);
TEST_ASSIGN(backupData, restoreBackupSet(), "get backup set");
TEST_RESULT_STR_Z(backupData.backupSet, "20161219-212741F_20161219-212918I", "default to latest backup set");
TEST_RESULT_UINT(backupData.repoIdx, 0, "repo1 chosen because of priority order");
TEST_RESULT_LOG(
"P00 WARN: unable to find backup set with stop time less than '2016-12-19 16:27:30-0500', latest backup set will be"
" used");
"P00 WARN: unable to find backup set with stop time less than '2016-12-19 16:27:30-0500', repo1: latest backup set"
" will be used");
// Request repo2 - latest from repo2 will be chosen
hrnCfgArgRawZ(argList, cfgOptRepo, "2");
harnessCfgLoad(cfgCmdRestore, argList);
TEST_ASSIGN(backupData, restoreBackupSet(), "get backup set");
TEST_RESULT_STR_Z(backupData.backupSet, "20201212-201243F", "default to latest backup set");
TEST_RESULT_UINT(backupData.repoIdx, 1, "repo2 chosen because repo option set");
TEST_RESULT_LOG(
"P00 WARN: unable to find backup set with stop time less than '2016-12-19 16:27:30-0500', repo2: latest backup set"
" will be used");
// Switch paths so newest on repo1
argList = strLstNew();
strLstAddZ(argList, "--stanza=test1");
strLstAdd(argList, strNewFmt("--repo1-path=%s", strZ(repoPath)));
strLstAdd(argList, strNewFmt("--pg1-path=%s", strZ(pgPath)));
strLstAddZ(argList, "--type=time");
strLstAddZ(argList, "--target=Tue, 15 Nov 1994 12:45:26");
hrnCfgArgRawZ(argList, cfgOptStanza ,"test1");
hrnCfgArgKeyRaw(argList, cfgOptRepoPath, 1, repoPath2);
hrnCfgArgKeyRaw(argList, cfgOptRepoPath, 2, repoPath);
hrnCfgArgKeyRaw(argList, cfgOptPgPath, 1, pgPath);
hrnCfgArgRawZ(argList, cfgOptType, "time");
hrnCfgArgRawZ(argList, cfgOptTarget, "2016-12-19 16:27:30-0500");
harnessCfgLoad(cfgCmdRestore, argList);
TEST_RESULT_STR_Z(restoreBackupSet(infoBackup), "20161219-212741F_20161219-212918I", "time invalid format, default latest");
TEST_ASSIGN(backupData, restoreBackupSet(), "get backup set");
TEST_RESULT_STR_Z(backupData.backupSet, "20201212-201243F", "default to latest backup set");
TEST_RESULT_UINT(backupData.repoIdx, 0, "repo1 chosen because of priority order");
TEST_RESULT_LOG(
"P00 WARN: unable to find backup set with stop time less than '2016-12-19 16:27:30-0500', repo1: latest backup set"
" will be used");
argList = strLstNew();
hrnCfgArgRawZ(argList, cfgOptStanza ,"test1");
hrnCfgArgKeyRaw(argList, cfgOptRepoPath, 1, repoPath);
hrnCfgArgKeyRaw(argList, cfgOptRepoPath, 2, repoPath2);
hrnCfgArgKeyRaw(argList, cfgOptPgPath, 1, pgPath);
hrnCfgArgRawZ(argList, cfgOptType, "time");
hrnCfgArgRawZ(argList, cfgOptTarget, "Tue, 15 Nov 1994 12:45:26");
harnessCfgLoad(cfgCmdRestore, argList);
TEST_ASSIGN(backupData, restoreBackupSet(), "get backup set");
TEST_RESULT_STR_Z(backupData.backupSet, "20161219-212741F_20161219-212918I", "time invalid format, default latest");
TEST_RESULT_UINT(backupData.repoIdx, 0, "repo1 chosen because of priority order");
TEST_RESULT_LOG(
"P00 WARN: automatic backup set selection cannot be performed with provided time 'Tue, 15 Nov 1994 12:45:26',"
" latest backup set will be used\n"
" HINT: time format must be YYYY-MM-DD HH:MM:SS with optional msec and optional timezone"
" (+/- HH or HHMM or HH:MM) - if timezone is omitted, local time is assumed (for UTC use +00)");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("target time, multi repo, no candidates found");
argList = strLstNew();
hrnCfgArgRawZ(argList, cfgOptStanza ,"test1");
hrnCfgArgKeyRaw(argList, cfgOptRepoPath, 1, repoPath);
hrnCfgArgKeyRaw(argList, cfgOptRepoPath, 2, repoPath2);
hrnCfgArgKeyRaw(argList, cfgOptPgPath, 1, pgPath);
hrnCfgArgRawZ(argList, cfgOptType, "time");
hrnCfgArgRawZ(argList, cfgOptTarget, "2016-12-19 16:27:30-0500");
harnessCfgLoad(cfgCmdRestore, argList);
// Write out backup.info with no current backups to repo1 and repo2
HRN_INFO_PUT(storageRepoIdxWrite(0), INFO_BACKUP_PATH_FILE, TEST_RESTORE_BACKUP_INFO_DB);
HRN_INFO_PUT(storageRepoIdxWrite(1), INFO_BACKUP_PATH_FILE, TEST_RESTORE_BACKUP_INFO_DB);
TEST_ERROR_FMT(restoreBackupSet(), BackupSetInvalidError, "no backup set found to restore");
TEST_RESULT_LOG(
"P00 WARN: repo1: [BackupSetInvalidError] no backup sets to restore\n"
"P00 WARN: repo2: [BackupSetInvalidError] no backup sets to restore");
}
// *****************************************************************************************************************************
@ -1665,6 +1771,7 @@ testRun(void)
{
const String *pgPath = strNewFmt("%s/pg", testPath());
const String *repoPath = strNewFmt("%s/repo", testPath());
const String *repoPathEncrpyt = strNewFmt("%s/repo-encrypt", testPath());
// Set log level to detail
harnessLogLevelSet(logLevelDetail);
@ -1691,20 +1798,17 @@ testRun(void)
TEST_ERROR(cmdRestore(), HostInvalidError, "restore command must be run on the PostgreSQL host");
// Write backup info
// -------------------------------------------------------------------------------------------------------------------------
storagePutP(
storageNewWriteP(storageRepoWrite(), INFO_BACKUP_PATH_FILE_STR),
harnessInfoChecksumZ(TEST_RESTORE_BACKUP_INFO "\n" TEST_RESTORE_BACKUP_INFO_DB));
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("full restore without delta");
TEST_TITLE("full restore without delta, multi-repo");
argList = strLstNew();
strLstAddZ(argList, "--stanza=test1");
strLstAdd(argList, strNewFmt("--repo1-path=%s", strZ(repoPath)));
hrnCfgArgKeyRaw(argList, cfgOptRepoPath, 1, repoPath);
hrnCfgArgKeyRaw(argList, cfgOptRepoPath, 2, repoPathEncrpyt);
strLstAdd(argList, strNewFmt("--pg1-path=%s", strZ(pgPath)));
strLstAddZ(argList, "--set=20161219-212741F");
hrnCfgArgKeyRawZ(argList, cfgOptRepoCipherType, 2, CIPHER_TYPE_AES_256_CBC);
hrnCfgEnvKeyRawZ(cfgOptRepoCipherPass, 2, TEST_CIPHER_PASS);
harnessCfgLoad(cfgCmdRestore, argList);
#define TEST_LABEL "20161219-212741F"
@ -1743,7 +1847,13 @@ testRun(void)
.mode = 0600, .group = groupName(), .user = userName(),
.checksumSha1 = "797e375b924134687cbf9eacd37a4355f3d825e4"});
storagePutP(
storageNewWriteP(storageRepoWrite(), STRDEF(TEST_REPO_PATH PG_FILE_PGVERSION)), BUFSTRDEF(PG_VERSION_84_STR "\n"));
storageNewWriteP(
storageRepoIdxWrite(0), STRDEF(TEST_REPO_PATH PG_FILE_PGVERSION)), BUFSTRDEF(PG_VERSION_84_STR "\n"));
// Store the file also to the encrypted repo
HRN_STORAGE_PUT(
storageRepoIdxWrite(1), TEST_REPO_PATH PG_FILE_PGVERSION, BUFSTRDEF(PG_VERSION_84_STR "\n"),
.cipherType = cipherTypeAes256Cbc, .cipherPass = TEST_CIPHER_PASS_ARCHIVE);
// pg_tblspc/1
manifestTargetAdd(
@ -1782,13 +1892,43 @@ testRun(void)
manifestSave(
manifest,
storageWriteIo(
storageNewWriteP(storageRepoWrite(),
storageNewWriteP(storageRepoIdxWrite(0),
strNew(STORAGE_REPO_BACKUP "/" TEST_LABEL "/" BACKUP_MANIFEST_FILE))));
// Read the manifest, set a cipher passphrase and store it to the encrypted repo
Manifest *manifestEncrypted = manifestLoadFile(
storageRepoIdxWrite(0), strNew(STORAGE_REPO_BACKUP "/" TEST_LABEL "/" BACKUP_MANIFEST_FILE), cipherTypeNone, NULL);
manifestCipherSubPassSet(manifestEncrypted, STRDEF(TEST_CIPHER_PASS_ARCHIVE));
// Open file for write
IoWrite *write = storageWriteIo(
storageNewWriteP(
storageRepoIdxWrite(1),
strNew(STORAGE_REPO_BACKUP "/" TEST_LABEL "/" BACKUP_MANIFEST_FILE)));
// Add encryption filter and save the encrypted manifest
#define TEST_CIPHER_PASS_MANIFEST "backpass"
cipherBlockFilterGroupAdd(
ioWriteFilterGroup(write), cipherType(cfgOptionIdxStr(cfgOptRepoCipherType, 1)), cipherModeEncrypt,
STRDEF(TEST_CIPHER_PASS_MANIFEST));
manifestSave(manifestEncrypted, write);
// Write backup.info to the encrypted repo
HRN_INFO_PUT(
storageRepoIdxWrite(1), INFO_BACKUP_PATH_FILE, TEST_RESTORE_BACKUP_INFO "\n[cipher]\ncipher-pass=\""
TEST_CIPHER_PASS_MANIFEST "\"\n\n" TEST_RESTORE_BACKUP_INFO_DB, .cipherType = cipherTypeAes256Cbc);
TEST_RESULT_VOID(cmdRestore(), "successful restore");
TEST_RESULT_LOG(
"P00 INFO: restore backup set 20161219-212741F\n"
strZ(strNewFmt(
"P00 WARN: repo1: [FileMissingError] unable to load info file"
" '%s/repo/backup/test1/backup.info' or '%s/repo/backup/test1/backup.info.copy':\n"
" FileMissingError: unable to open missing file '%s/repo/backup/test1/backup.info' for read\n"
" FileMissingError: unable to open missing file '%s/repo/backup/test1/backup.info.copy' for read\n"
" HINT: backup.info cannot be opened and is required to perform a backup.\n"
" HINT: has a stanza-create been performed?\n"
"P00 INFO: repo2: restore backup set 20161219-212741F\n"
"P00 DETAIL: check '{[path]}/pg' exists\n"
"P00 DETAIL: check '{[path]}/ts/1' exists\n"
"P00 DETAIL: update mode for '{[path]}/pg' to 0700\n"
@ -1796,14 +1936,14 @@ testRun(void)
"P00 DETAIL: create path '{[path]}/pg/pg_tblspc'\n"
"P00 DETAIL: create symlink '{[path]}/pg/pg_tblspc/1' to '{[path]}/ts/1'\n"
"P00 DETAIL: create path '{[path]}/pg/pg_tblspc/1/16384'\n"
"P01 INFO: restore file {[path]}/pg/PG_VERSION (4B, 100%) checksum 797e375b924134687cbf9eacd37a4355f3d825e4\n"
"P01 INFO: restore file {[path]}/pg/PG_VERSION (4B, 100%%) checksum 797e375b924134687cbf9eacd37a4355f3d825e4\n"
"P00 INFO: write {[path]}/pg/recovery.conf\n"
"P00 DETAIL: sync path '{[path]}/pg'\n"
"P00 DETAIL: sync path '{[path]}/pg/pg_tblspc'\n"
"P00 DETAIL: sync path '{[path]}/pg/pg_tblspc/1'\n"
"P00 DETAIL: sync path '{[path]}/pg/pg_tblspc/1/16384'\n"
"P00 WARN: backup does not contain 'global/pg_control' -- cluster will not start\n"
"P00 DETAIL: sync path '{[path]}/pg/global'");
"P00 DETAIL: sync path '{[path]}/pg/global'", testPath(), testPath(), testPath(), testPath())));
// Remove recovery.conf before file comparison since it will have a new timestamp. Make sure it existed, though.
storageRemoveP(storagePgWrite(), PG_FILE_RECOVERYCONF_STR, .errorOnMissing = true);
@ -1827,13 +1967,19 @@ testRun(void)
argList = strLstNew();
strLstAddZ(argList, "--stanza=test1");
strLstAdd(argList, strNewFmt("--repo1-path=%s", strZ(repoPath)));
hrnCfgArgKeyRaw(argList, cfgOptRepoPath, 2, repoPathEncrpyt);
strLstAdd(argList, strNewFmt("--pg1-path=%s", strZ(pgPath)));
strLstAddZ(argList, "--type=preserve");
strLstAddZ(argList, "--set=20161219-212741F");
strLstAddZ(argList, "--" CFGOPT_DELTA);
strLstAddZ(argList, "--force");
hrnCfgArgKeyRawZ(argList, cfgOptRepoCipherType, 2, CIPHER_TYPE_AES_256_CBC);
hrnCfgEnvKeyRawZ(cfgOptRepoCipherPass, 2, TEST_CIPHER_PASS);
harnessCfgLoad(cfgCmdRestore, argList);
// Store backup.info to repo1 - repo1 will be selected because of the priority order
HRN_INFO_PUT(storageRepoIdxWrite(0), INFO_BACKUP_PATH_FILE, TEST_RESTORE_BACKUP_INFO "\n" TEST_RESTORE_BACKUP_INFO_DB);
// Make sure existing backup.manifest file is ignored
storagePutP(storageNewWriteP(storagePgWrite(), BACKUP_MANIFEST_FILE_STR), NULL);
@ -1898,7 +2044,7 @@ testRun(void)
cmdRestore();
TEST_RESULT_LOG(
"P00 INFO: restore backup set 20161219-212741F\n"
"P00 INFO: repo1: restore backup set 20161219-212741F\n"
"P00 DETAIL: check '{[path]}/pg' exists\n"
"P00 DETAIL: check '{[path]}/ts/1' exists\n"
"P00 INFO: remove invalid files/links/paths from '{[path]}/pg'\n"
@ -1940,6 +2086,10 @@ testRun(void)
strNewBuf(storageGetP(storageNewReadP(storagePg(), STRDEF(PG_FILE_PGVERSION)))), "BOG\n",
"check PG_VERSION was not restored");
// Cleanup
hrnCfgEnvKeyRemoveRaw(cfgOptRepoCipherPass, 2);
storagePathRemoveP(storageRepoIdxWrite(1), NULL, .recurse = true);
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("full restore with force");
@ -1955,7 +2105,7 @@ testRun(void)
cmdRestore();
TEST_RESULT_LOG(
"P00 INFO: restore backup set 20161219-212741F\n"
"P00 INFO: repo1: restore backup set 20161219-212741F\n"
"P00 DETAIL: check '{[path]}/pg' exists\n"
"P00 DETAIL: check '{[path]}/ts/1' exists\n"
"P00 INFO: remove invalid files/links/paths from '{[path]}/pg'\n"
@ -2268,7 +2418,7 @@ testRun(void)
TEST_RESULT_VOID(cmdRestore(), "successful restore");
TEST_RESULT_LOG(
"P00 INFO: restore backup set 20161219-212741F_20161219-212918I\n"
"P00 INFO: repo1: restore backup set 20161219-212741F_20161219-212918I\n"
"P00 INFO: map link 'pg_hba.conf' to '../config/pg_hba.conf'\n"
"P00 INFO: map link 'pg_wal' to '../wal'\n"
"P00 INFO: map link 'postgresql.conf' to '../config/postgresql.conf'\n"
@ -2377,7 +2527,7 @@ testRun(void)
TEST_RESULT_VOID(cmdRestore(), "successful restore");
TEST_RESULT_LOG(
"P00 INFO: restore backup set 20161219-212741F_20161219-212918I\n"
"P00 INFO: repo2: restore backup set 20161219-212741F_20161219-212918I\n"
"P00 INFO: map link 'pg_hba.conf' to '../config/pg_hba.conf'\n"
"P00 INFO: map link 'pg_wal' to '../wal'\n"
"P00 INFO: map link 'postgresql.conf' to '../config/postgresql.conf'\n"

21
test/src/module/config/loadTest.c
View File

@ -76,13 +76,26 @@ testRun(void)
argList = strLstNew();
hrnCfgArgRawZ(argList, cfgOptStanza, "test");
hrnCfgArgRawZ(argList, cfgOptRepo, "3");
hrnCfgArgKeyRawZ(argList, cfgOptRepoRetentionDiff, 4, "4");
hrnCfgArgKeyRawZ(argList, cfgOptRepoRetentionDiff, 3, "3");
hrnCfgArgKeyRawZ(argList, cfgOptRepoPath, 4, "/repo4");
hrnCfgArgKeyRawZ(argList, cfgOptRepoPath, 3, "/repo4");
hrnCfgArgRawZ(argList, cfgOptPgPath, "/pg1");
hrnCfgArgKeyRawZ(argList, cfgOptRepoPath, 1, "/repo1");
hrnCfgArgKeyRawZ(argList, cfgOptRepoHost, 2, "host2");
TEST_ERROR(
harnessCfgLoad(cfgCmdExpire, argList), OptionInvalidValueError,
"local repo3 and repo4 paths are both '/var/lib/pgbackrest' but must be different");
harnessCfgLoad(cfgCmdRestore, argList), OptionInvalidValueError,
"local repo3 and repo4 paths are both '/repo4' but must be different");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("repo can be specified for backup");
argList = strLstNew();
hrnCfgArgRawZ(argList, cfgOptStanza, "test");
hrnCfgArgRawZ(argList, cfgOptRepo, "1");
hrnCfgArgKeyRawZ(argList, cfgOptRepoPath, 1, "/repo1");
hrnCfgArgKeyRawZ(argList, cfgOptRepoRetentionFull, 1, "1");
hrnCfgArgKeyRawZ(argList, cfgOptPgPath, 1, "/pg1");
harnessCfgLoad(cfgCmdBackup, argList);
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("local default repo paths for cifs repo type must be different");