You've already forked pgbackrest
mirror of
https://github.com/pgbackrest/pgbackrest.git
synced 2025-07-13 01:00:23 +02:00
Verify recovery target timeline.
If the user picks an invalid timeline (or the default is invalid) they will not discover it until after the restore is complete and recovery starts. In that case they'll receive a message like this: FATAL: requested timeline 2 is not a child of this server's history DETAIL: Latest checkpoint is at 0/7000028 on timeline 1, but in the history of the requested timeline, the server forked off from that timeline at 0/600AA20. This message generally causes confusion unless one is familiar with it. In this case 1) a standby was promoted creating a new timeline 2) a new backup was made from the primary 3) the new backup was restored but could not follow the new timeline because the backup was made after the new timeline forked off. Since PostgreSQL 12 following the latest timeline has been the default so this error has become common in split brain situations. Improve pgBackRest to read the history files and provide better error messages. Now this error is thrown before the restore starts: ERROR: [058]: target timeline 2 forked from backup timeline 1 at 0/600aa20 which is before backup lsn of 0/7000028 HINT: was the target timeline created by accidentally promoting a standby? HINT: was the target timeline created by testing a restore without --archive-mode=off? HINT: was the backup made after the target timeline was created? This saves time since it happens before the restore and gives more information about what has gone wrong. If the backup timeline is not an ancestor of the target timeline the error message is: ERROR: [058]: backup timeline 6, lsn 0/4ffffff is not in the history of target timeline B HINT: was the target timeline created by promoting from a timeline < latest? This situation should be rare but can happen during complex recovery scenarios where the user is explicitly setting the target time.
This commit is contained in:
@ -25,6 +25,19 @@
|
||||
</release-item>
|
||||
</release-bug-list>
|
||||
|
||||
<release-feature-list>
|
||||
<release-item>
|
||||
<github-pull-request id="2534"/>
|
||||
|
||||
<release-item-contributor-list>
|
||||
<release-item-contributor id="david.steele"/>
|
||||
<release-item-reviewer id="stefan.fercot"/>
|
||||
</release-item-contributor-list>
|
||||
|
||||
<p>Verify recovery target timeline.</p>
|
||||
</release-item>
|
||||
</release-feature-list>
|
||||
|
||||
<release-improvement-list>
|
||||
<release-item>
|
||||
<github-pull-request id="2512"/>
|
||||
|
@ -11,6 +11,7 @@ Restore Command
|
||||
#include "command/restore/file.h"
|
||||
#include "command/restore/protocol.h"
|
||||
#include "command/restore/restore.h"
|
||||
#include "command/restore/timeline.h"
|
||||
#include "common/crypto/cipherBlock.h"
|
||||
#include "common/debug.h"
|
||||
#include "common/log.h"
|
||||
@ -18,6 +19,7 @@ Restore Command
|
||||
#include "common/user.h"
|
||||
#include "config/config.h"
|
||||
#include "config/exec.h"
|
||||
#include "info/infoArchive.h"
|
||||
#include "info/infoBackup.h"
|
||||
#include "info/manifest.h"
|
||||
#include "postgres/interface.h"
|
||||
@ -2366,6 +2368,23 @@ cmdRestore(void)
|
||||
strNewFmt(STORAGE_REPO_BACKUP "/%s/" BACKUP_MANIFEST_FILE, strZ(backupData.backupSet)), backupData.repoCipherType,
|
||||
backupData.backupCipherPass);
|
||||
|
||||
// Verify that the selected timeline is valid for the backup -- including current and latest timelines
|
||||
if (manifestData(jobData.manifest)->backupOptionOnline)
|
||||
{
|
||||
const ManifestData *const data = manifestData(jobData.manifest);
|
||||
const InfoArchive *const archiveInfo = infoArchiveLoadFile(
|
||||
storageRepoIdx(backupData.repoIdx), INFO_ARCHIVE_PATH_FILE_STR,
|
||||
cfgOptionIdxStrId(cfgOptRepoCipherType, backupData.repoIdx),
|
||||
cfgOptionIdxStrNull(cfgOptRepoCipherPass, backupData.repoIdx));
|
||||
|
||||
timelineVerify(
|
||||
storageRepoIdx(backupData.repoIdx),
|
||||
strNewFmt("%s-%u", strZ(pgVersionToStr(data->pgVersion)), data->pgId), data->pgVersion,
|
||||
cvtZToUIntBase(strZ(strSubN(data->archiveStart, 0, 8)), 16), pgLsnFromStr(data->lsnStart),
|
||||
cfgOptionStrId(cfgOptType), cfgOptionStrNull(cfgOptTargetTimeline),
|
||||
cfgOptionIdxStrId(cfgOptRepoCipherType, backupData.repoIdx), infoArchiveCipherPass(archiveInfo));
|
||||
}
|
||||
|
||||
// Remotes (if any) are no longer needed since the rest of the repository reads will be done by the local processes
|
||||
protocolFree();
|
||||
|
||||
|
294
src/command/restore/timeline.c
Normal file
294
src/command/restore/timeline.c
Normal file
@ -0,0 +1,294 @@
|
||||
/***********************************************************************************************************************************
|
||||
Timeline Management
|
||||
***********************************************************************************************************************************/
|
||||
#include "build.auto.h"
|
||||
|
||||
#include <string.h>
|
||||
#include <sys/stat.h>
|
||||
#include <time.h>
|
||||
#include <unistd.h>
|
||||
|
||||
#include "command/restore/timeline.h"
|
||||
#include "common/crypto/cipherBlock.h"
|
||||
#include "common/log.h"
|
||||
#include "config/config.h"
|
||||
#include "postgres/interface.h"
|
||||
#include "postgres/version.h"
|
||||
#include "storage/helper.h"
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Parse history file
|
||||
***********************************************************************************************************************************/
|
||||
typedef struct HistoryItem
|
||||
{
|
||||
unsigned int timeline; // Parent timeline
|
||||
uint64_t lsn; // Boundary lsn
|
||||
} HistoryItem;
|
||||
|
||||
static List *
|
||||
historyParse(const String *const history)
|
||||
{
|
||||
FUNCTION_LOG_BEGIN(logLevelDebug);
|
||||
FUNCTION_LOG_PARAM(STRING, history);
|
||||
FUNCTION_LOG_END();
|
||||
|
||||
List *const result = lstNewP(sizeof(HistoryItem));
|
||||
|
||||
MEM_CONTEXT_TEMP_BEGIN()
|
||||
{
|
||||
const StringList *const historyList = strLstNewSplitZ(history, "\n");
|
||||
|
||||
for (unsigned int historyIdx = 0; historyIdx < strLstSize(historyList); historyIdx++)
|
||||
{
|
||||
const String *const line = strTrim(strLstGet(historyList, historyIdx));
|
||||
|
||||
// Skip empty lines
|
||||
if (strSize(line) == 0)
|
||||
continue;
|
||||
|
||||
// Skip comments. PostgreSQL does not write comments into history files but it does allow for them.
|
||||
if (strBeginsWithZ(line, "#"))
|
||||
continue;
|
||||
|
||||
// Split line
|
||||
const StringList *const split = strLstNewSplitZ(line, "\t");
|
||||
|
||||
if (strLstSize(split) < 2)
|
||||
THROW_FMT(FormatError, "invalid history line format '%s'", strZ(strLstGet(historyList, historyIdx)));
|
||||
|
||||
const HistoryItem historyItem =
|
||||
{
|
||||
.timeline = cvtZToUInt(strZ(strLstGet(split, 0))),
|
||||
.lsn = pgLsnFromStr(strLstGet(split, 1)),
|
||||
};
|
||||
|
||||
ASSERT(historyItem.lsn > 0);
|
||||
ASSERT(historyItem.timeline > 0);
|
||||
|
||||
lstAdd(result, &historyItem);
|
||||
}
|
||||
}
|
||||
MEM_CONTEXT_TEMP_END();
|
||||
|
||||
FUNCTION_LOG_RETURN(LIST, result);
|
||||
}
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Load history file
|
||||
***********************************************************************************************************************************/
|
||||
static List *
|
||||
historyLoad(
|
||||
const Storage *const storageRepo, const String *const archiveId, const unsigned int timeline, const CipherType cipherType,
|
||||
const String *const cipherPass)
|
||||
{
|
||||
FUNCTION_LOG_BEGIN(logLevelDebug);
|
||||
FUNCTION_LOG_PARAM(STORAGE, storageRepo);
|
||||
FUNCTION_LOG_PARAM(STRING, archiveId);
|
||||
FUNCTION_LOG_PARAM(UINT, timeline);
|
||||
FUNCTION_LOG_PARAM(STRING_ID, cipherType);
|
||||
FUNCTION_LOG_PARAM(STRING, cipherPass);
|
||||
FUNCTION_LOG_END();
|
||||
|
||||
List *result;
|
||||
|
||||
MEM_CONTEXT_TEMP_BEGIN()
|
||||
{
|
||||
const String *const historyFile = strNewFmt(STORAGE_REPO_ARCHIVE "/%s/%08X.history", strZ(archiveId), timeline);
|
||||
StorageRead *const storageRead = storageNewReadP(storageRepo, historyFile);
|
||||
cipherBlockFilterGroupAdd(ioReadFilterGroup(storageReadIo(storageRead)), cipherType, cipherModeDecrypt, cipherPass);
|
||||
const Buffer *const history = storageGetP(storageRead);
|
||||
|
||||
TRY_BEGIN()
|
||||
{
|
||||
result = lstMove(historyParse(strNewBuf(history)), memContextPrior());
|
||||
}
|
||||
CATCH_ANY()
|
||||
{
|
||||
THROW_FMT(
|
||||
FormatError, "invalid timeline '%X' at '%s': %s", timeline, strZ(storagePathP(storageRepo, historyFile)),
|
||||
errorMessage());
|
||||
}
|
||||
TRY_END();
|
||||
}
|
||||
MEM_CONTEXT_TEMP_END();
|
||||
|
||||
FUNCTION_LOG_RETURN(LIST, result);
|
||||
}
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Find latest timeline
|
||||
***********************************************************************************************************************************/
|
||||
static unsigned int
|
||||
timelineLatest(const Storage *const storageRepo, const String *const archiveId, const unsigned int timelineCurrent)
|
||||
{
|
||||
FUNCTION_LOG_BEGIN(logLevelDebug);
|
||||
FUNCTION_LOG_PARAM(STORAGE, storageRepo);
|
||||
FUNCTION_LOG_PARAM(STRING, archiveId);
|
||||
FUNCTION_LOG_PARAM(UINT, timelineCurrent);
|
||||
FUNCTION_LOG_END();
|
||||
|
||||
ASSERT(storageRepo != NULL);
|
||||
|
||||
unsigned int result = timelineCurrent;
|
||||
|
||||
MEM_CONTEXT_TEMP_BEGIN()
|
||||
{
|
||||
const StringList *const historyList = strLstSort(
|
||||
storageListP(
|
||||
storageRepo, strNewFmt(STORAGE_REPO_ARCHIVE "/%s", strZ(archiveId)), .expression = STRDEF("[0-F]{8}\\.history")),
|
||||
sortOrderDesc);
|
||||
|
||||
if (!strLstEmpty(historyList))
|
||||
{
|
||||
const unsigned int timelineLatest = cvtZToUIntBase(strZ(strSubN(strLstGet(historyList, 0), 0, 8)), 16);
|
||||
|
||||
if (timelineLatest > timelineCurrent)
|
||||
result = timelineLatest;
|
||||
}
|
||||
}
|
||||
MEM_CONTEXT_TEMP_END();
|
||||
|
||||
FUNCTION_LOG_RETURN(UINT, result);
|
||||
}
|
||||
|
||||
/**********************************************************************************************************************************/
|
||||
STRING_STATIC(TIMELINE_CURRENT_STR, "current");
|
||||
STRING_STATIC(TIMELINE_LATEST_STR, "latest");
|
||||
|
||||
FN_EXTERN void
|
||||
timelineVerify(
|
||||
const Storage *const storageRepo, const String *const archiveId, const unsigned int pgVersion,
|
||||
const unsigned int timelineBackup, const uint64_t lsnBackup, const StringId recoveryType, const String *timelineTargetStr,
|
||||
const CipherType cipherType, const String *const cipherPass)
|
||||
{
|
||||
FUNCTION_LOG_BEGIN(logLevelDebug);
|
||||
FUNCTION_LOG_PARAM(STORAGE, storageRepo);
|
||||
FUNCTION_LOG_PARAM(STRING, archiveId);
|
||||
FUNCTION_LOG_PARAM(UINT, pgVersion);
|
||||
FUNCTION_LOG_PARAM(UINT, timelineBackup);
|
||||
FUNCTION_LOG_PARAM(UINT64, lsnBackup);
|
||||
FUNCTION_LOG_PARAM(STRING_ID, recoveryType);
|
||||
FUNCTION_LOG_PARAM(STRING, timelineTargetStr);
|
||||
FUNCTION_LOG_PARAM(STRING_ID, cipherType);
|
||||
FUNCTION_LOG_PARAM(STRING, cipherPass);
|
||||
FUNCTION_LOG_END();
|
||||
|
||||
ASSERT(storageRepo != NULL);
|
||||
ASSERT(archiveId != NULL);
|
||||
ASSERT(pgVersion != 0);
|
||||
ASSERT(timelineBackup > 0);
|
||||
ASSERT(lsnBackup > 0);
|
||||
|
||||
// If timeline target is not specified then set based on the default by PostgreSQL version. Force timeline to current when the
|
||||
// recovery type is immediate since no timeline switch will be needed but PostgreSQL will error if there is a problem with the
|
||||
// latest timeline.
|
||||
if (timelineTargetStr == NULL)
|
||||
{
|
||||
if (pgVersion <= PG_VERSION_11 || recoveryType == CFGOPTVAL_TYPE_IMMEDIATE)
|
||||
timelineTargetStr = TIMELINE_CURRENT_STR;
|
||||
else
|
||||
timelineTargetStr = TIMELINE_LATEST_STR;
|
||||
}
|
||||
|
||||
// If target timeline is not current then verify that the timeline is valid based on backup timeline and lsn. Following the
|
||||
// current/backup timeline is guaranteed as long as the WAL is available -- no history file required.
|
||||
if (!strEq(timelineTargetStr, TIMELINE_CURRENT_STR))
|
||||
{
|
||||
MEM_CONTEXT_TEMP_BEGIN()
|
||||
{
|
||||
// Determine timeline target in order to load the history file. If target timeline is latest scan the history files to
|
||||
// find the most recent. If a target timeline is a number then parse it.
|
||||
unsigned int timelineTarget;
|
||||
|
||||
if (strEq(timelineTargetStr, TIMELINE_LATEST_STR))
|
||||
timelineTarget = timelineLatest(storageRepo, archiveId, timelineBackup);
|
||||
else
|
||||
{
|
||||
TRY_BEGIN()
|
||||
{
|
||||
if (strBeginsWithZ(timelineTargetStr, "0x"))
|
||||
timelineTarget = cvtZToUIntBase(strZ(timelineTargetStr) + 2, 16);
|
||||
else
|
||||
timelineTarget = cvtZToUInt(strZ(timelineTargetStr));
|
||||
}
|
||||
CATCH_ANY()
|
||||
{
|
||||
THROW_FMT(DbMismatchError, "invalid target timeline '%s'", strZ(timelineTargetStr));
|
||||
}
|
||||
TRY_END();
|
||||
}
|
||||
|
||||
// Only proceed if target timeline is not the backup timeline
|
||||
if (timelineTarget != timelineBackup)
|
||||
{
|
||||
// Search through the history for the target timeline to make sure it includes the backup timeline. This follows
|
||||
// the logic in PostgreSQL's readTimeLineHistory() and tliOfPointInHistory() but is a bit simplified for our usage.
|
||||
const List *const historyList = historyLoad(storageRepo, archiveId, timelineTarget, cipherType, cipherPass);
|
||||
unsigned int timelineFound = 0;
|
||||
|
||||
for (unsigned int historyIdx = 0; historyIdx < lstSize(historyList); historyIdx++)
|
||||
{
|
||||
const HistoryItem *const historyItem = lstGet(historyList, historyIdx);
|
||||
|
||||
// If backup timeline exists in the target timeline's history before the fork LSN then restore can proceed
|
||||
if (historyItem->timeline == timelineBackup && lsnBackup < historyItem->lsn)
|
||||
{
|
||||
timelineFound = timelineBackup;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Error when the timeline was not found or not found in the expected range
|
||||
if (timelineFound == 0)
|
||||
{
|
||||
uint64_t lsnFound = 0;
|
||||
|
||||
// Check if timeline exists but did not fork off when expected
|
||||
for (unsigned int historyIdx = 0; historyIdx < lstSize(historyList); historyIdx++)
|
||||
{
|
||||
const HistoryItem *const historyItem = lstGet(historyList, historyIdx);
|
||||
|
||||
if (historyItem->timeline == timelineBackup)
|
||||
{
|
||||
timelineFound = timelineBackup;
|
||||
lsnFound = historyItem->lsn;
|
||||
}
|
||||
}
|
||||
|
||||
// The timeline was found but it forked off earlier than the backup LSN. This covers the common case where a
|
||||
// standby is promoted by accident, which creates a new timeline, and then the user tries to restore a backup
|
||||
// from the original primary that was made after the new timeline was created. Since PostgreSQL >= 12 will
|
||||
// always try to follow the latest timeline this restore will fail since there is no path to the target
|
||||
// timeline.
|
||||
if (timelineFound != 0)
|
||||
{
|
||||
ASSERT(lsnFound != 0);
|
||||
|
||||
THROW_FMT(
|
||||
DbMismatchError,
|
||||
"target timeline %X forked from backup timeline %X at %s which is before backup lsn of %s\n"
|
||||
"HINT: was the target timeline created by accidentally promoting a standby?\n"
|
||||
"HINT: was the target timeline created by testing a restore without --archive-mode=off?\n"
|
||||
"HINT: was the backup made after the target timeline was created?",
|
||||
timelineTarget, timelineBackup, strZ(pgLsnToStr(lsnFound)), strZ(pgLsnToStr(lsnBackup)));
|
||||
}
|
||||
// Else the backup timeline was not found at all. This less common case occurs when, for example, a restore from
|
||||
// timeline 4 results in timeline 7 being created on promotion because 5 and 6 have already been created by
|
||||
// prior promotions and then the user tries to restore a backup from timeline 5 with a target of timeline 7.
|
||||
// There is no relationship between timeline 5 and 7 in this case at any LSN range.
|
||||
else
|
||||
{
|
||||
THROW_FMT(
|
||||
DbMismatchError,
|
||||
"backup timeline %X, lsn %s is not in the history of target timeline %X\n"
|
||||
"HINT: was the target timeline created by promoting from a timeline < latest?",
|
||||
timelineBackup, strZ(pgLsnToStr(lsnBackup)), timelineTarget);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
MEM_CONTEXT_TEMP_END();
|
||||
}
|
||||
|
||||
FUNCTION_LOG_RETURN_VOID();
|
||||
}
|
18
src/command/restore/timeline.h
Normal file
18
src/command/restore/timeline.h
Normal file
@ -0,0 +1,18 @@
|
||||
/***********************************************************************************************************************************
|
||||
Timeline Management
|
||||
***********************************************************************************************************************************/
|
||||
#ifndef COMMAND_RESTORE_TIMELINE_H
|
||||
#define COMMAND_RESTORE_TIMELINE_H
|
||||
|
||||
#include "common/crypto/common.h"
|
||||
#include "storage/storage.h"
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Functions
|
||||
***********************************************************************************************************************************/
|
||||
// Verify that target timeline is valid for a backup
|
||||
FN_EXTERN void timelineVerify(
|
||||
const Storage *storageRepo, const String *archiveId, unsigned int pgVersion, unsigned int timelineBackup,
|
||||
uint64_t lsnBackup, StringId recoveryType, const String *timelineTargetStr, CipherType cipherType, const String *cipherPass);
|
||||
|
||||
#endif
|
@ -159,6 +159,7 @@ src_pgbackrest = [
|
||||
'command/restore/file.c',
|
||||
'command/restore/protocol.c',
|
||||
'command/restore/restore.c',
|
||||
'command/restore/timeline.c',
|
||||
'command/remote/remote.c',
|
||||
'command/server/ping.c',
|
||||
'command/server/server.c',
|
||||
|
@ -1155,6 +1155,10 @@ src/command/restore/restore.c:
|
||||
class: core
|
||||
type: c
|
||||
|
||||
src/command/restore/timeline.c:
|
||||
class: core
|
||||
type: c
|
||||
|
||||
src/command/restore/restore.h:
|
||||
class: core
|
||||
type: c/h
|
||||
|
@ -948,7 +948,7 @@ unit:
|
||||
|
||||
# ----------------------------------------------------------------------------------------------------------------------------
|
||||
- name: restore
|
||||
total: 14
|
||||
total: 15
|
||||
|
||||
coverage:
|
||||
- command/restore/blockChecksum
|
||||
@ -956,6 +956,7 @@ unit:
|
||||
- command/restore/file
|
||||
- command/restore/protocol
|
||||
- command/restore/restore
|
||||
- command/restore/timeline
|
||||
|
||||
include:
|
||||
- common/user
|
||||
|
@ -465,6 +465,30 @@ hrnHostPgStop(HrnHost *const this)
|
||||
FUNCTION_HARNESS_RETURN_VOID();
|
||||
}
|
||||
|
||||
/**********************************************************************************************************************************/
|
||||
void
|
||||
hrnHostPgPromote(HrnHost *const this)
|
||||
{
|
||||
FUNCTION_HARNESS_BEGIN();
|
||||
FUNCTION_HARNESS_PARAM(HRN_HOST, this);
|
||||
FUNCTION_HARNESS_END();
|
||||
|
||||
ASSERT(this != NULL);
|
||||
ASSERT(hrnHostIsPg(this));
|
||||
|
||||
MEM_CONTEXT_TEMP_BEGIN()
|
||||
{
|
||||
// Promote pg
|
||||
const String *const command = strNewFmt(
|
||||
"%s/pg_ctl promote -D '%s' -s -w", strZ(hrnHostPgBinPath(this)), strZ(hrnHostPgDataPath(this)));
|
||||
|
||||
hrnHostExecP(this, command);
|
||||
}
|
||||
MEM_CONTEXT_TEMP_END();
|
||||
|
||||
FUNCTION_HARNESS_RETURN_VOID();
|
||||
}
|
||||
|
||||
/**********************************************************************************************************************************/
|
||||
PackRead *
|
||||
hrnHostSql(HrnHost *const this, const String *const statement, const PgClientQueryResult resultType)
|
||||
|
@ -327,7 +327,7 @@ typedef struct HrnHostExecBrParam
|
||||
|
||||
String *hrnHostExecBr(HrnHost *this, const char *command, HrnHostExecBrParam param);
|
||||
|
||||
// Create/start/stop Pg cluster
|
||||
// Create/start/stop/promote pg cluster
|
||||
#define HRN_HOST_PG_CREATE(this) \
|
||||
do \
|
||||
{ \
|
||||
@ -358,6 +358,16 @@ void hrnHostPgStart(HrnHost *this);
|
||||
|
||||
void hrnHostPgStop(HrnHost *this);
|
||||
|
||||
#define HRN_HOST_PG_PROMOTE(this) \
|
||||
do \
|
||||
{ \
|
||||
TEST_RESULT_INFO_FMT("%s: promote pg cluster", strZ(hrnHostName(this))); \
|
||||
hrnHostPgPromote(this); \
|
||||
} \
|
||||
while (0)
|
||||
|
||||
void hrnHostPgPromote(HrnHost *this);
|
||||
|
||||
// Query
|
||||
PackRead *hrnHostSql(HrnHost *this, const String *statement, const PgClientQueryResult resultType);
|
||||
|
||||
|
@ -1871,6 +1871,8 @@ testRun(void)
|
||||
|
||||
Manifest *manifest = testManifestMinimal(STRDEF("20161219-212741F"), PG_VERSION_12, STRDEF("/pg"));
|
||||
manifest->pub.data.backupOptionOnline = true;
|
||||
manifest->pub.data.archiveStart = strNewZ("000000010000000000000007");
|
||||
manifest->pub.data.lsnStart = strNewZ("0/7000028");
|
||||
|
||||
// -------------------------------------------------------------------------------------------------------------------------
|
||||
TEST_TITLE("error when standby_mode setting is present");
|
||||
@ -2049,6 +2051,107 @@ testRun(void)
|
||||
TEST_RESULT_LOG("P00 INFO: write updated " TEST_PATH "/pg/postgresql.auto.conf");
|
||||
}
|
||||
|
||||
// *****************************************************************************************************************************
|
||||
if (testBegin("timeline*()"))
|
||||
{
|
||||
StringList *argList = strLstNew();
|
||||
hrnCfgArgRawZ(argList, cfgOptStanza, "test1");
|
||||
hrnCfgArgRaw(argList, cfgOptRepoPath, STRDEF(TEST_PATH "/repo"));
|
||||
hrnCfgArgRawZ(argList, cfgOptPgPath, TEST_PATH "/pg");
|
||||
HRN_CFG_LOAD(cfgCmdRestore, argList);
|
||||
|
||||
// -------------------------------------------------------------------------------------------------------------------------
|
||||
TEST_TITLE("timelineVerify()");
|
||||
|
||||
TEST_RESULT_VOID(
|
||||
timelineVerify(
|
||||
storageRepoIdx(0), STRDEF("17-1"), PG_VERSION_11, 1, 0xA1, CFGOPTVAL_TYPE_DEFAULT, NULL, cipherTypeNone, NULL),
|
||||
"follow current timeline because of version");
|
||||
TEST_RESULT_VOID(
|
||||
timelineVerify(
|
||||
storageRepoIdx(0), STRDEF("17-1"), PG_VERSION_11, 1, 0xA1, CFGOPTVAL_TYPE_DEFAULT, STRDEF("latest"), cipherTypeNone,
|
||||
NULL),
|
||||
"follow latest timeline as requested");
|
||||
TEST_RESULT_VOID(
|
||||
timelineVerify(
|
||||
storageRepoIdx(0), STRDEF("17-1"), PG_VERSION_12, 1, 0xA1, CFGOPTVAL_TYPE_DEFAULT, NULL, cipherTypeNone, NULL),
|
||||
"follow latest timeline because of version");
|
||||
TEST_RESULT_VOID(
|
||||
timelineVerify(
|
||||
storageRepoIdx(0), STRDEF("17-1"), PG_VERSION_12, 1, 0xA1, CFGOPTVAL_TYPE_DEFAULT, STRDEF("current"),
|
||||
cipherTypeNone, NULL),
|
||||
"follow current timeline as requested");
|
||||
TEST_RESULT_VOID(
|
||||
timelineVerify(
|
||||
storageRepoIdx(0), STRDEF("17-1"), PG_VERSION_12, 1, 0xA1, CFGOPTVAL_TYPE_DEFAULT, STRDEF("1"), cipherTypeNone,
|
||||
NULL),
|
||||
"follow requested timeline (same as current)");
|
||||
TEST_RESULT_VOID(
|
||||
timelineVerify(
|
||||
storageRepoIdx(0), STRDEF("17-1"), PG_VERSION_12, 0x10, 0xA1, CFGOPTVAL_TYPE_DEFAULT, STRDEF("0x10"),
|
||||
cipherTypeNone, NULL),
|
||||
"follow requested hex timeline (same as current)");
|
||||
TEST_ERROR(
|
||||
timelineVerify(
|
||||
storageRepoIdx(0), STRDEF("17-1"), PG_VERSION_12, 0x10, 0xA1, CFGOPTVAL_TYPE_DEFAULT, STRDEF("bogus"),
|
||||
cipherTypeNone, NULL),
|
||||
DbMismatchError, "invalid target timeline 'bogus'");
|
||||
|
||||
HRN_STORAGE_PUT_Z(storageTest, "repo/archive/test1/17-1/00000009.history", "8");
|
||||
TEST_ERROR(
|
||||
timelineVerify(
|
||||
storageRepoIdx(0), STRDEF("17-1"), PG_VERSION_12, 8, 0xA1, CFGOPTVAL_TYPE_DEFAULT, STRDEF("9"), cipherTypeNone,
|
||||
NULL),
|
||||
FormatError,
|
||||
"invalid timeline '9' at '" TEST_PATH "/repo/archive/test1/17-1/00000009.history':"
|
||||
" invalid history line format '8'");
|
||||
|
||||
HRN_STORAGE_PUT_Z(
|
||||
storageTest, "repo/archive/test1/17-1/0000000A.history",
|
||||
"# comment\n"
|
||||
"8\t0/4000000\tcomment\n"
|
||||
"9\t0/5000000\tcomment\n");
|
||||
TEST_RESULT_VOID(
|
||||
timelineVerify(
|
||||
storageRepoIdx(0), STRDEF("17-1"), PG_VERSION_12, 10, 0x4FFFFFF, CFGOPTVAL_TYPE_DEFAULT, NULL, cipherTypeNone,
|
||||
NULL),
|
||||
"follow current timeline");
|
||||
TEST_RESULT_VOID(
|
||||
timelineVerify(
|
||||
storageRepoIdx(0), STRDEF("17-1"), PG_VERSION_12, 9, 0x4FFFFFF, CFGOPTVAL_TYPE_IMMEDIATE, NULL, cipherTypeNone,
|
||||
NULL),
|
||||
"follow current timeline (based on type immediate)");
|
||||
TEST_RESULT_VOID(
|
||||
timelineVerify(
|
||||
storageRepoIdx(0), STRDEF("17-1"), PG_VERSION_12, 9, 0x4FFFFFF, CFGOPTVAL_TYPE_DEFAULT, NULL, cipherTypeNone, NULL),
|
||||
"follow latest timeline");
|
||||
TEST_RESULT_VOID(
|
||||
timelineVerify(
|
||||
storageRepoIdx(0), STRDEF("17-1"), PG_VERSION_12, 9, 0x4FFFFFF, CFGOPTVAL_TYPE_DEFAULT, STRDEF("10"),
|
||||
cipherTypeNone, NULL),
|
||||
"target timeline found");
|
||||
TEST_ERROR(
|
||||
timelineVerify(
|
||||
storageRepoIdx(0), STRDEF("17-1"), PG_VERSION_12, 9, 0x6000000, CFGOPTVAL_TYPE_DEFAULT, STRDEF("10"),
|
||||
cipherTypeNone, NULL),
|
||||
DbMismatchError,
|
||||
"target timeline A forked from backup timeline 9 at 0/5000000 which is before backup lsn of 0/6000000\n"
|
||||
"HINT: was the target timeline created by accidentally promoting a standby?\n"
|
||||
"HINT: was the target timeline created by testing a restore without --archive-mode=off?\n"
|
||||
"HINT: was the backup made after the target timeline was created?");
|
||||
|
||||
HRN_STORAGE_PUT_Z(
|
||||
storageTest, "repo/archive/test1/17-1/0000000B.history",
|
||||
"7\t0/4000000\tcomment\n"
|
||||
"8\t0/5000000\tcomment\n");
|
||||
TEST_ERROR(
|
||||
timelineVerify(
|
||||
storageRepoIdx(0), STRDEF("17-1"), PG_VERSION_12, 6, 0x4FFFFFF, CFGOPTVAL_TYPE_DEFAULT, STRDEF("11"),
|
||||
cipherTypeNone, NULL),
|
||||
DbMismatchError, "backup timeline 6, lsn 0/4ffffff is not in the history of target timeline B\n"
|
||||
"HINT: was the target timeline created by promoting from a timeline < latest?");
|
||||
}
|
||||
|
||||
// *****************************************************************************************************************************
|
||||
if (testBegin("cmdRestore()"))
|
||||
{
|
||||
@ -2111,6 +2214,8 @@ testRun(void)
|
||||
manifest->pub.data.backupTimestampStart = 1482182860;
|
||||
manifest->pub.data.backupTimestampCopyStart = 1482182861; // So file timestamps should be less than this
|
||||
manifest->pub.data.backupOptionOnline = true;
|
||||
manifest->pub.data.archiveStart = strNewZ("000000010000000000000007");
|
||||
manifest->pub.data.lsnStart = strNewZ("0/7000028");
|
||||
|
||||
// Data directory
|
||||
HRN_MANIFEST_TARGET_ADD(manifest, .name = MANIFEST_TARGET_PGDATA, .path = strZ(pgPath));
|
||||
@ -2169,6 +2274,11 @@ testRun(void)
|
||||
storageRepoIdxWrite(1), INFO_BACKUP_PATH_FILE, TEST_RESTORE_BACKUP_INFO "\n[cipher]\ncipher-pass=\""
|
||||
TEST_CIPHER_PASS_MANIFEST "\"\n\n" TEST_RESTORE_BACKUP_INFO_DB, .cipherType = cipherTypeAes256Cbc);
|
||||
|
||||
// Write archive.info to the encrypted repo
|
||||
InfoArchive *infoArchive = infoArchiveNew(PG_VERSION_95, 6569239123849665679, STRDEF(TEST_CIPHER_PASS_ARCHIVE));
|
||||
infoArchiveSaveFile(
|
||||
infoArchive, storageRepoIdxWrite(1), INFO_ARCHIVE_PATH_FILE_STR, cipherTypeAes256Cbc, STRDEF(TEST_CIPHER_PASS));
|
||||
|
||||
TEST_RESULT_VOID(cmdRestore(), "successful restore");
|
||||
|
||||
TEST_RESULT_LOG(
|
||||
@ -2204,6 +2314,23 @@ testRun(void)
|
||||
"pg_tblspc/\n",
|
||||
.level = storageInfoLevelBasic, .includeDot = true);
|
||||
|
||||
// -------------------------------------------------------------------------------------------------------------------------
|
||||
TEST_TITLE("error on invalid timeline");
|
||||
|
||||
// Store backup.info to repo1 - repo1 will be selected because of the priority order
|
||||
HRN_INFO_PUT(storageRepoIdxWrite(0), INFO_BACKUP_PATH_FILE, TEST_RESTORE_BACKUP_INFO "\n" TEST_RESTORE_BACKUP_INFO_DB);
|
||||
|
||||
// Store archive.info to repo1 - repo1 will be selected because of the priority order
|
||||
infoArchive = infoArchiveNew(PG_VERSION_95, 6569239123849665679, NULL);
|
||||
infoArchiveSaveFile(infoArchive, storageRepoIdxWrite(0), INFO_ARCHIVE_PATH_FILE_STR, cipherTypeNone, NULL);
|
||||
|
||||
hrnCfgArgRawZ(argList, cfgOptTargetTimeline, "0xff");
|
||||
HRN_CFG_LOAD(cfgCmdRestore, argList);
|
||||
|
||||
TEST_ERROR(
|
||||
cmdRestore(), FileMissingError,
|
||||
"unable to open missing file '" TEST_PATH "/repo/archive/test1/9.5-0/000000FF.history' for read");
|
||||
|
||||
// -------------------------------------------------------------------------------------------------------------------------
|
||||
TEST_TITLE("full restore with delta force");
|
||||
|
||||
@ -2223,9 +2350,6 @@ testRun(void)
|
||||
// Munge PGDATA mode so it gets fixed
|
||||
HRN_STORAGE_MODE(storagePg(), NULL, 0777);
|
||||
|
||||
// Store backup.info to repo1 - repo1 will be selected because of the priority order
|
||||
HRN_INFO_PUT(storageRepoIdxWrite(0), INFO_BACKUP_PATH_FILE, TEST_RESTORE_BACKUP_INFO "\n" TEST_RESTORE_BACKUP_INFO_DB);
|
||||
|
||||
// Make sure existing backup.manifest file is ignored
|
||||
HRN_STORAGE_PUT_EMPTY(storagePgWrite(), BACKUP_MANIFEST_FILE);
|
||||
|
||||
@ -2483,6 +2607,8 @@ testRun(void)
|
||||
manifest->pub.data.backupType = backupTypeIncr;
|
||||
manifest->pub.data.blockIncr = true;
|
||||
manifest->pub.data.backupTimestampCopyStart = 1482182861; // So file timestamps should be less than this
|
||||
manifest->pub.data.archiveStart = strNewZ("000000010000000000000007");
|
||||
manifest->pub.data.lsnStart = strNewZ("0/7000028");
|
||||
|
||||
manifest->pub.referenceList = strLstNew();
|
||||
strLstAddZ(manifest->pub.referenceList, TEST_LABEL_FULL);
|
||||
|
@ -8,6 +8,7 @@ Real Integration Test
|
||||
#include "info/infoBackup.h"
|
||||
#include "postgres/interface.h"
|
||||
#include "postgres/version.h"
|
||||
#include "storage/helper.h"
|
||||
|
||||
#include "common/harnessErrorRetry.h"
|
||||
#include "common/harnessHost.h"
|
||||
@ -179,6 +180,17 @@ testRun(void)
|
||||
|
||||
HRN_HOST_PG_START(pg2);
|
||||
|
||||
// Promote the standby to create a new timeline that can be used to test timeline verification. Once the new timeline
|
||||
// has been created restore again to get the standby back on the same timeline as the primary.
|
||||
HRN_HOST_PG_PROMOTE(pg2);
|
||||
TEST_STORAGE_EXISTS(
|
||||
hrnHostRepo1Storage(repo),
|
||||
zNewFmt("archive/" HRN_STANZA "/%s-1/00000002.history", strZ(pgVersionToStr(hrnHostPgVersion()))), .timeout = 5000);
|
||||
|
||||
HRN_HOST_PG_STOP(pg2);
|
||||
TEST_HOST_BR(pg2, CFGCMD_RESTORE, .option = zNewFmt("%s --delta --target-timeline=current", option));
|
||||
HRN_HOST_PG_START(pg2);
|
||||
|
||||
// Check standby
|
||||
TEST_HOST_BR(pg2, CFGCMD_CHECK);
|
||||
|
||||
@ -308,13 +320,24 @@ testRun(void)
|
||||
TEST_LOG("name target = " TEST_RESTORE_POINT);
|
||||
|
||||
// -------------------------------------------------------------------------------------------------------------------------
|
||||
TEST_TITLE("primary restore (default target)");
|
||||
TEST_TITLE("primary restore fails on timeline verification");
|
||||
{
|
||||
// Stop the cluster
|
||||
HRN_HOST_PG_STOP(pg1);
|
||||
|
||||
// Restore
|
||||
TEST_HOST_BR(pg1, CFGCMD_RESTORE, .option = zNewFmt("--force --repo=%u", hrnHostRepoTotal()));
|
||||
// Restore fails because timeline 2 was created before the backup selected for restore. Specify target timeline latest
|
||||
// because PostgreSQL < 12 defaults to current.
|
||||
TEST_HOST_BR(
|
||||
pg1, CFGCMD_RESTORE, .option = "--delta --target-timeline=latest", .resultExpect = errorTypeCode(&DbMismatchError));
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------------------------------------------------------
|
||||
TEST_TITLE("primary restore (default target)");
|
||||
{
|
||||
// Restore on current timeline to skip the invalid timeline unrelated to the backup
|
||||
TEST_HOST_BR(
|
||||
pg1, CFGCMD_RESTORE,
|
||||
.option = zNewFmt("--force --target-timeline=current --repo=%u", hrnHostRepoTotal()));
|
||||
HRN_HOST_PG_START(pg1);
|
||||
|
||||
// Check that backup recovered to the expected target
|
||||
@ -330,7 +353,8 @@ testRun(void)
|
||||
// Stop the cluster and try again
|
||||
HRN_HOST_PG_STOP(pg1);
|
||||
|
||||
// Restore
|
||||
// Restore immediate and promote -- this avoids checking the invalid timeline since immediate recovery is always along
|
||||
// the current timeline. The promotion will create a new timeline so subsequent tests will pass timeline verification.
|
||||
TEST_HOST_BR(pg1, CFGCMD_RESTORE, .option = "--delta --type=immediate --target-action=promote --db-exclude=exclude_me");
|
||||
HRN_HOST_PG_START(pg1);
|
||||
|
||||
|
Reference in New Issue
Block a user