db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2024-12-12 10:04:14 +02:00
Code Issues Releases Activity

Add pg-version-force option for fork integration.

Browse Source 
Forks may update pg_control version or WAL magic without affecting the structures that pgBackRest depends on.

This option forces pgBackRest to treat a cluster as the specified version when it cannot be automatically identified.
This commit is contained in:
Stefan Fercot 2023-03-09 02:23:15 +01:00 committed by GitHub
parent 2fa7e53c5d
commit 740c2258e3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
21 changed files with 438 additions and 72 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
doc/xml/release.xml
View File

@ -66,6 +66,22 @@
<p>Keep only one all-default group index.</p>
</release-item>
</release-improvement-list>
<release-development-list>
<release-item>
<commit subject="Allow control version and WAL magic to be overridden in test harness."/>
<commit subject="Add pg-version-force option for fork integration.">
<github-pull-request id="2012"/>
</commit>
<release-item-contributor-list>
<release-item-contributor id="stefan.fercot"/>
<release-item-reviewer id="david.steele"/>
</release-item-contributor-list>
<p>Add <br-option>pg-version-force</br-option> option for fork integration.</p>
</release-item>
</release-development-list>
</release-core-list>
<release-doc-list>

16
src/build/config/config.yaml
View File

@ -1541,6 +1541,22 @@ option:
depend:
option: pg-path
pg-version-force:
section: stanza
type: string
internal: true
required: false
command:
archive-get: {}
archive-push: {}
backup: {}
check: {}
restore: {}
stanza-create: {}
stanza-delete: {}
stanza-upgrade: {}
verify: {}
# Repository options
#---------------------------------------------------------------------------------------------------------------------------------
repo:

12
src/build/help/help.xml
View File

@ -1714,6 +1714,18 @@
<example>25</example>
</config-key>
<config-key id="pg-version-force" name="Force PostgreSQL version">
<summary>Force <postgres/> version.</summary>
<text>
<p>The specified <postgres/> version will be used instead of the version automatically detected by reading <file>pg_control</file> or WAL headers. This is mainly useful for <postgres/> forks or development versions where those values are different from the release version. The version reported by <postgres/> via `server_version_num` must match the forced version.</p>
<admonition type="warning">Be cautious when using this option because <file>pg_control</file> and WAL headers will still be read with expected format for the specified version, i.e. the format from the official open-source version of <postgres/>. If the fork or development version changes the format of the fields that <backrest/> depends on it will lead to unexpected behavior.</admonition>
</text>
<example>15</example>
</config-key>
</config-key-list>
</config-section>
</config-section-list>

4
src/command/archive/get/get.c
View File

@ -376,7 +376,7 @@ archiveGetCheck(const StringList *archiveRequestList)
StringList *warnList = strLstNew();
// Get pg control info
PgControl controlInfo = pgControlFromFile(storagePg());
PgControl controlInfo = pgControlFromFile(storagePg(), cfgOptionStrNull(cfgOptPgVersionForce));
// Build list of repos/archiveIds where WAL may be found
List *cacheRepoList = lstNewP(sizeof(ArchiveGetFindCacheRepo));
@ -724,7 +724,7 @@ cmdArchiveGet(void)
false))
{
// Get control info
PgControl pgControl = pgControlFromFile(storagePg());
PgControl pgControl = pgControlFromFile(storagePg(), cfgOptionStrNull(cfgOptPgVersionForce));
// Create the queue
storagePathCreateP(storageSpoolWrite(), STORAGE_SPOOL_ARCHIVE_IN_STR);

2
src/command/archive/push/file.c
View File

@ -129,7 +129,7 @@ archivePushFile(
// If this is a segment compare archive version and systemId to the WAL header
if (headerCheck && isSegment)
{
PgWal walInfo = pgWalFromFile(walSource, storageLocal());
PgWal walInfo = pgWalFromFile(walSource, storageLocal(), cfgOptionStrNull(cfgOptPgVersionForce));
if (walInfo.version != pgVersion || walInfo.systemId != pgSystemId)
{

2
src/command/archive/push/push.c
View File

@ -229,7 +229,7 @@ archivePushCheck(bool pgPathSet)
if (pgPathSet)
{
// Get info from pg_control
PgControl pgControl = pgControlFromFile(storagePg());
PgControl pgControl = pgControlFromFile(storagePg(), cfgOptionStrNull(cfgOptPgVersionForce));
result.pgVersion = pgControl.version;
result.pgSystemId = pgControl.systemId;
}

2
src/command/backup/backup.c
View File

@ -215,7 +215,7 @@ backupInit(const InfoBackup *infoBackup)
}
// Else get pg_control info directly from the file
else
pgControl = pgControlFromFile(storagePgIdx(result->pgIdxPrimary));
pgControl = pgControlFromFile(storagePgIdx(result->pgIdxPrimary), cfgOptionStrNull(cfgOptPgVersionForce));
// Add primary info
result->storagePrimary = storagePgIdx(result->pgIdxPrimary);

2
src/command/stanza/common.c
View File

@ -57,7 +57,7 @@ pgValidate(void)
}
// If the database is not online, assume that pg1 is the primary
else
result = pgControlFromFile(storagePg());
result = pgControlFromFile(storagePg(), cfgOptionStrNull(cfgOptPgVersionForce));
}
MEM_CONTEXT_TEMP_END();

3
src/command/verify/verify.c
View File

@ -723,7 +723,8 @@ verifyArchive(VerifyJobData *const jobData)
strZ(strLstGet(jobData->walFileList, 0))),
jobData->walCipherPass);
PgWal walInfo = pgWalFromBuffer(storageGetP(walRead, .exactSize = PG_WAL_HEADER_SIZE));
PgWal walInfo = pgWalFromBuffer(
storageGetP(walRead, .exactSize = PG_WAL_HEADER_SIZE), cfgOptionStrNull(cfgOptPgVersionForce));
archiveResult->pgWalInfo.size = walInfo.size;
archiveResult->pgWalInfo.version = walInfo.version;

4
src/config/config.auto.h
View File

@ -95,6 +95,7 @@ Option constants
#define CFGOPT_ONLINE "online"
#define CFGOPT_OUTPUT "output"
#define CFGOPT_PG "pg"
#define CFGOPT_PG_VERSION_FORCE "pg-version-force"
#define CFGOPT_PROCESS "process"
#define CFGOPT_PROCESS_MAX "process-max"
#define CFGOPT_PROTOCOL_TIMEOUT "protocol-timeout"
@ -130,7 +131,7 @@ Option constants
#define CFGOPT_TYPE "type"
#define CFGOPT_VERBOSE "verbose"
#define CFG_OPTION_TOTAL 160
#define CFG_OPTION_TOTAL 161
/***********************************************************************************************************************************
Option value constants
@ -429,6 +430,7 @@ typedef enum
cfgOptPgPort,
cfgOptPgSocketPath,
cfgOptPgUser,
cfgOptPgVersionForce,
cfgOptProcess,
cfgOptProcessMax,
cfgOptProtocolTimeout,

51
src/config/parse.auto.c.inc
View File

@ -3978,6 +3978,56 @@ static const ParseRuleOption parseRuleOption[CFG_OPTION_TOTAL] =
), // opt/pg-user
), // opt/pg-user
// -----------------------------------------------------------------------------------------------------------------------------
PARSE_RULE_OPTION // opt/pg-version-force
( // opt/pg-version-force
PARSE_RULE_OPTION_NAME("pg-version-force"), // opt/pg-version-force
PARSE_RULE_OPTION_TYPE(cfgOptTypeString), // opt/pg-version-force
PARSE_RULE_OPTION_RESET(true), // opt/pg-version-force
PARSE_RULE_OPTION_REQUIRED(false), // opt/pg-version-force
PARSE_RULE_OPTION_SECTION(cfgSectionStanza), // opt/pg-version-force
// opt/pg-version-force
PARSE_RULE_OPTION_COMMAND_ROLE_MAIN_VALID_LIST // opt/pg-version-force
( // opt/pg-version-force
PARSE_RULE_OPTION_COMMAND(cfgCmdArchiveGet) // opt/pg-version-force
PARSE_RULE_OPTION_COMMAND(cfgCmdArchivePush) // opt/pg-version-force
PARSE_RULE_OPTION_COMMAND(cfgCmdBackup) // opt/pg-version-force
PARSE_RULE_OPTION_COMMAND(cfgCmdCheck) // opt/pg-version-force
PARSE_RULE_OPTION_COMMAND(cfgCmdRestore) // opt/pg-version-force
PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaCreate) // opt/pg-version-force
PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaDelete) // opt/pg-version-force
PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaUpgrade) // opt/pg-version-force
PARSE_RULE_OPTION_COMMAND(cfgCmdVerify) // opt/pg-version-force
), // opt/pg-version-force
// opt/pg-version-force
PARSE_RULE_OPTION_COMMAND_ROLE_ASYNC_VALID_LIST // opt/pg-version-force
( // opt/pg-version-force
PARSE_RULE_OPTION_COMMAND(cfgCmdArchiveGet) // opt/pg-version-force
PARSE_RULE_OPTION_COMMAND(cfgCmdArchivePush) // opt/pg-version-force
), // opt/pg-version-force
// opt/pg-version-force
PARSE_RULE_OPTION_COMMAND_ROLE_LOCAL_VALID_LIST // opt/pg-version-force
( // opt/pg-version-force
PARSE_RULE_OPTION_COMMAND(cfgCmdArchiveGet) // opt/pg-version-force
PARSE_RULE_OPTION_COMMAND(cfgCmdArchivePush) // opt/pg-version-force
PARSE_RULE_OPTION_COMMAND(cfgCmdBackup) // opt/pg-version-force
PARSE_RULE_OPTION_COMMAND(cfgCmdRestore) // opt/pg-version-force
PARSE_RULE_OPTION_COMMAND(cfgCmdVerify) // opt/pg-version-force
), // opt/pg-version-force
// opt/pg-version-force
PARSE_RULE_OPTION_COMMAND_ROLE_REMOTE_VALID_LIST // opt/pg-version-force
( // opt/pg-version-force
PARSE_RULE_OPTION_COMMAND(cfgCmdArchiveGet) // opt/pg-version-force
PARSE_RULE_OPTION_COMMAND(cfgCmdArchivePush) // opt/pg-version-force
PARSE_RULE_OPTION_COMMAND(cfgCmdBackup) // opt/pg-version-force
PARSE_RULE_OPTION_COMMAND(cfgCmdCheck) // opt/pg-version-force
PARSE_RULE_OPTION_COMMAND(cfgCmdRestore) // opt/pg-version-force
PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaCreate) // opt/pg-version-force
PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaDelete) // opt/pg-version-force
PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaUpgrade) // opt/pg-version-force
PARSE_RULE_OPTION_COMMAND(cfgCmdVerify) // opt/pg-version-force
), // opt/pg-version-force
), // opt/pg-version-force
// -----------------------------------------------------------------------------------------------------------------------------
PARSE_RULE_OPTION // opt/process
( // opt/process
PARSE_RULE_OPTION_NAME("process"), // opt/process
@ -9458,6 +9508,7 @@ static const uint8_t optionResolveOrder[] =
cfgOptPgPort, // opt-resolve-order
cfgOptPgSocketPath, // opt-resolve-order
cfgOptPgUser, // opt-resolve-order
cfgOptPgVersionForce, // opt-resolve-order
cfgOptProcess, // opt-resolve-order
cfgOptProcessMax, // opt-resolve-order
cfgOptProtocolTimeout, // opt-resolve-order

6
src/db/db.c
View File

@ -323,7 +323,7 @@ dbOpen(Db *this)
this->pub.standby = dbIsInRecovery(this);
// Get control file
this->pub.pgControl = pgControlFromFile(this->storage);
this->pub.pgControl = pgControlFromFile(this->storage, cfgOptionStrNull(cfgOptPgVersionForce));
}
MEM_CONTEXT_TEMP_END();
@ -441,7 +441,7 @@ dbBackupStart(Db *const this, const bool startFast, const bool stopAuto, const b
// Make sure the backup start checkpoint was written to pg_control. This helps ensure that we have a consistent view of the
// storage with PostgreSQL.
const PgControl pgControl = pgControlFromFile(this->storage);
const PgControl pgControl = pgControlFromFile(this->storage, cfgOptionStrNull(cfgOptPgVersionForce));
const String *const lsnStart = pckReadStrP(read);
if (pgControl.checkpoint < pgLsnFromStr(lsnStart))
@ -733,7 +733,7 @@ dbReplayWait(Db *const this, const String *const targetLsn, const uint32_t targe
}
// Reload pg_control in case timeline was updated by the checkpoint
this->pub.pgControl = pgControlFromFile(this->storage);
this->pub.pgControl = pgControlFromFile(this->storage, cfgOptionStrNull(cfgOptPgVersionForce));
// Check that the timeline matches the primary
if (dbPgControl(this).timeline != targetTimeline)

98
src/postgres/interface.c
View File

@ -111,7 +111,13 @@ pgInterfaceVersion(unsigned int pgVersion)
// If the version was not found then error
if (result == NULL)
THROW_FMT(AssertError, "invalid " PG_NAME " version %u", pgVersion);
{
THROW_FMT(
VersionNotSupportedError,
"invalid " PG_NAME " version %u\n"
"HINT: is this version of PostgreSQL supported?",
pgVersion);
}
FUNCTION_TEST_RETURN_TYPE_CONST_P(PgInterface, result);
}
@ -172,10 +178,11 @@ pgWalSegmentSizeCheck(unsigned int pgVersion, unsigned int walSegmentSize)
/**********************************************************************************************************************************/
static PgControl
pgControlFromBuffer(const Buffer *controlFile)
pgControlFromBuffer(const Buffer *controlFile, const String *const pgVersionForce)
{
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(BUFFER, controlFile);
FUNCTION_LOG_PARAM(STRING, pgVersionForce);
FUNCTION_LOG_END();
ASSERT(controlFile != NULL);
@ -183,28 +190,35 @@ pgControlFromBuffer(const Buffer *controlFile)
// Search for the version of PostgreSQL that uses this control file
const PgInterface *interface = NULL;
for (unsigned int interfaceIdx = 0; interfaceIdx < LENGTH_OF(pgInterface); interfaceIdx++)
if (pgVersionForce != NULL)
interface = pgInterfaceVersion(pgVersionFromStr(pgVersionForce));
else
{
if (pgInterface[interfaceIdx].controlIs(bufPtrConst(controlFile)))
for (unsigned int interfaceIdx = 0; interfaceIdx < LENGTH_OF(pgInterface); interfaceIdx++)
{
interface = &pgInterface[interfaceIdx];
break;
if (pgInterface[interfaceIdx].controlIs(bufPtrConst(controlFile)))
{
interface = &pgInterface[interfaceIdx];
break;
}
}
// If the version was not found then error with the control and catalog version that were found
if (interface == NULL)
{
const PgControlCommon *controlCommon = (const PgControlCommon *)bufPtrConst(controlFile);
THROW_FMT(
VersionNotSupportedError,
"unexpected control version = %u and catalog version = %u\n"
"HINT: is this version of PostgreSQL supported?",
controlCommon->controlVersion, controlCommon->catalogVersion);
}
}
// If the version was not found then error with the control and catalog version that were found
if (interface == NULL)
{
const PgControlCommon *controlCommon = (const PgControlCommon *)bufPtrConst(controlFile);
THROW_FMT(
VersionNotSupportedError,
"unexpected control version = %u and catalog version = %u\n"
"HINT: is this version of PostgreSQL supported?",
controlCommon->controlVersion, controlCommon->catalogVersion);
}
// Get info from the control file
ASSERT(pgVersionForce != NULL || interface->controlIs(bufPtrConst(controlFile)));
PgControl result = interface->control(bufPtrConst(controlFile));
result.version = interface->version;
@ -219,10 +233,11 @@ pgControlFromBuffer(const Buffer *controlFile)
}
FN_EXTERN PgControl
pgControlFromFile(const Storage *storage)
pgControlFromFile(const Storage *storage, const String *const pgVersionForce)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(STORAGE, storage);
FUNCTION_LOG_PARAM(STRING, pgVersionForce);
FUNCTION_LOG_END();
ASSERT(storage != NULL);
@ -235,7 +250,7 @@ pgControlFromFile(const Storage *storage)
Buffer *controlFile = storageGetP(
storageNewReadP(storage, STRDEF(PG_PATH_GLOBAL "/" PG_FILE_PGCONTROL)), .exactSize = PG_CONTROL_DATA_SIZE);
result = pgControlFromBuffer(controlFile);
result = pgControlFromBuffer(controlFile, pgVersionForce);
}
MEM_CONTEXT_TEMP_END();
@ -267,10 +282,11 @@ typedef struct PgWalCommon
/**********************************************************************************************************************************/
FN_EXTERN PgWal
pgWalFromBuffer(const Buffer *walBuffer)
pgWalFromBuffer(const Buffer *walBuffer, const String *const pgVersionForce)
{
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(BUFFER, walBuffer);
FUNCTION_LOG_PARAM(STRING, pgVersionForce);
FUNCTION_LOG_END();
ASSERT(walBuffer != NULL);
@ -282,26 +298,33 @@ pgWalFromBuffer(const Buffer *walBuffer)
// Search for the version of PostgreSQL that uses this WAL magic
const PgInterface *interface = NULL;
for (unsigned int interfaceIdx = 0; interfaceIdx < LENGTH_OF(pgInterface); interfaceIdx++)
if (pgVersionForce != NULL)
interface = pgInterfaceVersion(pgVersionFromStr(pgVersionForce));
else
{
if (pgInterface[interfaceIdx].walIs(bufPtrConst(walBuffer)))
for (unsigned int interfaceIdx = 0; interfaceIdx < LENGTH_OF(pgInterface); interfaceIdx++)
{
interface = &pgInterface[interfaceIdx];
break;
if (pgInterface[interfaceIdx].walIs(bufPtrConst(walBuffer)))
{
interface = &pgInterface[interfaceIdx];
break;
}
}
// If the version was not found then error with the magic that was found
if (interface == NULL)
{
THROW_FMT(
VersionNotSupportedError,
"unexpected WAL magic %u\n"
"HINT: is this version of PostgreSQL supported?",
((const PgWalCommon *)bufPtrConst(walBuffer))->magic);
}
}
// If the version was not found then error with the magic that was found
if (interface == NULL)
{
THROW_FMT(
VersionNotSupportedError,
"unexpected WAL magic %u\n"
"HINT: is this version of PostgreSQL supported?",
((const PgWalCommon *)bufPtrConst(walBuffer))->magic);
}
// Get info from the control file
ASSERT(pgVersionForce != NULL || interface->walIs(bufPtrConst(walBuffer)));
PgWal result = interface->wal(bufPtrConst(walBuffer));
result.version = interface->version;
@ -312,10 +335,11 @@ pgWalFromBuffer(const Buffer *walBuffer)
}
FN_EXTERN PgWal
pgWalFromFile(const String *walFile, const Storage *storage)
pgWalFromFile(const String *walFile, const Storage *storage, const String *const pgVersionForce)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(STRING, walFile);
FUNCTION_LOG_PARAM(STRING, pgVersionForce);
FUNCTION_LOG_END();
ASSERT(walFile != NULL);
@ -327,7 +351,7 @@ pgWalFromFile(const String *walFile, const Storage *storage)
// Read WAL segment header
Buffer *walBuffer = storageGetP(storageNewReadP(storage, walFile), .exactSize = PG_WAL_HEADER_SIZE);
result = pgWalFromBuffer(walBuffer);
result = pgWalFromBuffer(walBuffer, pgVersionForce);
}
MEM_CONTEXT_TEMP_END();

6
src/postgres/interface.h
View File

@ -130,7 +130,7 @@ FN_EXTERN bool pgDbIsSystem(const String *name);
FN_EXTERN bool pgDbIsSystemId(unsigned int id);
// Get info from pg_control
FN_EXTERN PgControl pgControlFromFile(const Storage *storage);
FN_EXTERN PgControl pgControlFromFile(const Storage *storage, const String *pgVersionForce);
// Get the control version for a PostgreSQL version
FN_EXTERN uint32_t pgControlVersion(unsigned int pgVersion);
@ -140,8 +140,8 @@ FN_EXTERN unsigned int pgVersionFromStr(const String *version);
FN_EXTERN String *pgVersionToStr(unsigned int version);
// Get info from WAL header
FN_EXTERN PgWal pgWalFromFile(const String *walFile, const Storage *storage);
FN_EXTERN PgWal pgWalFromBuffer(const Buffer *walBuffer);
FN_EXTERN PgWal pgWalFromFile(const String *walFile, const Storage *storage, const String *pgVersionForce);
FN_EXTERN PgWal pgWalFromBuffer(const Buffer *walBuffer, const String *pgVersionForce);
// Get the tablespace identifier used to distinguish versions in a tablespace directory, e.g. PG_15_202209061
FN_EXTERN String *pgTablespaceId(unsigned int pgVersion, unsigned int pgCatalogVersion);

2
src/postgres/interface/version.intern.h
View File

@ -61,7 +61,6 @@ Read the version specific pg_control into a general data structure
pgInterfaceControl##version(const unsigned char *controlFile) \
{ \
ASSERT(controlFile != NULL); \
ASSERT(pgInterfaceControlIs##version(controlFile)); \
\
return (PgControl) \
{ \
@ -123,7 +122,6 @@ Read the version specific WAL header into a general data structure
pgInterfaceWal##version(const unsigned char *walFile) \
{ \
ASSERT(walFile != NULL); \
ASSERT(pgInterfaceWalIs##version(walFile)); \
\
return (PgWal) \
{ \

24
test/src/module/command/archiveGetTest.c
View File

@ -832,6 +832,30 @@ testRun(void)
TEST_RESULT_LOG("P00 INFO: found 01ABCDEF01ABCDEF01ABCDEF in the repo1: 10-1 archive");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("get WAL segment with a modified control/catalog version");
// Modify control/catalog version and use the --pg-version option
HRN_PG_CONTROL_OVERRIDE_PUT(storagePgWrite(), PG_VERSION_10, 1501, .catalogVersion = 202211111);
TEST_ERROR(
cmdArchiveGet(), VersionNotSupportedError,
"unexpected control version = 1501 and catalog version = 202211111\n"
"HINT: is this version of PostgreSQL supported?");
StringList *argListTemp = strLstDup(argList);
hrnCfgArgRawZ(argListTemp, cfgOptPgVersionForce, "10");
HRN_CFG_LOAD(cfgCmdArchivePush, argListTemp);
TEST_RESULT_INT(cmdArchiveGet(), 0, "get");
TEST_RESULT_LOG("P00 INFO: found 01ABCDEF01ABCDEF01ABCDEF in the repo1: 10-1 archive");
// Reset control file and command options
HRN_PG_CONTROL_PUT(storagePgWrite(), PG_VERSION_10);
HRN_CFG_LOAD(cfgCmdArchiveGet, argList);
// Clean-up pg_wal directory
TEST_RESULT_UINT(storageInfoP(storagePg(), STRDEF("pg_wal/RECOVERYXLOG")).size, 16 * 1024 * 1024, "check size");
TEST_STORAGE_LIST(storagePgWrite(), "pg_wal", "RECOVERYXLOG\n", .remove = true);

35
test/src/module/command/archivePushTest.c
View File

@ -453,6 +453,41 @@ testRun(void)
" HINT: this is valid in some recovery scenarios but may also indicate a problem.\n"
"P00 INFO: pushed WAL file '000000010000000100000002' to the archive");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("push WAL with modified control/catalog/magic using the --pg-version option");
argListTemp = strLstDup(argList);
strLstAddZ(argListTemp, "pg_wal/000000010000000100000003");
HRN_CFG_LOAD(cfgCmdArchivePush, argListTemp);
memset(bufPtr(walBuffer1), 0, bufSize(walBuffer1));
HRN_PG_WAL_OVERRIDE_TO_BUFFER(walBuffer1, PG_VERSION_11, 999);
HRN_STORAGE_PUT(storagePgWrite(), "pg_wal/000000010000000100000003", walBuffer1);
TEST_ERROR(
cmdArchivePush(), VersionNotSupportedError,
"unexpected WAL magic 999\n"
"HINT: is this version of PostgreSQL supported?");
HRN_PG_CONTROL_OVERRIDE_PUT(storagePgWrite(), PG_VERSION_11, 1501, .catalogVersion = 202211111);
TEST_ERROR(
cmdArchivePush(), VersionNotSupportedError,
"unexpected control version = 1501 and catalog version = 202211111\n"
"HINT: is this version of PostgreSQL supported?");
argListTemp = strLstDup(argList);
hrnCfgArgRawZ(argListTemp, cfgOptPgVersionForce, "11");
strLstAddZ(argListTemp, "pg_wal/000000010000000100000003");
HRN_CFG_LOAD(cfgCmdArchivePush, argListTemp);
TEST_RESULT_VOID(cmdArchivePush(), "push the WAL segment with a modified control/catalog version");
TEST_RESULT_LOG(
"P00 INFO: pushed WAL file '000000010000000100000003' to the archive");
// Reset control file
HRN_PG_CONTROL_PUT(storagePgWrite(), PG_VERSION_11);
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("multiple repos, one encrypted");

14
test/src/module/command/backupTest.c
View File

@ -467,6 +467,7 @@ typedef struct TestBackupPqScriptParam
const char *cipherPass; // Cipher pass
unsigned int walTotal; // Total WAL to write
unsigned int timeline; // Timeline to use for WAL files
const String *pgVersionForce; // PG version to use when control/catalog not found
} TestBackupPqScriptParam;
#define testBackupPqScriptP(pgVersion, backupStartTime, ...) \
@ -482,7 +483,7 @@ testBackupPqScript(unsigned int pgVersion, time_t backupTimeStart, TestBackupPqS
param.timeline = param.timeline == 0 ? 1 : param.timeline;
// Read pg_control to get info about the cluster
PgControl pgControl = pgControlFromFile(storagePg());
PgControl pgControl = pgControlFromFile(storagePg(), param.pgVersionForce);
// Set archive timeout really small to save time on errors
cfgOptionSet(cfgOptArchiveTimeout, cfgSourceParam, varNewInt64(100));
@ -3429,7 +3430,7 @@ testRun(void)
}
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("online 11 full backup with tablespaces, bundles and annotations");
TEST_TITLE("online 11 full backup with tablespaces, bundles, annotations and unexpected pg_control/pg_catalog version");
backupTimeStart = BACKUP_EPOCH + 2400000;
@ -3448,8 +3449,14 @@ testRun(void)
hrnCfgArgRawBool(argList, cfgOptResume, false);
hrnCfgArgRawZ(argList, cfgOptAnnotation, "extra key=this is an annotation");
hrnCfgArgRawZ(argList, cfgOptAnnotation, "source=this is another annotation");
hrnCfgArgRawZ(argList, cfgOptPgVersionForce, "11");
HRN_CFG_LOAD(cfgCmdBackup, argList);
// Create pg_control with unexpected catalog and control version
HRN_PG_CONTROL_OVERRIDE_PUT(
storagePgWrite(), PG_VERSION_11, 1501, .catalogVersion = 202211110, .pageChecksum = true,
.walSegmentSize = 1024 * 1024);
// Set to a smaller values than the defaults allow
cfgOptionSet(cfgOptRepoBundleSize, cfgSourceParam, VARINT64(PG_PAGE_SIZE_DEFAULT));
cfgOptionSet(cfgOptRepoBundleLimit, cfgSourceParam, VARINT64(PG_PAGE_SIZE_DEFAULT));
@ -3476,7 +3483,8 @@ testRun(void)
HRN_STORAGE_PUT(storagePgWrite(), "bigish.dat", bigish, .timeModified = 1500000001);
// Run backup
testBackupPqScriptP(PG_VERSION_11, backupTimeStart, .walCompressType = compressTypeGz, .walTotal = 2);
testBackupPqScriptP(
PG_VERSION_11, backupTimeStart, .walCompressType = compressTypeGz, .walTotal = 2, .pgVersionForce = STRDEF("11"));
TEST_RESULT_VOID(hrnCmdBackup(), "backup");
TEST_RESULT_LOG(

110
test/src/module/command/stanzaTest.c
View File

@ -512,6 +512,41 @@ testRun(void)
TEST_RESULT_LOG(
"P00 WARN: option --force is no longer supported\n"
"P00 INFO: stanza-create for stanza 'db' on repo1");
// Clean-up file in archive directory
HRN_STORAGE_REMOVE(storageRepoIdxWrite(0), STORAGE_REPO_ARCHIVE "/somefile", .comment = "remove old test file");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("stanza-create forcing another PG version");
argList = strLstDup(argListBase);
hrnCfgArgRawZ(argList, cfgOptPgVersionForce, "15");
HRN_CFG_LOAD(cfgCmdStanzaCreate, argList);
HRN_PG_CONTROL_OVERRIDE_PUT(storagePgWrite(), PG_VERSION_15, 1501, .catalogVersion = 202211111);
TEST_RESULT_VOID(cmdStanzaCreate(), "stanza create - forcing another PG version");
TEST_RESULT_LOG("P00 INFO: stanza-create for stanza 'db' on repo1");
HRN_INFO_PUT(
storageHrn, "test.info",
"[db]\n"
"db-catalog-version=202211111\n"
"db-control-version=1300\n"
"db-id=1\n"
"db-system-id=" HRN_PG_SYSTEMID_15_Z "\n"
"db-version=\"15\"\n"
"\n"
"[db:history]\n"
"1={\"db-catalog-version\":202211111,\"db-control-version\":1300,\"db-system-id\":" HRN_PG_SYSTEMID_15_Z
",\"db-version\":\"15\"}\n",
.comment = "put backup info to test file, control version for PG 15, catalog version forced from pg_control");
TEST_RESULT_BOOL(
bufEq(
storageGetP(storageNewReadP(storageRepoIdx(0), INFO_BACKUP_PATH_FILE_STR)),
storageGetP(storageNewReadP(storageHrn, STRDEF("test.info")))),
true, "test and stanza backup info files are equal");
}
// *****************************************************************************************************************************
@ -885,6 +920,81 @@ testRun(void)
storageGetP(storageNewReadP(storageRepoIdx(0), INFO_BACKUP_PATH_FILE_STR)),
storageGetP(storageNewReadP(storageHrn, STRDEF("test.info")))),
true, "test and stanza backup info files are equal");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("stanza-upgrade forcing another PG version");
argList = strLstDup(argListBase);
hrnCfgArgRawZ(argList, cfgOptPgVersionForce, "15");
HRN_CFG_LOAD(cfgCmdStanzaUpgrade, argList);
HRN_PG_CONTROL_OVERRIDE_PUT(storagePgWrite(), PG_VERSION_15, 1501, .catalogVersion = 202211111);
HRN_INFO_PUT(
storageRepoIdxWrite(0), INFO_BACKUP_PATH_FILE,
"[db]\n"
"db-catalog-version=201608131\n"
"db-control-version=960\n"
"db-id=1\n"
"db-system-id=6569239123849665999\n"
"db-version=\"9.6\"\n"
"\n"
"[db:history]\n"
"1={\"db-catalog-version\":201608131,\"db-control-version\":960,\"db-system-id\":6569239123849665999"
",\"db-version\":\"9.6\"}\n");
HRN_INFO_PUT(
storageRepoIdxWrite(0), INFO_ARCHIVE_PATH_FILE,
"[db]\n"
"db-id=1\n"
"db-system-id=6569239123849665999\n"
"db-version=\"9.6\"\n"
"\n"
"[db:history]\n"
"1={\"db-id\":6569239123849665999,\"db-version\":\"9.6\"}\n");
TEST_RESULT_VOID(cmdStanzaUpgrade(), "stanza upgrade - forcing another PG version");
TEST_RESULT_LOG("P00 INFO: stanza-upgrade for stanza 'db' on repo1");
HRN_INFO_PUT(
storageHrn, "test.info",
"[db]\n"
"db-id=2\n"
"db-system-id=" HRN_PG_SYSTEMID_15_Z "\n"
"db-version=\"15\"\n"
"\n"
"[db:history]\n"
"1={\"db-id\":6569239123849665999,\"db-version\":\"9.6\"}\n"
"2={\"db-id\":" HRN_PG_SYSTEMID_15_Z ",\"db-version\":\"15\"}\n",
.comment = "put archive info to test file");
TEST_RESULT_BOOL(
bufEq(
storageGetP(storageNewReadP(storageRepoIdx(0), INFO_ARCHIVE_PATH_FILE_STR)),
storageGetP(storageNewReadP(storageHrn, STRDEF("test.info")))),
true, "test and stanza archive info files are equal");
HRN_INFO_PUT(
storageHrn, "test.info",
"[db]\n"
"db-catalog-version=202211111\n"
"db-control-version=1300\n"
"db-id=2\n"
"db-system-id=" HRN_PG_SYSTEMID_15_Z "\n"
"db-version=\"15\"\n"
"\n"
"[db:history]\n"
"1={\"db-catalog-version\":201608131,\"db-control-version\":960,\"db-system-id\":6569239123849665999"
",\"db-version\":\"9.6\"}\n"
"2={\"db-catalog-version\":202211111,\"db-control-version\":1300,\"db-system-id\":" HRN_PG_SYSTEMID_15_Z
",\"db-version\":\"15\"}\n",
.comment = "put backup info to test file, control version for PG 15, catalog version forced from pg_control");
TEST_RESULT_BOOL(
bufEq(
storageGetP(storageNewReadP(storageRepoIdx(0), INFO_BACKUP_PATH_FILE_STR)),
storageGetP(storageNewReadP(storageHrn, STRDEF("test.info")))),
true, "test and stanza backup info files are equal");
}
// *****************************************************************************************************************************

30
test/src/module/command/verifyTest.c
View File

@ -1798,21 +1798,45 @@ testRun(void)
"P00 DETAIL: archiveId: 11-2, wal start: 000000020000000700000FFE, wal stop: 000000020000000700000FFE");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("none output, verbose, with no verify failures");
TEST_TITLE("invalid WAL magic");
// Create WAL file with just header info and small WAL size
walBuffer = bufNew((size_t)(1024 * 1024));
bufUsedSet(walBuffer, bufSize(walBuffer));
memset(bufPtr(walBuffer), 0, bufSize(walBuffer));
HRN_PG_WAL_OVERRIDE_TO_BUFFER(walBuffer, PG_VERSION_11, 999, .size = 1024 * 1024);
const char *walBufferSha2 = strZ(strNewEncode(encodingHex, cryptoHashOne(hashTypeSha1, walBuffer)));
HRN_STORAGE_PUT(
storageRepoIdxWrite(0),
zNewFmt(STORAGE_REPO_ARCHIVE "/11-2/0000000200000007/000000020000000700000FFD-%s", walBufferSha2), walBuffer,
.comment = "invalid WAL magic");
HRN_CFG_LOAD(cfgCmdVerify, argList);
TEST_ERROR(
verifyProcess(cfgOptionBool(cfgOptVerbose)), VersionNotSupportedError,
"unexpected WAL magic 999\n"
"HINT: is this version of PostgreSQL supported?");
TEST_RESULT_LOG(
"P00 DETAIL: no backups exist in the repo");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("none output, verbose, override WAL magic, with no verify failures");
hrnCfgArgRawZ(argList, cfgOptVerbose, "y");
hrnCfgArgRawZ(argList, cfgOptPgVersionForce, PG_VERSION_11_STR);
HRN_CFG_LOAD(cfgCmdVerify, argList);
TEST_RESULT_STR_Z(
verifyProcess(cfgOptionBool(cfgOptVerbose)),
"stanza: db\n"
"status: ok\n"
" archiveId: 11-2, total WAL checked: 1, total valid WAL: 1\n"
" archiveId: 11-2, total WAL checked: 2, total valid WAL: 2\n"
" missing: 0, checksum invalid: 0, size invalid: 0, other: 0\n"
" backup: none found",
"verify none output, verbose, with no failures");
TEST_RESULT_LOG(
"P00 DETAIL: no backups exist in the repo\n"
"P00 DETAIL: archiveId: 11-2, wal start: 000000020000000700000FFE, wal stop: 000000020000000700000FFE");
"P00 DETAIL: archiveId: 11-2, wal start: 000000020000000700000FFD, wal stop: 000000020000000700000FFE");
}
// *****************************************************************************************************************************

71
test/src/module/postgres/interfaceTest.c
View File

@ -49,7 +49,10 @@ testRun(void)
// *****************************************************************************************************************************
if (testBegin("pgControlVersion()"))
{
TEST_ERROR(pgControlVersion(70300), AssertError, "invalid PostgreSQL version 70300");
TEST_ERROR(
pgControlVersion(70300), VersionNotSupportedError,
"invalid PostgreSQL version 70300\n"
"HINT: is this version of PostgreSQL supported?");
TEST_RESULT_UINT(pgControlVersion(PG_VERSION_93), 937, "9.3 control version");
TEST_RESULT_UINT(pgControlVersion(PG_VERSION_11), 1100, "11 control version");
}
@ -64,11 +67,11 @@ testRun(void)
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("unknown control version");
HRN_PG_CONTROL_OVERRIDE_PUT(storageTest, PG_VERSION_15, 1501, .catalogVersion = 202211110);
HRN_PG_CONTROL_OVERRIDE_PUT(storageTest, PG_VERSION_15, 1501, .catalogVersion = 202211111);
TEST_ERROR(
pgControlFromFile(storageTest), VersionNotSupportedError,
"unexpected control version = 1501 and catalog version = 202211110\n"
pgControlFromFile(storageTest, NULL), VersionNotSupportedError,
"unexpected control version = 1501 and catalog version = 202211111\n"
"HINT: is this version of PostgreSQL supported?");
// -------------------------------------------------------------------------------------------------------------------------
@ -77,7 +80,7 @@ testRun(void)
.walSegmentSize = 1024 * 1024);
PgControl info = {0};
TEST_ASSIGN(info, pgControlFromFile(storageTest), "get control info v11");
TEST_ASSIGN(info, pgControlFromFile(storageTest, NULL), "get control info v11");
TEST_RESULT_UINT(info.systemId, 0xFACEFACE, " check system id");
TEST_RESULT_UINT(info.version, PG_VERSION_11, " check version");
TEST_RESULT_UINT(info.catalogVersion, 201809051, " check catalog version");
@ -88,24 +91,48 @@ testRun(void)
HRN_PG_CONTROL_PUT(storageTest, PG_VERSION_93, .walSegmentSize = 1024 * 1024);
TEST_ERROR(
pgControlFromFile(storageTest), FormatError, "wal segment size is 1048576 but must be 16777216 for PostgreSQL <= 10");
pgControlFromFile(storageTest, NULL), FormatError,
"wal segment size is 1048576 but must be 16777216 for PostgreSQL <= 10");
// -------------------------------------------------------------------------------------------------------------------------
HRN_PG_CONTROL_PUT(storageTest, PG_VERSION_95, .pageSize = 32 * 1024);
TEST_ERROR(pgControlFromFile(storageTest), FormatError, "page size is 32768 but must be 8192");
TEST_ERROR(pgControlFromFile(storageTest, NULL), FormatError, "page size is 32768 but must be 8192");
// -------------------------------------------------------------------------------------------------------------------------
HRN_PG_CONTROL_PUT(
storageTest, PG_VERSION_93, .systemId = 0xEFEFEFEFEF, .catalogVersion = hrnPgCatalogVersion(PG_VERSION_93),
.checkpoint = 0xAABBAABBEEFFEEFF, .timeline = 88);
TEST_ASSIGN(info, pgControlFromFile(storageTest), "get control info v90");
TEST_ASSIGN(info, pgControlFromFile(storageTest, NULL), "get control info v90");
TEST_RESULT_UINT(info.systemId, 0xEFEFEFEFEF, " check system id");
TEST_RESULT_UINT(info.version, PG_VERSION_93, " check version");
TEST_RESULT_UINT(info.catalogVersion, 201306121, " check catalog version");
TEST_RESULT_UINT(info.checkpoint, 0xAABBAABBEEFFEEFF, "check checkpoint");
TEST_RESULT_UINT(info.timeline, 88, "check timeline");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("force control version");
HRN_PG_CONTROL_OVERRIDE_PUT(
storageTest, PG_VERSION_15, 1501, .systemId = 0xEFEFEFEFEF, .catalogVersion = 202211111,
.checkpoint = 0xAABBAABBEEFFEEFF, .timeline = 88);
TEST_ERROR(
pgControlFromFile(storageTest, NULL), VersionNotSupportedError,
"unexpected control version = 1501 and catalog version = 202211111\n"
"HINT: is this version of PostgreSQL supported?");
TEST_ERROR(
pgControlFromFile(storageTest, STRDEF("99")), VersionNotSupportedError,
"invalid PostgreSQL version 990000\n"
"HINT: is this version of PostgreSQL supported?");
TEST_ASSIGN(info, pgControlFromFile(storageTest, STRDEF(PG_VERSION_15_STR)), "get control info v90");
TEST_RESULT_UINT(info.systemId, 0xEFEFEFEFEF, "check system id");
TEST_RESULT_UINT(info.version, PG_VERSION_15, "check version");
TEST_RESULT_UINT(info.catalogVersion, 202211111, "check catalog version");
TEST_RESULT_UINT(info.checkpoint, 0xAABBAABBEEFFEEFF, "check checkpoint");
TEST_RESULT_UINT(info.timeline, 88, "check timeline");
}
// *****************************************************************************************************************************
@ -187,7 +214,7 @@ testRun(void)
HRN_PG_WAL_OVERRIDE_TO_BUFFER(result, PG_VERSION_15, 777);
TEST_ERROR(
pgWalFromBuffer(result), VersionNotSupportedError,
pgWalFromBuffer(result, NULL), VersionNotSupportedError,
"unexpected WAL magic 777\n"
"HINT: is this version of PostgreSQL supported?");
@ -196,7 +223,7 @@ testRun(void)
((PgWalCommon *)bufPtr(result))->flag = 0;
TEST_ERROR(pgWalFromBuffer(result), FormatError, "first page header in WAL file is expected to be in long format");
TEST_ERROR(pgWalFromBuffer(result, NULL), FormatError, "first page header in WAL file is expected to be in long format");
// -------------------------------------------------------------------------------------------------------------------------
memset(bufPtr(result), 0, bufSize(result));
@ -204,7 +231,7 @@ testRun(void)
storagePutP(storageNewWriteP(storageTest, walFile), result);
PgWal info = {0};
TEST_ASSIGN(info, pgWalFromFile(walFile, storageTest), "get wal info v11");
TEST_ASSIGN(info, pgWalFromFile(walFile, storageTest, NULL), "get wal info v11");
TEST_RESULT_UINT(info.systemId, 0xECAFECAF, " check system id");
TEST_RESULT_UINT(info.version, PG_VERSION_11, " check version");
TEST_RESULT_UINT(info.size, PG_WAL_SEGMENT_SIZE_DEFAULT * 2, " check size");
@ -213,17 +240,35 @@ testRun(void)
memset(bufPtr(result), 0, bufSize(result));
HRN_PG_WAL_TO_BUFFER(result, PG_VERSION_96, .systemId = 0xEAEAEAEA, .size = PG_WAL_SEGMENT_SIZE_DEFAULT * 2);
TEST_ERROR(pgWalFromBuffer(result), FormatError, "wal segment size is 33554432 but must be 16777216 for PostgreSQL <= 10");
TEST_ERROR(
pgWalFromBuffer(result, NULL), FormatError, "wal segment size is 33554432 but must be 16777216 for PostgreSQL <= 10");
// -------------------------------------------------------------------------------------------------------------------------
memset(bufPtr(result), 0, bufSize(result));
HRN_PG_WAL_TO_BUFFER(result, PG_VERSION_93, .systemId = 0xEAEAEAEA, .size = PG_WAL_SEGMENT_SIZE_DEFAULT);
storagePutP(storageNewWriteP(storageTest, walFile), result);
TEST_ASSIGN(info, pgWalFromFile(walFile, storageTest), "get wal info v9.3");
TEST_ASSIGN(info, pgWalFromFile(walFile, storageTest, NULL), "get wal info v9.3");
TEST_RESULT_UINT(info.systemId, 0xEAEAEAEA, " check system id");
TEST_RESULT_UINT(info.version, PG_VERSION_93, " check version");
TEST_RESULT_UINT(info.size, PG_WAL_SEGMENT_SIZE_DEFAULT, " check size");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("force WAL version");
memset(bufPtr(result), 0, bufSize(result));
HRN_PG_WAL_OVERRIDE_TO_BUFFER(result, PG_VERSION_15, 777, .systemId = 0xFAFAFAFA, .size = PG_WAL_SEGMENT_SIZE_DEFAULT);
storagePutP(storageNewWriteP(storageTest, walFile), result);
TEST_ERROR(
pgWalFromBuffer(result, NULL), VersionNotSupportedError,
"unexpected WAL magic 777\n"
"HINT: is this version of PostgreSQL supported?");
TEST_ASSIGN(info, pgWalFromFile(walFile, storageTest, STRDEF(PG_VERSION_15_STR)), "force wal info v15");
TEST_RESULT_UINT(info.systemId, 0xFAFAFAFA, "check system id");
TEST_RESULT_UINT(info.version, PG_VERSION_15, " check version");
TEST_RESULT_UINT(info.size, PG_WAL_SEGMENT_SIZE_DEFAULT, " check size");
}
// *****************************************************************************************************************************