db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2025-07-17 01:12:23 +02:00
Code Issues Releases Activity

Add archive-header-check option.

Browse Source 
Enabled by default, this option checks the WAL header against the PostgreSQL version and system identifier to ensure that the WAL is being copied to the correct stanza. This is in addition to checking pg_control against the stanza and verifying that WAL is being copied from the same PostgreSQL data directory where pg_control is located.

Therefore, disabling this check is fairly safe but should only be done when required, e.g. if the WAL is encrypted.
This commit is contained in:
David Steele
2021-03-25 15:33:50 -04:00
committed by GitHub
parent 01b8e2258f
commit b6106f3c1f
12 changed files with 150 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
doc/xml/reference.xml
View File

@ -748,6 +748,17 @@
<example>y</example>
</config-key>
<!-- ======================================================================================================= -->
<config-key id="archive-header-check" name="Check WAL Headers">
<summary>Check PostgreSQL version/id in WAL headers.</summary>
<text>Enabled by default, this option checks the WAL header against the <postgres/> version and system identifier to ensure that the WAL is being copied to the correct stanza. This is in addition to checking <file>pg_control</file> against the stanza and verifying that WAL is being copied from the same <postgres/> data directory where <file>pg_control</file> is located.
Therefore, disabling this check is fairly safe but should only be done when needed, e.g. if the WAL is encrypted.</text>
<example>n</example>
</config-key>
<!-- ======================================================================================================= -->
<config-key id="archive-mode-check" name="Check Archive Mode">
<summary>Check the <postgres/> <setting>archive_mode</setting> setting.</summary>

11
doc/xml/release.xml
View File

@ -90,6 +90,17 @@
<p>GCS support for repository storage.</p>
</release-item>
<release-item>
<release-item-contributor-list>
<release-item-ideator id="hans.jurgen.schonig"/>
<release-item-contributor id="david.steele"/>
<release-item-reviewer id="stephen.frost"/>
<release-item-reviewer id="cynthia.shang"/>
</release-item-contributor-list>
<p>Add <br-option>archive-header-check</br-option> option.</p>
</release-item>
</release-feature-list>
<release-improvement-list>

10
src/build/config/config.yaml
View File

@ -1016,6 +1016,16 @@ option:
command-role:
default: {}
archive-header-check:
section: global
type: boolean
default: true
command:
archive-push: {}
command-role:
default: {}
async: {}
archive-mode-check:
section: global
type: boolean

7
src/command/archive/push/file.c
View File

@ -95,11 +95,12 @@ archivePushFileIo(ArchivePushFileIoType type, IoWrite *write, const Buffer *buff
/**********************************************************************************************************************************/
ArchivePushFileResult
archivePushFile(
const String *walSource, unsigned int pgVersion, uint64_t pgSystemId, const String *archiveFile, CompressType compressType,
int compressLevel, const List *const repoList, const StringList *const priorErrorList)
const String *walSource, bool headerCheck, unsigned int pgVersion, uint64_t pgSystemId, const String *archiveFile,
CompressType compressType, int compressLevel, const List *const repoList, const StringList *const priorErrorList)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(STRING, walSource);
FUNCTION_LOG_PARAM(BOOL, headerCheck);
FUNCTION_LOG_PARAM(UINT, pgVersion);
FUNCTION_LOG_PARAM(UINT64, pgSystemId);
FUNCTION_LOG_PARAM(STRING, archiveFile);
@ -124,7 +125,7 @@ archivePushFile(
bool isSegment = walIsSegment(archiveFile);
// If this is a segment compare archive version and systemId to the WAL header
if (isSegment)
if (headerCheck && isSegment)
{
PgWal walInfo = pgWalFromFile(walSource, storageLocal());

4
src/command/archive/push/file.h
View File

@ -31,7 +31,7 @@ typedef struct ArchivePushFileResult
// Copy a file from the source to the archive
ArchivePushFileResult archivePushFile(
const String *walSource, unsigned int pgVersion, uint64_t pgSystemId, const String *archiveFile, CompressType compressType,
int compressLevel, const List *const repoList, const StringList *const priorErrorList);
const String *walSource, bool headerCheck, unsigned int pgVersion, uint64_t pgSystemId, const String *archiveFile,
CompressType compressType, int compressLevel, const List *const repoList, const StringList *const priorErrorList);
#endif

11
src/command/archive/push/protocol.c
View File

@ -33,8 +33,8 @@ archivePushFileProtocol(const VariantList *paramList, ProtocolServer *server)
{
// Build the repo data list
List *repoList = lstNewP(sizeof(ArchivePushFileRepoData));
unsigned int repoListSize = varUIntForce(varLstGet(paramList, 7));
unsigned int paramIdx = 8;
unsigned int repoListSize = varUIntForce(varLstGet(paramList, 8));
unsigned int paramIdx = 9;
for (unsigned int repoListIdx = 0; repoListIdx < repoListSize; repoListIdx++)
{
@ -53,9 +53,10 @@ archivePushFileProtocol(const VariantList *paramList, ProtocolServer *server)
// Push the file
ArchivePushFileResult fileResult = archivePushFile(
varStr(varLstGet(paramList, 0)), varUIntForce(varLstGet(paramList, 1)), varUInt64(varLstGet(paramList, 2)),
varStr(varLstGet(paramList, 3)), (CompressType)varUIntForce(varLstGet(paramList, 4)),
varIntForce(varLstGet(paramList, 5)), repoList, strLstNewVarLst(varVarLst(varLstGet(paramList, 6))));
varStr(varLstGet(paramList, 0)), varBool(varLstGet(paramList, 1)), varUIntForce(varLstGet(paramList, 2)),
varUInt64(varLstGet(paramList, 3)), varStr(varLstGet(paramList, 4)),
(CompressType)varUIntForce(varLstGet(paramList, 5)), varIntForce(varLstGet(paramList, 6)), repoList,
strLstNewVarLst(varVarLst(varLstGet(paramList, 7))));
// Return result
VariantList *result = varLstNew();

3
src/command/archive/push/push.c
View File

@ -412,7 +412,7 @@ cmdArchivePush(void)
// Push the file to the archive
ArchivePushFileResult fileResult = archivePushFile(
walFile, archiveInfo.pgVersion, archiveInfo.pgSystemId, archiveFile,
walFile, cfgOptionBool(cfgOptArchiveHeaderCheck), archiveInfo.pgVersion, archiveInfo.pgSystemId, archiveFile,
compressTypeEnum(cfgOptionStr(cfgOptCompressType)), cfgOptionInt(cfgOptCompressLevel), archiveInfo.repoList,
archiveInfo.errorList);
@ -462,6 +462,7 @@ archivePushAsyncCallback(void *data, unsigned int clientIdx)
ProtocolCommand *command = protocolCommandNew(PROTOCOL_COMMAND_ARCHIVE_PUSH_FILE_STR);
protocolCommandParamAdd(command, VARSTR(strNewFmt("%s/%s", strZ(jobData->walPath), strZ(walFile))));
protocolCommandParamAdd(command, VARBOOL(cfgOptionBool(cfgOptArchiveHeaderCheck)));
protocolCommandParamAdd(command, VARUINT(jobData->archiveInfo.pgVersion));
protocolCommandParamAdd(command, VARUINT64(jobData->archiveInfo.pgSystemId));
protocolCommandParamAdd(command, VARSTR(walFile));

33
src/command/help/help.auto.c
View File

@ -840,6 +840,39 @@ static const unsigned char helpDataPack[] =
0x6C, 0x69, 0x65, 0x72, 0x20, 0x69, 0x73, 0x20, 0x61, 0x20, 0x70, 0x6F, 0x77, 0x65, 0x72, 0x20, 0x6F, 0x66, 0x20, 0x31,
0x30, 0x32, 0x34, 0x2E,
// archive-header-check option
// -------------------------------------------------------------------------------------------------------------------------
pckTypeStr << 4 | 0x0B, 0x06, // Section
0x62, 0x61, 0x63, 0x6B, 0x75, 0x70,
pckTypeStr << 4 | 0x08, 0x2B, // Summary
0x43, 0x68, 0x65, 0x63, 0x6B, 0x20, 0x50, 0x6F, 0x73, 0x74, 0x67, 0x72, 0x65, 0x53, 0x51, 0x4C, 0x20, 0x76, 0x65, 0x72,
0x73, 0x69, 0x6F, 0x6E, 0x2F, 0x69, 0x64, 0x20, 0x69, 0x6E, 0x20, 0x57, 0x41, 0x4C, 0x20, 0x68, 0x65, 0x61, 0x64, 0x65,
0x72, 0x73, 0x2E,
pckTypeStr << 4 | 0x08, 0xC5, 0x03, // Description
0x45, 0x6E, 0x61, 0x62, 0x6C, 0x65, 0x64, 0x20, 0x62, 0x79, 0x20, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6C, 0x74, 0x2C, 0x20,
0x74, 0x68, 0x69, 0x73, 0x20, 0x6F, 0x70, 0x74, 0x69, 0x6F, 0x6E, 0x20, 0x63, 0x68, 0x65, 0x63, 0x6B, 0x73, 0x20, 0x74,
0x68, 0x65, 0x20, 0x57, 0x41, 0x4C, 0x20, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x20, 0x61, 0x67, 0x61, 0x69, 0x6E, 0x73,
0x74, 0x20, 0x74, 0x68, 0x65, 0x20, 0x50, 0x6F, 0x73, 0x74, 0x67, 0x72, 0x65, 0x53, 0x51, 0x4C, 0x20, 0x76, 0x65, 0x72,
0x73, 0x69, 0x6F, 0x6E, 0x20, 0x61, 0x6E, 0x64, 0x20, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6D, 0x20, 0x69, 0x64, 0x65, 0x6E,
0x74, 0x69, 0x66, 0x69, 0x65, 0x72, 0x20, 0x74, 0x6F, 0x20, 0x65, 0x6E, 0x73, 0x75, 0x72, 0x65, 0x20, 0x74, 0x68, 0x61,
0x74, 0x20, 0x74, 0x68, 0x65, 0x20, 0x57, 0x41, 0x4C, 0x20, 0x69, 0x73, 0x20, 0x62, 0x65, 0x69, 0x6E, 0x67, 0x20, 0x63,
0x6F, 0x70, 0x69, 0x65, 0x64, 0x20, 0x74, 0x6F, 0x20, 0x74, 0x68, 0x65, 0x20, 0x63, 0x6F, 0x72, 0x72, 0x65, 0x63, 0x74,
0x20, 0x73, 0x74, 0x61, 0x6E, 0x7A, 0x61, 0x2E, 0x20, 0x54, 0x68, 0x69, 0x73, 0x20, 0x69, 0x73, 0x20, 0x69, 0x6E, 0x20,
0x61, 0x64, 0x64, 0x69, 0x74, 0x69, 0x6F, 0x6E, 0x20, 0x74, 0x6F, 0x20, 0x63, 0x68, 0x65, 0x63, 0x6B, 0x69, 0x6E, 0x67,
0x20, 0x70, 0x67, 0x5F, 0x63, 0x6F, 0x6E, 0x74, 0x72, 0x6F, 0x6C, 0x20, 0x61, 0x67, 0x61, 0x69, 0x6E, 0x73, 0x74, 0x20,
0x74, 0x68, 0x65, 0x20, 0x73, 0x74, 0x61, 0x6E, 0x7A, 0x61, 0x20, 0x61, 0x6E, 0x64, 0x20, 0x76, 0x65, 0x72, 0x69, 0x66,
0x79, 0x69, 0x6E, 0x67, 0x20, 0x74, 0x68, 0x61, 0x74, 0x20, 0x57, 0x41, 0x4C, 0x20, 0x69, 0x73, 0x20, 0x62, 0x65, 0x69,
0x6E, 0x67, 0x20, 0x63, 0x6F, 0x70, 0x69, 0x65, 0x64, 0x20, 0x66, 0x72, 0x6F, 0x6D, 0x20, 0x74, 0x68, 0x65, 0x20, 0x73,
0x61, 0x6D, 0x65, 0x20, 0x50, 0x6F, 0x73, 0x74, 0x67, 0x72, 0x65, 0x53, 0x51, 0x4C, 0x20, 0x64, 0x61, 0x74, 0x61, 0x20,
0x64, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6F, 0x72, 0x79, 0x20, 0x77, 0x68, 0x65, 0x72, 0x65, 0x20, 0x70, 0x67, 0x5F, 0x63,
0x6F, 0x6E, 0x74, 0x72, 0x6F, 0x6C, 0x20, 0x69, 0x73, 0x20, 0x6C, 0x6F, 0x63, 0x61, 0x74, 0x65, 0x64, 0x2E, 0x0A, 0x0A,
0x54, 0x68, 0x65, 0x72, 0x65, 0x66, 0x6F, 0x72, 0x65, 0x2C, 0x20, 0x64, 0x69, 0x73, 0x61, 0x62, 0x6C, 0x69, 0x6E, 0x67,
0x20, 0x74, 0x68, 0x69, 0x73, 0x20, 0x63, 0x68, 0x65, 0x63, 0x6B, 0x20, 0x69, 0x73, 0x20, 0x66, 0x61, 0x69, 0x72, 0x6C,
0x79, 0x20, 0x73, 0x61, 0x66, 0x65, 0x20, 0x62, 0x75, 0x74, 0x20, 0x73, 0x68, 0x6F, 0x75, 0x6C, 0x64, 0x20, 0x6F, 0x6E,
0x6C, 0x79, 0x20, 0x62, 0x65, 0x20, 0x64, 0x6F, 0x6E, 0x65, 0x20, 0x77, 0x68, 0x65, 0x6E, 0x20, 0x6E, 0x65, 0x65, 0x64,
0x65, 0x64, 0x2C, 0x20, 0x65, 0x2E, 0x67, 0x2E, 0x20, 0x69, 0x66, 0x20, 0x74, 0x68, 0x65, 0x20, 0x57, 0x41, 0x4C, 0x20,
0x69, 0x73, 0x20, 0x65, 0x6E, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x2E,
// archive-mode option
// -------------------------------------------------------------------------------------------------------------------------
pckTypeStr << 4 | 0x0B, 0x07, // Section

1
src/config/config.auto.c
View File

@ -261,6 +261,7 @@ STRING_EXTERN(CFGOPT_ARCHIVE_ASYNC_STR, CFGOPT_ARCHI
STRING_EXTERN(CFGOPT_ARCHIVE_CHECK_STR, CFGOPT_ARCHIVE_CHECK);
STRING_EXTERN(CFGOPT_ARCHIVE_COPY_STR, CFGOPT_ARCHIVE_COPY);
STRING_EXTERN(CFGOPT_ARCHIVE_GET_QUEUE_MAX_STR, CFGOPT_ARCHIVE_GET_QUEUE_MAX);
STRING_EXTERN(CFGOPT_ARCHIVE_HEADER_CHECK_STR, CFGOPT_ARCHIVE_HEADER_CHECK);
STRING_EXTERN(CFGOPT_ARCHIVE_MODE_STR, CFGOPT_ARCHIVE_MODE);
STRING_EXTERN(CFGOPT_ARCHIVE_MODE_CHECK_STR, CFGOPT_ARCHIVE_MODE_CHECK);
STRING_EXTERN(CFGOPT_ARCHIVE_PUSH_QUEUE_MAX_STR, CFGOPT_ARCHIVE_PUSH_QUEUE_MAX);

5
src/config/config.auto.h
View File

@ -68,6 +68,8 @@ Option constants
STRING_DECLARE(CFGOPT_ARCHIVE_COPY_STR);
#define CFGOPT_ARCHIVE_GET_QUEUE_MAX "archive-get-queue-max"
STRING_DECLARE(CFGOPT_ARCHIVE_GET_QUEUE_MAX_STR);
#define CFGOPT_ARCHIVE_HEADER_CHECK "archive-header-check"
STRING_DECLARE(CFGOPT_ARCHIVE_HEADER_CHECK_STR);
#define CFGOPT_ARCHIVE_MODE "archive-mode"
STRING_DECLARE(CFGOPT_ARCHIVE_MODE_STR);
#define CFGOPT_ARCHIVE_MODE_CHECK "archive-mode-check"
@ -209,7 +211,7 @@ Option constants
#define CFGOPT_TYPE "type"
STRING_DECLARE(CFGOPT_TYPE_STR);
#define CFG_OPTION_TOTAL 128
#define CFG_OPTION_TOTAL 129
/***********************************************************************************************************************************
Command enum
@ -257,6 +259,7 @@ typedef enum
cfgOptArchiveCheck,
cfgOptArchiveCopy,
cfgOptArchiveGetQueueMax,
cfgOptArchiveHeaderCheck,
cfgOptArchiveMode,
cfgOptArchiveModeCheck,
cfgOptArchivePushQueueMax,

40
src/config/parse.auto.c
View File

@ -406,6 +406,30 @@ static const ParseRuleOption parseRuleOption[CFG_OPTION_TOTAL] =
),
),
// -----------------------------------------------------------------------------------------------------------------------------
PARSE_RULE_OPTION
(
PARSE_RULE_OPTION_NAME("archive-header-check"),
PARSE_RULE_OPTION_TYPE(cfgOptTypeBoolean),
PARSE_RULE_OPTION_REQUIRED(true),
PARSE_RULE_OPTION_SECTION(cfgSectionGlobal),
PARSE_RULE_OPTION_COMMAND_ROLE_DEFAULT_VALID_LIST
(
PARSE_RULE_OPTION_COMMAND(cfgCmdArchivePush)
),
PARSE_RULE_OPTION_COMMAND_ROLE_ASYNC_VALID_LIST
(
PARSE_RULE_OPTION_COMMAND(cfgCmdArchivePush)
),
PARSE_RULE_OPTION_OPTIONAL_LIST
(
PARSE_RULE_OPTION_OPTIONAL_DEFAULT("1"),
),
),
// -----------------------------------------------------------------------------------------------------------------------------
PARSE_RULE_OPTION
(
@ -6528,6 +6552,21 @@ static const struct option optionList[] =
.val = PARSE_OPTION_FLAG | PARSE_RESET_FLAG | cfgOptArchiveGetQueueMax,
},
// archive-header-check option
// -----------------------------------------------------------------------------------------------------------------------------
{
.name = "archive-header-check",
.val = PARSE_OPTION_FLAG | cfgOptArchiveHeaderCheck,
},
{
.name = "no-archive-header-check",
.val = PARSE_OPTION_FLAG | PARSE_NEGATE_FLAG | cfgOptArchiveHeaderCheck,
},
{
.name = "reset-archive-header-check",
.val = PARSE_OPTION_FLAG | PARSE_RESET_FLAG | cfgOptArchiveHeaderCheck,
},
// archive-mode option
// -----------------------------------------------------------------------------------------------------------------------------
{
@ -10555,6 +10594,7 @@ static const ConfigOption optionResolveOrder[] =
cfgOptStanza,
cfgOptArchiveAsync,
cfgOptArchiveGetQueueMax,
cfgOptArchiveHeaderCheck,
cfgOptArchiveMode,
cfgOptArchivePushQueueMax,
cfgOptArchiveTimeout,

27
test/src/module/command/archivePushTest.c
View File

@ -327,6 +327,7 @@ testRun(void)
memset(bufPtr(walBuffer1), 0, bufSize(walBuffer1));
pgWalTestToBuffer((PgWal){.version = PG_VERSION_11, .systemId = 0xECAFECAFECAFECAF}, walBuffer1);
const char *walBuffer1Sha1 = strZ(bufHex(cryptoHashOne(HASH_TYPE_SHA1_STR, walBuffer1)));
storagePutP(storageNewWriteP(storagePgWrite(), strNew("pg_wal/000000010000000100000001")), walBuffer1);
@ -335,14 +336,37 @@ testRun(void)
"WAL file '{[path]}/pg/pg_wal/000000010000000100000001' version 11, system-id 17055110554209741999 do not match"
" stanza version 11, system-id 18072658121562454734");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("push by ignoring the invalid header");
argListTemp = strLstDup(argList);
hrnCfgArgRawNegate(argListTemp, cfgOptArchiveHeaderCheck);
strLstAddZ(argListTemp, "pg_wal/000000010000000100000001");
harnessCfgLoad(cfgCmdArchivePush, argListTemp);
TEST_RESULT_VOID(cmdArchivePush(), "push the WAL segment");
harnessLogResult("P00 INFO: pushed WAL file '000000010000000100000001' to the archive");
TEST_RESULT_BOOL(
storageExistsP(
storageRepoIdx(0),
strNewFmt(STORAGE_REPO_ARCHIVE "/11-1/000000010000000100000001-%s.gz", walBuffer1Sha1)),
true, "check repo for WAL file");
TEST_STORAGE_REMOVE(
storageRepoIdxWrite(0), strZ(strNewFmt(STORAGE_REPO_ARCHIVE "/11-1/000000010000000100000001-%s.gz", walBuffer1Sha1)));
// Generate valid WAL and push them
// -------------------------------------------------------------------------------------------------------------------------
argListTemp = strLstDup(argList);
strLstAddZ(argListTemp, "pg_wal/000000010000000100000001");
harnessCfgLoad(cfgCmdArchivePush, argListTemp);
memset(bufPtr(walBuffer1), 0, bufSize(walBuffer1));
pgWalTestToBuffer((PgWal){.version = PG_VERSION_11, .systemId = 0xFACEFACEFACEFACE}, walBuffer1);
// Check sha1 checksum against fixed values once to make sure they are not getting munged. After this we'll calculate them
// directly from the buffers to reduce the cost of maintaining checksums.
const char *walBuffer1Sha1 = TEST_64BIT() ?
walBuffer1Sha1 = TEST_64BIT() ?
(TEST_BIG_ENDIAN() ? "1c5f963d720bb199d7935dbd315447ea2ec3feb2" : "aae7591a1dbc58f21d0d004886075094f622e6dd") :
"28a13fd8cf6fcd9f9a8108aed4c8bcc58040863a";
@ -452,6 +476,7 @@ testRun(void)
// -------------------------------------------------------------------------------------------------------------------------
VariantList *paramList = varLstNew();
varLstAdd(paramList, varNewStr(strNewFmt("%s/pg/pg_wal/000000010000000100000002", testPath())));
varLstAdd(paramList, varNewBool(true));
varLstAdd(paramList, varNewUInt64(PG_VERSION_11));
varLstAdd(paramList, varNewUInt64(0xFACEFACEFACEFACE));
varLstAdd(paramList, varNewStrZ("000000010000000100000002"));