db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2024-12-12 10:04:14 +02:00
Code Issues Releases Activity

Multi-stanza check command.

Browse Source 
Check command now checks multiple stanzas when the stanza option is omitted.

The stanza list is extracted from the current configuration rather than scanning the repository like the info command. Scanning the repository is a problem because configuration for each stanza may not be present in the current configuration. Since this functionality is new for check there is no regression.

Add a new section to the user guide to cover multi-stanza configuration and provide additional coverage for this feature.

Also fix a small issue in the parser when an indexed option has a dependency on a non-indexed option. There were no examples of this case in the previous configuration.
This commit is contained in:
David Steele 2023-08-07 17:03:09 +01:00 committed by GitHub
parent 995a8e9669
commit 1141dc2070
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
24 changed files with 516 additions and 86 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
doc/xml/release.xml
View File

@ -17,6 +17,17 @@
<release date="XXXX-XX-XX" version="2.48dev" title="Under Development">
<release-core-list>
<release-improvement-list>
<release-item>
<github-pull-request id="2057"/>
<release-item-contributor-list>
<release-item-contributor id="david.steele"/>
<release-item-reviewer id="stephen.frost"/>
</release-item-contributor-list>
<p>Multi-stanza check command.</p>
</release-item>
<release-item>
<github-pull-request id="2072"/>

173
doc/xml/user-guide.xml
View File

@ -220,6 +220,12 @@
<variable key="host-pg2-image">{[host-image]}</variable>
<variable key="host-pg2-mount">{[host-mount]}</variable>
<variable key="host-pgalt-id">pgalt</variable>
<variable key="host-pgalt">pg-alt</variable>
<variable key="host-pgalt-user">{[host-pg1-user]}</variable>
<variable key="host-pgalt-image">{[host-image]}</variable>
<variable key="host-pgalt-mount">{[host-mount]}</variable>
<variable key="host-repo1-id">repo1</variable>
<variable key="host-repo1">repository</variable>
<variable key="host-repo1-user">{[host-user]}</variable>
@ -2720,7 +2726,7 @@
<backrest-config-option if="{[protocol-tls]}" section="demo" key="pg1-host-cert-file">/etc/pgbackrest/cert/client.crt</backrest-config-option>
<backrest-config-option if="{[protocol-tls]}" section="demo" key="pg1-host-key-file">/etc/pgbackrest/cert/client.key</backrest-config-option>
<backrest-config-option if="{[protocol-tls]}" section="global" key="tls-server-auth">pgbackrest-client=demo</backrest-config-option>
<backrest-config-option if="{[protocol-tls]}" section="global" key="tls-server-auth">pgbackrest-client=*</backrest-config-option>
<backrest-config-option if="{[protocol-tls]}" section="global" key="tls-server-address">*</backrest-config-option>
<backrest-config-option if="{[protocol-tls]}" section="global" key="tls-server-ca-file">/etc/pgbackrest/cert/ca.crt</backrest-config-option>
<backrest-config-option if="{[protocol-tls]}" section="global" key="tls-server-cert-file">/etc/pgbackrest/cert/server.crt</backrest-config-option>
@ -3431,6 +3437,171 @@
</section>
</section>
<!-- ======================================================================================================================= -->
<section id="multi-stanza" depend="/repo-host/perform-backup">
<title>Multiple Stanzas</title>
<p><backrest/> supports multiple stanzas. The most common usage is sharing a <host>{[host-repo1]}</host> host among multiple stanzas.</p>
<!-- =================================================================================================================== -->
<section id="installation">
<title>Installation</title>
<p>A new host named <host>{[host-pgalt]}</host> is created to run the new primary.</p>
<host-add id="{[host-pgalt-id]}" name="{[host-pgalt]}" user="{[host-pgalt-user]}" image="{[host-pgalt-image]}" os="{[os-type]}" mount="{[host-pgalt-mount]}" option="{[host-mem]} {[host-option]}"/>
<block id="br-install">
<block-variable-replace key="br-install-host">{[host-pgalt]}</block-variable-replace>
<block-variable-replace key="br-install-user">postgres</block-variable-replace>
<block-variable-replace key="br-install-group">postgres</block-variable-replace>
</block>
</section>
<!-- =================================================================================================================== -->
<section if="{[protocol-ssh]}" id="setup-ssh">
<title>Setup Passwordless SSH</title>
<block id="setup-ssh-intro">
<!-- ??? Bogus variable is set because the syntax currently requires at least one -->
<block-variable-replace key="bogus"></block-variable-replace>
</block>
<block id="setup-ssh">
<block-variable-replace key="setup-ssh-host">{[host-pgalt]}</block-variable-replace>
<block-variable-replace key="setup-ssh-user">postgres</block-variable-replace>
<block-variable-replace key="setup-ssh-user-home-path">{[pg-home-path]}</block-variable-replace>
</block>
</section>
<!-- =================================================================================================================== -->
<section id="configuration">
<title>Configuration</title>
<p><backrest/> configuration is nearly identical to <host>{[host-pg1]}</host> except that the <id>demo-alt</id> stanza will be used so backups and archive will be stored in a separate location.</p>
<backrest-config host="{[host-pgalt]}" file="{[backrest-config-demo]}">
<title>Configure <backrest/> on the new primary</title>
<backrest-config-option section="demo-alt" key="pg1-path">{[pg-path]}</backrest-config-option>
<backrest-config-option section="global" key="repo1-host">{[host-repo1]}</backrest-config-option>
<backrest-config-option if="{[protocol-tls]}" section="global" key="repo1-host-type">tls</backrest-config-option>
<backrest-config-option if="{[protocol-tls]}" section="global" key="repo1-host-ca-file">/etc/pgbackrest/cert/ca.crt</backrest-config-option>
<backrest-config-option if="{[protocol-tls]}" section="global" key="repo1-host-cert-file">/etc/pgbackrest/cert/client.crt</backrest-config-option>
<backrest-config-option if="{[protocol-tls]}" section="global" key="repo1-host-key-file">/etc/pgbackrest/cert/client.key</backrest-config-option>
<backrest-config-option if="{[protocol-tls]}" section="global" key="tls-server-auth">pgbackrest-client=demo-alt</backrest-config-option>
<backrest-config-option if="{[protocol-tls]}" section="global" key="tls-server-address">*</backrest-config-option>
<backrest-config-option if="{[protocol-tls]}" section="global" key="tls-server-ca-file">/etc/pgbackrest/cert/ca.crt</backrest-config-option>
<backrest-config-option if="{[protocol-tls]}" section="global" key="tls-server-cert-file">/etc/pgbackrest/cert/server.crt</backrest-config-option>
<backrest-config-option if="{[protocol-tls]}" section="global" key="tls-server-key-file">/etc/pgbackrest/cert/server.key</backrest-config-option>
<backrest-config-option section="global" key="log-level-file">detail</backrest-config-option>
<backrest-config-option section="global" key="log-level-stderr">off</backrest-config-option>
<backrest-config-option section="global" key="log-timestamp">n</backrest-config-option>
</backrest-config>
<backrest-config host="{[host-repo1]}" owner="{[br-user]}:{[br-group]}" file="{[backrest-config-demo]}">
<title>Configure <br-option>pg1-host</br-option>/<br-option>pg1-host-user</br-option> and <br-option>pg1-path</br-option></title>
<backrest-config-option section="demo-alt" key="pg1-path">{[pg-path]}</backrest-config-option>
<backrest-config-option section="demo-alt" key="pg1-host">{[host-pgalt]}</backrest-config-option>
<backrest-config-option if="{[protocol-tls]}" section="demo-alt" key="pg1-host-type">tls</backrest-config-option>
<backrest-config-option if="{[protocol-tls]}" section="demo-alt" key="pg1-host-ca-file">/etc/pgbackrest/cert/ca.crt</backrest-config-option>
<backrest-config-option if="{[protocol-tls]}" section="demo-alt" key="pg1-host-cert-file">/etc/pgbackrest/cert/client.crt</backrest-config-option>
<backrest-config-option if="{[protocol-tls]}" section="demo-alt" key="pg1-host-key-file">/etc/pgbackrest/cert/client.key</backrest-config-option>
</backrest-config>
<block if="{[protocol-tls]}" id="setup-tls">
<block-variable-replace key="setup-tls-host">{[host-pgalt]}</block-variable-replace>
<block-variable-replace key="setup-tls-user">postgres</block-variable-replace>
<block-variable-replace key="setup-tls-group">postgres</block-variable-replace>
</block>
</section>
<!-- =================================================================================================================== -->
<section id="setup-demo-cluster">
<title>Setup Demo Cluster</title>
<execute-list host="{[host-pgalt]}">
<title>Create the demo cluster</title>
<execute user="postgres">
<exe-cmd>
{[pg-bin-path]}/initdb
-D {[pg-path]} -k -A peer
</exe-cmd>
</execute>
<execute if="{[os-type-is-debian]}" user="root" output="y" filter="n">
<exe-cmd>{[pg-cluster-create]}</exe-cmd>
</execute>
<execute user="root" show="n" user-force="y">
<exe-cmd>cat /root/postgresql.common.conf >> {[postgres-config-demo]}</exe-cmd>
</execute>
</execute-list>
<postgres-config host="{[host-pgalt]}" file="{[postgres-config-demo]}">
<title>Configure <postgres/> settings</title>
<postgres-config-option key="archive_command">'{[project-exe]} {[dash]}-stanza=demo-alt archive-push %p'</postgres-config-option>
<postgres-config-option key="archive_mode">on</postgres-config-option>
<postgres-config-option key="wal_level">{[wal-level]}</postgres-config-option>
<postgres-config-option key="max_wal_senders">3</postgres-config-option>
<postgres-config-option if="{[os-type-is-rhel]}" key="log_filename">'postgresql.log'</postgres-config-option>
</postgres-config>
<execute-list host="{[host-pgalt]}">
<title>Start the {[postgres-cluster-demo]} cluster</title>
<execute user="root">
<exe-cmd>{[pg-cluster-restart]}</exe-cmd>
</execute>
<execute user="postgres" show="n">
<exe-cmd>{[pg-cluster-wait]}</exe-cmd>
</execute>
</execute-list>
</section>
<!-- =================================================================================================================== -->
<section id="create-stanza">
<title>Create the Stanza and Check Configuration</title>
<p>The <cmd>stanza-create</cmd> command must be run to initialize the stanza. It is recommended that the <cmd>check</cmd> command be run after <cmd>stanza-create</cmd> to ensure archiving and backups are properly configured.</p>
<execute-list host="{[host-pgalt]}">
<title>Create the stanza and check the configuration</title>
<execute user="postgres" output="y">
<exe-cmd>{[project-exe]} {[dash]}-stanza=demo-alt {[dash]}-log-level-console=info stanza-create</exe-cmd>
<exe-highlight>completed successfully</exe-highlight>
</execute>
<execute user="postgres" output="y">
<exe-cmd>{[project-exe]} {[dash]}-log-level-console=info check</exe-cmd>
<exe-highlight>check stanza | successfully archived to </exe-highlight>
</execute>
</execute-list>
<p>If the <cmd>check</cmd> command is run from the <host>{[host-repo1]}</host> host then all stanzas will be checked.</p>
<execute-list host="{[host-repo1]}">
<title>Check the configuration for all stanzas</title>
<execute user="{[br-user]}" output="y">
<exe-cmd>{[project-exe]} {[dash]}-log-level-console=info check</exe-cmd>
<exe-highlight>check stanza | successfully archived to </exe-highlight>
</execute>
</execute-list>
</section>
</section>
<!-- ======================================================================================================================= -->
<section id="async-archiving" depend="/replication">
<title>Asynchronous Archiving</title>

7
src/build/config/config.yaml
View File

@ -368,7 +368,8 @@ option:
archive-get: {}
archive-push: {}
backup: {}
check: {}
check:
required: false
expire: {}
info:
required: false
@ -1544,7 +1545,9 @@ option:
archive-push:
required: false
backup: {}
check: {}
check:
depend:
option: stanza
manifest:
required: false
restore: {}

55
src/command/check/check.c
View File

@ -10,6 +10,8 @@ Check Command
#include "common/log.h"
#include "common/memContext.h"
#include "config/config.h"
#include "config/load.h"
#include "config/parse.h"
#include "db/helper.h"
#include "info/infoArchive.h"
#include "postgres/interface.h"
@ -176,15 +178,54 @@ cmdCheck(void)
MEM_CONTEXT_TEMP_BEGIN()
{
// Get the primary/standby connections (standby is only required if backup from standby is enabled)
DbGetResult dbGroup = dbGet(false, false, false);
// Build stanza list based on whether a stanza was specified or not
StringList *stanzaList;
bool stanzaSpecified = cfgOptionTest(cfgOptStanza);
if (dbGroup.standby == NULL && dbGroup.primary == NULL)
THROW(ConfigError, "no database found\nHINT: check indexed pg-path/pg-host configurations");
if (stanzaSpecified)
{
stanzaList = strLstNew();
strLstAdd(stanzaList, cfgOptionStr(cfgOptStanza));
}
else
{
stanzaList = cfgParseStanzaList();
const unsigned int pgPathDefinedTotal = checkManifest();
checkStandby(dbGroup, pgPathDefinedTotal);
checkPrimary(dbGroup);
if (strLstSize(stanzaList) == 0)
{
LOG_WARN(
"no stanzas found to check\n"
"HINT: are there non-empty stanza sections in the configuration?");
}
}
// Iterate stanzas
for (unsigned int stanzaIdx = 0; stanzaIdx < strLstSize(stanzaList); stanzaIdx++)
{
// Switch stanza if required
if (!stanzaSpecified)
{
const String *const stanza = strLstGet(stanzaList, stanzaIdx);
LOG_INFO_FMT("check stanza '%s'", strZ(stanza));
// Free storage and protocol cache
storageHelperFree();
protocolFree();
// Reload config with new stanza
cfgLoadStanza(stanza);
}
// Get the primary/standby connections (standby is only required if backup from standby is enabled)
DbGetResult dbGroup = dbGet(false, false, false);
if (dbGroup.standby == NULL && dbGroup.primary == NULL)
THROW(ConfigError, "no database found\nHINT: check indexed pg-path/pg-host configurations");
const unsigned int pgPathDefinedTotal = checkManifest();
checkStandby(dbGroup, pgPathDefinedTotal);
checkPrimary(dbGroup);
}
}
MEM_CONTEXT_TEMP_END();

14
src/common/ini.c
View File

@ -297,6 +297,20 @@ iniSectionKeyIsList(const Ini *const this, const String *const section, const St
FUNCTION_TEST_RETURN(BOOL, varType(result) == varTypeVariantList);
}
/**********************************************************************************************************************************/
FN_EXTERN StringList *
iniSectionList(const Ini *const this)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(INI, this);
FUNCTION_TEST_END();
ASSERT(this != NULL);
ASSERT(this->store != NULL);
FUNCTION_TEST_RETURN(STRING_LIST, strLstNewVarLst(kvKeyList(this->store)));
}
/**********************************************************************************************************************************/
FN_EXTERN StringList *
iniSectionKeyList(const Ini *const this, const String *const section)

3
src/common/ini.h
View File

@ -68,6 +68,9 @@ FN_EXTERN StringList *iniGetList(const Ini *this, const String *section, const S
// The key's value is a list
FN_EXTERN bool iniSectionKeyIsList(const Ini *this, const String *section, const String *key);
// List of sections
FN_EXTERN StringList *iniSectionList(const Ini *const this);
// List of keys for a section
FN_EXTERN StringList *iniSectionKeyList(const Ini *this, const String *section);

58
src/config/load.c
View File

@ -25,6 +25,15 @@ Configuration Load
#include "storage/posix/storage.h"
#include "storage/sftp/storage.h"
/***********************************************************************************************************************************
Local variables
***********************************************************************************************************************************/
static struct ConfigLoadLocal
{
unsigned int argListSize; // Argument list size
const char **argList; // Argument list
} configLoadLocal;
/***********************************************************************************************************************************
Load log settings
***********************************************************************************************************************************/
@ -437,17 +446,24 @@ cfgLoadLogFile(void)
/**********************************************************************************************************************************/
FN_EXTERN void
cfgLoad(unsigned int argListSize, const char *argList[])
cfgLoad(const unsigned int argListSize, const char *argList[])
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(UINT, argListSize);
FUNCTION_LOG_PARAM(CHARPY, argList);
FUNCTION_LOG_END();
ASSERT(argListSize > 0);
ASSERT(argList != NULL);
// Store arguments
configLoadLocal.argListSize = argListSize;
configLoadLocal.argList = argList;
MEM_CONTEXT_TEMP_BEGIN()
{
// Parse config from command line and config file
cfgParseP(storageLocal(), argListSize, argList);
cfgParseP(storageLocal(), configLoadLocal.argListSize, configLoadLocal.argList);
// Initialize dry-run mode for storage when valid for the current command
storageHelperDryRunInit(cfgOptionValid(cfgOptDryRun) && cfgOptionBool(cfgOptDryRun));
@ -517,3 +533,41 @@ cfgLoad(unsigned int argListSize, const char *argList[])
FUNCTION_LOG_RETURN_VOID();
}
/**********************************************************************************************************************************/
FN_EXTERN void
cfgLoadStanza(const String *const stanza)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(STRING, stanza);
FUNCTION_LOG_END();
ASSERT(stanza != NULL);
ASSERT(configLoadLocal.argListSize > 0);
MEM_CONTEXT_TEMP_BEGIN()
{
// Store the exec id so it can be preserved after reload
const Variant *const execId = varNewStr(cfgOptionStr(cfgOptExecId));
// Make a copy of the arguments and add the stanza (this assumes the stanza was not originally specified)
StringList *const argListNew = strLstNew();
for (unsigned int argListIdx = 0; argListIdx < configLoadLocal.argListSize; argListIdx++)
strLstAddZ(argListNew, configLoadLocal.argList[argListIdx]);
strLstAddFmt(argListNew, "--" CFGOPT_STANZA "=%s", strZ(stanza));
// Parse config from command line and config file
cfgParseP(storageLocal(), strLstSize(argListNew), strLstPtr(argListNew), .noConfigLoad = true, .noResetLogLevel = true);
// Update options that have complex rules
cfgLoadUpdateOption();
// Set execId to prior value
cfgOptionSet(cfgOptExecId, cfgSourceParam, execId);
}
MEM_CONTEXT_TEMP_END();
FUNCTION_LOG_RETURN_VOID();
}

3
src/config/load.h
View File

@ -14,6 +14,9 @@ Functions
// Load the configuration
FN_EXTERN void cfgLoad(unsigned int argListSize, const char *argList[]);
// Load the configuration using the specified stanza
FN_EXTERN void cfgLoadStanza(const String *stanza);
// Generate log file path and name. Only the command role is configurable here because log settings may vary between commands.
FN_EXTERN String *cfgLoadLogFileName(ConfigCommandRole commandRole);

14
src/config/parse.auto.c.inc
View File

@ -4020,6 +4020,19 @@ static const ParseRuleOption parseRuleOption[CFG_OPTION_TOTAL] =
), // opt/pg-path
// opt/pg-path
PARSE_RULE_OPTIONAL_NOT_REQUIRED(), // opt/pg-path
), // opt/pg-path
// opt/pg-path
PARSE_RULE_OPTIONAL_GROUP // opt/pg-path
( // opt/pg-path
PARSE_RULE_FILTER_CMD // opt/pg-path
( // opt/pg-path
PARSE_RULE_VAL_CMD(cfgCmdCheck), // opt/pg-path
), // opt/pg-path
// opt/pg-path
PARSE_RULE_OPTIONAL_DEPEND // opt/pg-path
( // opt/pg-path
PARSE_RULE_VAL_OPT(cfgOptStanza), // opt/pg-path
), // opt/pg-path
), // opt/pg-path
), // opt/pg-path
), // opt/pg-path
@ -9556,6 +9569,7 @@ static const ParseRuleOption parseRuleOption[CFG_OPTION_TOTAL] =
( // opt/stanza
PARSE_RULE_FILTER_CMD // opt/stanza
( // opt/stanza
PARSE_RULE_VAL_CMD(cfgCmdCheck), // opt/stanza
PARSE_RULE_VAL_CMD(cfgCmdInfo), // opt/stanza
PARSE_RULE_VAL_CMD(cfgCmdRepoCreate), // opt/stanza
PARSE_RULE_VAL_CMD(cfgCmdRepoGet), // opt/stanza

105
src/config/parse.c
View File

@ -56,6 +56,15 @@ Prefix for environment variables
// In some environments this will not be externed
extern char **environ;
/***********************************************************************************************************************************
Mem context and local variables
***********************************************************************************************************************************/
static struct ConfigParseLocal
{
MemContext *memContext; // Mem context
Ini *ini; // Parsed ini data
} configParseLocal;
/***********************************************************************************************************************************
Define how a command is parsed
***********************************************************************************************************************************/
@ -481,6 +490,43 @@ cfgParseCommandRoleName(const ConfigCommand commandId, const ConfigCommandRole c
FUNCTION_TEST_RETURN(STRING, result);
}
/**********************************************************************************************************************************/
FN_EXTERN StringList *
cfgParseStanzaList(void)
{
FUNCTION_TEST_VOID();
StringList *const result = strLstNew();
if (configParseLocal.ini != NULL)
{
MEM_CONTEXT_TEMP_BEGIN()
{
const StringList *const sectionList = strLstSort(iniSectionList(configParseLocal.ini), sortOrderAsc);
for (unsigned int sectionIdx = 0; sectionIdx < strLstSize(sectionList); sectionIdx++)
{
const String *const section = strLstGet(sectionList, sectionIdx);
// Skip global sections
if (strEqZ(section, CFGDEF_SECTION_GLOBAL) || strBeginsWithZ(section, CFGDEF_SECTION_GLOBAL ":"))
continue;
// Extract stanza
const StringList *const sectionPart = strLstNewSplitZ(section, ":");
ASSERT(strLstSize(sectionPart) <= 2);
strLstAddIfMissing(result, strLstGet(sectionPart, 0));
}
}
MEM_CONTEXT_TEMP_END();
}
strLstSort(result, sortOrderAsc);
FUNCTION_TEST_RETURN(STRING_LIST, result);
}
/***********************************************************************************************************************************
Find an option by name in the option list
***********************************************************************************************************************************/
@ -1046,7 +1092,8 @@ cfgParseOptionalFilterDepend(PackRead *const filter, const Config *const config,
// Get the depend option value
const ConfigOption dependId = (ConfigOption)pckReadU32P(filter);
ASSERT(config->option[dependId].index != NULL);
const ConfigOptionValue *const dependValue = &config->option[dependId].index[optionListIdx];
const ConfigOptionValue *const dependValue =
&config->option[dependId].index[parseRuleOption[dependId].group ? optionListIdx : 0];
// Is the dependency resolved?
if (dependValue->set)
@ -1468,8 +1515,23 @@ cfgParse(const Storage *const storage, const unsigned int argListSize, const cha
FUNCTION_LOG_PARAM(UINT, argListSize);
FUNCTION_LOG_PARAM(CHARPY, argList);
FUNCTION_LOG_PARAM(BOOL, param.noResetLogLevel);
FUNCTION_LOG_PARAM(BOOL, param.noConfigLoad);
FUNCTION_LOG_END();
// Initialize local mem context
if (configParseLocal.memContext == NULL)
{
MEM_CONTEXT_BEGIN(memContextTop())
{
MEM_CONTEXT_NEW_BEGIN(ConfigParse, .childQty = MEM_CONTEXT_QTY_MAX)
{
configParseLocal.memContext = MEM_CONTEXT_NEW();
}
MEM_CONTEXT_NEW_END();
}
MEM_CONTEXT_END();
}
MEM_CONTEXT_TEMP_BEGIN()
{
// Create the config struct
@ -1821,16 +1883,30 @@ cfgParse(const Storage *const storage, const unsigned int argListSize, const cha
// Phase 3: parse config file unless --no-config passed
// ---------------------------------------------------------------------------------------------------------------------
// Load the configuration file(s)
String *configString = cfgFileLoad(
storage, parseOptionList,
(const String *)&parseRuleValueStr[parseRuleValStrCFGOPTDEF_CONFIG_PATH_SP_QT_FS_QT_SP_PROJECT_CONFIG_FILE],
(const String *)&parseRuleValueStr[parseRuleValStrCFGOPTDEF_CONFIG_PATH_SP_QT_FS_QT_SP_PROJECT_CONFIG_INCLUDE_PATH],
PGBACKREST_CONFIG_ORIG_PATH_FILE_STR);
if (configString != NULL)
if (!param.noConfigLoad)
{
const Ini *const ini = iniNewP(ioBufferReadNew(BUFSTR(configString)), .store = true);
const String *const configString = cfgFileLoad(
storage, parseOptionList,
(const String *)&parseRuleValueStr[parseRuleValStrCFGOPTDEF_CONFIG_PATH_SP_QT_FS_QT_SP_PROJECT_CONFIG_FILE],
(const String *)&parseRuleValueStr[
parseRuleValStrCFGOPTDEF_CONFIG_PATH_SP_QT_FS_QT_SP_PROJECT_CONFIG_INCLUDE_PATH],
PGBACKREST_CONFIG_ORIG_PATH_FILE_STR);
iniFree(configParseLocal.ini);
configParseLocal.ini = NULL;
if (configString != NULL)
{
MEM_CONTEXT_BEGIN(configParseLocal.memContext)
{
configParseLocal.ini = iniNewP(ioBufferReadNew(BUFSTR(configString)), .store = true);
}
MEM_CONTEXT_END();
}
}
if (configParseLocal.ini != NULL)
{
// Get the stanza name
String *stanza = NULL;
@ -1853,7 +1929,7 @@ cfgParse(const Storage *const storage, const unsigned int argListSize, const cha
for (unsigned int sectionIdx = 0; sectionIdx < strLstSize(sectionList); sectionIdx++)
{
String *section = strLstGet(sectionList, sectionIdx);
StringList *keyList = iniSectionKeyList(ini, section);
const StringList *const keyList = iniSectionKeyList(configParseLocal.ini, section);
KeyValue *optionFound = kvNew();
// Loop through keys to search for options
@ -1938,7 +2014,7 @@ cfgParse(const Storage *const storage, const unsigned int argListSize, const cha
optionValue->source = cfgSourceConfig;
// Process list
if (iniSectionKeyIsList(ini, section, key))
if (iniSectionKeyIsList(configParseLocal.ini, section, key))
{
// Error if the option cannot be specified multiple times
if (!parseRuleOption[option.id].multi)
@ -1948,12 +2024,12 @@ cfgParse(const Storage *const storage, const unsigned int argListSize, const cha
cfgParseOptionKeyIdxName(option.id, option.keyIdx));
}
optionValue->valueList = iniGetList(ini, section, key);
optionValue->valueList = iniGetList(configParseLocal.ini, section, key);
}
else
{
// Get the option value
const String *value = iniGet(ini, section, key);
const String *value = iniGet(configParseLocal.ini, section, key);
if (strSize(value) == 0)
{
@ -2164,7 +2240,8 @@ cfgParse(const Storage *const storage, const unsigned int argListSize, const cha
// Get depend option id and name
ConfigOption dependId = pckReadU32P(filter);
const String *dependOptionName = STR(cfgParseOptionKeyIdxName(dependId, optionKeyIdx));
const String *dependOptionName = STR(
cfgParseOptionKeyIdxName(dependId, parseRuleOption[dependId].group ? optionKeyIdx : 0));
// If depend value is not set
ASSERT(config->option[dependId].index != NULL);

4
src/config/parse.h
View File

@ -44,6 +44,8 @@ typedef struct CfgParseParam
{
VAR_PARAM_HEADER;
bool noResetLogLevel; // Do not reset log level
bool noConfigLoad; // Do not reload the config file
const String *stanza; // Load config as stanza
} CfgParseParam;
#define cfgParseP(storage, argListSize, argList, ...) \
@ -105,6 +107,8 @@ FN_EXTERN ConfigOptionDataType cfgParseOptionDataType(ConfigOption optionId);
// Is the option required?
FN_EXTERN bool cfgParseOptionRequired(ConfigCommand commandId, ConfigOption optionId);
FN_EXTERN StringList *cfgParseStanzaList(void);
// Is the option valid for the command?
FN_EXTERN bool cfgParseOptionValid(ConfigCommand commandId, ConfigCommandRole commandRoleId, ConfigOption optionId);

14
src/storage/helper.c
View File

@ -568,3 +568,17 @@ storageSpoolWrite(void)
FUNCTION_TEST_RETURN_CONST(STORAGE, storageHelper.storageSpoolWrite);
}
/**********************************************************************************************************************************/
FN_EXTERN void
storageHelperFree(void)
{
FUNCTION_TEST_VOID();
if (storageHelper.memContext != NULL)
memContextFree(storageHelper.memContext);
storageHelper = (struct StorageHelperLocal){.memContext = NULL, .helperList = storageHelper.helperList};
FUNCTION_TEST_RETURN_VOID();
}

3
src/storage/helper.h
View File

@ -62,4 +62,7 @@ FN_EXTERN const Storage *storageRepoWrite(void);
FN_EXTERN const Storage *storageSpool(void);
FN_EXTERN const Storage *storageSpoolWrite(void);
// Free cached storage objects
FN_EXTERN void storageHelperFree(void);
#endif

4
test/code-count/file-type.yaml
View File

@ -2379,10 +2379,6 @@ test/src/common/harnessStorageHelper.c:
class: test/harness
type: c
test/src/common/harnessStorageHelper.h:
class: test/harness
type: c/h
test/src/common/harnessTest.c:
class: test/harness
type: c

6
test/define.yaml
View File

@ -420,11 +420,10 @@ unit:
# ----------------------------------------------------------------------------------------------------------------------------
- name: lock
total: 3
harness: config
harness:
name: storageHelper
name: config
shim:
storage/helper: ~
config/load: ~
harness:
name: lock
shim:
@ -832,7 +831,6 @@ unit:
# ----------------------------------------------------------------------------------------------------------------------------
- name: check
total: 3
containerReq: true
coverage:
- command/check/common

2
test/lib/pgBackRestTest/Env/Host/HostBackupTest.pm
View File

@ -775,7 +775,7 @@ sub check
' --config=' . $self->backrestConfig() .
(defined($$oParam{iTimeout}) ? " --archive-timeout=$$oParam{iTimeout}" : '') .
(defined($$oParam{strOptionalParam}) ? " $$oParam{strOptionalParam}" : '') .
' --stanza=' . $self->stanza() . ' check',
(!defined($oParam->{bStanza}) || $oParam->{bStanza} ? ' --stanza=' . $self->stanza() : '') . ' check',
{strComment => $strComment, iExpectedExitStatus => $$oParam{iExpectedExitStatus}, bLogOutput => $self->synthetic()});
# Return from function and log return values if any

2
test/lib/pgBackRestTest/Module/Real/RealAllTest.pm
View File

@ -155,7 +155,7 @@ sub run
# --------------------------------------------------------------------------------------------------------------------------
my $strComment = 'verify check command runs successfully';
$oHostDbPrimary->check($strComment, {iTimeout => 10});
$oHostDbPrimary->check($strComment, {iTimeout => 10, bStanza => false});
# Also run check on the backup host when present
if ($bHostBackup)

12
test/src/common/harnessConfig.c
View File

@ -19,9 +19,13 @@ Harness for Loading Test Configurations
#include "common/harnessDebug.h"
#include "common/harnessLock.h"
#include "common/harnessLog.h"
#include "common/harnessStorageHelper.h"
#include "common/harnessTest.h"
/***********************************************************************************************************************************
Include shimmed C modules
***********************************************************************************************************************************/
{[SHIM_MODULE]}
/**********************************************************************************************************************************/
void
hrnCfgLoad(ConfigCommand commandId, const StringList *argListParam, const HrnCfgLoadParam param)
@ -79,7 +83,11 @@ hrnCfgLoad(ConfigCommand commandId, const StringList *argListParam, const HrnCfg
}
// Free objects in storage helper
hrnStorageHelperFree();
storageHelperFree();
// Store config so it can be reloaded with a stanza
configLoadLocal.argListSize = strLstSize(argList);
configLoadLocal.argList = strLstPtr(argList);
// Parse config
cfgParseP(storageLocal(), strLstSize(argList), strLstPtr(argList), .noResetLogLevel = true);

25
test/src/common/harnessStorageHelper.c
View File

@ -1,25 +0,0 @@
/***********************************************************************************************************************************
Storage Helper Test Harness
***********************************************************************************************************************************/
#include "build.auto.h"
#include "common/harnessStorageHelper.h"
/***********************************************************************************************************************************
Include shimmed C modules
***********************************************************************************************************************************/
{[SHIM_MODULE]}
/**********************************************************************************************************************************/
void
hrnStorageHelperFree(void)
{
FUNCTION_TEST_VOID();
if (storageHelper.memContext != NULL)
memContextFree(storageHelper.memContext);
storageHelper = (struct StorageHelperLocal){.memContext = NULL, .helperList = storageHelper.helperList};
FUNCTION_TEST_RETURN_VOID();
}

17
test/src/common/harnessStorageHelper.h
View File

@ -1,17 +0,0 @@
/***********************************************************************************************************************************
Storage Helper Test Harness
Helper functions for testing the storage helper.
***********************************************************************************************************************************/
#ifndef TEST_COMMON_HARNESS_STORAGE_HELPER_H
#define TEST_COMMON_HARNESS_STORAGE_HELPER_H
#include "storage/helper.h"
/***********************************************************************************************************************************
Functions
***********************************************************************************************************************************/
// Free all storage helper objects. This should be done on any config load to ensure that stanza changes are honored.
void hrnStorageHelperFree(void);
#endif

39
test/src/module/command/checkTest.c
View File

@ -6,11 +6,13 @@ Test Check Command
#include "info/infoBackup.h"
#include "postgres/version.h"
#include "storage/helper.h"
#include "storage/posix/storage.h"
#include "common/harnessConfig.h"
#include "common/harnessInfo.h"
#include "common/harnessPostgres.h"
#include "common/harnessPq.h"
#include "common/harnessStorage.h"
/***********************************************************************************************************************************
Test Run
@ -20,6 +22,9 @@ testRun(void)
{
FUNCTION_HARNESS_VOID();
// Create default storage object for testing
const Storage *const storageTest = storagePosixNewP(TEST_PATH_STR, .write = true);
// PQfinish() is strictly checked
harnessPqScriptStrictSet(true);
@ -272,9 +277,13 @@ testRun(void)
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("multi-repo - primary database only, WAL not found");
HRN_STORAGE_PUT_Z(
storageTest, "pgbackrest.conf",
"[test1]\n"
"pg1-path=" TEST_PATH "/pg\n");
argList = strLstNew();
hrnCfgArgRawZ(argList, cfgOptStanza, "test1");
hrnCfgArgRawZ(argList, cfgOptPgPath, TEST_PATH "/pg");
hrnCfgArgRawZ(argList, cfgOptConfig, TEST_PATH "/pgbackrest.conf");
hrnCfgArgRawZ(argList, cfgOptRepoPath, TEST_PATH "/repo");
hrnCfgArgKeyRawZ(argList, cfgOptRepoPath, 2, TEST_PATH "/repo2");
hrnCfgArgRawZ(argList, cfgOptArchiveTimeout, ".5");
@ -282,7 +291,7 @@ testRun(void)
// Create stanza files on repo2
HRN_INFO_PUT(
storageRepoIdxWrite(1), INFO_ARCHIVE_PATH_FILE,
storageTest, "repo2/archive/test1/" INFO_ARCHIVE_FILE,
"[db]\n"
"db-id=1\n"
"db-system-id=" HRN_PG_SYSTEMID_15_Z "\n"
@ -291,7 +300,7 @@ testRun(void)
"[db:history]\n"
"1={\"db-id\":" HRN_PG_SYSTEMID_15_Z ",\"db-version\":\"15\"}\n");
HRN_INFO_PUT(
storageRepoIdxWrite(1), INFO_BACKUP_PATH_FILE,
storageTest, "repo2/backup/test1/" INFO_BACKUP_FILE,
"[db]\n"
"db-catalog-version=202209061\n"
"db-control-version=1300\n"
@ -320,13 +329,35 @@ testRun(void)
"HINT: check the PostgreSQL server log for errors.\n"
"HINT: run the 'start' command if the stanza was previously stopped.");
TEST_RESULT_LOG(
"P00 INFO: check stanza 'test1'\n"
"P00 INFO: check repo1 configuration (primary)\n"
"P00 INFO: check repo2 configuration (primary)\n"
"P00 INFO: check repo1 archive for WAL (primary)");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("no stanzas in config file");
HRN_STORAGE_PUT_Z(
storageTest, "pgbackrest.conf",
"[test1]\n");
HRN_CFG_LOAD(cfgCmdCheck, argList);
TEST_RESULT_VOID(cmdCheck(), "check");
TEST_RESULT_LOG(
"P00 WARN: no stanzas found to check\n"
" HINT: are there non-empty stanza sections in the configuration?");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("multi-repo - WAL segment switch performed once for all repos");
argList = strLstNew();
hrnCfgArgRawZ(argList, cfgOptStanza, "test1");
hrnCfgArgRawZ(argList, cfgOptPgPath, TEST_PATH "/pg");
hrnCfgArgRawZ(argList, cfgOptRepoPath, TEST_PATH "/repo");
hrnCfgArgKeyRawZ(argList, cfgOptRepoPath, 2, TEST_PATH "/repo2");
hrnCfgArgRawZ(argList, cfgOptArchiveTimeout, ".5");
HRN_CFG_LOAD(cfgCmdCheck, argList);
// Create WAL segment
Buffer *buffer = bufNew(16 * 1024 * 1024);
memset(bufPtr(buffer), 0, bufSize(buffer));

1
test/src/module/common/iniTest.c
View File

@ -146,6 +146,7 @@ testRun(void)
TEST_RESULT_STRLST_Z(iniGetList(ini, STRDEF("global"), STRDEF("repeat")), "1\n2\n", "key list");
TEST_RESULT_PTR(iniGetList(ini, STRDEF("globalx"), STRDEF("repeat2")), NULL, "null key list");
TEST_RESULT_STRLST_Z(iniSectionList(ini), "global\ndb\n", "sections");
TEST_RESULT_STRLST_Z(iniSectionKeyList(ini, STRDEF("global")), "compress\nrepeat\n", "section keys");
TEST_RESULT_STRLST_Z(iniSectionKeyList(ini, STRDEF("bogus")), NULL, "empty section keys");

6
test/src/module/config/loadTest.c
View File

@ -706,6 +706,12 @@ testRun(void)
TEST_RESULT_BOOL(socketLocal.keepAlive, false, "check socketLocal.keepAlive");
TEST_RESULT_UINT(ioTimeoutMs(), 60000, "check io timeout");
String *execId = strDup(cfgOptionStr(cfgOptExecId));
TEST_RESULT_VOID(cfgLoadStanza(STRDEF("test1")), "load config with stanza");
TEST_RESULT_STR_Z(cfgOptionStr(cfgOptStanza), "test1", "check stanza");
TEST_RESULT_STR(cfgOptionStr(cfgOptExecId), execId, "check execId");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("umask is reset, neutral-umask=y");

24
test/src/module/config/parseTest.c
View File

@ -100,6 +100,11 @@ testRun(void)
const String *configIncludePath = STRDEF(TEST_PATH "/conf.d");
HRN_SYSTEM_FMT("mkdir -m 750 %s", strZ(configIncludePath));
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("no stanzas loaded yet");
TEST_RESULT_STRLST_Z(cfgParseStanzaList(), NULL, "stanza list");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("check old config file constants");
@ -621,8 +626,8 @@ testRun(void)
strLstAddZ(argList, TEST_BACKREST_EXE);
strLstAddZ(argList, "-bogus");
TEST_ERROR(
cfgParseP(storageTest, strLstSize(argList), strLstPtr(argList), .noResetLogLevel = true), OptionInvalidError,
"option '-bogus' must begin with --");
cfgParseP(storageTest, strLstSize(argList), strLstPtr(argList), .noResetLogLevel = true),
OptionInvalidError, "option '-bogus' must begin with --");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("error when option argument is invalid");
@ -1261,6 +1266,17 @@ testRun(void)
cfgParseP(storageTest, strLstSize(argList), strLstPtr(argList), .noResetLogLevel = true), OptionInvalidError,
"option 'target-exclusive' not valid without option 'type' in ('lsn', 'time', 'xid')");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("dependent option missing (indexed option depends on non-indexed option");
argList = strLstNew();
strLstAddZ(argList, TEST_BACKREST_EXE);
hrnCfgArgRawZ(argList, cfgOptPgPath, "/path/to/db");
strLstAddZ(argList, CFGCMD_CHECK);
TEST_ERROR(
cfgParseP(storageTest, strLstSize(argList), strLstPtr(argList), .noResetLogLevel = true), OptionInvalidError,
"option 'pg1-path' not valid without option 'stanza'");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("option invalid for command");
@ -1453,6 +1469,9 @@ testRun(void)
TEST_ERROR(
cfgParseP(storageTest, strLstSize(argList), strLstPtr(argList), .noResetLogLevel = true), OptionInvalidError,
"configuration file contains duplicate options ('db-path', 'pg1-path') in section '[db]'");
TEST_ERROR(
cfgParseP(storageTest, strLstSize(argList), strLstPtr(argList), .noResetLogLevel = true, .noConfigLoad = true),
OptionInvalidError, "configuration file contains duplicate options ('db-path', 'pg1-path') in section '[db]'");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("config file - option set multiple times");
@ -1711,6 +1730,7 @@ testRun(void)
"P00 WARN: configuration file contains invalid option 'backup-standb'\n"
"P00 WARN: configuration file contains stanza-only option 'pg1-path' in global section 'global'");
TEST_RESULT_STRLST_Z(cfgParseStanzaList(), "db\n", "stanza list");
TEST_RESULT_STR_Z(jsonFromVar(varNewVarLst(cfgCommandJobRetry())), "[0,33000,33000]", "custom job retries");
TEST_RESULT_BOOL(cfgOptionIdxTest(cfgOptPgHost, 0), false, "pg1-host is not set (command line reset override)");
TEST_RESULT_BOOL(cfgOptionIdxReset(cfgOptPgHost, 0), true, "pg1-host was reset");