db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2025-07-17 01:12:23 +02:00
Code Issues Releases Activity

Split cfgLoad() into multiple functions to make testing easier.

Browse Source 
Mainly this helps with unit tests that need to do log expect testing. Add harnessCfgLoad() test function, which allows a new config to be loaded for unit testing without resetting log functions, opening a log file, or taking locks.
This commit is contained in:
David Steele
2018-04-13 16:05:52 -04:00
parent 49fc737cd0
commit 2a4ff2f904
11 changed files with 221 additions and 176 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
doc/xml/release.xml
View File

@ -47,6 +47,10 @@
<p>Storage object improvements. Convert all functions to variadic functions. Enforce read-only storage. Add <code>storageLocalWrite()</code> helper function. Add <code>storageExists()</code>, <code>storagePathCreate()</code>, and <code>storageRemove()</code>. Add <code>StorageFile</code> object and <code>storageOpenRead()</code>/<code>storageOpenWrite()</code>. Abstract Posix driver code into a separate module. Add <code>storagePathRemove()</code> and use it in the Perl Posix driver.</p>
</release-item>
<release-item>
<p>Split <code>cfgLoad()</code> into multiple functions to make testing easier. Mainly this helps with unit tests that need to do log expect testing.</p>
</release-item>
<release-item>
<p>Allow <code>MemContext</code> objects to be copied to a new parent. This makes it easier to create objects and then copy them to another context when they are complete without having to worry about freeing them on error. Update <code>List</code>, <code>StringList</code>, and <code>Buffer</code> to allow moves. Update <code>Ini</code> and <code>Storage</code> to take advantage of moves.</p>
</release-item>
@ -128,6 +132,10 @@
<p>Move <id>archive-stop</id> and <id>expire</id> tests to the <id>mock</id> module. These are mock integration tests so they should be grouped with the other mock integration tests.</p>
</release-item>
<release-item>
<p>Add <code>harnessCfgLoad()</code> test function, which allows a new config to be loaded for unit testing without resetting log functions, opening a log file, or taking locks.</p>
</release-item>
<release-item>
<p>Include VM type in <id>gcov</id> path to avoid conflicts between VMs with different architectures.</p>
</release-item>

1
libc/LibC.xs
View File

@ -47,6 +47,7 @@ These includes are from the src directory. There is no Perl-specific code in th
#include "config/config.h"
#include "config/define.h"
#include "config/load.h"
#include "config/parse.h"
#include "perl/config.h"
#include "postgres/pageChecksum.h"
#include "storage/driver/posix.h"

6
libc/xs/config/configTest.xs
View File

@ -19,7 +19,11 @@ CODE:
// This should run in a temp context but for some reason getopt_long gets upset when if gets called again after the previous
// arg list being freed. So, this is a memory leak but it is only used for testing, not production.
StringList *paramList = strLstNewSplitZ(strCat(strNew("pgbackrest|"), parseParam), "|");
cfgLoadParam(strLstSize(paramList), strLstPtr(paramList), strNew(backrestBin));
// Don't use cfgLoad() because it has a lot of side effects that we don't want
configParse(strLstSize(paramList), strLstPtr(paramList));
cfgExeSet(strNew(backrestBin));
cfgLoadUpdateOption();
String *result = perlOptionJson();

315
src/config/load.c
View File

@ -45,17 +45,158 @@ cfgLoadLogSetting()
logInit(logLevelConsole, logLevelStdErr, logLevelFile, logTimestamp);
}
/***********************************************************************************************************************************
Update options that have complex rules
***********************************************************************************************************************************/
void
cfgLoadUpdateOption()
{
// Set default for repo-host-cmd
if (cfgOptionValid(cfgOptRepoHost) && cfgOptionTest(cfgOptRepoHost) &&
cfgOptionSource(cfgOptRepoHostCmd) == cfgSourceDefault)
{
cfgOptionDefaultSet(cfgOptRepoHostCmd, varNewStr(cfgExe()));
}
// Set default for pg-host-cmd
if (cfgOptionValid(cfgOptPgHostCmd))
{
for (unsigned int optionIdx = 0; optionIdx < cfgOptionIndexTotal(cfgOptPgHost); optionIdx++)
{
if (cfgOptionTest(cfgOptPgHost + optionIdx) && cfgOptionSource(cfgOptPgHostCmd + optionIdx) == cfgSourceDefault)
cfgOptionDefaultSet(cfgOptPgHostCmd + optionIdx, varNewStr(cfgExe()));
}
}
// Protocol timeout should be greater than db timeout
if (cfgOptionTest(cfgOptDbTimeout) && cfgOptionTest(cfgOptProtocolTimeout) &&
cfgOptionDbl(cfgOptProtocolTimeout) <= cfgOptionDbl(cfgOptDbTimeout))
{
// If protocol-timeout is default then increase it to be greater than db-timeout
if (cfgOptionSource(cfgOptProtocolTimeout) == cfgSourceDefault)
cfgOptionSet(cfgOptProtocolTimeout, cfgSourceDefault, varNewDbl(cfgOptionDbl(cfgOptDbTimeout) + 30));
else
{
THROW(OptionInvalidValueError,
"'%f' is not valid for '%s' option\nHINT '%s' option (%f) should be greater than '%s' option (%f).",
cfgOptionDbl(cfgOptProtocolTimeout), cfgOptionName(cfgOptProtocolTimeout),
cfgOptionName(cfgOptProtocolTimeout), cfgOptionDbl(cfgOptProtocolTimeout), cfgOptionName(cfgOptDbTimeout),
cfgOptionDbl(cfgOptDbTimeout));
}
}
// Make sure that repo and pg host settings are not both set - cannot both be remote
if (cfgOptionValid(cfgOptPgHost) && cfgOptionValid(cfgOptRepoHost))
{
bool pgHostFound = false;
for (unsigned int optionIdx = 0; optionIdx < cfgOptionIndexTotal(cfgOptPgHost); optionIdx++)
{
if (cfgOptionTest(cfgOptPgHost + optionIdx))
{
pgHostFound = true;
break;
}
}
// If a pg-host was found, see if a repo-host is configured
if (pgHostFound == true)
{
for (unsigned int optionIdx = 0; optionIdx < cfgOptionIndexTotal(cfgOptRepoHost); optionIdx++)
{
if (cfgOptionTest(cfgOptRepoHost + optionIdx))
THROW(ConfigError, "pg and repo hosts cannot both be configured as remote");
}
}
}
// Warn when repo-retention-full is not set on a configured repo
if (cfgOptionValid(cfgOptRepoRetentionFull))
{
for (unsigned int optionIdx = 0; optionIdx < cfgOptionIndexTotal(cfgOptRepoType); optionIdx++)
{
// If the repo-type is defined, then see if corresponding retention-full is set
if (cfgOptionTest(cfgOptRepoType + optionIdx) && !cfgOptionTest(cfgOptRepoRetentionFull + optionIdx))
{
LOG_WARN("option %s is not set, the repository may run out of space\n"
"HINT: to retain full backups indefinitely (without warning), set option '%s' to the maximum.",
cfgOptionName(cfgOptRepoRetentionFull + optionIdx),
cfgOptionName(cfgOptRepoRetentionFull + optionIdx));
}
}
}
// If archive retention is valid for the command, then set archive settings
if (cfgOptionValid(cfgOptRepoRetentionArchive))
{
// For each possible repo, check and adjust the settings as appropriate
for (unsigned int optionIdx = 0; optionIdx < cfgOptionIndexTotal(cfgOptRepoType); optionIdx++)
{
const String *archiveRetentionType = cfgOptionStr(cfgOptRepoRetentionArchiveType + optionIdx);
const String *msgArchiveOff = strNewFmt("WAL segments will not be expired: option '%s=%s' but",
cfgOptionName(cfgOptRepoRetentionArchiveType), strPtr(archiveRetentionType));
// If the archive retention is not explicitly set then determine what it should be defaulted to
// to.
if (!cfgOptionTest(cfgOptRepoRetentionArchive + optionIdx))
{
// If repo-retention-archive-type is default, then if repo-retention-full is set, set the repo-retention-archive
// to this value, else ignore archiving
if (strEqZ(archiveRetentionType, CFGOPTVAL_TMP_REPO_RETENTION_ARCHIVE_TYPE_FULL))
{
if (cfgOptionTest(cfgOptRepoRetentionFull + optionIdx))
{
cfgOptionSet(cfgOptRepoRetentionArchive + optionIdx, cfgSourceDefault,
varNewInt(cfgOptionInt(cfgOptRepoRetentionFull + optionIdx)));
}
}
else if (strEqZ(archiveRetentionType, CFGOPTVAL_TMP_REPO_RETENTION_ARCHIVE_TYPE_DIFF))
{
// if repo-retention-diff is set then user must have set it
if (cfgOptionTest(cfgOptRepoRetentionDiff + optionIdx))
{
cfgOptionSet(cfgOptRepoRetentionArchive + optionIdx, cfgSourceDefault,
varNewInt(cfgOptionInt(cfgOptRepoRetentionDiff + optionIdx)));
}
else
{
LOG_WARN("%s neither option '%s' nor option '%s' is set", strPtr(msgArchiveOff),
cfgOptionName(cfgOptRepoRetentionArchive + optionIdx),
cfgOptionName(cfgOptRepoRetentionDiff + optionIdx));
}
}
else if (strEqZ(archiveRetentionType, CFGOPTVAL_TMP_REPO_RETENTION_ARCHIVE_TYPE_INCR))
{
LOG_WARN("%s option '%s' is not set", strPtr(msgArchiveOff),
cfgOptionName(cfgOptRepoRetentionArchive + optionIdx));
}
}
else
{
// If repo-retention-archive is set then check repo-retention-archive-type and issue a warning if the
// corresponding setting is UNDEF since UNDEF means backups will not be expired but they should be in the
// practice of setting this value even though expiring the archive itself is OK and will be performed.
if ((strEqZ(archiveRetentionType, CFGOPTVAL_TMP_REPO_RETENTION_ARCHIVE_TYPE_DIFF)) &&
(!cfgOptionTest(cfgOptRepoRetentionDiff + optionIdx)))
{
LOG_WARN("option '%s' is not set for '%s=%s'\n"
"HINT: to retain differential backups indefinitely (without warning), set option '%s' to the maximum.",
cfgOptionName(cfgOptRepoRetentionDiff + optionIdx),
cfgOptionName(cfgOptRepoRetentionArchiveType + optionIdx),
CFGOPTVAL_TMP_REPO_RETENTION_ARCHIVE_TYPE_DIFF,
cfgOptionName(cfgOptRepoRetentionDiff + optionIdx));
}
}
}
}
}
/***********************************************************************************************************************************
Load the configuration
***********************************************************************************************************************************/
void
cfgLoad(unsigned int argListSize, const char *argList[])
{
cfgLoadParam(argListSize, argList, NULL);
}
void
cfgLoadParam(unsigned int argListSize, const char *argList[], String *exe)
{
MEM_CONTEXT_TEMP_BEGIN()
{
@ -65,9 +206,17 @@ cfgLoadParam(unsigned int argListSize, const char *argList[], String *exe)
// Load the log settings
cfgLoadLogSetting();
// Only continue if a command was set. If no command is set then help will be displayed
// Neutralize the umask to make the repository file/path modes more consistent
if (cfgOptionValid(cfgOptNeutralUmask) && cfgOptionBool(cfgOptNeutralUmask))
umask(0000);
// If a command is set
if (cfgCommand() != cfgCmdNone)
{
// Acquire a lock if required
if (cfgLockRequired())
lockAcquire(cfgOptionStr(cfgOptLockPath), cfgOptionStr(cfgOptStanza), cfgLockType(), 0, true);
// Open the log file if this command logs to a file
if (cfgLogFile())
{
@ -79,156 +228,8 @@ cfgLoadParam(unsigned int argListSize, const char *argList[], String *exe)
// Begin the command
cmdBegin(true);
// If an exe was passed in then use it
if (exe != NULL)
cfgExeSet(exe);
// Set default for repo-host-cmd
if (cfgOptionValid(cfgOptRepoHost) && cfgOptionTest(cfgOptRepoHost) &&
cfgOptionSource(cfgOptRepoHostCmd) == cfgSourceDefault)
{
cfgOptionDefaultSet(cfgOptRepoHostCmd, varNewStr(cfgExe()));
}
// Set default for pg-host-cmd
if (cfgOptionValid(cfgOptPgHostCmd))
{
for (unsigned int optionIdx = 0; optionIdx < cfgOptionIndexTotal(cfgOptPgHost); optionIdx++)
{
if (cfgOptionTest(cfgOptPgHost + optionIdx) && cfgOptionSource(cfgOptPgHostCmd + optionIdx) == cfgSourceDefault)
cfgOptionDefaultSet(cfgOptPgHostCmd + optionIdx, varNewStr(cfgExe()));
}
}
if (cfgLockRequired())
lockAcquire(cfgOptionStr(cfgOptLockPath), cfgOptionStr(cfgOptStanza), cfgLockType(), 0, true);
}
// Neutralize the umask to make the repository file/path modes more consistent
if (cfgOptionValid(cfgOptNeutralUmask) && cfgOptionBool(cfgOptNeutralUmask))
umask(0000);
// Protocol timeout should be greater than db timeout
if (cfgOptionTest(cfgOptDbTimeout) && cfgOptionTest(cfgOptProtocolTimeout) &&
cfgOptionDbl(cfgOptProtocolTimeout) <= cfgOptionDbl(cfgOptDbTimeout))
{
// If protocol-timeout is default then increase it to be greater than db-timeout
if (cfgOptionSource(cfgOptProtocolTimeout) == cfgSourceDefault)
cfgOptionSet(cfgOptProtocolTimeout, cfgSourceDefault, varNewDbl(cfgOptionDbl(cfgOptDbTimeout) + 30));
else
{
THROW(OptionInvalidValueError,
"'%f' is not valid for '%s' option\nHINT '%s' option (%f) should be greater than '%s' option (%f).",
cfgOptionDbl(cfgOptProtocolTimeout), cfgOptionName(cfgOptProtocolTimeout),
cfgOptionName(cfgOptProtocolTimeout), cfgOptionDbl(cfgOptProtocolTimeout), cfgOptionName(cfgOptDbTimeout),
cfgOptionDbl(cfgOptDbTimeout));
}
}
// Make sure that repo and pg host settings are not both set - cannot both be remote
if (cfgOptionValid(cfgOptPgHost) && cfgOptionValid(cfgOptRepoHost))
{
bool pgHostFound = false;
for (unsigned int optionIdx = 0; optionIdx < cfgOptionIndexTotal(cfgOptPgHost); optionIdx++)
{
if (cfgOptionTest(cfgOptPgHost + optionIdx))
{
pgHostFound = true;
break;
}
}
// If a pg-host was found, see if a repo-host is configured
if (pgHostFound == true)
{
for (unsigned int optionIdx = 0; optionIdx < cfgOptionIndexTotal(cfgOptRepoHost); optionIdx++)
{
if (cfgOptionTest(cfgOptRepoHost + optionIdx))
THROW(ConfigError, "pg and repo hosts cannot both be configured as remote");
}
}
}
// Warn when repo-retention-full is not set on a configured repo
if (cfgOptionValid(cfgOptRepoRetentionFull))
{
for (unsigned int optionIdx = 0; optionIdx < cfgOptionIndexTotal(cfgOptRepoType); optionIdx++)
{
// If the repo-type is defined, then see if corresponding retention-full is set
if (cfgOptionTest(cfgOptRepoType + optionIdx) && !cfgOptionTest(cfgOptRepoRetentionFull + optionIdx))
{
LOG_WARN("option %s is not set, the repository may run out of space\n"
"HINT: to retain full backups indefinitely (without warning), set option '%s' to the maximum.",
cfgOptionName(cfgOptRepoRetentionFull + optionIdx),
cfgOptionName(cfgOptRepoRetentionFull + optionIdx));
}
}
}
// If archive retention is valid for the command, then set archive settings
if (cfgOptionValid(cfgOptRepoRetentionArchive))
{
// For each possible repo, check and adjust the settings as appropriate
for (unsigned int optionIdx = 0; optionIdx < cfgOptionIndexTotal(cfgOptRepoType); optionIdx++)
{
const String *archiveRetentionType = cfgOptionStr(cfgOptRepoRetentionArchiveType + optionIdx);
const String *msgArchiveOff = strNewFmt("WAL segments will not be expired: option '%s=%s' but",
cfgOptionName(cfgOptRepoRetentionArchiveType), strPtr(archiveRetentionType));
// If the archive retention is not explicitly set then determine what it should be defaulted to
// to.
if (!cfgOptionTest(cfgOptRepoRetentionArchive + optionIdx))
{
// If repo-retention-archive-type is default, then if repo-retention-full is set, set the repo-retention-archive
// to this value, else ignore archiving
if (strEqZ(archiveRetentionType, CFGOPTVAL_TMP_REPO_RETENTION_ARCHIVE_TYPE_FULL))
{
if (cfgOptionTest(cfgOptRepoRetentionFull + optionIdx))
{
cfgOptionSet(cfgOptRepoRetentionArchive + optionIdx, cfgSourceDefault,
varNewInt(cfgOptionInt(cfgOptRepoRetentionFull + optionIdx)));
}
}
else if (strEqZ(archiveRetentionType, CFGOPTVAL_TMP_REPO_RETENTION_ARCHIVE_TYPE_DIFF))
{
// if repo-retention-diff is set then user must have set it
if (cfgOptionTest(cfgOptRepoRetentionDiff + optionIdx))
{
cfgOptionSet(cfgOptRepoRetentionArchive + optionIdx, cfgSourceDefault,
varNewInt(cfgOptionInt(cfgOptRepoRetentionDiff + optionIdx)));
}
else
{
LOG_WARN("%s neither option '%s' nor option '%s' is set", strPtr(msgArchiveOff),
cfgOptionName(cfgOptRepoRetentionArchive + optionIdx),
cfgOptionName(cfgOptRepoRetentionDiff + optionIdx));
}
}
else if (strEqZ(archiveRetentionType, CFGOPTVAL_TMP_REPO_RETENTION_ARCHIVE_TYPE_INCR))
{
LOG_WARN("%s option '%s' is not set", strPtr(msgArchiveOff),
cfgOptionName(cfgOptRepoRetentionArchive + optionIdx));
}
}
else
{
// If repo-retention-archive is set then check repo-retention-archive-type and issue a warning if the
// corresponding setting is UNDEF since UNDEF means backups will not be expired but they should be in the
// practice of setting this value even though expiring the archive itself is OK and will be performed.
if ((strEqZ(archiveRetentionType, CFGOPTVAL_TMP_REPO_RETENTION_ARCHIVE_TYPE_DIFF)) &&
(!cfgOptionTest(cfgOptRepoRetentionDiff + optionIdx)))
{
LOG_WARN("option '%s' is not set for '%s=%s'\n"
"HINT: to retain differential backups indefinitely (without warning), set option '%s' to the maximum.",
cfgOptionName(cfgOptRepoRetentionDiff + optionIdx),
cfgOptionName(cfgOptRepoRetentionArchiveType + optionIdx),
CFGOPTVAL_TMP_REPO_RETENTION_ARCHIVE_TYPE_DIFF,
cfgOptionName(cfgOptRepoRetentionDiff + optionIdx));
}
}
}
// Update options that have complex rules
cfgLoadUpdateOption();
}
}
MEM_CONTEXT_TEMP_END();

2
src/config/load.h
View File

@ -8,7 +8,7 @@ Configuration Load
Functions
***********************************************************************************************************************************/
void cfgLoad(unsigned int argListSize, const char *argList[]);
void cfgLoadParam(unsigned int argListSize, const char *argList[], String *exe);
void cfgLoadLogSetting();
void cfgLoadUpdateOption();
#endif

3
test/lib/pgBackRestTest/Env/ConfigEnvTest.pm
View File

@ -186,9 +186,6 @@ sub configTestLoad
$self->testResult(
sub {configLoad(false, backrestBin(), cfgCommandName($iCommandId), \$strConfigJson)},
true, 'config load: ' . join(" ", @stryArg));
# Release any lock that was taken during configLoad.
lockRelease(false);
}
1;

21
test/src/common/harnessConfig.c Normal file
View File

@ -0,0 +1,21 @@
/***********************************************************************************************************************************
Harness for Loading Test Configurations
***********************************************************************************************************************************/
#include "common/logTest.h"
#include "config/load.h"
#include "config/parse.h"
/***********************************************************************************************************************************
Load a test configuration without any side effects
Log testing requires that log levels be set in a certain way but calls to cfgLoad() will reset that. Also there's no need to open
log files, acquire locks, etc.
***********************************************************************************************************************************/
void
harnessCfgLoad(unsigned int argListSize, const char *argList[])
{
configParse(argListSize, argList);
logInit(logLevelInfo, logLevelOff, logLevelOff, false);
cfgLoadUpdateOption();
}

8
test/src/common/harnessConfig.h Normal file
View File

@ -0,0 +1,8 @@
/***********************************************************************************************************************************
Harness for Loading Test Configurations
***********************************************************************************************************************************/
/***********************************************************************************************************************************
Functions
***********************************************************************************************************************************/
void harnessCfgLoad(unsigned int argListSize, const char *argList[]);

20
test/src/common/logTest.c
View File

@ -11,6 +11,11 @@ Log Test Harness
#ifndef NO_LOG
/***********************************************************************************************************************************
Has the log harness been init'd?
***********************************************************************************************************************************/
static bool harnessLogInit = false;
/***********************************************************************************************************************************
Name of file where logs are stored for testing
***********************************************************************************************************************************/
@ -23,13 +28,18 @@ Initialize log for testing
void
testLogInit()
{
logInit(logLevelInfo, logLevelOff, logLevelOff, false);
if (!harnessLogInit)
{
logInit(logLevelInfo, logLevelOff, logLevelOff, false);
stdoutFile = strNewFmt("%s/stdout.log", testPath());
logHandleStdOut = open(strPtr(stdoutFile), O_WRONLY | O_CREAT | O_TRUNC, 0640);
stdoutFile = strNewFmt("%s/stdout.log", testPath());
logHandleStdOut = open(strPtr(stdoutFile), O_WRONLY | O_CREAT | O_TRUNC, 0640);
stderrFile = strNewFmt("%s/stderr.log", testPath());
logHandleStdErr = open(strPtr(stderrFile), O_WRONLY | O_CREAT | O_TRUNC, 0640);
stderrFile = strNewFmt("%s/stderr.log", testPath());
logHandleStdErr = open(strPtr(stderrFile), O_WRONLY | O_CREAT | O_TRUNC, 0640);
harnessLogInit = true;
}
}
/***********************************************************************************************************************************

8
test/src/module/archive/pushTest.c
View File

@ -6,6 +6,8 @@ Test Archive Push Command
#include "config/load.h"
#include "version.h"
#include "common/harnessConfig.h"
/***********************************************************************************************************************************
Test Run
***********************************************************************************************************************************/
@ -22,8 +24,7 @@ testRun()
strLstAddZ(argList, "--archive-timeout=1");
strLstAddZ(argList, "--stanza=db");
strLstAddZ(argList, "archive-push");
cfgLoad(strLstSize(argList), strLstPtr(argList));
logInit(logLevelInfo, logLevelOff, logLevelOff, false);
harnessCfgLoad(strLstSize(argList), strLstPtr(argList));
// -------------------------------------------------------------------------------------------------------------------------
String *segment = strNew("000000010000000100000001");
@ -114,8 +115,7 @@ testRun()
strLstAddZ(argList, "--log-level-console=off");
strLstAddZ(argList, "--log-level-stderr=off");
strLstAdd(argList, strNewFmt("--pg1-path=%s/db", testPath()));
cfgLoad(strLstSize(argList), strLstPtr(argList));
logInit(logLevelInfo, logLevelOff, logLevelOff, false);
harnessCfgLoad(strLstSize(argList), strLstPtr(argList));
TEST_ERROR(
cmdArchivePush(), ArchiveTimeoutError,

5
test/src/module/config/loadTest.c
View File

@ -42,11 +42,6 @@ testRun()
TEST_RESULT_INT(logLevelStdErr, logLevelError, "stderr logging is error");
TEST_RESULT_INT(logLevelFile, logLevelOff, "file logging is off");
// -------------------------------------------------------------------------------------------------------------------------
TEST_RESULT_VOID(cfgLoadParam(strLstSize(argList), strLstPtr(argList), strNew("pgbackrest2")), "load local config");
TEST_RESULT_STR(strPtr(cfgExe()), "pgbackrest2", "check exe");
// -------------------------------------------------------------------------------------------------------------------------
argList = strLstNew();
strLstAdd(argList, strNew("pgbackrest"));