db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2025-06-18 23:57:33 +02:00
Code Issues Releases Activity

The check command is implemented partly in C.

Browse Source 
Implement switch WAL and archive check in C but leave the rest in Perl for now.

The main idea was to have some real integration tests for the new database code so the rest of the migration can wait.

Reviewed by Cynthia Shang.
This commit is contained in:
David Steele
2019-08-01 20:35:01 -04:00
parent e4901d50d5
commit 3d3003e9ca
11 changed files with 248 additions and 145 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
doc/xml/release.xml
View File

@ -59,6 +59,14 @@
<p>The <cmd>local</cmd> command for <cmd>backup</cmd> is implemented entirely in C.</p>
</release-item>
<release-item>
<release-item-contributor-list>
<release-item-reviewer id="cynthia.shang"/>
</release-item-contributor-list>
<p>The <cmd>check</cmd> command is implemented partly in C.</p>
</release-item>
</release-improvement-list>
<release-development-list>

4
doc/xml/user-guide.xml
View File

@ -1089,7 +1089,7 @@
<execute user="postgres" output="y">
<exe-cmd>{[project-exe]} {[dash]}-stanza={[postgres-cluster-demo]} {[dash]}-log-level-console=info check</exe-cmd>
<exe-highlight> successfully stored in the archive at </exe-highlight>
<exe-highlight> successfully archived to </exe-highlight>
</execute>
</execute-list>
@ -2756,7 +2756,7 @@
<execute user="postgres" output="y">
<exe-cmd>{[project-exe]} {[dash]}-stanza={[postgres-cluster-demo]} {[dash]}-log-level-console=info check</exe-cmd>
<exe-highlight>all other checks passed</exe-highlight>
<exe-highlight>because no primary was found</exe-highlight>
</execute>
</execute-list>
</section>

47
lib/pgBackRest/Check/Check.pm
View File

@ -159,57 +159,14 @@ sub process
};
}
# If able to get the archive id then force archiving and check the arrival of the archived WAL file with the time specified
if ($iResult == 0 && !$oDb->isStandby())
{
$strWalSegment = $oDb->walSwitch();
eval
{
$strArchiveFile = walSegmentFind(storageRepo(), $strArchiveId, $strWalSegment, $iArchiveTimeout);
return true;
}
# If this is a backrest error then capture the code and message else confess
or do
{
# Capture error information
$iResult = exceptionCode($EVAL_ERROR);
$strResultMessage = exceptionMessage($EVAL_ERROR);
};
}
# Reset the console logging
logLevelSet(undef, cfgOption(CFGOPT_LOG_LEVEL_CONSOLE));
}
# If the archiving was successful and backup.info check did not error in an unexpected way, then indicate success
# Else, log the error.
if ($iResult == 0)
{
if (!$oDb->isStandby())
{
&log(INFO,
"WAL segment ${strWalSegment} successfully stored in the archive at '" .
storageRepo()->pathGet(STORAGE_REPO_ARCHIVE . "/$strArchiveId/${strArchiveFile}") . "'");
}
else
{
&log(INFO, 'switch ' . $oDb->walId() . ' cannot be performed on the standby, all other checks passed successfully');
}
}
else
{
# Log the captured error
&log(ERROR, $strResultMessage, $iResult);
# If a WAL switch was attempted, then alert the user that the WAL that did not reach the archive
if (defined($strWalSegment) && !defined($strArchiveFile))
if ($iResult != 0)
{
&log(WARN,
"WAL segment ${strWalSegment} did not reach the archive:" . (defined($strArchiveId) ? $strArchiveId : '') . "\n" .
"HINT: Check the archive_command to ensure that all options are correct (especially --stanza).\n" .
"HINT: Check the PostgreSQL server log for errors.");
}
&log(ERROR, $strResultMessage, $iResult);
}
# Return from function and log return values if any

32
lib/pgBackRest/Db.pm
View File

@ -769,38 +769,6 @@ sub lsnId
return $self->{strDbVersion} >= PG_VERSION_10 ? 'lsn' : 'location';
}
####################################################################################################################################
# walSwitch
#
# Forces a switch to the next transaction log in order to archive the current log.
####################################################################################################################################
sub walSwitch
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my $strOperation = logDebugParam(__PACKAGE__ . '->walSwitch');
# Create a restore point to ensure current WAL will be archived. For versions <= 9.0 activity will need to be generated by
# the user if there have been no writes since the last WAL switch.
if ($self->{strDbVersion} >= PG_VERSION_91)
{
$self->executeSql("select pg_create_restore_point('" . PROJECT_NAME . " Archive Check')::text;");
}
my $strWalFileName = $self->executeSqlOne(
'select pg_' . $self->walId() . 'file_name from pg_' . $self->walId() . 'file_name(pg_switch_' . $self->walId() . '());');
&log(INFO, "switch WAL ${strWalFileName}");
# Return from function and log return values if any
return logDebugReturn
(
$strOperation,
{name => 'strWalFileName', value => $strWalFileName}
);
}
####################################################################################################################################
# isStandby
#

6
src/Makefile.in
View File

@ -54,6 +54,7 @@ SRCS = \
command/backup/common.c \
command/backup/file.c \
command/backup/pageChecksum.c \
command/check/check.c \
command/backup/protocol.c \
command/expire/expire.c \
command/help/help.c \
@ -234,6 +235,9 @@ command/backup/pageChecksum.o: command/backup/pageChecksum.c build.auto.h comman
command/backup/protocol.o: command/backup/protocol.c build.auto.h command/backup/file.h command/backup/protocol.h common/assert.h common/crypto/common.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/io.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h protocol/server.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h
$(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c command/backup/protocol.c -o command/backup/protocol.o
command/check/check.o: command/check/check.c build.auto.h command/archive/common.h command/check/check.h common/assert.h common/crypto/common.h common/debug.h common/error.auto.h common/error.h common/ini.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h db/db.h db/helper.h info/info.h info/infoArchive.h info/infoPg.h postgres/client.h protocol/client.h protocol/command.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h
$(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c command/check/check.c -o command/check/check.o
command/command.o: command/command.c build.auto.h common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/http/client.h common/io/http/header.h common/io/http/query.h common/io/read.h common/io/tls/client.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h version.h
$(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c command/command.c -o command/command.o
@ -453,7 +457,7 @@ info/infoManifest.o: info/infoManifest.c build.auto.h common/error.auto.h common
info/infoPg.o: info/infoPg.c build.auto.h common/assert.h common/crypto/common.h common/debug.h common/error.auto.h common/error.h common/ini.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/log.h common/logLevel.h common/macro.h common/memContext.h common/object.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/json.h common/type/keyValue.h common/type/list.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h info/info.h info/infoPg.h postgres/interface.h postgres/version.h storage/helper.h storage/info.h storage/read.h storage/storage.h storage/write.h
$(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c info/infoPg.c -o info/infoPg.o
main.o: main.c build.auto.h command/archive/get/get.h command/archive/push/push.h command/command.h command/expire/expire.h command/help/help.h command/info/info.h command/local/local.h command/remote/remote.h command/storage/list.h common/assert.h common/debug.h common/error.auto.h common/error.h common/exit.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h config/load.h perl/exec.h postgres/interface.h version.h
main.o: main.c build.auto.h command/archive/get/get.h command/archive/push/push.h command/check/check.h command/command.h command/expire/expire.h command/help/help.h command/info/info.h command/local/local.h command/remote/remote.h command/storage/list.h common/assert.h common/debug.h common/error.auto.h common/error.h common/exit.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h config/load.h perl/exec.h postgres/interface.h version.h
$(CC) $(CPPFLAGS) $(CFLAGS) $(CMAKE) -c main.c -o main.o
perl/config.o: perl/config.c build.auto.h common/assert.h common/debug.h common/error.auto.h common/error.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/time.h common/type/buffer.h common/type/convert.h common/type/json.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h

77
src/command/check/check.c Normal file
View File

@ -0,0 +1,77 @@
/***********************************************************************************************************************************
Check Command
***********************************************************************************************************************************/
#include "build.auto.h"
#include "command/archive/common.h"
#include "command/check/check.h"
#include "common/debug.h"
#include "common/log.h"
#include "common/memContext.h"
#include "config/config.h"
#include "db/helper.h"
#include "info/infoArchive.h"
#include "storage/helper.h"
/***********************************************************************************************************************************
Perform standard checks
***********************************************************************************************************************************/
void
cmdCheck(void)
{
FUNCTION_LOG_VOID(logLevelDebug);
MEM_CONTEXT_TEMP_BEGIN()
{
// Get the repo storage in case it is remote and encryption settings need to be pulled down
storageRepo();
// Attempt to load the archive info file
InfoArchive *archiveInfo = infoArchiveNewLoad(
storageRepo(), INFO_ARCHIVE_PATH_FILE_STR, cipherType(cfgOptionStr(cfgOptRepoCipherType)),
cfgOptionStr(cfgOptRepoCipherPass));
const String *archiveId = infoArchiveId(archiveInfo);
// Get the primary/standby connections (standby is only required if backup from standby is enabled)
DbGetResult dbGroup = dbGet(false, false);
// Free the standby connection immediately since we don't need it for anything
dbFree(dbGroup.standby);
// Perform a WAL switch and make sure the WAL is archived if a primary was found
if (dbGroup.primary != NULL)
{
// Perform WAL switch
const String *walSegment = dbWalSwitch(dbGroup.primary);
dbFree(dbGroup.primary);
// Wait for the WAL to appear in the repo
TimeMSec archiveTimeout = (TimeMSec)(cfgOptionDbl(cfgOptArchiveTimeout) * MSEC_PER_SEC);
const String *walSegmentFile = walSegmentFind(storageRepo(), archiveId, walSegment, archiveTimeout);
if (walSegmentFile != NULL)
{
LOG_INFO(
"WAL segment %s successfully archived to '%s'", strPtr(walSegment),
strPtr(
storagePath(
storageRepo(), strNewFmt(STORAGE_REPO_ARCHIVE "/%s/%s", strPtr(archiveId), strPtr(walSegmentFile)))));
}
else
{
THROW_FMT(
ArchiveTimeoutError,
"WAL segment %s was not archived before the %" PRIu64 "ms timeout\n"
"HINT: Check the archive_command to ensure that all options are correct (especially --stanza).\n"
"HINT: Check the PostgreSQL server log for errors.",
strPtr(walSegment), archiveTimeout);
}
}
else
LOG_INFO("switch wal not performed because no primary was found");
}
MEM_CONTEXT_TEMP_END();
FUNCTION_LOG_RETURN_VOID();
}

12
src/command/check/check.h Normal file
View File

@ -0,0 +1,12 @@
/***********************************************************************************************************************************
Check Command
***********************************************************************************************************************************/
#ifndef COMMAND_CHECK_CHECK_H
#define COMMAND_CHECK_CHECK_H
/***********************************************************************************************************************************
Functions
***********************************************************************************************************************************/
void cmdCheck(void);
#endif

3
src/main.c
View File

@ -9,6 +9,7 @@ Main
#include "command/archive/get/get.h"
#include "command/archive/push/push.h"
#include "command/check/check.h"
#include "command/command.h"
#include "command/expire/expire.h"
#include "command/help/help.h"
@ -125,7 +126,9 @@ main(int argListSize, const char *argList[])
// -----------------------------------------------------------------------------------------------------------------
case cfgCmdCheck:
{
// Functionality is currently split between Perl and C
perlExec();
cmdCheck();
break;
}

67
src/perl/embed.auto.c
View File

@ -3326,52 +3326,12 @@ static const EmbeddedModule embeddedModule[] =
"};\n"
"}\n"
"\n\n"
"if ($iResult == 0 && !$oDb->isStandby())\n"
"{\n"
"$strWalSegment = $oDb->walSwitch();\n"
"\n"
"eval\n"
"{\n"
"$strArchiveFile = walSegmentFind(storageRepo(), $strArchiveId, $strWalSegment, $iArchiveTimeout);\n"
"return true;\n"
"}\n"
"\n"
"or do\n"
"{\n"
"\n"
"$iResult = exceptionCode($EVAL_ERROR);\n"
"$strResultMessage = exceptionMessage($EVAL_ERROR);\n"
"};\n"
"}\n"
"\n\n"
"logLevelSet(undef, cfgOption(CFGOPT_LOG_LEVEL_CONSOLE));\n"
"}\n"
"\n\n\n"
"if ($iResult == 0)\n"
"{\n"
"if (!$oDb->isStandby())\n"
"{\n"
"&log(INFO,\n"
"\"WAL segment ${strWalSegment} successfully stored in the archive at '\" .\n"
"storageRepo()->pathGet(STORAGE_REPO_ARCHIVE . \"/$strArchiveId/${strArchiveFile}\") . \"'\");\n"
"}\n"
"else\n"
"{\n"
"&log(INFO, 'switch ' . $oDb->walId() . ' cannot be performed on the standby, all other checks passed successfully');\n"
"}\n"
"}\n"
"else\n"
"{\n"
"\n"
"&log(ERROR, $strResultMessage, $iResult);\n"
"\n\n"
"if (defined($strWalSegment) && !defined($strArchiveFile))\n"
"if ($iResult != 0)\n"
"{\n"
"&log(WARN,\n"
"\"WAL segment ${strWalSegment} did not reach the archive:\" . (defined($strArchiveId) ? $strArchiveId : '') . \"\\n\" .\n"
"\"HINT: Check the archive_command to ensure that all options are correct (especially --stanza).\\n\" .\n"
"\"HINT: Check the PostgreSQL server log for errors.\");\n"
"}\n"
"&log(ERROR, $strResultMessage, $iResult);\n"
"}\n"
"\n\n"
"return logDebugReturn\n"
@ -7517,29 +7477,6 @@ static const EmbeddedModule embeddedModule[] =
"return $self->{strDbVersion} >= PG_VERSION_10 ? 'lsn' : 'location';\n"
"}\n"
"\n\n\n\n\n\n"
"sub walSwitch\n"
"{\n"
"my $self = shift;\n"
"\n\n"
"my $strOperation = logDebugParam(__PACKAGE__ . '->walSwitch');\n"
"\n\n\n"
"if ($self->{strDbVersion} >= PG_VERSION_91)\n"
"{\n"
"$self->executeSql(\"select pg_create_restore_point('\" . PROJECT_NAME . \" Archive Check')::text;\");\n"
"}\n"
"\n"
"my $strWalFileName = $self->executeSqlOne(\n"
"'select pg_' . $self->walId() . 'file_name from pg_' . $self->walId() . 'file_name(pg_switch_' . $self->walId() . '());');\n"
"\n"
"&log(INFO, \"switch WAL ${strWalFileName}\");\n"
"\n\n"
"return logDebugReturn\n"
"(\n"
"$strOperation,\n"
"{name => 'strWalFileName', value => $strWalFileName}\n"
");\n"
"}\n"
"\n\n\n\n\n\n"
"sub isStandby\n"
"{\n"
"my $self = shift;\n"

7
test/define.yaml
View File

@ -611,6 +611,13 @@ unit:
include:
- storage/storage
# ----------------------------------------------------------------------------------------------------------------------------
- name: check
total: 1
coverage:
command/check/check: full
# ----------------------------------------------------------------------------------------------------------------------------
- name: command
total: 1

130
test/src/module/command/checkTest.c Normal file
View File

@ -0,0 +1,130 @@
/***********************************************************************************************************************************
Test Check Command
***********************************************************************************************************************************/
#include "postgres/version.h"
#include "storage/helper.h"
#include "storage/storage.intern.h"
#include "common/harnessConfig.h"
#include "common/harnessInfo.h"
#include "common/harnessPq.h"
/***********************************************************************************************************************************
Test Run
***********************************************************************************************************************************/
void
testRun(void)
{
FUNCTION_HARNESS_VOID();
// *****************************************************************************************************************************
if (testBegin("cmdCheck()"))
{
String *pg1Path = strNewFmt("--pg1-path=%s/pg1", testPath());
StringList *argList = strLstNew();
strLstAddZ(argList, "pgbackrest");
strLstAddZ(argList, "--stanza=test1");
strLstAdd(argList, pg1Path);
strLstAdd(argList, strNewFmt("--repo1-path=%s/repo", testPath()));
strLstAddZ(argList, "--archive-timeout=.5");
strLstAddZ(argList, "check");
harnessCfgLoad(strLstSize(argList), strLstPtr(argList));
TEST_ERROR_FMT(
cmdCheck(), FileMissingError,
"unable to load info file '%s/repo/archive/test1/archive.info' or '%s/repo/archive/test1/archive.info.copy':\n"
"FileMissingError: " STORAGE_ERROR_READ_MISSING "\n"
"FileMissingError: " STORAGE_ERROR_READ_MISSING "\n"
"HINT: archive.info cannot be opened but is required to push/get WAL segments.\n"
"HINT: is archive_command configured correctly in postgresql.conf?\n"
"HINT: has a stanza-create been performed?\n"
"HINT: use --no-archive-check to disable archive checks during backup if you have an alternate archiving scheme.",
testPath(), testPath(), strPtr(strNewFmt("%s/repo/archive/test1/archive.info", testPath())),
strPtr(strNewFmt("%s/repo/archive/test1/archive.info.copy", testPath())));
// Create archive.info file
storagePutNP(
storageNewWriteNP(storageRepoWrite(), INFO_ARCHIVE_PATH_FILE_STR),
harnessInfoChecksum(
strNew(
"[db]\n"
"db-id=1\n"
"db-system-id=6569239123849665679\n"
"db-version=\"9.2\"\n"
"\n"
"[db:history]\n"
"1={\"db-id\":6569239123849665679,\"db-version\":\"9.2\"}\n")));
// Single primary
// -------------------------------------------------------------------------------------------------------------------------
// Error when WAL segment not found
harnessPqScriptSet((HarnessPq [])
{
HRNPQ_MACRO_OPEN_92(1, "dbname='postgres' port=5432", strPtr(pg1Path), false),
HRNPQ_MACRO_CREATE_RESTORE_POINT(1, "1/1"),
HRNPQ_MACRO_WAL_SWITCH(1, "xlog", "000000010000000100000001"),
HRNPQ_MACRO_CLOSE(1),
HRNPQ_MACRO_DONE()
});
TEST_ERROR(
cmdCheck(), ArchiveTimeoutError,
"WAL segment 000000010000000100000001 was not archived before the 500ms timeout\n"
"HINT: Check the archive_command to ensure that all options are correct (especially --stanza).\n"
"HINT: Check the PostgreSQL server log for errors.");
// Create WAL segment
Buffer *buffer = bufNew(16 * 1024 * 1024);
memset(bufPtr(buffer), 0, bufSize(buffer));
bufUsedSet(buffer, bufSize(buffer));
// WAL segment is found
harnessPqScriptSet((HarnessPq [])
{
HRNPQ_MACRO_OPEN_92(1, "dbname='postgres' port=5432", strPtr(pg1Path), false),
HRNPQ_MACRO_CREATE_RESTORE_POINT(1, "1/1"),
HRNPQ_MACRO_WAL_SWITCH(1, "xlog", "000000010000000100000001"),
HRNPQ_MACRO_CLOSE(1),
HRNPQ_MACRO_DONE()
});
storagePutNP(
storageNewWriteNP(
storageRepoWrite(),
strNew(STORAGE_REPO_ARCHIVE "/9.2-1/000000010000000100000001-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")),
buffer);
TEST_RESULT_VOID(cmdCheck(), "check");
harnessLogResult(
strPtr(
strNewFmt(
"P00 INFO: WAL segment 000000010000000100000001 successfully archived to '%s/repo/archive/test1/9.2-1/"
"0000000100000001/000000010000000100000001-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'",
testPath())));
// Single standby
// -------------------------------------------------------------------------------------------------------------------------
argList = strLstNew();
strLstAddZ(argList, "pgbackrest");
strLstAddZ(argList, "--stanza=test1");
strLstAdd(argList, pg1Path);
strLstAdd(argList, strNewFmt("--repo1-path=%s/repo", testPath()));
strLstAddZ(argList, "--archive-timeout=.5");
strLstAddZ(argList, "check");
harnessCfgLoad(strLstSize(argList), strLstPtr(argList));
// Set script
harnessPqScriptSet((HarnessPq [])
{
HRNPQ_MACRO_OPEN_92(1, "dbname='postgres' port=5432", strPtr(pg1Path), true),
HRNPQ_MACRO_CLOSE(1),
HRNPQ_MACRO_DONE()
});
TEST_RESULT_VOID(cmdCheck(), "check");
harnessLogResult("P00 INFO: switch wal not performed because no primary was found");
}
FUNCTION_HARNESS_RESULT_VOID();
}