db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2024-12-12 10:04:14 +02:00
Code Issues Releases Activity

Add timeout to walSegmentFind().

Browse Source 
Keep trying to locate the WAL segment until timeout.  This is useful for the check and backup commands which must wait for segments to arrive in the archive.
This commit is contained in:
David Steele 2019-07-31 19:58:57 -04:00
parent 03b28da1ca
commit 893ae24284
5 changed files with 63 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

49
src/command/archive/common.c
View File

@ -298,12 +298,13 @@ The file name can have several things appended such as a hash, compression exten
have multiple files that match the segment, though more than one match is not a good thing.
***********************************************************************************************************************************/
String *
walSegmentFind(const Storage *storage, const String *archiveId, const String *walSegment)
walSegmentFind(const Storage *storage, const String *archiveId, const String *walSegment, TimeMSec timeout)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(STORAGE, storage);
FUNCTION_LOG_PARAM(STRING, archiveId);
FUNCTION_LOG_PARAM(STRING, walSegment);
FUNCTION_LOG_PARAM(TIME_MSEC, timeout);
FUNCTION_LOG_END();
ASSERT(storage != NULL);
@ -315,30 +316,36 @@ walSegmentFind(const Storage *storage, const String *archiveId, const String *wa
MEM_CONTEXT_TEMP_BEGIN()
{
// Get a list of all WAL segments that match
StringList *list = storageListP(
storage, strNewFmt(STORAGE_REPO_ARCHIVE "/%s/%s", strPtr(archiveId), strPtr(strSubN(walSegment, 0, 16))),
.expression = strNewFmt("^%s%s-[0-f]{40}(\\.gz){0,1}$", strPtr(strSubN(walSegment, 0, 24)),
walIsPartial(walSegment) ? WAL_SEGMENT_PARTIAL_EXT : ""), .nullOnMissing = true);
Wait *wait = timeout > 0 ? waitNew(timeout) : NULL;
// If there are results
if (list != NULL && strLstSize(list) > 0)
do
{
// Error if there is more than one match
if (strLstSize(list) > 1)
{
THROW_FMT(
ArchiveDuplicateError,
"duplicates found in archive for WAL segment %s: %s\n"
"HINT: are multiple primaries archiving to this stanza?",
strPtr(walSegment), strPtr(strLstJoin(strLstSort(list, sortOrderAsc), ", ")));
}
// Get a list of all WAL segments that match
StringList *list = storageListP(
storage, strNewFmt(STORAGE_REPO_ARCHIVE "/%s/%s", strPtr(archiveId), strPtr(strSubN(walSegment, 0, 16))),
.expression = strNewFmt("^%s%s-[0-f]{40}(\\.gz){0,1}$", strPtr(strSubN(walSegment, 0, 24)),
walIsPartial(walSegment) ? WAL_SEGMENT_PARTIAL_EXT : ""), .nullOnMissing = true);
// Copy file name of WAL segment found into the calling context
memContextSwitch(MEM_CONTEXT_OLD());
result = strDup(strLstGet(list, 0));
memContextSwitch(MEM_CONTEXT_TEMP());
// If there are results
if (list != NULL && strLstSize(list) > 0)
{
// Error if there is more than one match
if (strLstSize(list) > 1)
{
THROW_FMT(
ArchiveDuplicateError,
"duplicates found in archive for WAL segment %s: %s\n"
"HINT: are multiple primaries archiving to this stanza?",
strPtr(walSegment), strPtr(strLstJoin(strLstSort(list, sortOrderAsc), ", ")));
}
// Copy file name of WAL segment found into the calling context
memContextSwitch(MEM_CONTEXT_OLD());
result = strDup(strLstGet(list, 0));
memContextSwitch(MEM_CONTEXT_TEMP());
}
}
while (result == NULL && wait != NULL && waitMore(wait));
}
MEM_CONTEXT_TEMP_END();

2
src/command/archive/common.h
View File

@ -65,7 +65,7 @@ void archiveAsyncStatusErrorWrite(ArchiveMode archiveMode, const String *walSegm
bool walIsPartial(const String *walSegment);
bool walIsSegment(const String *walSegment);
String *walPath(const String *walFile, const String *pgPath, const String *command);
String *walSegmentFind(const Storage *storage, const String *archiveId, const String *walSegment);
String *walSegmentFind(const Storage *storage, const String *archiveId, const String *walSegment, TimeMSec timeout);
String *walSegmentNext(const String *walSegment, size_t walSegmentSize, unsigned int pgVersion);
StringList *walSegmentRange(const String *walSegmentBegin, size_t walSegmentSize, unsigned int pgVersion, unsigned int range);

2
src/command/archive/get/file.c
View File

@ -68,7 +68,7 @@ archiveGetCheck(const String *archiveFile, CipherType cipherType, const String *
// If a WAL segment search among the possible file names
if (walIsSegment(archiveFile))
{
String *walSegmentFile = walSegmentFind(storageRepo(), archiveId, archiveFile);
String *walSegmentFile = walSegmentFind(storageRepo(), archiveId, archiveFile, 0);
if (walSegmentFile != NULL)
{

2
src/command/archive/push/file.c
View File

@ -80,7 +80,7 @@ archivePushFile(
const String *walSegmentChecksum = varStr(ioFilterGroupResult(ioReadFilterGroup(read), CRYPTO_HASH_FILTER_TYPE_STR));
// If the wal segment already exists in the repo then compare checksums
walSegmentFile = walSegmentFind(storageRepo(), archiveId, archiveFile);
walSegmentFile = walSegmentFind(storageRepo(), archiveId, archiveFile, 0);
if (walSegmentFile != NULL)
{

44
test/src/module/command/archiveCommonTest.c
View File

@ -7,6 +7,7 @@ Test Archive Common
#include "storage/posix/storage.h"
#include "common/harnessConfig.h"
#include "common/harnessFork.h"
/***********************************************************************************************************************************
Test Run
@ -196,20 +197,39 @@ testRun(void)
strLstAddZ(argList, "archive-get");
harnessCfgLoad(strLstSize(argList), strLstPtr(argList));
TEST_RESULT_PTR(walSegmentFind(storageRepo(), strNew("9.6-2"), strNew("123456781234567812345678")), NULL, "no path");
TEST_RESULT_PTR(walSegmentFind(storageRepo(), strNew("9.6-2"), strNew("123456781234567812345678"), 0), NULL, "no path");
storagePathCreateNP(storageTest, strNew("archive/db/9.6-2/1234567812345678"));
TEST_RESULT_PTR(walSegmentFind(storageRepo(), strNew("9.6-2"), strNew("123456781234567812345678")), NULL, "no segment");
TEST_RESULT_PTR(walSegmentFind(storageRepo(), strNew("9.6-2"), strNew("123456781234567812345678"), 0), NULL, "no segment");
TEST_RESULT_PTR(
walSegmentFind(storageRepo(), strNew("9.6-2"), strNew("123456781234567812345678"), 500), NULL,
"no segment after 500ms");
storagePutNP(
storageNewWriteNP(
storageTest,
strNew("archive/db/9.6-2/1234567812345678/123456781234567812345678-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")),
NULL);
// Check timeout by making the wal segment appear after 250ms
HARNESS_FORK_BEGIN()
{
HARNESS_FORK_CHILD_BEGIN(0, false)
{
sleepMSec(250);
TEST_RESULT_STR(
strPtr(walSegmentFind(storageRepo(), strNew("9.6-2"), strNew("123456781234567812345678"))),
"123456781234567812345678-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "found segment");
storagePutNP(
storageNewWriteNP(
storageTest,
strNew(
"archive/db/9.6-2/1234567812345678/123456781234567812345678-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")),
NULL);
}
HARNESS_FORK_CHILD_END();
HARNESS_FORK_PARENT_BEGIN()
{
TEST_RESULT_STR(
strPtr(walSegmentFind(storageRepo(), strNew("9.6-2"), strNew("123456781234567812345678"), 1000)),
"123456781234567812345678-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "found segment");
}
HARNESS_FORK_PARENT_END();
}
HARNESS_FORK_END();
storagePutNP(
storageNewWriteNP(
@ -218,7 +238,7 @@ testRun(void)
NULL);
TEST_ERROR(
walSegmentFind(storageRepo(), strNew("9.6-2"), strNew("123456781234567812345678")),
walSegmentFind(storageRepo(), strNew("9.6-2"), strNew("123456781234567812345678"), 0),
ArchiveDuplicateError,
"duplicates found in archive for WAL segment 123456781234567812345678:"
" 123456781234567812345678-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
@ -226,7 +246,7 @@ testRun(void)
"\nHINT: are multiple primaries archiving to this stanza?");
TEST_RESULT_STR(
walSegmentFind(storageRepo(), strNew("9.6-2"), strNew("123456781234567812345678.partial")), NULL,
walSegmentFind(storageRepo(), strNew("9.6-2"), strNew("123456781234567812345678.partial"), 0), NULL,
"did not find partial segment");
}