1
0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2025-07-15 01:04:37 +02:00

Add timeout to walSegmentFind().

Keep trying to locate the WAL segment until timeout.  This is useful for the check and backup commands which must wait for segments to arrive in the archive.
This commit is contained in:
David Steele
2019-07-31 19:58:57 -04:00
parent 03b28da1ca
commit 893ae24284
5 changed files with 63 additions and 36 deletions

View File

@ -298,12 +298,13 @@ The file name can have several things appended such as a hash, compression exten
have multiple files that match the segment, though more than one match is not a good thing. have multiple files that match the segment, though more than one match is not a good thing.
***********************************************************************************************************************************/ ***********************************************************************************************************************************/
String * String *
walSegmentFind(const Storage *storage, const String *archiveId, const String *walSegment) walSegmentFind(const Storage *storage, const String *archiveId, const String *walSegment, TimeMSec timeout)
{ {
FUNCTION_LOG_BEGIN(logLevelDebug); FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(STORAGE, storage); FUNCTION_LOG_PARAM(STORAGE, storage);
FUNCTION_LOG_PARAM(STRING, archiveId); FUNCTION_LOG_PARAM(STRING, archiveId);
FUNCTION_LOG_PARAM(STRING, walSegment); FUNCTION_LOG_PARAM(STRING, walSegment);
FUNCTION_LOG_PARAM(TIME_MSEC, timeout);
FUNCTION_LOG_END(); FUNCTION_LOG_END();
ASSERT(storage != NULL); ASSERT(storage != NULL);
@ -314,6 +315,10 @@ walSegmentFind(const Storage *storage, const String *archiveId, const String *wa
String *result = NULL; String *result = NULL;
MEM_CONTEXT_TEMP_BEGIN() MEM_CONTEXT_TEMP_BEGIN()
{
Wait *wait = timeout > 0 ? waitNew(timeout) : NULL;
do
{ {
// Get a list of all WAL segments that match // Get a list of all WAL segments that match
StringList *list = storageListP( StringList *list = storageListP(
@ -340,6 +345,8 @@ walSegmentFind(const Storage *storage, const String *archiveId, const String *wa
memContextSwitch(MEM_CONTEXT_TEMP()); memContextSwitch(MEM_CONTEXT_TEMP());
} }
} }
while (result == NULL && wait != NULL && waitMore(wait));
}
MEM_CONTEXT_TEMP_END(); MEM_CONTEXT_TEMP_END();
FUNCTION_LOG_RETURN(STRING, result); FUNCTION_LOG_RETURN(STRING, result);

View File

@ -65,7 +65,7 @@ void archiveAsyncStatusErrorWrite(ArchiveMode archiveMode, const String *walSegm
bool walIsPartial(const String *walSegment); bool walIsPartial(const String *walSegment);
bool walIsSegment(const String *walSegment); bool walIsSegment(const String *walSegment);
String *walPath(const String *walFile, const String *pgPath, const String *command); String *walPath(const String *walFile, const String *pgPath, const String *command);
String *walSegmentFind(const Storage *storage, const String *archiveId, const String *walSegment); String *walSegmentFind(const Storage *storage, const String *archiveId, const String *walSegment, TimeMSec timeout);
String *walSegmentNext(const String *walSegment, size_t walSegmentSize, unsigned int pgVersion); String *walSegmentNext(const String *walSegment, size_t walSegmentSize, unsigned int pgVersion);
StringList *walSegmentRange(const String *walSegmentBegin, size_t walSegmentSize, unsigned int pgVersion, unsigned int range); StringList *walSegmentRange(const String *walSegmentBegin, size_t walSegmentSize, unsigned int pgVersion, unsigned int range);

View File

@ -68,7 +68,7 @@ archiveGetCheck(const String *archiveFile, CipherType cipherType, const String *
// If a WAL segment search among the possible file names // If a WAL segment search among the possible file names
if (walIsSegment(archiveFile)) if (walIsSegment(archiveFile))
{ {
String *walSegmentFile = walSegmentFind(storageRepo(), archiveId, archiveFile); String *walSegmentFile = walSegmentFind(storageRepo(), archiveId, archiveFile, 0);
if (walSegmentFile != NULL) if (walSegmentFile != NULL)
{ {

View File

@ -80,7 +80,7 @@ archivePushFile(
const String *walSegmentChecksum = varStr(ioFilterGroupResult(ioReadFilterGroup(read), CRYPTO_HASH_FILTER_TYPE_STR)); const String *walSegmentChecksum = varStr(ioFilterGroupResult(ioReadFilterGroup(read), CRYPTO_HASH_FILTER_TYPE_STR));
// If the wal segment already exists in the repo then compare checksums // If the wal segment already exists in the repo then compare checksums
walSegmentFile = walSegmentFind(storageRepo(), archiveId, archiveFile); walSegmentFile = walSegmentFind(storageRepo(), archiveId, archiveFile, 0);
if (walSegmentFile != NULL) if (walSegmentFile != NULL)
{ {

View File

@ -7,6 +7,7 @@ Test Archive Common
#include "storage/posix/storage.h" #include "storage/posix/storage.h"
#include "common/harnessConfig.h" #include "common/harnessConfig.h"
#include "common/harnessFork.h"
/*********************************************************************************************************************************** /***********************************************************************************************************************************
Test Run Test Run
@ -196,20 +197,39 @@ testRun(void)
strLstAddZ(argList, "archive-get"); strLstAddZ(argList, "archive-get");
harnessCfgLoad(strLstSize(argList), strLstPtr(argList)); harnessCfgLoad(strLstSize(argList), strLstPtr(argList));
TEST_RESULT_PTR(walSegmentFind(storageRepo(), strNew("9.6-2"), strNew("123456781234567812345678")), NULL, "no path"); TEST_RESULT_PTR(walSegmentFind(storageRepo(), strNew("9.6-2"), strNew("123456781234567812345678"), 0), NULL, "no path");
storagePathCreateNP(storageTest, strNew("archive/db/9.6-2/1234567812345678")); storagePathCreateNP(storageTest, strNew("archive/db/9.6-2/1234567812345678"));
TEST_RESULT_PTR(walSegmentFind(storageRepo(), strNew("9.6-2"), strNew("123456781234567812345678")), NULL, "no segment"); TEST_RESULT_PTR(walSegmentFind(storageRepo(), strNew("9.6-2"), strNew("123456781234567812345678"), 0), NULL, "no segment");
TEST_RESULT_PTR(
walSegmentFind(storageRepo(), strNew("9.6-2"), strNew("123456781234567812345678"), 500), NULL,
"no segment after 500ms");
// Check timeout by making the wal segment appear after 250ms
HARNESS_FORK_BEGIN()
{
HARNESS_FORK_CHILD_BEGIN(0, false)
{
sleepMSec(250);
storagePutNP( storagePutNP(
storageNewWriteNP( storageNewWriteNP(
storageTest, storageTest,
strNew("archive/db/9.6-2/1234567812345678/123456781234567812345678-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")), strNew(
"archive/db/9.6-2/1234567812345678/123456781234567812345678-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")),
NULL); NULL);
}
HARNESS_FORK_CHILD_END();
HARNESS_FORK_PARENT_BEGIN()
{
TEST_RESULT_STR( TEST_RESULT_STR(
strPtr(walSegmentFind(storageRepo(), strNew("9.6-2"), strNew("123456781234567812345678"))), strPtr(walSegmentFind(storageRepo(), strNew("9.6-2"), strNew("123456781234567812345678"), 1000)),
"123456781234567812345678-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "found segment"); "123456781234567812345678-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "found segment");
}
HARNESS_FORK_PARENT_END();
}
HARNESS_FORK_END();
storagePutNP( storagePutNP(
storageNewWriteNP( storageNewWriteNP(
@ -218,7 +238,7 @@ testRun(void)
NULL); NULL);
TEST_ERROR( TEST_ERROR(
walSegmentFind(storageRepo(), strNew("9.6-2"), strNew("123456781234567812345678")), walSegmentFind(storageRepo(), strNew("9.6-2"), strNew("123456781234567812345678"), 0),
ArchiveDuplicateError, ArchiveDuplicateError,
"duplicates found in archive for WAL segment 123456781234567812345678:" "duplicates found in archive for WAL segment 123456781234567812345678:"
" 123456781234567812345678-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" " 123456781234567812345678-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
@ -226,7 +246,7 @@ testRun(void)
"\nHINT: are multiple primaries archiving to this stanza?"); "\nHINT: are multiple primaries archiving to this stanza?");
TEST_RESULT_STR( TEST_RESULT_STR(
walSegmentFind(storageRepo(), strNew("9.6-2"), strNew("123456781234567812345678.partial")), NULL, walSegmentFind(storageRepo(), strNew("9.6-2"), strNew("123456781234567812345678.partial"), 0), NULL,
"did not find partial segment"); "did not find partial segment");
} }