db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2025-07-05 00:28:52 +02:00
Code Issues Releases Activity

Migrate walIsPartial(), walIsSegment(), and walSegmentFind() from Perl to C.

Browse Source 
Also refactor regular expression defines to make them more reusable.
This commit is contained in:
David Steele
2018-09-07 08:00:18 -07:00
parent 9660076093
commit 5bdaa35fa5
6 changed files with 180 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
doc/xml/release.xml
View File

@ -38,6 +38,10 @@
<release-item>
<p>Add helper for repository storage.</p>
</release-item>
<release-item>
<p>Migrate <code>walIsPartial()</code>, <code>walIsSegment()</code>, and <code>walSegmentFind()</code> from Perl to C.</p>
</release-item>
</release-development-list>
</release-core-list>
</release>

2
src/Makefile
View File

@ -140,7 +140,7 @@ install: pgbackrest
####################################################################################################################################
# Compile rules
####################################################################################################################################
command/archive/common.o: command/archive/common.c command/archive/common.h common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/log.h common/logLevel.h common/memContext.h common/stackTrace.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h common/wait.h postgres/version.h storage/driver/posix/driverRead.h storage/driver/posix/driverWrite.h storage/fileRead.h storage/fileWrite.h storage/helper.h storage/info.h storage/storage.h version.h
command/archive/common.o: command/archive/common.c command/archive/common.h common/assert.h common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/log.h common/logLevel.h common/memContext.h common/regExp.h common/stackTrace.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h common/wait.h postgres/version.h protocol/storage/helper.h storage/driver/posix/driverRead.h storage/driver/posix/driverWrite.h storage/fileRead.h storage/fileWrite.h storage/helper.h storage/info.h storage/storage.h version.h
$(CC) $(CFLAGS) -c command/archive/common.c -o command/archive/common.o
command/archive/get/get.o: command/archive/get/get.c command/archive/common.h command/command.h common/assert.h common/debug.h common/error.auto.h common/error.h common/fork.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/regExp.h common/stackTrace.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h common/wait.h config/config.auto.h config/config.h config/define.auto.h config/define.h config/load.h perl/exec.h postgres/info.h storage/driver/posix/driverRead.h storage/driver/posix/driverWrite.h storage/fileRead.h storage/fileWrite.h storage/helper.h storage/info.h storage/storage.h version.h

98
src/command/archive/common.c
View File

@ -10,8 +10,10 @@ Archive Push Command
#include "common/debug.h"
#include "common/log.h"
#include "common/memContext.h"
#include "common/regExp.h"
#include "common/wait.h"
#include "postgres/version.h"
#include "protocol/storage/helper.h"
#include "storage/helper.h"
/***********************************************************************************************************************************
@ -111,6 +113,102 @@ archiveAsyncStatus(ArchiveMode archiveMode, const String *walSegment, bool confe
FUNCTION_DEBUG_RESULT(BOOL, result);
}
/***********************************************************************************************************************************
Is the segment partial?
***********************************************************************************************************************************/
bool
walIsPartial(const String *walSegment)
{
FUNCTION_DEBUG_BEGIN(logLevelTrace);
FUNCTION_DEBUG_PARAM(STRING, walSegment);
FUNCTION_DEBUG_ASSERT(walSegment != NULL);
FUNCTION_DEBUG_ASSERT(walIsSegment(walSegment));
FUNCTION_DEBUG_END();
FUNCTION_DEBUG_RESULT(BOOL, strEndsWithZ(walSegment, WAL_SEGMENT_PARTIAL_EXT));
}
/***********************************************************************************************************************************
Is the file a segment or some other file (e.g. .history, .backup, etc)
***********************************************************************************************************************************/
bool
walIsSegment(const String *walSegment)
{
FUNCTION_DEBUG_BEGIN(logLevelTrace);
FUNCTION_DEBUG_PARAM(STRING, walSegment);
FUNCTION_DEBUG_ASSERT(walSegment != NULL);
FUNCTION_DEBUG_END();
// Create the regular expression to identify WAL segments if it does not already exist
static RegExp *regExpSegment = NULL;
if (regExpSegment == NULL)
{
MEM_CONTEXT_BEGIN(memContextTop())
{
regExpSegment = regExpNew(strNew(WAL_SEGMENT_PARTIAL_REGEXP));
}
MEM_CONTEXT_END();
}
FUNCTION_DEBUG_RESULT(BOOL, regExpMatch(regExpSegment, walSegment));
}
/***********************************************************************************************************************************
Find a WAL segment in the repository
The file name can have several things appended such as a hash, compression extension, and partial extension so it is possible to
have multiple files that match the segment, though more than one match is not a good thing.
***********************************************************************************************************************************/
String *
walSegmentFind(const Storage *storage, const String *archiveId, const String *walSegment)
{
FUNCTION_DEBUG_BEGIN(logLevelDebug);
FUNCTION_DEBUG_PARAM(STORAGE, storage);
FUNCTION_DEBUG_PARAM(STRING, archiveId);
FUNCTION_DEBUG_PARAM(STRING, walSegment);
FUNCTION_DEBUG_ASSERT(storage != NULL);
FUNCTION_DEBUG_ASSERT(archiveId != NULL);
FUNCTION_DEBUG_ASSERT(walSegment != NULL);
FUNCTION_DEBUG_ASSERT(walIsSegment(walSegment));
FUNCTION_DEBUG_END();
String *result = NULL;
MEM_CONTEXT_TEMP_BEGIN()
{
// Get a list of all WAL segments that match
StringList *list = storageListP(
storage, strNewFmt(STORAGE_REPO_ARCHIVE "/%s/%s", strPtr(archiveId), strPtr(strSubN(walSegment, 0, 16))),
.expression = strNewFmt("^%s%s-[0-f]{40}(\\.gz){0,1}$", strPtr(strSubN(walSegment, 0, 24)),
walIsPartial(walSegment) ? WAL_SEGMENT_PARTIAL_EXT : ""));
// If there are results
if (list != NULL && strLstSize(list) > 0)
{
// Error if there is more than one match
if (strLstSize(list) > 1)
{
THROW_FMT(
ArchiveDuplicateError,
"duplicates found in archive for WAL segment %s: %s\n"
"HINT: are multiple primaries archiving to this stanza?",
strPtr(walSegment), strPtr(strLstJoin(strLstSort(list, sortOrderAsc), ", ")));
}
// Copy file name of WAL segment found into the calling context
memContextSwitch(MEM_CONTEXT_OLD());
result = strDup(strLstGet(list, 0));
}
}
MEM_CONTEXT_TEMP_END();
FUNCTION_DEBUG_RESULT(STRING, result);
}
/***********************************************************************************************************************************
Get the next WAL segment given a WAL segment and WAL segment size
***********************************************************************************************************************************/

17
src/command/archive/common.h
View File

@ -18,12 +18,22 @@ typedef enum
} ArchiveMode;
#include "common/type/stringList.h"
#include "storage/storage.h"
/***********************************************************************************************************************************
WAL segment constants
***********************************************************************************************************************************/
// Only match on a WAL segment without checksum appended
#define WAL_SEGMENT_REGEXP "^[0-F]{24}$"
// Extension for partial segments
#define WAL_SEGMENT_PARTIAL_EXT ".partial"
// Match when the first 24 characters match a WAL segment
#define WAL_SEGMENT_PREFIX_REGEXP "^[0-F]{24}"
// Match on a WAL segment without checksum appended
#define WAL_SEGMENT_REGEXP WAL_SEGMENT_PREFIX_REGEXP "$"
// Match on a WAL segment with partial allowed
#define WAL_SEGMENT_PARTIAL_REGEXP WAL_SEGMENT_PREFIX_REGEXP "(\\.partial){0,1}$"
// Defines the size of standard WAL segment name -- this should never changed
#define WAL_SEGMENT_NAME_SIZE ((unsigned int)24)
@ -36,6 +46,9 @@ Functions
***********************************************************************************************************************************/
bool archiveAsyncStatus(ArchiveMode archiveMode, const String *walSegment, bool confessOnError);
bool walIsPartial(const String *walSegment);
bool walIsSegment(const String *walSegment);
String *walSegmentFind(const Storage *storage, const String *archiveId, const String *walSegment);
String *walSegmentNext(const String *walSegment, size_t walSegmentSize, unsigned int pgVersion);
StringList *walSegmentRange(const String *walSegmentBegin, size_t walSegmentSize, unsigned int pgVersion, unsigned int range);

2
test/define.yaml
View File

@ -593,7 +593,7 @@ unit:
test:
# ----------------------------------------------------------------------------------------------------------------------------
- name: common
total: 3
total: 6
coverage:
command/archive/common: full

61
test/src/module/archive/commonTest.c
View File

@ -5,6 +5,8 @@ Test Archive Common
#include "common/harnessConfig.h"
#include "protocol/storage/helper.h"
/***********************************************************************************************************************************
Test Run
***********************************************************************************************************************************/
@ -13,6 +15,9 @@ testRun(void)
{
FUNCTION_HARNESS_VOID();
// Create default storage object for testing
Storage *storageTest = storageNewP(strNew(testPath()), .write = true);
// *****************************************************************************************************************************
if (testBegin("archiveAsyncStatus()"))
{
@ -98,6 +103,62 @@ testRun(void)
unlink(strPtr(storagePathNP(storageSpool(), strNewFmt(STORAGE_SPOOL_ARCHIVE_OUT "/%s.error", strPtr(segment)))));
}
// *****************************************************************************************************************************
if (testBegin("walIsPartial()"))
{
TEST_RESULT_BOOL(walIsPartial(strNew("000000010000000100000001")), false, "not partial");
TEST_RESULT_BOOL(walIsPartial(strNew("FFFFFFFFFFFFFFFFFFFFFFFF.partial")), true, "partial");
}
// *****************************************************************************************************************************
if (testBegin("walIsSegment()"))
{
TEST_RESULT_BOOL(walIsSegment(strNew("000000010000000100000001")), true, "wal segment");
TEST_RESULT_BOOL(walIsSegment(strNew("FFFFFFFFFFFFFFFFFFFFFFFF.partial")), true, "partial wal segment");
TEST_RESULT_BOOL(walIsSegment(strNew("0000001A.history")), false, "history file");
}
// *****************************************************************************************************************************
if (testBegin("walSegmentFind()"))
{
// Load configuration to set repo-path and stanza
StringList *argList = strLstNew();
strLstAddZ(argList, "pgbackrest");
strLstAddZ(argList, "--stanza=db");
strLstAdd(argList, strNewFmt("--repo-path=%s", testPath()));
strLstAddZ(argList, "archive-get");
harnessCfgLoad(strLstSize(argList), strLstPtr(argList));
TEST_RESULT_PTR(walSegmentFind(storageRepo(), strNew("9.6-2"), strNew("123456781234567812345678")), NULL, "no path");
storagePathCreateNP(storageTest, strNew("archive/db/9.6-2/1234567812345678"));
TEST_RESULT_PTR(walSegmentFind(storageRepo(), strNew("9.6-2"), strNew("123456781234567812345678")), NULL, "no segment");
storagePutNP(
storageNewWriteNP(
storageTest,
strNew("archive/db/9.6-2/1234567812345678/123456781234567812345678-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")),
NULL);
TEST_RESULT_STR(
strPtr(walSegmentFind(storageRepo(), strNew("9.6-2"), strNew("123456781234567812345678"))),
"123456781234567812345678-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "found segment");
storagePutNP(
storageNewWriteNP(
storageTest,
strNew("archive/db/9.6-2/1234567812345678/123456781234567812345678-bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb.gz")),
NULL);
TEST_ERROR(
walSegmentFind(storageRepo(), strNew("9.6-2"), strNew("123456781234567812345678")),
ArchiveDuplicateError,
"duplicates found in archive for WAL segment 123456781234567812345678:"
" 123456781234567812345678-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
", 123456781234567812345678-bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb.gz"
"\nHINT: are multiple primaries archiving to this stanza?");
}
// *****************************************************************************************************************************
if (testBegin("walSegmentNext()"))
{