db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2025-03-03 14:52:21 +02:00
Code Issues Releases Activity

Add helper for repository storage.

Browse Source 
Implement rules for generating paths within the archive part of the repository. Add a helper function, storageRepo(), to create the repository storage based on configuration settings.

The repository storage helper is located in the protocol module because it will support remote file systems in the future, just as the Perl version does.

Also, improve the existing helper functions a bit using string functions that were not available when they were written.
This commit is contained in:
David Steele 2018-09-07 07:58:08 -07:00
parent 960ad73298
commit 9660076093
8 changed files with 214 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
doc/xml/release.xml
View File

@ -34,6 +34,10 @@
<release-item>
<p>Info objects now parse JSON and use specified storage.</p>
</release-item>
<release-item>
<p>Add helper for repository storage.</p>
</release-item>
</release-development-list>
</release-core-list>
</release>

4
src/Makefile
View File

@ -110,6 +110,7 @@ SRCS = \
perl/exec.c \
postgres/info.c \
postgres/pageChecksum.c \
protocol/storage/helper.c \
storage/driver/posix/driver.c \
storage/driver/posix/driverFile.c \
storage/driver/posix/driverRead.c \
@ -310,6 +311,9 @@ postgres/info.o: postgres/info.c common/debug.h common/error.auto.h common/error
postgres/pageChecksum.o: postgres/pageChecksum.c common/assert.h common/debug.h common/error.auto.h common/error.h common/log.h common/logLevel.h common/stackTrace.h common/type/convert.h postgres/pageChecksum.h postgres/type.h
$(CC) $(CFLAGS) -funroll-loops -ftree-vectorize -c postgres/pageChecksum.c -o postgres/pageChecksum.o
protocol/storage/helper.o: protocol/storage/helper.c common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/lock.h common/log.h common/logLevel.h common/memContext.h common/regExp.h common/stackTrace.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h config/config.auto.h config/config.h config/define.auto.h config/define.h protocol/storage/helper.h storage/driver/posix/driverRead.h storage/driver/posix/driverWrite.h storage/fileRead.h storage/fileWrite.h storage/info.h storage/storage.h version.h
$(CC) $(CFLAGS) -c protocol/storage/helper.c -o protocol/storage/helper.o
storage/driver/posix/driver.o: storage/driver/posix/driver.c common/debug.h common/error.auto.h common/error.h common/io/filter/filter.h common/io/filter/group.h common/io/read.h common/io/write.h common/log.h common/logLevel.h common/memContext.h common/regExp.h common/stackTrace.h common/type/buffer.h common/type/convert.h common/type/keyValue.h common/type/string.h common/type/stringList.h common/type/variant.h common/type/variantList.h storage/driver/posix/driver.h storage/driver/posix/driverFile.h storage/driver/posix/driverRead.h storage/driver/posix/driverWrite.h storage/fileRead.h storage/fileWrite.h storage/info.h storage/storage.h version.h
$(CC) $(CFLAGS) -c storage/driver/posix/driver.c -o storage/driver/posix/driver.o

102
src/protocol/storage/helper.c Normal file
View File

@ -0,0 +1,102 @@
/***********************************************************************************************************************************
Protocol Storage Helper
***********************************************************************************************************************************/
#include <string.h>
#include "common/debug.h"
#include "common/memContext.h"
#include "common/regExp.h"
#include "config/config.h"
#include "protocol/storage/helper.h"
/***********************************************************************************************************************************
Local variables
***********************************************************************************************************************************/
struct
{
MemContext *memContext; // Mem context for protocol storage
Storage *storageRepo; // Repository read-only storage
String *stanza; // Stanza for storage
RegExp *walRegExp; // Regular expression for identifying wal files
} protocolStorageHelper;
/***********************************************************************************************************************************
Create the storage helper memory context
***********************************************************************************************************************************/
static void
protocolStorageHelperInit(void)
{
FUNCTION_TEST_VOID();
if (protocolStorageHelper.memContext == NULL)
{
MEM_CONTEXT_BEGIN(memContextTop())
{
protocolStorageHelper.memContext = memContextNew("protocolStorageHelper");
}
MEM_CONTEXT_END();
}
FUNCTION_TEST_RESULT_VOID();
}
/***********************************************************************************************************************************
Get a spool storage object
***********************************************************************************************************************************/
static String *
storageRepoPathExpression(const String *expression, const String *path)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(STRING, expression);
FUNCTION_TEST_PARAM(STRING, path);
FUNCTION_TEST_ASSERT(expression != NULL);
FUNCTION_TEST_END();
String *result = NULL;
if (strEqZ(expression, STORAGE_REPO_ARCHIVE))
{
result = strNewFmt("archive/%s", strPtr(protocolStorageHelper.stanza));
if (path != NULL)
{
StringList *pathSplit = strLstNewSplitZ(path, "/");
String *file = strLstSize(pathSplit) == 2 ? strLstGet(pathSplit, 1) : NULL;
if (file != NULL && regExpMatch(protocolStorageHelper.walRegExp, file))
strCatFmt(result, "/%s/%s/%s", strPtr(strLstGet(pathSplit, 0)), strPtr(strSubN(file, 0, 16)), strPtr(file));
else
strCatFmt(result, "/%s", strPtr(path));
}
}
else
THROW_FMT(AssertError, "invalid expression '%s'", strPtr(expression));
FUNCTION_TEST_RESULT(STRING, result);
}
/***********************************************************************************************************************************
Get a read-only repository storage object
***********************************************************************************************************************************/
const Storage *
storageRepo(void)
{
FUNCTION_TEST_VOID();
if (protocolStorageHelper.storageRepo == NULL)
{
protocolStorageHelperInit();
MEM_CONTEXT_BEGIN(protocolStorageHelper.memContext)
{
protocolStorageHelper.stanza = strDup(cfgOptionStr(cfgOptStanza));
protocolStorageHelper.walRegExp = regExpNew(strNew("^[0-F]{24}"));
protocolStorageHelper.storageRepo = storageNewP(
cfgOptionStr(cfgOptRepoPath), .pathExpressionFunction = storageRepoPathExpression);
}
MEM_CONTEXT_END();
}
FUNCTION_TEST_RESULT(STORAGE, protocolStorageHelper.storageRepo);
}

26
src/protocol/storage/helper.h Normal file
View File

@ -0,0 +1,26 @@
/***********************************************************************************************************************************
Protocol Storage Helper
***********************************************************************************************************************************/
#ifndef PROTOCOL_STORAGE_HELPER_H
#define PROTOCOL_STORAGE_HELPER_H
#include "storage/storage.h"
/***********************************************************************************************************************************
Repo storage path constants
***********************************************************************************************************************************/
#define STORAGE_REPO_ARCHIVE "<REPO:ARCHIVE>"
/***********************************************************************************************************************************
Repository storage types
***********************************************************************************************************************************/
#define STORAGE_TYPE_CIFS "cifs"
#define STORAGE_TYPE_POSIX "posix"
#define STORAGE_TYPE_S3 "s3"
/***********************************************************************************************************************************
Functions
***********************************************************************************************************************************/
const Storage *storageRepo(void);
#endif

6
src/storage/helper.c
View File

@ -94,7 +94,7 @@ storageLocalWrite(void)
/***********************************************************************************************************************************
Get a spool storage object
***********************************************************************************************************************************/
String *
static String *
storageSpoolPathExpression(const String *expression, const String *path)
{
FUNCTION_TEST_BEGIN();
@ -106,14 +106,14 @@ storageSpoolPathExpression(const String *expression, const String *path)
String *result = NULL;
if (strcmp(strPtr(expression), STORAGE_SPOOL_ARCHIVE_IN) == 0)
if (strEqZ(expression, STORAGE_SPOOL_ARCHIVE_IN))
{
if (path == NULL)
result = strNewFmt("archive/%s/in", strPtr(storageSpoolStanza));
else
result = strNewFmt("archive/%s/in/%s", strPtr(storageSpoolStanza), strPtr(path));
}
else if (strcmp(strPtr(expression), STORAGE_SPOOL_ARCHIVE_OUT) == 0)
else if (strEqZ(expression, STORAGE_SPOOL_ARCHIVE_OUT))
{
if (path == NULL)
result = strNewFmt("archive/%s/out", strPtr(storageSpoolStanza));

7
test/define.yaml
View File

@ -530,6 +530,13 @@ unit:
coverage:
Protocol/Base/Minion: partial
# ----------------------------------------------------------------------------------------------------------------------------
- name: storage-helper
total: 2
coverage:
protocol/storage/helper: full
# ----------------------------------------------------------------------------------------------------------------------------
- name: helper-perl
total: 2

67
test/src/module/protocol/storageHelperTest.c Normal file
View File

@ -0,0 +1,67 @@
/***********************************************************************************************************************************
Test Protocol Storage Helper
***********************************************************************************************************************************/
#include "common/harnessConfig.h"
/***********************************************************************************************************************************
Test Run
***********************************************************************************************************************************/
void
testRun(void)
{
FUNCTION_HARNESS_VOID();
String *writeFile = strNewFmt("%s/writefile", testPath());
// *****************************************************************************************************************************
if (testBegin("protocolStorageHelperInit()"))
{
TEST_RESULT_PTR(protocolStorageHelper.memContext, NULL, "mem context not created");
TEST_RESULT_VOID(protocolStorageHelperInit(), "create mem context");
TEST_RESULT_BOOL(protocolStorageHelper.memContext != NULL, true, "mem context created");
TEST_RESULT_VOID(protocolStorageHelperInit(), "reinit does nothing");
}
// *****************************************************************************************************************************
if (testBegin("storageRepo()"))
{
const Storage *storage = NULL;
// Load configuration to set repo-path and stanza
StringList *argList = strLstNew();
strLstAddZ(argList, "pgbackrest");
strLstAddZ(argList, "--stanza=db");
strLstAdd(argList, strNewFmt("--repo-path=%s", testPath()));
strLstAddZ(argList, "archive-get");
harnessCfgLoad(strLstSize(argList), strLstPtr(argList));
TEST_RESULT_PTR(protocolStorageHelper.storageRepo, NULL, "repo storage not cached");
TEST_ASSIGN(storage, storageRepo(), "new storage");
TEST_RESULT_PTR(protocolStorageHelper.storageRepo, storage, "repo storage cached");
TEST_RESULT_PTR(storageRepo(), storage, "get cached storage");
// -------------------------------------------------------------------------------------------------------------------------
TEST_ERROR(storagePathNP(storage, strNew("<BOGUS>/path")), AssertError, "invalid expression '<BOGUS>'");
TEST_ERROR(storageNewWriteNP(storage, writeFile), AssertError, "function debug assertion 'this->write' failed");
TEST_RESULT_STR(strPtr(storagePathNP(storage, NULL)), testPath(), "check base path");
TEST_RESULT_STR(
strPtr(storagePathNP(storage, strNew(STORAGE_REPO_ARCHIVE))), strPtr(strNewFmt("%s/archive/db", testPath())),
"check archive path");
TEST_RESULT_STR(
strPtr(storagePathNP(storage, strNew(STORAGE_REPO_ARCHIVE "/simple"))),
strPtr(strNewFmt("%s/archive/db/simple", testPath())), "check simple path");
TEST_RESULT_STR(
strPtr(storagePathNP(storage, strNew(STORAGE_REPO_ARCHIVE "/9.4-1/700000007000000070000000"))),
strPtr(strNewFmt("%s/archive/db/9.4-1/7000000070000000/700000007000000070000000", testPath())), "check segment path");
TEST_RESULT_STR(
strPtr(storagePathNP(storage, strNew(STORAGE_REPO_ARCHIVE "/9.4-1/00000008.history"))),
strPtr(strNewFmt("%s/archive/db/9.4-1/00000008.history", testPath())), "check history path");
TEST_RESULT_STR(
strPtr(storagePathNP(storage, strNew(STORAGE_REPO_ARCHIVE "/9.4-1/000000010000014C0000001A.00000028.backup"))),
strPtr(strNewFmt("%s/archive/db/9.4-1/000000010000014C/000000010000014C0000001A.00000028.backup", testPath())),
"check backup path");
}
FUNCTION_HARNESS_RESULT_VOID();
}

1
test/src/module/storage/helperTest.c
View File

@ -26,6 +26,7 @@ testRun(void)
TEST_ERROR(storageNewWriteNP(storage, writeFile), AssertError, "function debug assertion 'this->write' failed");
}
// -----------------------------------------------------------------------------------------------------------------------------
if (testBegin("storageLocalWrite()"))
{