db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2024-12-12 10:04:14 +02:00
Code Issues Releases Activity

Add configurable storage helpers to create repository storage.

Browse Source 
Remove the hardcoded storage helpers from storageRepoGet() except for the the built-in Posix helper and the special remote helper.

The goal is to make storage driver development a bit easier by isolating as much of the code as possible into the driver module. This also makes coverage reporting much simpler for additional drivers since they do not need to provide coverage for storage/helper.

Consolidate the CIFS tests into the Posix tests since CIFS is just a special case of the Posix.

Test all storage features in the Posix test so that other storage driver tests do not need to provide coverage for storage/storage.

Remove some dead code in the storage/s3 test.
This commit is contained in:
David Steele 2021-10-06 19:27:04 -04:00
parent cfd823355a
commit fb3f6928c9
20 changed files with 466 additions and 205 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/Makefile.in
View File

@ -148,10 +148,13 @@ SRCS = \
protocol/parallel.c \
protocol/parallelJob.c \
protocol/server.c \
storage/azure/helper.c \
storage/azure/read.c \
storage/azure/storage.c \
storage/azure/write.c \
storage/cifs/helper.c \
storage/cifs/storage.c \
storage/gcs/helper.c \
storage/gcs/read.c \
storage/gcs/storage.c \
storage/gcs/write.c \
@ -160,6 +163,7 @@ SRCS = \
storage/remote/protocol.c \
storage/remote/storage.c \
storage/remote/write.c \
storage/s3/helper.c \
storage/s3/read.c \
storage/s3/storage.c \
storage/s3/write.c

16
src/main.c
View File

@ -39,7 +39,11 @@ Main
#include "config/load.h"
#include "postgres/interface.h"
#include "protocol/helper.h"
#include "storage/azure/helper.h"
#include "storage/cifs/helper.h"
#include "storage/gcs/helper.h"
#include "storage/helper.h"
#include "storage/s3/helper.h"
#include "version.h"
/***********************************************************************************************************************************
@ -54,6 +58,18 @@ main(int argListSize, const char *argList[])
static const ErrorHandlerFunction errorHandlerList[] = {stackTraceClean, memContextClean};
errorHandlerSet(errorHandlerList, sizeof(errorHandlerList) / sizeof(ErrorHandlerFunction));
// Set storage helpers
static const StorageHelper storageHelperList[] =
{
STORAGE_AZURE_HELPER,
STORAGE_CIFS_HELPER,
STORAGE_GCS_HELPER,
STORAGE_S3_HELPER,
STORAGE_END_HELPER
};
storageHelperInit(storageHelperList);
#ifdef WITH_BACKTRACE
stackTraceInit(argList[0]);
#endif

48
src/storage/azure/helper.c Normal file
View File

@ -0,0 +1,48 @@
/***********************************************************************************************************************************
Azure Storage Helper
***********************************************************************************************************************************/
#include "build.auto.h"
#include "common/debug.h"
#include "common/io/io.h"
#include "common/log.h"
#include "config/config.h"
#include "storage/azure/helper.h"
/**********************************************************************************************************************************/
Storage *
storageAzureHelper(const unsigned int repoIdx, const bool write, StoragePathExpressionCallback pathExpressionCallback)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(UINT, repoIdx);
FUNCTION_LOG_PARAM(BOOL, write);
FUNCTION_LOG_PARAM_P(VOID, pathExpressionCallback);
FUNCTION_LOG_END();
ASSERT(cfgOptionIdxStrId(cfgOptRepoType, repoIdx) == STORAGE_AZURE_TYPE);
const String *endpoint = cfgOptionIdxStr(cfgOptRepoAzureEndpoint, repoIdx);
const String *const host = cfgOptionIdxStrNull(cfgOptRepoStorageHost, repoIdx);
StorageAzureUriStyle uriStyle = (StorageAzureUriStyle)cfgOptionIdxStrId(cfgOptRepoAzureUriStyle, repoIdx);
// If the host is set then set it as the endpoint. The host option is used to set path-style URIs when working with Azurite.
// This was ill-advised, so the uri-style option was added to allow the user to select the URI style used by the server.
// Preserve the old behavior when uri-style is defaulted.
if (host != NULL)
{
endpoint = host;
if (cfgOptionIdxSource(cfgOptRepoAzureUriStyle, repoIdx) == cfgSourceDefault)
uriStyle = storageAzureUriStylePath;
}
Storage *const result = storageAzureNew(
cfgOptionIdxStr(cfgOptRepoPath, repoIdx), write, pathExpressionCallback,
cfgOptionIdxStr(cfgOptRepoAzureContainer, repoIdx), cfgOptionIdxStr(cfgOptRepoAzureAccount, repoIdx),
(StorageAzureKeyType)cfgOptionIdxStrId(cfgOptRepoAzureKeyType, repoIdx),
cfgOptionIdxStr(cfgOptRepoAzureKey, repoIdx), STORAGE_AZURE_BLOCKSIZE_MIN, endpoint, uriStyle,
cfgOptionIdxUInt(cfgOptRepoStoragePort, repoIdx), ioTimeoutMs(), cfgOptionIdxBool(cfgOptRepoStorageVerifyTls, repoIdx),
cfgOptionIdxStrNull(cfgOptRepoStorageCaFile, repoIdx), cfgOptionIdxStrNull(cfgOptRepoStorageCaPath, repoIdx));
FUNCTION_LOG_RETURN(STORAGE, result);
}

19
src/storage/azure/helper.h Normal file
View File

@ -0,0 +1,19 @@
/***********************************************************************************************************************************
Azure Storage Helper
***********************************************************************************************************************************/
#ifndef STORAGE_AZURE_STORAGE_HELPER_H
#define STORAGE_AZURE_STORAGE_HELPER_H
#include "storage/azure/storage.h"
/***********************************************************************************************************************************
Functions
***********************************************************************************************************************************/
Storage *storageAzureHelper(unsigned int repoIdx, bool write, StoragePathExpressionCallback pathExpressionCallback);
/***********************************************************************************************************************************
Storage helper for StorageHelper array passed to storageHelperInit()
***********************************************************************************************************************************/
#define STORAGE_AZURE_HELPER {.type = STORAGE_AZURE_TYPE, .helper = storageAzureHelper}
#endif

28
src/storage/cifs/helper.c Normal file
View File

@ -0,0 +1,28 @@
/***********************************************************************************************************************************
CIFS Storage Helper
***********************************************************************************************************************************/
#include "build.auto.h"
#include "common/debug.h"
#include "common/log.h"
#include "config/config.h"
#include "storage/cifs/helper.h"
/**********************************************************************************************************************************/
Storage *
storageCifsHelper(const unsigned int repoIdx, const bool write, StoragePathExpressionCallback pathExpressionCallback)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(UINT, repoIdx);
FUNCTION_LOG_PARAM(BOOL, write);
FUNCTION_LOG_PARAM_P(VOID, pathExpressionCallback);
FUNCTION_LOG_END();
ASSERT(cfgOptionIdxStrId(cfgOptRepoType, repoIdx) == STORAGE_CIFS_TYPE);
Storage *const result = storageCifsNew(
cfgOptionIdxStr(cfgOptRepoPath, repoIdx), STORAGE_MODE_FILE_DEFAULT, STORAGE_MODE_PATH_DEFAULT, write,
pathExpressionCallback);
FUNCTION_LOG_RETURN(STORAGE, result);
}

19
src/storage/cifs/helper.h Normal file
View File

@ -0,0 +1,19 @@
/***********************************************************************************************************************************
CIFS Storage Helper
***********************************************************************************************************************************/
#ifndef STORAGE_CIFS_STORAGE_HELPER_H
#define STORAGE_CIFS_STORAGE_HELPER_H
#include "storage/cifs/storage.h"
/***********************************************************************************************************************************
Functions
***********************************************************************************************************************************/
Storage *storageCifsHelper(unsigned int repoIdx, bool write, StoragePathExpressionCallback pathExpressionCallback);
/***********************************************************************************************************************************
Storage helper for StorageHelper array passed to storageHelperInit()
***********************************************************************************************************************************/
#define STORAGE_CIFS_HELPER {.type = STORAGE_CIFS_TYPE, .helper = storageCifsHelper}
#endif

32
src/storage/gcs/helper.c Normal file
View File

@ -0,0 +1,32 @@
/***********************************************************************************************************************************
GCS Storage Helper
***********************************************************************************************************************************/
#include "build.auto.h"
#include "common/debug.h"
#include "common/io/io.h"
#include "common/log.h"
#include "config/config.h"
#include "storage/gcs/helper.h"
/**********************************************************************************************************************************/
Storage *
storageGcsHelper(const unsigned int repoIdx, const bool write, StoragePathExpressionCallback pathExpressionCallback)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(UINT, repoIdx);
FUNCTION_LOG_PARAM(BOOL, write);
FUNCTION_LOG_PARAM_P(VOID, pathExpressionCallback);
FUNCTION_LOG_END();
ASSERT(cfgOptionIdxStrId(cfgOptRepoType, repoIdx) == STORAGE_GCS_TYPE);
Storage *const result = storageGcsNew(
cfgOptionIdxStr(cfgOptRepoPath, repoIdx), write, pathExpressionCallback, cfgOptionIdxStr(cfgOptRepoGcsBucket, repoIdx),
(StorageGcsKeyType)cfgOptionIdxStrId(cfgOptRepoGcsKeyType, repoIdx), cfgOptionIdxStrNull(cfgOptRepoGcsKey, repoIdx),
STORAGE_GCS_CHUNKSIZE_DEFAULT, cfgOptionIdxStr(cfgOptRepoGcsEndpoint, repoIdx), ioTimeoutMs(),
cfgOptionIdxBool(cfgOptRepoStorageVerifyTls, repoIdx), cfgOptionIdxStrNull(cfgOptRepoStorageCaFile, repoIdx),
cfgOptionIdxStrNull(cfgOptRepoStorageCaPath, repoIdx));
FUNCTION_LOG_RETURN(STORAGE, result);
}

19
src/storage/gcs/helper.h Normal file
View File

@ -0,0 +1,19 @@
/***********************************************************************************************************************************
GCS Storage Helper
***********************************************************************************************************************************/
#ifndef STORAGE_GCS_STORAGE_HELPER_H
#define STORAGE_GCS_STORAGE_HELPER_H
#include "storage/gcs/storage.h"
/***********************************************************************************************************************************
Functions
***********************************************************************************************************************************/
Storage *storageGcsHelper(unsigned int repoIdx, bool write, StoragePathExpressionCallback pathExpressionCallback);
/***********************************************************************************************************************************
Storage helper for StorageHelper array passed to storageHelperInit()
***********************************************************************************************************************************/
#define STORAGE_GCS_HELPER {.type = STORAGE_GCS_TYPE, .helper = storageGcsHelper}
#endif

133
src/storage/helper.c
View File

@ -11,12 +11,8 @@ Storage Helper
#include "common/regExp.h"
#include "config/config.h"
#include "protocol/helper.h"
#include "storage/azure/storage.h"
#include "storage/cifs/storage.h"
#include "storage/gcs/storage.h"
#include "storage/posix/storage.h"
#include "storage/remote/storage.h"
#include "storage/s3/storage.h"
#include "storage/helper.h"
/***********************************************************************************************************************************
@ -41,10 +37,12 @@ Error message when writable storage is requested in dry-run mode
/***********************************************************************************************************************************
Local variables
***********************************************************************************************************************************/
static struct StorageHelper
static struct StorageHelperLocal
{
MemContext *memContext; // Mem context for storage helper
const StorageHelper *helperList; // List of helpers to create storage
Storage *storageLocal; // Local read-only storage
Storage *storageLocalWrite; // Local write storage
Storage **storagePg; // PostgreSQL read-only storage
@ -65,7 +63,7 @@ static struct StorageHelper
Create the storage helper memory context
***********************************************************************************************************************************/
static void
storageHelperInit(void)
storageHelperContextInit(void)
{
FUNCTION_TEST_VOID();
@ -85,6 +83,18 @@ storageHelperInit(void)
FUNCTION_TEST_RETURN_VOID();
}
/**********************************************************************************************************************************/
void storageHelperInit(const StorageHelper *const helperList)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM_P(VOID, helperList);
FUNCTION_TEST_END();
storageHelper.helperList = helperList;
FUNCTION_TEST_RETURN_VOID();
}
/**********************************************************************************************************************************/
void
storageHelperDryRunInit(bool dryRun)
@ -130,7 +140,7 @@ storageLocal(void)
if (storageHelper.storageLocal == NULL)
{
storageHelperInit();
storageHelperContextInit();
MEM_CONTEXT_BEGIN(storageHelper.memContext)
{
@ -149,7 +159,7 @@ storageLocalWrite(void)
if (storageHelper.storageLocalWrite == NULL)
{
storageHelperInit();
storageHelperContextInit();
MEM_CONTEXT_BEGIN(storageHelper.memContext)
{
@ -200,7 +210,7 @@ storagePgIdx(unsigned int pgIdx)
if (storageHelper.storagePg == NULL || storageHelper.storagePg[pgIdx] == NULL)
{
storageHelperInit();
storageHelperContextInit();
MEM_CONTEXT_BEGIN(storageHelper.memContext)
{
@ -235,7 +245,7 @@ storagePgIdxWrite(unsigned int pgIdx)
if (storageHelper.storagePgWrite == NULL || storageHelper.storagePgWrite[pgIdx] == NULL)
{
storageHelperInit();
storageHelperContextInit();
MEM_CONTEXT_BEGIN(storageHelper.memContext)
{
@ -353,95 +363,28 @@ storageRepoGet(unsigned int repoIdx, bool write)
// Use local storage
else
{
// Search for the helper
const StringId type = cfgOptionIdxStrId(cfgOptRepoType, repoIdx);
switch (type)
if (storageHelper.helperList != NULL)
{
// Use Azure storage
case STORAGE_AZURE_TYPE:
for (const StorageHelper *helper = storageHelper.helperList; helper->type != 0; helper++)
{
const String *endpoint = cfgOptionIdxStr(cfgOptRepoAzureEndpoint, repoIdx);
const String *const host = cfgOptionIdxStrNull(cfgOptRepoStorageHost, repoIdx);
StorageAzureUriStyle uriStyle = (StorageAzureUriStyle)cfgOptionIdxStrId(cfgOptRepoAzureUriStyle, repoIdx);
// If the host is set then set it as the endpoint. The host option is used to set path-style URIs when working with
// Azurite. This was ill-advised, so the uri-style option was added to allow the user to select the URI style used
// by the server. Preserve the old behavior when uri-style is defaulted.
if (host != NULL)
if (helper->type == type)
{
endpoint = host;
if (cfgOptionIdxSource(cfgOptRepoAzureUriStyle, repoIdx) == cfgSourceDefault)
uriStyle = storageAzureUriStylePath;
result = helper->helper(repoIdx, write, storageRepoPathExpression);
break;
}
result = storageAzureNew(
cfgOptionIdxStr(cfgOptRepoPath, repoIdx), write, storageRepoPathExpression,
cfgOptionIdxStr(cfgOptRepoAzureContainer, repoIdx), cfgOptionIdxStr(cfgOptRepoAzureAccount, repoIdx),
(StorageAzureKeyType)cfgOptionIdxStrId(cfgOptRepoAzureKeyType, repoIdx),
cfgOptionIdxStr(cfgOptRepoAzureKey, repoIdx), STORAGE_AZURE_BLOCKSIZE_MIN,
endpoint, uriStyle, cfgOptionIdxUInt(cfgOptRepoStoragePort, repoIdx), ioTimeoutMs(),
cfgOptionIdxBool(cfgOptRepoStorageVerifyTls, repoIdx), cfgOptionIdxStrNull(cfgOptRepoStorageCaFile, repoIdx),
cfgOptionIdxStrNull(cfgOptRepoStorageCaPath, repoIdx));
break;
}
}
// Use CIFS storage
case STORAGE_CIFS_TYPE:
result = storageCifsNew(
cfgOptionIdxStr(cfgOptRepoPath, repoIdx), STORAGE_MODE_FILE_DEFAULT, STORAGE_MODE_PATH_DEFAULT, write,
storageRepoPathExpression);
break;
// If no helper was found it try Posix
if (result == NULL)
{
CHECK(type == STORAGE_POSIX_TYPE);
// Use GCS storage
case STORAGE_GCS_TYPE:
result = storageGcsNew(
cfgOptionIdxStr(cfgOptRepoPath, repoIdx), write, storageRepoPathExpression,
cfgOptionIdxStr(cfgOptRepoGcsBucket, repoIdx),
(StorageGcsKeyType)cfgOptionIdxStrId(cfgOptRepoGcsKeyType, repoIdx),
cfgOptionIdxStrNull(cfgOptRepoGcsKey, repoIdx), STORAGE_GCS_CHUNKSIZE_DEFAULT,
cfgOptionIdxStr(cfgOptRepoGcsEndpoint, repoIdx), ioTimeoutMs(),
cfgOptionIdxBool(cfgOptRepoStorageVerifyTls, repoIdx), cfgOptionIdxStrNull(cfgOptRepoStorageCaFile, repoIdx),
cfgOptionIdxStrNull(cfgOptRepoStorageCaPath, repoIdx));
break;
// Use S3 storage
case STORAGE_S3_TYPE:
{
// Set the default port
unsigned int port = cfgOptionIdxUInt(cfgOptRepoStoragePort, repoIdx);
// Extract port from the endpoint and host if it is present
const String *const endPoint = cfgOptionIdxHostPort(cfgOptRepoS3Endpoint, repoIdx, &port);
const String *const host = cfgOptionIdxHostPort(cfgOptRepoStorageHost, repoIdx, &port);
// If the port option was set explicitly then use it in preference to appended ports
if (cfgOptionIdxSource(cfgOptRepoStoragePort, repoIdx) != cfgSourceDefault)
port = cfgOptionIdxUInt(cfgOptRepoStoragePort, repoIdx);
result = storageS3New(
cfgOptionIdxStr(cfgOptRepoPath, repoIdx), write, storageRepoPathExpression,
cfgOptionIdxStr(cfgOptRepoS3Bucket, repoIdx), endPoint,
(StorageS3UriStyle)cfgOptionIdxStrId(cfgOptRepoS3UriStyle, repoIdx),
cfgOptionIdxStr(cfgOptRepoS3Region, repoIdx), (StorageS3KeyType)cfgOptionIdxStrId(cfgOptRepoS3KeyType, repoIdx),
cfgOptionIdxStrNull(cfgOptRepoS3Key, repoIdx), cfgOptionIdxStrNull(cfgOptRepoS3KeySecret, repoIdx),
cfgOptionIdxStrNull(cfgOptRepoS3Token, repoIdx), cfgOptionIdxStrNull(cfgOptRepoS3Role, repoIdx),
STORAGE_S3_PARTSIZE_MIN, host, port, ioTimeoutMs(), cfgOptionIdxBool(cfgOptRepoStorageVerifyTls, repoIdx),
cfgOptionIdxStrNull(cfgOptRepoStorageCaFile, repoIdx), cfgOptionIdxStrNull(cfgOptRepoStorageCaPath, repoIdx));
break;
}
// Use Posix storage. Keep this as the default to prevent code churn.
default:
{
CHECK(type == STORAGE_POSIX_TYPE);
result = storagePosixNewP(
cfgOptionIdxStr(cfgOptRepoPath, repoIdx), .write = write, .pathExpressionFunction = storageRepoPathExpression);
break;
}
result = storagePosixNewP(
cfgOptionIdxStr(cfgOptRepoPath, repoIdx), .write = write, .pathExpressionFunction = storageRepoPathExpression);
}
}
@ -458,7 +401,7 @@ storageRepoIdx(unsigned int repoIdx)
if (storageHelper.storageRepo == NULL)
{
storageHelperInit();
storageHelperContextInit();
storageHelperStanzaInit(false);
storageHelperRepoInit();
@ -501,7 +444,7 @@ storageRepoIdxWrite(unsigned int repoIdx)
if (storageHelper.storageRepoWrite == NULL)
{
storageHelperInit();
storageHelperContextInit();
storageHelperStanzaInit(false);
storageHelperRepoInit();
@ -582,7 +525,7 @@ storageSpool(void)
if (storageHelper.storageSpool == NULL)
{
storageHelperInit();
storageHelperContextInit();
storageHelperStanzaInit(true);
MEM_CONTEXT_BEGIN(storageHelper.memContext)
@ -607,7 +550,7 @@ storageSpoolWrite(void)
if (storageHelper.storageSpoolWrite == NULL)
{
storageHelperInit();
storageHelperContextInit();
storageHelperStanzaInit(true);
MEM_CONTEXT_BEGIN(storageHelper.memContext)
@ -630,7 +573,7 @@ storageHelperFree(void)
if (storageHelper.memContext != NULL)
memContextFree(storageHelper.memContext);
storageHelper = (struct StorageHelper){.memContext = NULL};
storageHelper = (struct StorageHelperLocal){.memContext = NULL, .helperList = storageHelper.helperList};
FUNCTION_TEST_RETURN_VOID();
}

3
src/storage/helper.h
View File

@ -34,6 +34,9 @@ Functions
// prevent damage to the repository from an error in dry-run coding in the individual commands.
void storageHelperDryRunInit(bool dryRun);
// Initialize helpers to create storage other than built-in Posix
void storageHelperInit(const StorageHelper *helperList);
// Local storage object. Writable local storage should be used very sparingly. If writes are not needed then always use
// storageLocal() or a specific storage object instead.
const Storage *storageLocal(void);

45
src/storage/s3/helper.c Normal file
View File

@ -0,0 +1,45 @@
/***********************************************************************************************************************************
S3 Storage Helper
***********************************************************************************************************************************/
#include "build.auto.h"
#include "common/debug.h"
#include "common/io/io.h"
#include "common/log.h"
#include "config/config.h"
#include "storage/s3/helper.h"
/**********************************************************************************************************************************/
Storage *
storageS3Helper(const unsigned int repoIdx, const bool write, StoragePathExpressionCallback pathExpressionCallback)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(UINT, repoIdx);
FUNCTION_LOG_PARAM(BOOL, write);
FUNCTION_LOG_PARAM_P(VOID, pathExpressionCallback);
FUNCTION_LOG_END();
ASSERT(cfgOptionIdxStrId(cfgOptRepoType, repoIdx) == STORAGE_S3_TYPE);
// Set the default port
unsigned int port = cfgOptionIdxUInt(cfgOptRepoStoragePort, repoIdx);
// Extract port from the endpoint and host if it is present
const String *const endPoint = cfgOptionIdxHostPort(cfgOptRepoS3Endpoint, repoIdx, &port);
const String *const host = cfgOptionIdxHostPort(cfgOptRepoStorageHost, repoIdx, &port);
// If the port option was set explicitly then use it in preference to appended ports
if (cfgOptionIdxSource(cfgOptRepoStoragePort, repoIdx) != cfgSourceDefault)
port = cfgOptionIdxUInt(cfgOptRepoStoragePort, repoIdx);
Storage *const result = storageS3New(
cfgOptionIdxStr(cfgOptRepoPath, repoIdx), write, pathExpressionCallback, cfgOptionIdxStr(cfgOptRepoS3Bucket, repoIdx),
endPoint, (StorageS3UriStyle)cfgOptionIdxStrId(cfgOptRepoS3UriStyle, repoIdx), cfgOptionIdxStr(cfgOptRepoS3Region, repoIdx),
(StorageS3KeyType)cfgOptionIdxStrId(cfgOptRepoS3KeyType, repoIdx), cfgOptionIdxStrNull(cfgOptRepoS3Key, repoIdx),
cfgOptionIdxStrNull(cfgOptRepoS3KeySecret, repoIdx), cfgOptionIdxStrNull(cfgOptRepoS3Token, repoIdx),
cfgOptionIdxStrNull(cfgOptRepoS3Role, repoIdx), STORAGE_S3_PARTSIZE_MIN, host, port, ioTimeoutMs(),
cfgOptionIdxBool(cfgOptRepoStorageVerifyTls, repoIdx), cfgOptionIdxStrNull(cfgOptRepoStorageCaFile, repoIdx),
cfgOptionIdxStrNull(cfgOptRepoStorageCaPath, repoIdx));
FUNCTION_LOG_RETURN(STORAGE, result);
}

19
src/storage/s3/helper.h Normal file
View File

@ -0,0 +1,19 @@
/***********************************************************************************************************************************
S3 Storage Helper
***********************************************************************************************************************************/
#ifndef STORAGE_S3_STORAGE_HELPER_H
#define STORAGE_S3_STORAGE_HELPER_H
#include "storage/s3/storage.h"
/***********************************************************************************************************************************
Functions
***********************************************************************************************************************************/
Storage *storageS3Helper(unsigned int repoIdx, bool write, StoragePathExpressionCallback pathExpressionCallback);
/***********************************************************************************************************************************
Storage helper for StorageHelper array passed to storageHelperInit()
***********************************************************************************************************************************/
#define STORAGE_S3_HELPER {.type = STORAGE_S3_TYPE, .helper = storageS3Helper}
#endif

15
src/storage/storage.intern.h
View File

@ -234,6 +234,21 @@ typedef void StorageInterfacePathSync(void *thisVoid, const String *path, Storag
#define storageInterfacePathSyncP(thisVoid, path, ...) \
STORAGE_COMMON_INTERFACE(thisVoid).pathSync(thisVoid, path, (StorageInterfacePathSyncParam){VAR_PARAM_INIT, __VA_ARGS__})
/***********************************************************************************************************************************
Storage type and helper function struct
An array of this struct must be passed to storageHelperInit() to enable storage drivers other than built-in Posix.
***********************************************************************************************************************************/
typedef Storage *(*StorageHelperFunction)(unsigned int repoIdx, bool write, StoragePathExpressionCallback pathExpressionCallback);
typedef struct StorageHelper
{
StringId type; // StringId that identifies the storage driver
StorageHelperFunction helper; // Function that creates the storage
} StorageHelper;
#define STORAGE_END_HELPER {.type = 0}
/***********************************************************************************************************************************
Constructors
***********************************************************************************************************************************/

89
test/define.yaml
View File

@ -408,21 +408,11 @@ unit:
- protocol/command
- protocol/helper
- protocol/server
- storage/azure/read
- storage/azure/storage
- storage/azure/write
- storage/cifs/storage
- storage/gcs/read
- storage/gcs/storage
- storage/gcs/write
- storage/helper
- storage/remote/read
- storage/remote/protocol
- storage/remote/storage
- storage/remote/write
- storage/s3/read
- storage/s3/storage
- storage/s3/write
# ----------------------------------------------------------------------------------------------------------------------------
- name: lock
@ -523,11 +513,13 @@ unit:
test:
# ----------------------------------------------------------------------------------------------------------------------------
- name: posix
total: 21
total: 22
feature: STORAGE
harness: storage
coverage:
- storage/cifs/helper
- storage/cifs/storage
- storage/posix/read
- storage/posix/storage
- storage/posix/write
@ -536,46 +528,6 @@ unit:
- storage/storage
- storage/write
# ----------------------------------------------------------------------------------------------------------------------------
- name: gcs
total: 3
coverage:
- storage/gcs/read
- storage/gcs/storage
- storage/gcs/write
- storage/helper
include:
- storage/storage
- storage/write
# ----------------------------------------------------------------------------------------------------------------------------
- name: azure
total: 3
coverage:
- storage/azure/read
- storage/azure/storage
- storage/azure/write
- storage/helper
include:
- storage/storage
- storage/write
# ----------------------------------------------------------------------------------------------------------------------------
- name: cifs
total: 1
coverage:
- storage/cifs/storage
- storage/posix/storage
- storage/helper
include:
- storage/storage
# ----------------------------------------------------------------------------------------------------------------------------
- name: remote
total: 9
@ -592,18 +544,49 @@ unit:
- storage/read
- storage/write
# ----------------------------------------------------------------------------------------------------------------------------
- name: azure
total: 3
coverage:
- storage/azure/helper
- storage/azure/read
- storage/azure/storage
- storage/azure/write
include:
- storage/helper
- storage/storage
- storage/write
# ----------------------------------------------------------------------------------------------------------------------------
- name: gcs
total: 3
coverage:
- storage/gcs/helper
- storage/gcs/read
- storage/gcs/storage
- storage/gcs/write
include:
- storage/helper
- storage/storage
- storage/write
# ----------------------------------------------------------------------------------------------------------------------------
- name: s3
total: 2
coverage:
- storage/s3/helper
- storage/s3/read
- storage/s3/storage
- storage/s3/write
- storage/helper
- storage/storage
include:
- storage/helper
- storage/storage
- storage/write
# ********************************************************************************************************************************

8
test/src/common/harnessStorage.c
View File

@ -114,8 +114,14 @@ hrnStorageInfoListCallback(void *callbackData, const StorageInfo *info)
}
case storageTypeLink:
strCatFmt(data->content, "link, d=%s", strZ(info->linkDestination));
{
strCatZ(data->content, "link");
if (info->linkDestination != NULL)
strCatFmt(data->content, ", d=%s", strZ(info->linkDestination));
break;
}
case storageTypePath:
strCatZ(data->content, "path");

5
test/src/module/storage/azureTest.c
View File

@ -3,6 +3,7 @@ Test Azure Storage
***********************************************************************************************************************************/
#include "common/io/fdRead.h"
#include "common/io/fdWrite.h"
#include "storage/helper.h"
#include "common/harnessConfig.h"
#include "common/harnessFork.h"
@ -169,6 +170,10 @@ testRun(void)
{
FUNCTION_HARNESS_VOID();
// Set storage helper
static const StorageHelper storageHelperList[] = {STORAGE_AZURE_HELPER, STORAGE_END_HELPER};
storageHelperInit(storageHelperList);
// *****************************************************************************************************************************
if (testBegin("storageRepoGet()"))
{

51
test/src/module/storage/cifsTest.c
View File

@ -1,51 +0,0 @@
/***********************************************************************************************************************************
Test CIFS Storage
***********************************************************************************************************************************/
#include "common/harnessConfig.h"
/***********************************************************************************************************************************
Test Run
***********************************************************************************************************************************/
void
testRun(void)
{
FUNCTION_HARNESS_VOID();
// *****************************************************************************************************************************
if (testBegin("storageRepoGet() and StorageDriverCifs"))
{
// Load configuration
StringList *argList = strLstNew();
hrnCfgArgRawZ(argList, cfgOptStanza, "test");
hrnCfgArgRawZ(argList, cfgOptPgPath, "/path/to/pg");
hrnCfgArgRawZ(argList, cfgOptRepoType, "cifs");
hrnCfgArgRawZ(argList, cfgOptRepoPath, TEST_PATH);
HRN_CFG_LOAD(cfgCmdArchiveGet, argList);
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("storage configuration");
const Storage *storage = NULL;
TEST_ASSIGN(storage, storageRepoGet(0, true), "get cifs repo storage");
TEST_RESULT_UINT(storageType(storage), STORAGE_CIFS_TYPE, "check storage type");
TEST_RESULT_BOOL(storageFeature(storage, storageFeaturePath), true, "check path feature");
TEST_RESULT_BOOL(storageFeature(storage, storageFeatureCompress), true, "check compress feature");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("write object path sync false");
// Create a FileWrite object with path sync enabled and ensure that path sync is false in the write object
StorageWrite *file = NULL;
TEST_ASSIGN(file, storageNewWriteP(storage, STRDEF("somefile"), .noSyncPath = false), "new file write");
TEST_RESULT_BOOL(storageWriteSyncPath(file), false, "path sync is disabled");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("path sync result is noop");
// Test the path sync function -- pass a bogus path to ensure that this is a noop
TEST_RESULT_VOID(storagePathSyncP(storage, STRDEF(BOGUS_STR)), "path sync is a noop");
}
FUNCTION_HARNESS_RETURN_VOID();
}

5
test/src/module/storage/gcsTest.c
View File

@ -3,6 +3,7 @@ Test GCS Storage
***********************************************************************************************************************************/
#include "common/io/fdRead.h"
#include "common/io/fdWrite.h"
#include "storage/helper.h"
#include "common/harnessConfig.h"
#include "common/harnessFork.h"
@ -184,6 +185,10 @@ testRun(void)
{
FUNCTION_HARNESS_VOID();
// Set storage helper
static const StorageHelper storageHelperList[] = {STORAGE_GCS_HELPER, STORAGE_END_HELPER};
storageHelperInit(storageHelperList);
Storage *storageTest = storagePosixNewP(TEST_PATH_STR, .write = true);
// Get test host and ports

104
test/src/module/storage/posixTest.c
View File

@ -1,5 +1,5 @@
/***********************************************************************************************************************************
Test Posix Storage
Test Posix/CIFS Storage
***********************************************************************************************************************************/
#include "common/io/io.h"
#include "common/time.h"
@ -194,6 +194,14 @@ testRun(void)
TEST_RESULT_BOOL(storageInfoP(storagePosixNewP(FSLASH_STR), NULL).exists, true, "info for /");
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("info for / does not exist with no path feature");
Storage *storageRootNoPath = storagePosixNewP(FSLASH_STR);
storageRootNoPath->pub.interface.feature ^= 1 << storageFeaturePath;
TEST_RESULT_BOOL(storageInfoP(storageRootNoPath, NULL, .ignoreMissing = true).exists, false, "no info for /");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("directory does not exists");
@ -238,6 +246,26 @@ testRun(void)
TEST_RESULT_UINT(info.groupId, TEST_GROUP_ID, "check group id");
TEST_RESULT_STR(info.group, TEST_GROUP_STR, "check group");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("info basic - path");
storageTest->pub.interface.feature ^= 1 << storageFeatureInfoDetail;
TEST_ASSIGN(info, storageInfoP(storageTest, TEST_PATH_STR), "get path info");
TEST_RESULT_STR(info.name, NULL, "name is not set");
TEST_RESULT_BOOL(info.exists, true, "check exists");
TEST_RESULT_INT(info.type, storageTypePath, "check type");
TEST_RESULT_UINT(info.size, 0, "check size");
TEST_RESULT_INT(info.mode, 0, "check mode");
TEST_RESULT_INT(info.timeModified, 1555160000, "check mod time");
TEST_RESULT_STR(info.linkDestination, NULL, "no link destination");
TEST_RESULT_UINT(info.userId, 0, "check user id");
TEST_RESULT_STR(info.user, NULL, "check user");
TEST_RESULT_UINT(info.groupId, 0, "check group id");
TEST_RESULT_STR(info.group, NULL, "check group");
storageTest->pub.interface.feature ^= 1 << storageFeatureInfoDetail;
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("info - file");
@ -431,6 +459,32 @@ testRun(void)
". {path}\n",
"check content");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("path basic info - recurse");
storagePathCreateP(storageTest, STRDEF("pg/path"), .mode = 0700);
storagePutP(storageNewWriteP(storageTest, STRDEF("pg/path/file"), .modeFile = 0600), BUFSTRDEF("TESTDATA"));
callbackData.content = strNew();
storageTest->pub.interface.feature ^= 1 << storageFeatureInfoDetail;
TEST_RESULT_VOID(
storageInfoListP(
storageTest, STRDEF("pg"), hrnStorageInfoListCallback, &callbackData, .sortOrder = sortOrderDesc, .recurse = true),
"recurse descending");
TEST_RESULT_STR_Z(
callbackData.content,
"pipe {special}\n"
"path/file {file, s=8}\n"
"path {path}\n"
"link {link}\n"
"file {file, s=8}\n"
". {path}\n",
"check content");
storageTest->pub.interface.feature ^= 1 << storageFeatureInfoDetail;
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("path - filter");
@ -1474,5 +1528,53 @@ testRun(void)
TEST_ERROR(storageSpoolWrite(), AssertError, "stanza cannot be NULL for this storage object");
}
// *****************************************************************************************************************************
if (testBegin("storageRepoGet() and StorageDriverCifs"))
{
// Load configuration
StringList *argList = strLstNew();
hrnCfgArgRawZ(argList, cfgOptStanza, "test");
hrnCfgArgRawZ(argList, cfgOptPgPath, "/path/to/pg");
hrnCfgArgRawZ(argList, cfgOptRepoType, "cifs");
hrnCfgArgRawZ(argList, cfgOptRepoPath, TEST_PATH);
HRN_CFG_LOAD(cfgCmdArchiveGet, argList);
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("error on invalid storage type");
static const StorageHelper storageHelperListError[] = {{.type = STORAGE_POSIX_TYPE}, STORAGE_END_HELPER};
storageHelperInit(storageHelperListError);
TEST_ERROR(storageRepoGet(0, true), AssertError, "check 'type == STORAGE_POSIX_TYPE' failed");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("storage configuration");
// Set storage helper
static const StorageHelper storageHelperList[] = {STORAGE_CIFS_HELPER, STORAGE_END_HELPER};
storageHelperInit(storageHelperList);
const Storage *storage = NULL;
TEST_ASSIGN(storage, storageRepoGet(0, true), "get cifs repo storage");
TEST_RESULT_UINT(storageType(storage), STORAGE_CIFS_TYPE, "check storage type");
TEST_RESULT_BOOL(storageFeature(storage, storageFeaturePath), true, "check path feature");
TEST_RESULT_BOOL(storageFeature(storage, storageFeatureCompress), true, "check compress feature");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("write object path sync false");
// Create a FileWrite object with path sync enabled and ensure that path sync is false in the write object
StorageWrite *file = NULL;
TEST_ASSIGN(file, storageNewWriteP(storage, STRDEF("somefile"), .noSyncPath = false), "new file write");
TEST_RESULT_BOOL(storageWriteSyncPath(file), false, "path sync is disabled");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("path sync result is noop");
// Test the path sync function -- pass a bogus path to ensure that this is a noop
TEST_RESULT_VOID(storagePathSyncP(storage, STRDEF(BOGUS_STR)), "path sync is a noop");
}
FUNCTION_HARNESS_RETURN_VOID();
}

9
test/src/module/storage/s3Test.c
View File

@ -5,6 +5,7 @@ Test S3 Storage
#include "common/io/fdRead.h"
#include "common/io/fdWrite.h"
#include "storage/helper.h"
#include "version.h"
#include "common/harnessConfig.h"
@ -149,10 +150,6 @@ testResponse(IoWrite *write, TestResponseParam param)
case 200:
strCatZ(response, "OK");
break;
case 403:
strCatZ(response, "Forbidden");
break;
}
// End header
@ -205,6 +202,10 @@ testRun(void)
{
FUNCTION_HARNESS_VOID();
// Set storage helper
static const StorageHelper storageHelperList[] = {STORAGE_S3_HELPER, STORAGE_END_HELPER};
storageHelperInit(storageHelperList);
// Test strings
const String *path = STRDEF("/");
const String *bucket = STRDEF("bucket");