mirror of
https://github.com/pgbackrest/pgbackrest.git
synced 2025-01-18 04:58:51 +02:00
Update storage module to use StringIds.
Use StringIds for the storage types (e.g. STORAGE_S3_TYPE) and configuration settings, e.g. cfgOptS3KeyType. Also add new config functions and harness config functions to support StringIds.
This commit is contained in:
parent
aa72c19a83
commit
6cc521b6b2
@ -84,7 +84,12 @@
|
||||
</release-item>
|
||||
|
||||
<release-item>
|
||||
<github-pull-request id="1358"/>
|
||||
<commit subject="Add StringId type.">
|
||||
<github-pull-request id="1358"/>
|
||||
</commit>
|
||||
<commit subject="Update storage module to use StringIds.">
|
||||
<github-pull-request id="1379"/>
|
||||
</commit>
|
||||
|
||||
<release-item-contributor-list>
|
||||
<release-item-reviewer id="cynthia.shang"/>
|
||||
|
@ -21,23 +21,32 @@ cmdRepoCreate(void)
|
||||
|
||||
MEM_CONTEXT_TEMP_BEGIN()
|
||||
{
|
||||
if (strEq(storageType(storageRepo()), STORAGE_S3_TYPE_STR))
|
||||
{
|
||||
storageS3RequestP((StorageS3 *)storageDriver(storageRepoWrite()), HTTP_VERB_PUT_STR, FSLASH_STR);
|
||||
}
|
||||
else if (strEq(storageType(storageRepo()), STORAGE_AZURE_TYPE_STR))
|
||||
switch (storageType(storageRepo()))
|
||||
{
|
||||
case STORAGE_AZURE_TYPE:
|
||||
storageAzureRequestP(
|
||||
(StorageAzure *)storageDriver(storageRepoWrite()), HTTP_VERB_PUT_STR,
|
||||
.query = httpQueryAdd(httpQueryNewP(), AZURE_QUERY_RESTYPE_STR, AZURE_QUERY_VALUE_CONTAINER_STR));
|
||||
}
|
||||
else if (strEq(storageType(storageRepo()), STORAGE_GCS_TYPE_STR))
|
||||
break;
|
||||
|
||||
case STORAGE_GCS_TYPE:
|
||||
{
|
||||
KeyValue *kvContent = kvPut(kvNew(), GCS_JSON_NAME_VAR, VARSTR(cfgOptionStr(cfgOptRepoGcsBucket)));
|
||||
const KeyValue *const kvContent = kvPut(kvNew(), GCS_JSON_NAME_VAR, VARSTR(cfgOptionStr(cfgOptRepoGcsBucket)));
|
||||
|
||||
storageGcsRequestP(
|
||||
(StorageGcs *)storageDriver(storageRepoWrite()), HTTP_VERB_POST_STR, .noBucket = true,
|
||||
.content = BUFSTR(jsonFromKv(kvContent)));
|
||||
|
||||
break;
|
||||
}
|
||||
|
||||
case STORAGE_S3_TYPE:
|
||||
storageS3RequestP((StorageS3 *)storageDriver(storageRepoWrite()), HTTP_VERB_PUT_STR, FSLASH_STR);
|
||||
break;
|
||||
|
||||
// Other storage types do not require the repo to be created
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
MEM_CONTEXT_TEMP_END();
|
||||
|
@ -1058,6 +1058,47 @@ cfgOptionIdxStrNull(ConfigOption optionId, unsigned int optionIdx)
|
||||
FUNCTION_LOG_RETURN_CONST(STRING, varStr(cfgOptionIdxInternal(optionId, optionIdx, varTypeString, true)));
|
||||
}
|
||||
|
||||
// Helper to convert option String values to StringIds. Some options need 6-bit encoding while most work fine with 5-bit encoding.
|
||||
// At some point the config parser will work with StringIds directly and this code can be removed, but for now it protects the
|
||||
// callers from this logic and hopefully means no changes to the callers when the parser is updated.
|
||||
static StringId
|
||||
cfgOptionStrIdInternal(
|
||||
const ConfigOption optionId, const unsigned int optionIdx)
|
||||
{
|
||||
FUNCTION_TEST_BEGIN();
|
||||
FUNCTION_TEST_PARAM(ENUM, optionId);
|
||||
FUNCTION_TEST_PARAM(UINT, optionIdx);
|
||||
FUNCTION_TEST_END();
|
||||
|
||||
const String *const value = varStr(cfgOptionIdxInternal(optionId, optionIdx, varTypeString, false));
|
||||
|
||||
if (optionId == cfgOptRepoType)
|
||||
FUNCTION_TEST_RETURN(strIdFromStr(stringIdBit6, value));
|
||||
|
||||
FUNCTION_TEST_RETURN(strIdFromStr(stringIdBit5, value));
|
||||
}
|
||||
|
||||
StringId
|
||||
cfgOptionStrId(ConfigOption optionId)
|
||||
{
|
||||
FUNCTION_LOG_BEGIN(logLevelTrace);
|
||||
FUNCTION_LOG_PARAM(ENUM, optionId);
|
||||
FUNCTION_LOG_END();
|
||||
|
||||
FUNCTION_LOG_RETURN(STRING_ID, cfgOptionStrIdInternal(optionId, cfgOptionIdxDefault(optionId)));
|
||||
}
|
||||
|
||||
StringId
|
||||
cfgOptionIdxStrId(ConfigOption optionId, unsigned int optionIdx)
|
||||
{
|
||||
FUNCTION_LOG_BEGIN(logLevelTrace);
|
||||
FUNCTION_LOG_PARAM(ENUM, optionId);
|
||||
FUNCTION_LOG_PARAM(UINT, optionIdx);
|
||||
FUNCTION_LOG_END();
|
||||
|
||||
FUNCTION_LOG_RETURN(STRING_ID, cfgOptionStrIdInternal(optionId, optionIdx));
|
||||
}
|
||||
|
||||
/**********************************************************************************************************************************/
|
||||
void
|
||||
cfgOptionSet(ConfigOption optionId, ConfigSource source, const Variant *value)
|
||||
|
@ -9,6 +9,7 @@ config/parse.c sets the command and options and determines which options are val
|
||||
|
||||
#include "common/lock.h"
|
||||
#include "common/log.h"
|
||||
#include "common/type/stringId.h"
|
||||
#include "common/type/stringList.h"
|
||||
#include "config/config.auto.h"
|
||||
|
||||
@ -127,6 +128,8 @@ const VariantList *cfgOptionLst(ConfigOption optionId);
|
||||
const VariantList *cfgOptionIdxLst(ConfigOption optionId, unsigned int optionIdx);
|
||||
const String *cfgOptionStr(ConfigOption optionId);
|
||||
const String *cfgOptionIdxStr(ConfigOption optionId, unsigned int optionIdx);
|
||||
StringId cfgOptionStrId(ConfigOption optionId);
|
||||
StringId cfgOptionIdxStrId(ConfigOption optionId, unsigned int optionIdx);
|
||||
const String *cfgOptionStrNull(ConfigOption optionId);
|
||||
const String *cfgOptionIdxStrNull(ConfigOption optionId, unsigned int optionIdx);
|
||||
unsigned int cfgOptionUInt(ConfigOption optionId);
|
||||
|
@ -86,9 +86,9 @@ cfgLoadUpdateOption(void)
|
||||
for (unsigned int optionIdx = 0; optionIdx < cfgOptionGroupIdxTotal(cfgOptGrpRepo); optionIdx++)
|
||||
{
|
||||
// If the repo is local and either posix or cifs
|
||||
if (!(cfgOptionIdxTest(cfgOptRepoHost, optionIdx)) &&
|
||||
(strEq(cfgOptionIdxStr(cfgOptRepoType, optionIdx), STORAGE_POSIX_TYPE_STR) ||
|
||||
strEq(cfgOptionIdxStr(cfgOptRepoType, optionIdx), STORAGE_CIFS_TYPE_STR)))
|
||||
if (!cfgOptionIdxTest(cfgOptRepoHost, optionIdx) &&
|
||||
(cfgOptionIdxStrId(cfgOptRepoType, optionIdx) == STORAGE_POSIX_TYPE ||
|
||||
cfgOptionIdxStrId(cfgOptRepoType, optionIdx) == STORAGE_CIFS_TYPE))
|
||||
{
|
||||
// Ensure a local repo does not have the same path as another local repo of the same type
|
||||
for (unsigned int repoIdx = 0; repoIdx < cfgOptionGroupIdxTotal(cfgOptGrpRepo); repoIdx++)
|
||||
|
@ -132,7 +132,7 @@ storageReadAzureNew(StorageAzure *storage, const String *name, bool ignoreMissin
|
||||
|
||||
.interface = (StorageReadInterface)
|
||||
{
|
||||
.type = STORAGE_AZURE_TYPE_STR,
|
||||
.type = STORAGE_AZURE_TYPE,
|
||||
.name = strDup(name),
|
||||
.ignoreMissing = ignoreMissing,
|
||||
|
||||
|
@ -21,11 +21,6 @@ Azure Storage
|
||||
#include "storage/azure/storage.intern.h"
|
||||
#include "storage/azure/write.h"
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Storage type
|
||||
***********************************************************************************************************************************/
|
||||
STRING_EXTERN(STORAGE_AZURE_TYPE_STR, STORAGE_AZURE_TYPE);
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Azure http headers
|
||||
***********************************************************************************************************************************/
|
||||
@ -699,7 +694,7 @@ storageAzureNew(
|
||||
FUNCTION_LOG_PARAM(FUNCTIONP, pathExpressionFunction);
|
||||
FUNCTION_LOG_PARAM(STRING, container);
|
||||
FUNCTION_TEST_PARAM(STRING, account);
|
||||
FUNCTION_LOG_PARAM(ENUM, keyType);
|
||||
FUNCTION_LOG_PARAM(STRING_ID, keyType);
|
||||
FUNCTION_TEST_PARAM(STRING, key);
|
||||
FUNCTION_LOG_PARAM(SIZE, blockSize);
|
||||
FUNCTION_LOG_PARAM(STRING, host);
|
||||
@ -756,7 +751,7 @@ storageAzureNew(
|
||||
// Generate starting file id
|
||||
cryptoRandomBytes((unsigned char *)&driver->fileId, sizeof(driver->fileId));
|
||||
|
||||
this = storageNew(STORAGE_AZURE_TYPE_STR, path, 0, 0, write, pathExpressionFunction, driver, driver->interface);
|
||||
this = storageNew(STORAGE_AZURE_TYPE, path, 0, 0, write, pathExpressionFunction, driver, driver->interface);
|
||||
}
|
||||
MEM_CONTEXT_NEW_END();
|
||||
|
||||
|
@ -9,21 +9,17 @@ Azure Storage
|
||||
/***********************************************************************************************************************************
|
||||
Storage type
|
||||
***********************************************************************************************************************************/
|
||||
#define STORAGE_AZURE_TYPE "azure"
|
||||
STRING_DECLARE(STORAGE_AZURE_TYPE_STR);
|
||||
#define STORAGE_AZURE_TYPE STRID6("azure", 0x54956811)
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Key type
|
||||
***********************************************************************************************************************************/
|
||||
typedef enum
|
||||
{
|
||||
storageAzureKeyTypeShared,
|
||||
storageAzureKeyTypeSas,
|
||||
storageAzureKeyTypeShared = STRID5("shared", 0x85905130),
|
||||
storageAzureKeyTypeSas = STRID5("sas", 0x4c330),
|
||||
} StorageAzureKeyType;
|
||||
|
||||
#define STORAGE_AZURE_KEY_TYPE_SHARED "shared"
|
||||
#define STORAGE_AZURE_KEY_TYPE_SAS "sas"
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Defaults
|
||||
***********************************************************************************************************************************/
|
||||
|
@ -289,7 +289,7 @@ storageWriteAzureNew(StorageAzure *storage, const String *name, uint64_t fileId,
|
||||
|
||||
.interface = (StorageWriteInterface)
|
||||
{
|
||||
.type = STORAGE_AZURE_TYPE_STR,
|
||||
.type = STORAGE_AZURE_TYPE,
|
||||
.name = strDup(name),
|
||||
.atomic = true,
|
||||
.createPath = true,
|
||||
|
@ -10,11 +10,6 @@ CIFS Storage
|
||||
#include "storage/cifs/storage.h"
|
||||
#include "storage/posix/storage.intern.h"
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Storage type
|
||||
***********************************************************************************************************************************/
|
||||
STRING_EXTERN(STORAGE_CIFS_TYPE_STR, STORAGE_CIFS_TYPE);
|
||||
|
||||
/**********************************************************************************************************************************/
|
||||
Storage *
|
||||
storageCifsNew(
|
||||
@ -29,5 +24,5 @@ storageCifsNew(
|
||||
FUNCTION_LOG_END();
|
||||
|
||||
FUNCTION_LOG_RETURN(
|
||||
STORAGE, storagePosixNewInternal(STORAGE_CIFS_TYPE_STR, path, modeFile, modePath, write, pathExpressionFunction, false));
|
||||
STORAGE, storagePosixNewInternal(STORAGE_CIFS_TYPE, path, modeFile, modePath, write, pathExpressionFunction, false));
|
||||
}
|
||||
|
@ -9,8 +9,7 @@ CIFS Storage
|
||||
/***********************************************************************************************************************************
|
||||
Storage type
|
||||
***********************************************************************************************************************************/
|
||||
#define STORAGE_CIFS_TYPE "cifs"
|
||||
STRING_DECLARE(STORAGE_CIFS_TYPE_STR);
|
||||
#define STORAGE_CIFS_TYPE STRID6("cifs", 0x4c62431)
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Constructors
|
||||
|
@ -139,7 +139,7 @@ storageReadGcsNew(StorageGcs *storage, const String *name, bool ignoreMissing)
|
||||
|
||||
.interface = (StorageReadInterface)
|
||||
{
|
||||
.type = STORAGE_GCS_TYPE_STR,
|
||||
.type = STORAGE_GCS_TYPE,
|
||||
.name = strDup(name),
|
||||
.ignoreMissing = ignoreMissing,
|
||||
|
||||
|
@ -27,11 +27,6 @@ GCS Storage
|
||||
#include "storage/gcs/write.h"
|
||||
#include "storage/posix/storage.h"
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Storage type
|
||||
***********************************************************************************************************************************/
|
||||
STRING_EXTERN(STORAGE_GCS_TYPE_STR, STORAGE_GCS_TYPE);
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
HTTP headers
|
||||
***********************************************************************************************************************************/
|
||||
@ -867,7 +862,7 @@ storageGcsNew(
|
||||
FUNCTION_LOG_PARAM(BOOL, write);
|
||||
FUNCTION_LOG_PARAM(FUNCTIONP, pathExpressionFunction);
|
||||
FUNCTION_LOG_PARAM(STRING, bucket);
|
||||
FUNCTION_LOG_PARAM(ENUM, keyType);
|
||||
FUNCTION_LOG_PARAM(STRING_ID, keyType);
|
||||
FUNCTION_TEST_PARAM(STRING, key);
|
||||
FUNCTION_LOG_PARAM(SIZE, chunkSize);
|
||||
FUNCTION_LOG_PARAM(STRING, endpoint);
|
||||
@ -943,7 +938,7 @@ storageGcsNew(
|
||||
driver->queryRedactList = strLstNew();
|
||||
strLstAdd(driver->queryRedactList, GCS_QUERY_UPLOAD_ID_STR);
|
||||
|
||||
this = storageNew(STORAGE_GCS_TYPE_STR, path, 0, 0, write, pathExpressionFunction, driver, driver->interface);
|
||||
this = storageNew(STORAGE_GCS_TYPE, path, 0, 0, write, pathExpressionFunction, driver, driver->interface);
|
||||
}
|
||||
MEM_CONTEXT_NEW_END();
|
||||
|
||||
|
@ -9,21 +9,17 @@ GCS Storage
|
||||
/***********************************************************************************************************************************
|
||||
Storage type
|
||||
***********************************************************************************************************************************/
|
||||
#define STORAGE_GCS_TYPE "gcs"
|
||||
STRING_DECLARE(STORAGE_GCS_TYPE_STR);
|
||||
#define STORAGE_GCS_TYPE STRID6("gcs", 0x130c71)
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Key type
|
||||
***********************************************************************************************************************************/
|
||||
typedef enum
|
||||
{
|
||||
storageGcsKeyTypeService,
|
||||
storageGcsKeyTypeToken,
|
||||
storageGcsKeyTypeService = STRID5("service", 0x1469b48b30),
|
||||
storageGcsKeyTypeToken = STRID5("token", 0xe2adf40),
|
||||
} StorageGcsKeyType;
|
||||
|
||||
#define STORAGE_GCS_KEY_TYPE_SERVICE "service"
|
||||
#define STORAGE_GCS_KEY_TYPE_TOKEN "token"
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Defaults
|
||||
***********************************************************************************************************************************/
|
||||
|
@ -342,7 +342,7 @@ storageWriteGcsNew(StorageGcs *storage, const String *name, size_t chunkSize)
|
||||
|
||||
.interface = (StorageWriteInterface)
|
||||
{
|
||||
.type = STORAGE_GCS_TYPE_STR,
|
||||
.type = STORAGE_GCS_TYPE,
|
||||
.name = strDup(name),
|
||||
.atomic = true,
|
||||
.createPath = true,
|
||||
|
@ -352,77 +352,77 @@ storageRepoGet(unsigned int repoIdx, bool write)
|
||||
// Use local storage
|
||||
else
|
||||
{
|
||||
const String *type = cfgOptionIdxStr(cfgOptRepoType, repoIdx);
|
||||
const StringId type = cfgOptionIdxStrId(cfgOptRepoType, repoIdx);
|
||||
|
||||
// Use Azure storage
|
||||
if (strEqZ(type, STORAGE_AZURE_TYPE))
|
||||
switch (type)
|
||||
{
|
||||
result = storageAzureNew(
|
||||
cfgOptionIdxStr(cfgOptRepoPath, repoIdx), write, storageRepoPathExpression,
|
||||
cfgOptionIdxStr(cfgOptRepoAzureContainer, repoIdx), cfgOptionIdxStr(cfgOptRepoAzureAccount, repoIdx),
|
||||
strEqZ(cfgOptionIdxStr(cfgOptRepoAzureKeyType, repoIdx), STORAGE_AZURE_KEY_TYPE_SHARED) ?
|
||||
storageAzureKeyTypeShared : storageAzureKeyTypeSas,
|
||||
cfgOptionIdxStr(cfgOptRepoAzureKey, repoIdx), STORAGE_AZURE_BLOCKSIZE_MIN,
|
||||
cfgOptionIdxStrNull(cfgOptRepoStorageHost, repoIdx), cfgOptionIdxStr(cfgOptRepoAzureEndpoint, repoIdx),
|
||||
cfgOptionIdxUInt(cfgOptRepoStoragePort, repoIdx), ioTimeoutMs(),
|
||||
cfgOptionIdxBool(cfgOptRepoStorageVerifyTls, repoIdx), cfgOptionIdxStrNull(cfgOptRepoStorageCaFile, repoIdx),
|
||||
cfgOptionIdxStrNull(cfgOptRepoStorageCaPath, repoIdx));
|
||||
}
|
||||
// Use CIFS storage
|
||||
else if (strEqZ(type, STORAGE_CIFS_TYPE))
|
||||
{
|
||||
result = storageCifsNew(
|
||||
cfgOptionIdxStr(cfgOptRepoPath, repoIdx), STORAGE_MODE_FILE_DEFAULT, STORAGE_MODE_PATH_DEFAULT, write,
|
||||
storageRepoPathExpression);
|
||||
}
|
||||
// Use GCS storage
|
||||
else if (strEqZ(type, STORAGE_GCS_TYPE))
|
||||
{
|
||||
result = storageGcsNew(
|
||||
cfgOptionIdxStr(cfgOptRepoPath, repoIdx), write, storageRepoPathExpression,
|
||||
cfgOptionIdxStr(cfgOptRepoGcsBucket, repoIdx),
|
||||
strEqZ(cfgOptionIdxStr(cfgOptRepoGcsKeyType, repoIdx), STORAGE_GCS_KEY_TYPE_SERVICE) ?
|
||||
storageGcsKeyTypeService : storageGcsKeyTypeToken,
|
||||
cfgOptionIdxStr(cfgOptRepoGcsKey, repoIdx), STORAGE_GCS_CHUNKSIZE_DEFAULT,
|
||||
cfgOptionIdxStr(cfgOptRepoGcsEndpoint, repoIdx), ioTimeoutMs(),
|
||||
cfgOptionIdxBool(cfgOptRepoStorageVerifyTls, repoIdx),
|
||||
cfgOptionIdxStrNull(cfgOptRepoStorageCaFile, repoIdx), cfgOptionIdxStrNull(cfgOptRepoStorageCaPath, repoIdx));
|
||||
}
|
||||
// Use Posix storage
|
||||
else if (strEqZ(type, STORAGE_POSIX_TYPE))
|
||||
{
|
||||
result = storagePosixNewP(
|
||||
cfgOptionIdxStr(cfgOptRepoPath, repoIdx), .write = write, .pathExpressionFunction = storageRepoPathExpression);
|
||||
}
|
||||
// Use S3 storage
|
||||
else
|
||||
{
|
||||
// Storage must be S3
|
||||
CHECK(strEqZ(type, STORAGE_S3_TYPE));
|
||||
// Use Azure storage
|
||||
case STORAGE_AZURE_TYPE:
|
||||
result = storageAzureNew(
|
||||
cfgOptionIdxStr(cfgOptRepoPath, repoIdx), write, storageRepoPathExpression,
|
||||
cfgOptionIdxStr(cfgOptRepoAzureContainer, repoIdx), cfgOptionIdxStr(cfgOptRepoAzureAccount, repoIdx),
|
||||
(StorageAzureKeyType)cfgOptionIdxStrId(cfgOptRepoAzureKeyType, repoIdx),
|
||||
cfgOptionIdxStr(cfgOptRepoAzureKey, repoIdx), STORAGE_AZURE_BLOCKSIZE_MIN,
|
||||
cfgOptionIdxStrNull(cfgOptRepoStorageHost, repoIdx), cfgOptionIdxStr(cfgOptRepoAzureEndpoint, repoIdx),
|
||||
cfgOptionIdxUInt(cfgOptRepoStoragePort, repoIdx), ioTimeoutMs(),
|
||||
cfgOptionIdxBool(cfgOptRepoStorageVerifyTls, repoIdx), cfgOptionIdxStrNull(cfgOptRepoStorageCaFile, repoIdx),
|
||||
cfgOptionIdxStrNull(cfgOptRepoStorageCaPath, repoIdx));
|
||||
break;
|
||||
|
||||
// Set the default port
|
||||
unsigned int port = cfgOptionIdxUInt(cfgOptRepoStoragePort, repoIdx);
|
||||
// Use CIFS storage
|
||||
case STORAGE_CIFS_TYPE:
|
||||
result = storageCifsNew(
|
||||
cfgOptionIdxStr(cfgOptRepoPath, repoIdx), STORAGE_MODE_FILE_DEFAULT, STORAGE_MODE_PATH_DEFAULT, write,
|
||||
storageRepoPathExpression);
|
||||
break;
|
||||
|
||||
// Extract port from the endpoint and host if it is present
|
||||
const String *endPoint = cfgOptionIdxHostPort(cfgOptRepoS3Endpoint, repoIdx, &port);
|
||||
const String *host = cfgOptionIdxHostPort(cfgOptRepoStorageHost, repoIdx, &port);
|
||||
// Use GCS storage
|
||||
case STORAGE_GCS_TYPE:
|
||||
result = storageGcsNew(
|
||||
cfgOptionIdxStr(cfgOptRepoPath, repoIdx), write, storageRepoPathExpression,
|
||||
cfgOptionIdxStr(cfgOptRepoGcsBucket, repoIdx),
|
||||
(StorageGcsKeyType)cfgOptionIdxStrId(cfgOptRepoGcsKeyType, repoIdx), cfgOptionIdxStr(cfgOptRepoGcsKey, repoIdx),
|
||||
STORAGE_GCS_CHUNKSIZE_DEFAULT, cfgOptionIdxStr(cfgOptRepoGcsEndpoint, repoIdx), ioTimeoutMs(),
|
||||
cfgOptionIdxBool(cfgOptRepoStorageVerifyTls, repoIdx), cfgOptionIdxStrNull(cfgOptRepoStorageCaFile, repoIdx),
|
||||
cfgOptionIdxStrNull(cfgOptRepoStorageCaPath, repoIdx));
|
||||
break;
|
||||
|
||||
// If the port option was set explicitly then use it in preference to appended ports
|
||||
if (cfgOptionIdxSource(cfgOptRepoStoragePort, repoIdx) != cfgSourceDefault)
|
||||
port = cfgOptionIdxUInt(cfgOptRepoStoragePort, repoIdx);
|
||||
// Use S3 storage
|
||||
case STORAGE_S3_TYPE:
|
||||
{
|
||||
// Set the default port
|
||||
unsigned int port = cfgOptionIdxUInt(cfgOptRepoStoragePort, repoIdx);
|
||||
|
||||
result = storageS3New(
|
||||
cfgOptionIdxStr(cfgOptRepoPath, repoIdx), write, storageRepoPathExpression,
|
||||
cfgOptionIdxStr(cfgOptRepoS3Bucket, repoIdx), endPoint,
|
||||
strEqZ(cfgOptionIdxStr(cfgOptRepoS3UriStyle, repoIdx), STORAGE_S3_URI_STYLE_HOST) ?
|
||||
storageS3UriStyleHost : storageS3UriStylePath,
|
||||
cfgOptionIdxStr(cfgOptRepoS3Region, repoIdx),
|
||||
strEqZ(cfgOptionIdxStr(cfgOptRepoS3KeyType, repoIdx), STORAGE_S3_KEY_TYPE_SHARED) ?
|
||||
storageS3KeyTypeShared : storageS3KeyTypeAuto,
|
||||
cfgOptionIdxStrNull(cfgOptRepoS3Key, repoIdx), cfgOptionIdxStrNull(cfgOptRepoS3KeySecret, repoIdx),
|
||||
cfgOptionIdxStrNull(cfgOptRepoS3Token, repoIdx), cfgOptionIdxStrNull(cfgOptRepoS3Role, repoIdx),
|
||||
STORAGE_S3_PARTSIZE_MIN, host, port, ioTimeoutMs(), cfgOptionIdxBool(cfgOptRepoStorageVerifyTls, repoIdx),
|
||||
cfgOptionIdxStrNull(cfgOptRepoStorageCaFile, repoIdx), cfgOptionIdxStrNull(cfgOptRepoStorageCaPath, repoIdx));
|
||||
// Extract port from the endpoint and host if it is present
|
||||
const String *const endPoint = cfgOptionIdxHostPort(cfgOptRepoS3Endpoint, repoIdx, &port);
|
||||
const String *const host = cfgOptionIdxHostPort(cfgOptRepoStorageHost, repoIdx, &port);
|
||||
|
||||
// If the port option was set explicitly then use it in preference to appended ports
|
||||
if (cfgOptionIdxSource(cfgOptRepoStoragePort, repoIdx) != cfgSourceDefault)
|
||||
port = cfgOptionIdxUInt(cfgOptRepoStoragePort, repoIdx);
|
||||
|
||||
result = storageS3New(
|
||||
cfgOptionIdxStr(cfgOptRepoPath, repoIdx), write, storageRepoPathExpression,
|
||||
cfgOptionIdxStr(cfgOptRepoS3Bucket, repoIdx), endPoint,
|
||||
(StorageS3UriStyle)cfgOptionIdxStrId(cfgOptRepoS3UriStyle, repoIdx),
|
||||
cfgOptionIdxStr(cfgOptRepoS3Region, repoIdx), (StorageS3KeyType)cfgOptionIdxStrId(cfgOptRepoS3KeyType, repoIdx),
|
||||
cfgOptionIdxStrNull(cfgOptRepoS3Key, repoIdx), cfgOptionIdxStrNull(cfgOptRepoS3KeySecret, repoIdx),
|
||||
cfgOptionIdxStrNull(cfgOptRepoS3Token, repoIdx), cfgOptionIdxStrNull(cfgOptRepoS3Role, repoIdx),
|
||||
STORAGE_S3_PARTSIZE_MIN, host, port, ioTimeoutMs(), cfgOptionIdxBool(cfgOptRepoStorageVerifyTls, repoIdx),
|
||||
cfgOptionIdxStrNull(cfgOptRepoStorageCaFile, repoIdx), cfgOptionIdxStrNull(cfgOptRepoStorageCaPath, repoIdx));
|
||||
|
||||
break;
|
||||
}
|
||||
|
||||
// Use Posix storage. Keep this as the default to prevent code churn.
|
||||
default:
|
||||
{
|
||||
CHECK(type == STORAGE_POSIX_TYPE);
|
||||
|
||||
result = storagePosixNewP(
|
||||
cfgOptionIdxStr(cfgOptRepoPath, repoIdx), .write = write, .pathExpressionFunction = storageRepoPathExpression);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -234,7 +234,7 @@ storageReadPosixNew(StoragePosix *storage, const String *name, bool ignoreMissin
|
||||
|
||||
.interface = (StorageReadInterface)
|
||||
{
|
||||
.type = STORAGE_POSIX_TYPE_STR,
|
||||
.type = STORAGE_POSIX_TYPE,
|
||||
.name = strDup(name),
|
||||
.ignoreMissing = ignoreMissing,
|
||||
.limit = varDup(limit),
|
||||
|
@ -21,11 +21,6 @@ Posix Storage
|
||||
#include "storage/posix/storage.intern.h"
|
||||
#include "storage/posix/write.h"
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Storage type
|
||||
***********************************************************************************************************************************/
|
||||
STRING_EXTERN(STORAGE_POSIX_TYPE_STR, STORAGE_POSIX_TYPE);
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Define PATH_MAX if it is not defined
|
||||
***********************************************************************************************************************************/
|
||||
@ -567,11 +562,11 @@ static const StorageInterface storageInterfacePosix =
|
||||
|
||||
Storage *
|
||||
storagePosixNewInternal(
|
||||
const String *type, const String *path, mode_t modeFile, mode_t modePath, bool write,
|
||||
StringId type, const String *path, mode_t modeFile, mode_t modePath, bool write,
|
||||
StoragePathExpressionCallback pathExpressionFunction, bool pathSync)
|
||||
{
|
||||
FUNCTION_LOG_BEGIN(logLevelDebug);
|
||||
FUNCTION_LOG_PARAM(STRING, type);
|
||||
FUNCTION_LOG_PARAM(STRING_ID, type);
|
||||
FUNCTION_LOG_PARAM(STRING, path);
|
||||
FUNCTION_LOG_PARAM(MODE, modeFile);
|
||||
FUNCTION_LOG_PARAM(MODE, modePath);
|
||||
@ -580,7 +575,7 @@ storagePosixNewInternal(
|
||||
FUNCTION_LOG_PARAM(BOOL, pathSync);
|
||||
FUNCTION_LOG_END();
|
||||
|
||||
ASSERT(type != NULL);
|
||||
ASSERT(type != 0);
|
||||
ASSERT(path != NULL);
|
||||
ASSERT(modeFile != 0);
|
||||
ASSERT(modePath != 0);
|
||||
@ -606,7 +601,7 @@ storagePosixNewInternal(
|
||||
driver->interface.pathSync = NULL;
|
||||
|
||||
// If this is a posix driver then add link features
|
||||
if (strEq(type, STORAGE_POSIX_TYPE_STR))
|
||||
if (type == STORAGE_POSIX_TYPE)
|
||||
driver->interface.feature |=
|
||||
1 << storageFeatureHardLink | 1 << storageFeatureSymLink | 1 << storageFeaturePathSync |
|
||||
1 << storageFeatureInfoDetail;
|
||||
@ -632,6 +627,6 @@ storagePosixNew(const String *path, StoragePosixNewParam param)
|
||||
FUNCTION_LOG_RETURN(
|
||||
STORAGE,
|
||||
storagePosixNewInternal(
|
||||
STORAGE_POSIX_TYPE_STR, path, param.modeFile == 0 ? STORAGE_MODE_FILE_DEFAULT : param.modeFile,
|
||||
STORAGE_POSIX_TYPE, path, param.modeFile == 0 ? STORAGE_MODE_FILE_DEFAULT : param.modeFile,
|
||||
param.modePath == 0 ? STORAGE_MODE_PATH_DEFAULT : param.modePath, param.write, param.pathExpressionFunction, true));
|
||||
}
|
||||
|
@ -14,8 +14,7 @@ typedef struct StoragePosix StoragePosix;
|
||||
/***********************************************************************************************************************************
|
||||
Storage type
|
||||
***********************************************************************************************************************************/
|
||||
#define STORAGE_POSIX_TYPE "posix"
|
||||
STRING_DECLARE(STORAGE_POSIX_TYPE_STR);
|
||||
#define STORAGE_POSIX_TYPE STRID6("posix", 0x182533d01)
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Constructors
|
||||
|
@ -11,7 +11,7 @@ Posix Storage Internal
|
||||
Constructors
|
||||
***********************************************************************************************************************************/
|
||||
Storage *storagePosixNewInternal(
|
||||
const String *type, const String *path, mode_t modeFile, mode_t modePath, bool write,
|
||||
StringId type, const String *path, mode_t modeFile, mode_t modePath, bool write,
|
||||
StoragePathExpressionCallback pathExpressionFunction, bool pathSync);
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
|
@ -260,7 +260,7 @@ storageWritePosixNew(
|
||||
|
||||
.interface = (StorageWriteInterface)
|
||||
{
|
||||
.type = STORAGE_POSIX_TYPE_STR,
|
||||
.type = STORAGE_POSIX_TYPE,
|
||||
.name = strDup(name),
|
||||
.atomic = atomic,
|
||||
.createPath = createPath,
|
||||
|
@ -60,6 +60,6 @@ String *
|
||||
storageReadToLog(const StorageRead *this)
|
||||
{
|
||||
return strNewFmt(
|
||||
"{type: %s, name: %s, ignoreMissing: %s}", strZ(storageReadType(this)), strZ(strToLog(storageReadName(this))),
|
||||
"{type: %s, name: %s, ignoreMissing: %s}", strZ(strIdToStr(storageReadType(this))), strZ(strToLog(storageReadName(this))),
|
||||
cvtBoolToConstZ(storageReadIgnoreMissing(this)));
|
||||
}
|
||||
|
@ -11,6 +11,7 @@ typedef struct StorageRead StorageRead;
|
||||
|
||||
#include "common/io/read.h"
|
||||
#include "common/type/object.h"
|
||||
#include "common/type/stringId.h"
|
||||
#include "storage/read.intern.h"
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
@ -61,7 +62,7 @@ storageReadName(const StorageRead *const this)
|
||||
}
|
||||
|
||||
// Get file type
|
||||
__attribute__((always_inline)) static inline const String *
|
||||
__attribute__((always_inline)) static inline StringId
|
||||
storageReadType(const StorageRead *const this)
|
||||
{
|
||||
return THIS_PUB(StorageRead)->interface->type;
|
||||
|
@ -11,7 +11,7 @@ Constructors
|
||||
***********************************************************************************************************************************/
|
||||
typedef struct StorageReadInterface
|
||||
{
|
||||
const String *type;
|
||||
StringId type; // Storage type
|
||||
const String *name;
|
||||
bool compressible; // Is this file compressible?
|
||||
unsigned int compressLevel; // Level to use for compression
|
||||
|
@ -207,7 +207,7 @@ storageReadRemoteNew(
|
||||
|
||||
.interface = (StorageReadInterface)
|
||||
{
|
||||
.type = STORAGE_REMOTE_TYPE_STR,
|
||||
.type = STORAGE_REMOTE_TYPE,
|
||||
.name = strDup(name),
|
||||
.compressible = compressible,
|
||||
.compressLevel = compressLevel,
|
||||
|
@ -13,11 +13,6 @@ Remote Storage
|
||||
#include "storage/remote/storage.intern.h"
|
||||
#include "storage/remote/write.h"
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Storage type
|
||||
***********************************************************************************************************************************/
|
||||
STRING_EXTERN(STORAGE_REMOTE_TYPE_STR, STORAGE_REMOTE_TYPE);
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Object type
|
||||
***********************************************************************************************************************************/
|
||||
@ -412,8 +407,7 @@ storageRemoteNew(
|
||||
}
|
||||
MEM_CONTEXT_TEMP_END();
|
||||
|
||||
this = storageNew(
|
||||
STORAGE_REMOTE_TYPE_STR, path, modeFile, modePath, write, pathExpressionFunction, driver, driver->interface);
|
||||
this = storageNew(STORAGE_REMOTE_TYPE, path, modeFile, modePath, write, pathExpressionFunction, driver, driver->interface);
|
||||
}
|
||||
MEM_CONTEXT_NEW_END();
|
||||
|
||||
|
@ -10,8 +10,7 @@ Remote Storage
|
||||
/***********************************************************************************************************************************
|
||||
Storage type
|
||||
***********************************************************************************************************************************/
|
||||
#define STORAGE_REMOTE_TYPE "remote"
|
||||
STRING_DECLARE(STORAGE_REMOTE_TYPE_STR);
|
||||
#define STORAGE_REMOTE_TYPE STRID6("remote", 0x1543cd1521)
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Constructors
|
||||
|
@ -210,7 +210,7 @@ storageWriteRemoteNew(
|
||||
|
||||
.interface = (StorageWriteInterface)
|
||||
{
|
||||
.type = STORAGE_REMOTE_TYPE_STR,
|
||||
.type = STORAGE_REMOTE_TYPE,
|
||||
.name = strDup(name),
|
||||
.atomic = atomic,
|
||||
.compressible = compressible,
|
||||
|
@ -135,7 +135,7 @@ storageReadS3New(StorageS3 *storage, const String *name, bool ignoreMissing)
|
||||
|
||||
.interface = (StorageReadInterface)
|
||||
{
|
||||
.type = STORAGE_S3_TYPE_STR,
|
||||
.type = STORAGE_S3_TYPE,
|
||||
.name = strDup(name),
|
||||
.ignoreMissing = ignoreMissing,
|
||||
|
||||
|
@ -21,11 +21,6 @@ S3 Storage
|
||||
#include "storage/s3/storage.intern.h"
|
||||
#include "storage/s3/write.h"
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Storage type
|
||||
***********************************************************************************************************************************/
|
||||
STRING_EXTERN(STORAGE_S3_TYPE_STR, STORAGE_S3_TYPE);
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Defaults
|
||||
***********************************************************************************************************************************/
|
||||
@ -111,7 +106,7 @@ struct StorageS3
|
||||
|
||||
const String *bucket; // Bucket to store data in
|
||||
const String *region; // e.g. us-east-1
|
||||
StorageS3KeyType keyType; // Key type (shared or temp)
|
||||
StorageS3KeyType keyType; // Key type (e.g. storageS3KeyTypeShared)
|
||||
String *accessKey; // Access key
|
||||
String *secretAccessKey; // Secret access key
|
||||
String *securityToken; // Security token, if any
|
||||
@ -946,9 +941,9 @@ storageS3New(
|
||||
FUNCTION_LOG_PARAM(FUNCTIONP, pathExpressionFunction);
|
||||
FUNCTION_LOG_PARAM(STRING, bucket);
|
||||
FUNCTION_LOG_PARAM(STRING, endPoint);
|
||||
FUNCTION_LOG_PARAM(ENUM, uriStyle);
|
||||
FUNCTION_LOG_PARAM(STRING_ID, uriStyle);
|
||||
FUNCTION_LOG_PARAM(STRING, region);
|
||||
FUNCTION_LOG_PARAM(ENUM, keyType);
|
||||
FUNCTION_LOG_PARAM(STRING_ID, keyType);
|
||||
FUNCTION_TEST_PARAM(STRING, accessKey);
|
||||
FUNCTION_TEST_PARAM(STRING, secretAccessKey);
|
||||
FUNCTION_TEST_PARAM(STRING, securityToken);
|
||||
@ -1016,8 +1011,7 @@ storageS3New(
|
||||
strLstAdd(driver->headerRedactList, S3_HEADER_DATE_STR);
|
||||
strLstAdd(driver->headerRedactList, S3_HEADER_TOKEN_STR);
|
||||
|
||||
this = storageNew(
|
||||
STORAGE_S3_TYPE_STR, path, 0, 0, write, pathExpressionFunction, driver, driver->interface);
|
||||
this = storageNew(STORAGE_S3_TYPE, path, 0, 0, write, pathExpressionFunction, driver, driver->interface);
|
||||
}
|
||||
MEM_CONTEXT_NEW_END();
|
||||
|
||||
|
@ -9,33 +9,26 @@ S3 Storage
|
||||
/***********************************************************************************************************************************
|
||||
Storage type
|
||||
***********************************************************************************************************************************/
|
||||
#define STORAGE_S3_TYPE "s3"
|
||||
STRING_DECLARE(STORAGE_S3_TYPE_STR);
|
||||
#define STORAGE_S3_TYPE STRID6("s3", 0x7d31)
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Key type
|
||||
***********************************************************************************************************************************/
|
||||
typedef enum
|
||||
{
|
||||
storageS3KeyTypeShared,
|
||||
storageS3KeyTypeAuto,
|
||||
storageS3KeyTypeShared = STRID5("shared", 0x85905130),
|
||||
storageS3KeyTypeAuto = STRID5("auto", 0x7d2a10),
|
||||
} StorageS3KeyType;
|
||||
|
||||
#define STORAGE_S3_KEY_TYPE_SHARED "shared"
|
||||
#define STORAGE_S3_KEY_TYPE_AUTO "auto"
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
URI style
|
||||
***********************************************************************************************************************************/
|
||||
typedef enum
|
||||
{
|
||||
storageS3UriStyleHost,
|
||||
storageS3UriStylePath,
|
||||
storageS3UriStyleHost = STRID5("host", 0xa4de80),
|
||||
storageS3UriStylePath = STRID5("path", 0x450300),
|
||||
} StorageS3UriStyle;
|
||||
|
||||
#define STORAGE_S3_URI_STYLE_HOST "host"
|
||||
#define STORAGE_S3_URI_STYLE_PATH "path"
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Defaults
|
||||
***********************************************************************************************************************************/
|
||||
|
@ -279,7 +279,7 @@ storageWriteS3New(StorageS3 *storage, const String *name, size_t partSize)
|
||||
|
||||
.interface = (StorageWriteInterface)
|
||||
{
|
||||
.type = STORAGE_S3_TYPE_STR,
|
||||
.type = STORAGE_S3_TYPE,
|
||||
.name = strDup(name),
|
||||
.atomic = true,
|
||||
.createPath = true,
|
||||
|
@ -32,11 +32,11 @@ struct Storage
|
||||
/**********************************************************************************************************************************/
|
||||
Storage *
|
||||
storageNew(
|
||||
const String *type, const String *path, mode_t modeFile, mode_t modePath, bool write,
|
||||
StringId type, const String *path, mode_t modeFile, mode_t modePath, bool write,
|
||||
StoragePathExpressionCallback pathExpressionFunction, void *driver, StorageInterface interface)
|
||||
{
|
||||
FUNCTION_LOG_BEGIN(logLevelTrace);
|
||||
FUNCTION_LOG_PARAM(STRING, type);
|
||||
FUNCTION_LOG_PARAM(STRING_ID, type);
|
||||
FUNCTION_LOG_PARAM(STRING, path);
|
||||
FUNCTION_LOG_PARAM(MODE, modeFile);
|
||||
FUNCTION_LOG_PARAM(MODE, modePath);
|
||||
@ -46,7 +46,7 @@ storageNew(
|
||||
FUNCTION_LOG_PARAM(STORAGE_INTERFACE, interface);
|
||||
FUNCTION_LOG_END();
|
||||
|
||||
ASSERT(type != NULL);
|
||||
ASSERT(type != 0);
|
||||
ASSERT(strSize(path) >= 1 && strZ(path)[0] == '/');
|
||||
ASSERT(driver != NULL);
|
||||
ASSERT(interface.info != NULL);
|
||||
@ -575,8 +575,8 @@ storageMove(const Storage *this, StorageRead *source, StorageWrite *destination)
|
||||
ASSERT(source != NULL);
|
||||
ASSERT(destination != NULL);
|
||||
ASSERT(!storageReadIgnoreMissing(source));
|
||||
ASSERT(strEq(storageType(this), storageReadType(source)));
|
||||
ASSERT(strEq(storageReadType(source), storageWriteType(destination)));
|
||||
ASSERT(storageType(this) == storageReadType(source));
|
||||
ASSERT(storageReadType(source) == storageWriteType(destination));
|
||||
|
||||
MEM_CONTEXT_TEMP_BEGIN()
|
||||
{
|
||||
@ -926,5 +926,6 @@ String *
|
||||
storageToLog(const Storage *this)
|
||||
{
|
||||
return strNewFmt(
|
||||
"{type: %s, path: %s, write: %s}", strZ(storageType(this)), strZ(strToLog(this->path)), cvtBoolToConstZ(this->write));
|
||||
"{type: %s, path: %s, write: %s}", strZ(strIdToStr(storageType(this))), strZ(strToLog(this->path)),
|
||||
cvtBoolToConstZ(this->write));
|
||||
}
|
||||
|
@ -257,7 +257,7 @@ storageFeature(const Storage *const this, const StorageFeature feature)
|
||||
}
|
||||
|
||||
// Storage type (posix, cifs, etc.)
|
||||
__attribute__((always_inline)) static inline const String *
|
||||
__attribute__((always_inline)) static inline StringId
|
||||
storageType(const Storage *const this)
|
||||
{
|
||||
return THIS_PUB(Storage)->type;
|
||||
|
@ -260,7 +260,7 @@ typedef struct StorageInterface
|
||||
storageNew(type, path, modeFile, modePath, write, pathExpressionFunction, driver, (StorageInterface){__VA_ARGS__})
|
||||
|
||||
Storage *storageNew(
|
||||
const String *type, const String *path, mode_t modeFile, mode_t modePath, bool write,
|
||||
StringId type, const String *path, mode_t modeFile, mode_t modePath, bool write,
|
||||
StoragePathExpressionCallback pathExpressionFunction, void *driver, StorageInterface interface);
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
@ -284,7 +284,7 @@ Getters/Setters
|
||||
***********************************************************************************************************************************/
|
||||
typedef struct StoragePub
|
||||
{
|
||||
const String *type; // Storage type
|
||||
StringId type; // Storage type
|
||||
void *driver; // Storage driver
|
||||
StorageInterface interface; // Storage interface
|
||||
} StoragePub;
|
||||
|
@ -62,7 +62,7 @@ storageWriteToLog(const StorageWrite *this)
|
||||
{
|
||||
return strNewFmt(
|
||||
"{type: %s, name: %s, modeFile: %04o, modePath: %04o, createPath: %s, syncFile: %s, syncPath: %s, atomic: %s}",
|
||||
strZ(storageWriteType(this)), strZ(strToLog(storageWriteName(this))), storageWriteModeFile(this),
|
||||
strZ(strIdToStr(storageWriteType(this))), strZ(strToLog(storageWriteName(this))), storageWriteModeFile(this),
|
||||
storageWriteModePath(this), cvtBoolToConstZ(storageWriteCreatePath(this)), cvtBoolToConstZ(storageWriteSyncFile(this)),
|
||||
cvtBoolToConstZ(storageWriteSyncPath(this)), cvtBoolToConstZ(storageWriteAtomic(this)));
|
||||
}
|
||||
|
@ -13,6 +13,7 @@ typedef struct StorageWrite StorageWrite;
|
||||
#include "common/type/buffer.h"
|
||||
#include "common/type/object.h"
|
||||
#include "common/type/string.h"
|
||||
#include "common/type/stringId.h"
|
||||
#include "storage/write.intern.h"
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
@ -93,7 +94,7 @@ storageWriteSyncPath(const StorageWrite *const this)
|
||||
}
|
||||
|
||||
// File type
|
||||
__attribute__((always_inline)) static inline const String *
|
||||
__attribute__((always_inline)) static inline StringId
|
||||
storageWriteType(const StorageWrite *const this)
|
||||
{
|
||||
return THIS_PUB(StorageWrite)->interface->type;
|
||||
|
@ -17,7 +17,7 @@ Constructors
|
||||
***********************************************************************************************************************************/
|
||||
typedef struct StorageWriteInterface
|
||||
{
|
||||
const String *type;
|
||||
StringId type; // Storage type
|
||||
const String *name;
|
||||
|
||||
bool atomic;
|
||||
|
@ -160,6 +160,21 @@ hrnCfgArgKeyRawZ(StringList *argList, ConfigOption optionId, unsigned optionKey,
|
||||
strLstAdd(argList, strNewFmt("--%s=%s", cfgParseOptionKeyIdxName(optionId, optionKey - 1), value));
|
||||
}
|
||||
|
||||
void
|
||||
hrnCfgArgRawStrId(StringList *argList, ConfigOption optionId, StringId value)
|
||||
{
|
||||
hrnCfgArgKeyRawStrId(argList, optionId, 1, value);
|
||||
}
|
||||
|
||||
void
|
||||
hrnCfgArgKeyRawStrId(StringList *argList, ConfigOption optionId, unsigned optionKey, StringId value)
|
||||
{
|
||||
char buffer[STRID_MAX + 1];
|
||||
strIdToZ(value, buffer);
|
||||
|
||||
hrnCfgArgKeyRawZ(argList, optionId, optionKey, buffer);
|
||||
}
|
||||
|
||||
void
|
||||
hrnCfgArgRawBool(StringList *argList, ConfigOption optionId, bool value)
|
||||
{
|
||||
|
@ -49,6 +49,9 @@ void hrnCfgArgKeyRawFmt(StringList *argList, ConfigOption optionId, unsigned opt
|
||||
void hrnCfgArgRawZ(StringList *argList, ConfigOption optionId, const char *value);
|
||||
void hrnCfgArgKeyRawZ(StringList *argList, ConfigOption optionId, unsigned optionKey, const char *value);
|
||||
|
||||
void hrnCfgArgRawStrId(StringList *argList, ConfigOption optionId, StringId value);
|
||||
void hrnCfgArgKeyRawStrId(StringList *argList, ConfigOption optionId, unsigned optionKey, StringId);
|
||||
|
||||
void hrnCfgArgRawBool(StringList *argList, ConfigOption optionId, bool value);
|
||||
void hrnCfgArgKeyRawBool(StringList *argList, ConfigOption optionId, unsigned optionKey, bool value);
|
||||
|
||||
|
@ -111,8 +111,8 @@ testRun(void)
|
||||
|
||||
argList = strLstNew();
|
||||
hrnCfgArgRawZ(argList, cfgOptRepo, "2");
|
||||
hrnCfgArgKeyRawZ(argList, cfgOptRepoType, 1, STORAGE_CIFS_TYPE);
|
||||
hrnCfgArgKeyRawZ(argList, cfgOptRepoType, 2, STORAGE_CIFS_TYPE);
|
||||
hrnCfgArgKeyRawStrId(argList, cfgOptRepoType, 1, STORAGE_CIFS_TYPE);
|
||||
hrnCfgArgKeyRawStrId(argList, cfgOptRepoType, 2, STORAGE_CIFS_TYPE);
|
||||
TEST_ERROR(
|
||||
harnessCfgLoad(cfgCmdInfo, argList), OptionInvalidValueError,
|
||||
"local repo1 and repo2 paths are both '/var/lib/pgbackrest' but must be different");
|
||||
@ -122,8 +122,8 @@ testRun(void)
|
||||
|
||||
argList = strLstNew();
|
||||
hrnCfgArgRawZ(argList, cfgOptRepo, "1");
|
||||
hrnCfgArgKeyRawZ(argList, cfgOptRepoType, 1, STORAGE_POSIX_TYPE);
|
||||
hrnCfgArgKeyRawZ(argList, cfgOptRepoType, 2, STORAGE_CIFS_TYPE);
|
||||
hrnCfgArgKeyRawStrId(argList, cfgOptRepoType, 1, STORAGE_POSIX_TYPE);
|
||||
hrnCfgArgKeyRawStrId(argList, cfgOptRepoType, 2, STORAGE_CIFS_TYPE);
|
||||
hrnCfgArgKeyRawZ(argList, cfgOptRepoType, 3, "s3");
|
||||
hrnCfgArgKeyRawZ(argList, cfgOptRepoS3Bucket, 3, "cool-bucket");
|
||||
hrnCfgArgKeyRawZ(argList, cfgOptRepoS3Region, 3, "region");
|
||||
|
@ -1200,9 +1200,11 @@ testRun(void)
|
||||
TEST_RESULT_BOOL(cfgOptionNegate(cfgOptConfig), true, " config is negated");
|
||||
TEST_RESULT_INT(cfgOptionSource(cfgOptStanza), cfgSourceParam, " stanza is source param");
|
||||
TEST_RESULT_STR_Z(cfgOptionStr(cfgOptStanza), "db", " stanza is set");
|
||||
TEST_RESULT_UINT(cfgOptionStrId(cfgOptStanza), strIdFromZ(stringIdBit5, "db"), " stanza is set");
|
||||
TEST_RESULT_INT(cfgOptionSource(cfgOptStanza), cfgSourceParam, " stanza is source param");
|
||||
TEST_RESULT_STR_Z(cfgOptionIdxStr(cfgOptPgPath, 0), "/path/to/db", " pg1-path is set");
|
||||
TEST_RESULT_INT(cfgOptionSource(cfgOptPgPath), cfgSourceParam, " pg1-path is source param");
|
||||
TEST_RESULT_UINT(cfgOptionIdxStrId(cfgOptRepoType, 0), strIdFromZ(stringIdBit6, "s3"), " repo-type is set");
|
||||
TEST_RESULT_STR_Z(cfgOptionStr(cfgOptRepoS3KeySecret), "xxx", " repo1-s3-secret is set");
|
||||
TEST_RESULT_INT(cfgOptionSource(cfgOptRepoS3KeySecret), cfgSourceConfig, " repo1-s3-secret is source env");
|
||||
TEST_RESULT_BOOL(cfgOptionBool(cfgOptOnline), false, " online is not set");
|
||||
|
@ -169,7 +169,7 @@ testRun(void)
|
||||
|
||||
storageHelper.storageRepo = memNew(sizeof(Storage *));
|
||||
storageHelper.storageRepo[0] = storageNew(
|
||||
STRDEF("TEST"), STRDEF("/"), 0, 0, false, NULL, &driver, driver.interface);
|
||||
strIdFromZ(stringIdBit6, "test"), STRDEF("/"), 0, 0, false, NULL, &driver, driver.interface);
|
||||
|
||||
// Setup handler for remote storage protocol
|
||||
IoRead *read = ioFdReadNew(strNew("storage server read"), HARNESS_FORK_CHILD_READ(), 60000);
|
||||
|
@ -252,7 +252,8 @@ testRun(void)
|
||||
driver.interface.info = storageTestManifestNewBuildInfo;
|
||||
driver.interface.infoList = storageTestManifestNewBuildInfoList;
|
||||
|
||||
Storage *storagePg = storageNew(STRDEF("TEST"), STRDEF("/pg"), 0, 0, false, NULL, &driver, driver.interface);
|
||||
const Storage *const storagePg = storageNew(
|
||||
strIdFromZ(stringIdBit6, "test"), STRDEF("/pg"), 0, 0, false, NULL, &driver, driver.interface);
|
||||
|
||||
// -------------------------------------------------------------------------------------------------------------------------
|
||||
TEST_TITLE("build manifest");
|
||||
|
@ -178,7 +178,7 @@ testRun(void)
|
||||
|
||||
StringList *argList = strLstNew();
|
||||
strLstAddZ(argList, "--" CFGOPT_STANZA "=test");
|
||||
hrnCfgArgRawZ(argList, cfgOptRepoType, STORAGE_AZURE_TYPE);
|
||||
hrnCfgArgRawStrId(argList, cfgOptRepoType, STORAGE_AZURE_TYPE);
|
||||
hrnCfgArgRawZ(argList, cfgOptRepoPath, "/repo");
|
||||
hrnCfgArgRawZ(argList, cfgOptRepoAzureContainer, TEST_CONTAINER);
|
||||
hrnCfgEnvRawZ(cfgOptRepoAzureAccount, TEST_ACCOUNT);
|
||||
@ -285,7 +285,7 @@ testRun(void)
|
||||
|
||||
StringList *argList = strLstNew();
|
||||
strLstAddZ(argList, "--" CFGOPT_STANZA "=test");
|
||||
hrnCfgArgRawZ(argList, cfgOptRepoType, STORAGE_AZURE_TYPE);
|
||||
hrnCfgArgRawStrId(argList, cfgOptRepoType, STORAGE_AZURE_TYPE);
|
||||
hrnCfgArgRawZ(argList, cfgOptRepoPath, "/");
|
||||
hrnCfgArgRawZ(argList, cfgOptRepoAzureContainer, TEST_CONTAINER);
|
||||
hrnCfgArgRaw(argList, cfgOptRepoStorageHost, hrnServerHost());
|
||||
@ -729,7 +729,7 @@ testRun(void)
|
||||
|
||||
hrnServerScriptClose(service);
|
||||
|
||||
hrnCfgArgRawZ(argList, cfgOptRepoAzureKeyType, STORAGE_AZURE_KEY_TYPE_SAS);
|
||||
hrnCfgArgRawStrId(argList, cfgOptRepoAzureKeyType, storageAzureKeyTypeSas);
|
||||
hrnCfgEnvRawZ(cfgOptRepoAzureKey, TEST_KEY_SAS);
|
||||
harnessCfgLoad(cfgCmdArchivePush, argList);
|
||||
|
||||
|
@ -24,7 +24,7 @@ testRun(void)
|
||||
|
||||
const Storage *storage = NULL;
|
||||
TEST_ASSIGN(storage, storageRepoGet(0, true), "get cifs repo storage");
|
||||
TEST_RESULT_STR_Z(storageType(storage), "cifs", "check storage type");
|
||||
TEST_RESULT_UINT(storageType(storage), STORAGE_CIFS_TYPE, "check storage type");
|
||||
TEST_RESULT_BOOL(storageFeature(storage, storageFeaturePath), true, " check path feature");
|
||||
TEST_RESULT_BOOL(storageFeature(storage, storageFeatureCompress), true, " check compress feature");
|
||||
|
||||
|
@ -199,10 +199,10 @@ testRun(void)
|
||||
|
||||
StringList *argList = strLstNew();
|
||||
strLstAddZ(argList, "--" CFGOPT_STANZA "=test");
|
||||
hrnCfgArgRawZ(argList, cfgOptRepoType, STORAGE_GCS_TYPE);
|
||||
hrnCfgArgRawStrId(argList, cfgOptRepoType, STORAGE_GCS_TYPE);
|
||||
hrnCfgArgRawZ(argList, cfgOptRepoPath, "/repo");
|
||||
hrnCfgArgRawZ(argList, cfgOptRepoGcsBucket, TEST_BUCKET);
|
||||
hrnCfgArgRawZ(argList, cfgOptRepoGcsKeyType, STORAGE_GCS_KEY_TYPE_TOKEN);
|
||||
hrnCfgArgRawStrId(argList, cfgOptRepoGcsKeyType, storageGcsKeyTypeToken);
|
||||
hrnCfgEnvRawZ(cfgOptRepoGcsKey, TEST_TOKEN);
|
||||
harnessCfgLoad(cfgCmdArchivePush, argList);
|
||||
|
||||
@ -310,7 +310,7 @@ testRun(void)
|
||||
|
||||
StringList *argList = strLstNew();
|
||||
strLstAddZ(argList, "--" CFGOPT_STANZA "=test");
|
||||
hrnCfgArgRawZ(argList, cfgOptRepoType, STORAGE_GCS_TYPE);
|
||||
hrnCfgArgRawStrId(argList, cfgOptRepoType, STORAGE_GCS_TYPE);
|
||||
hrnCfgArgRawZ(argList, cfgOptRepoPath, "/");
|
||||
hrnCfgArgRawZ(argList, cfgOptRepoGcsBucket, TEST_BUCKET);
|
||||
hrnCfgArgRawFmt(argList, cfgOptRepoGcsEndpoint, "%s:%u", strZ(hrnServerHost()), hrnServerPort(0));
|
||||
|
@ -104,7 +104,7 @@ testRun(void)
|
||||
|
||||
TEST_RESULT_PTR(storageInterface(storageTest).info, storageTest->pub.interface.info, " check interface");
|
||||
TEST_RESULT_PTR(storageDriver(storageTest), storageTest->pub.driver, " check driver");
|
||||
TEST_RESULT_STR(storageType(storageTest), storageTest->pub.type, " check type");
|
||||
TEST_RESULT_UINT(storageType(storageTest), storageTest->pub.type, " check type");
|
||||
TEST_RESULT_BOOL(storageFeature(storageTest, storageFeaturePath), true, " check path feature");
|
||||
TEST_RESULT_BOOL(storageFeature(storageTest, storageFeatureCompress), true, " check compress feature");
|
||||
}
|
||||
@ -960,7 +960,7 @@ testRun(void)
|
||||
|
||||
TEST_RESULT_BOOL(ioReadOpen(storageReadIo(file)), true, " open file");
|
||||
TEST_RESULT_STR(storageReadName(file), fileName, " check file name");
|
||||
TEST_RESULT_STR_Z(storageReadType(file), "posix", " check file type");
|
||||
TEST_RESULT_UINT(storageReadType(file), STORAGE_POSIX_TYPE, " check file type");
|
||||
TEST_RESULT_UINT(varUInt64(storageReadLimit(file)), 44, " check limit");
|
||||
|
||||
TEST_RESULT_VOID(ioRead(storageReadIo(file), outBuffer), " load data");
|
||||
@ -1068,7 +1068,7 @@ testRun(void)
|
||||
// -------------------------------------------------------------------------------------------------------------------------
|
||||
TEST_ASSIGN(file, storageNewWriteP(storageTest, fileName), "new write file");
|
||||
TEST_RESULT_STR(storageWriteName(file), fileName, " check file name");
|
||||
TEST_RESULT_STR_Z(storageWriteType(file), "posix", " check file type");
|
||||
TEST_RESULT_UINT(storageWriteType(file), STORAGE_POSIX_TYPE, " check file type");
|
||||
TEST_RESULT_VOID(ioWriteOpen(storageWriteIo(file)), " open file");
|
||||
|
||||
// Rename the file back to original name from tmp -- this will cause the rename in close to fail
|
||||
|
@ -86,9 +86,9 @@ testRequest(IoWrite *write, Storage *s3, const char *verb, const char *path, Tes
|
||||
if (s3 != NULL)
|
||||
{
|
||||
if (driver->uriStyle == storageS3UriStyleHost)
|
||||
strCatFmt(request, "host:bucket." S3_TEST_HOST "\r\n");
|
||||
else
|
||||
strCatFmt(request, "host:" S3_TEST_HOST "\r\n");
|
||||
strCatFmt(request, "host:bucket." S3_TEST_HOST "\r\n");
|
||||
else
|
||||
strCatFmt(request, "host:" S3_TEST_HOST "\r\n");
|
||||
}
|
||||
else
|
||||
strCatFmt(request, "host:%s\r\n", strZ(hrnServerHost()));
|
||||
@ -442,7 +442,7 @@ testRun(void)
|
||||
argList = strLstDup(commonArgList);
|
||||
hrnCfgArgRawFmt(argList, cfgOptRepoStorageHost, "%s:%u", strZ(host), port);
|
||||
hrnCfgArgRaw(argList, cfgOptRepoS3Role, credRole);
|
||||
hrnCfgArgRawZ(argList, cfgOptRepoS3KeyType, STORAGE_S3_KEY_TYPE_AUTO);
|
||||
hrnCfgArgRawStrId(argList, cfgOptRepoS3KeyType, storageS3KeyTypeAuto);
|
||||
harnessCfgLoad(cfgCmdArchivePush, argList);
|
||||
|
||||
s3 = storageRepoGet(0, true);
|
||||
@ -1013,7 +1013,7 @@ testRun(void)
|
||||
hrnServerScriptClose(service);
|
||||
|
||||
argList = strLstDup(commonArgList);
|
||||
hrnCfgArgRawZ(argList, cfgOptRepoS3UriStyle, STORAGE_S3_URI_STYLE_PATH);
|
||||
hrnCfgArgRawStrId(argList, cfgOptRepoS3UriStyle, storageS3UriStylePath);
|
||||
hrnCfgArgRaw(argList, cfgOptRepoStorageHost, host);
|
||||
hrnCfgArgRawFmt(argList, cfgOptRepoStoragePort, "%u", port);
|
||||
hrnCfgEnvRemoveRaw(cfgOptRepoS3Token);
|
||||
|
Loading…
x
Reference in New Issue
Block a user