db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2024-12-12 10:04:14 +02:00
Code Issues Releases Activity

Allow upload chunk size to be configured for object stores.

Browse Source 
There are performance benefits to increasing the upload chunk size as long as the tradeoff with additional memory consumption is acceptable.

Make the chunk size configurable for S3, GCS, and Azure, but don't attempt to do any validation of the chunk size beyond some sane limits. The defaults remain as is for each storage type to avoid any unintentional regressions.
This commit is contained in:
David Steele 2022-08-24 09:33:26 -04:00 committed by GitHub
parent 37b4592e52
commit b05d31f531
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 317 additions and 136 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
src/build/config/config.yaml
View File

@ -2245,3 +2245,17 @@ option:
deprecate:
repo?-azure-port: {}
repo?-s3-port: {}
repo-storage-upload-chunk-size:
section: global
group: repo
type: size
required: false
allow-range: [64KiB, 1TiB]
command: repo-type
depend:
option: repo-type
list:
- azure
- gcs
- s3

22
src/build/help/help.xml
View File

@ -972,6 +972,28 @@
<example>9000</example>
</config-key>
<config-key id="repo-storage-upload-chunk-size" name="Repository Storage Upload Chunk Size">
<summary>Repository storage upload chunk size.</summary>
<text>
<p>Object stores such as S3 allow files to be uploaded in chunks when the file is too large to be stored in memory. Even if the file can be stored in memory, it is more memory efficient to limit the amount of memory used for uploads.</p>
<p>A larger chunk size will generally lead to better performance because it will minimize upload requests and allow more files to be uploaded in a single request rather than in chunks. The disadvantage is that memory usage will be higher and because the chunk buffer must be allocated per process, larger <br-option>process-max</br-option> values will lead to more memory being consumed overall.</p>
<p>Default chunk sizes by repo type:</p>
<list>
<list-item><id>azure</id> - 4MiB</list-item>
<list-item><id>gcs</id> - 4MiB</list-item>
<list-item><id>s3</id> - 5MiB</list-item>
</list>
<p>Note that valid chunk sizes vary by storage type and by platform. For example, <proper>AWS S3</proper> has a minimum chunk size of 5MiB but <proper>S3</proper> clones may accept lower values. Terminology for chunk size varies by storage type, so when searching min/max values use <quote>part size</quote> for <proper>AWS S3</proper>, <quote>chunk size</quote> for <proper>GCS</proper>, and <quote>block size</quote> for <proper>Azure</proper>. No attempt is made to validate configured chunk sizes so selecting an invalid value will lead to errors from the storage service or undefined behavior.</p>
</text>
<example>16MiB</example>
</config-key>
<config-key id="repo-storage-verify-tls" name="Repository Storage Certificate Verify">
<summary>Repository storage certificate verify.</summary>

3
src/config/config.auto.h
View File

@ -128,7 +128,7 @@ Option constants
#define CFGOPT_TYPE "type"
#define CFGOPT_VERBOSE "verbose"
#define CFG_OPTION_TOTAL 155
#define CFG_OPTION_TOTAL 156
/***********************************************************************************************************************************
Option value constants
@ -483,6 +483,7 @@ typedef enum
cfgOptRepoStorageCaPath,
cfgOptRepoStorageHost,
cfgOptRepoStoragePort,
cfgOptRepoStorageUploadChunkSize,
cfgOptRepoStorageVerifyTls,
cfgOptRepoType,
cfgOptResume,

14
src/config/load.c
View File

@ -288,6 +288,20 @@ cfgLoadUpdateOption(void)
}
}
// Set default upload chunk size if not set
if (cfgOptionValid(cfgOptRepoStorageUploadChunkSize))
{
for (unsigned int repoIdx = 0; repoIdx < cfgOptionGroupIdxTotal(cfgOptGrpRepo); repoIdx++)
{
if (!cfgOptionIdxTest(cfgOptRepoStorageUploadChunkSize, repoIdx))
{
cfgOptionIdxSet(
cfgOptRepoStorageUploadChunkSize, repoIdx, cfgSourceDefault,
VARINT64((cfgOptionIdxStrId(cfgOptRepoType, repoIdx) == CFGOPTVAL_REPO_TYPE_S3 ? 5 : 4) * 1024 * 1024));
}
}
}
// Set pg-host-port/repo-host-port default when pg-host-type/repo-host-type is tls. ??? This should be handled in the parser but
// it requires a default that depends on another option value and that is not currently possible.
#define HOST_PORT_TLS 8432

85
src/config/parse.auto.c.inc
View File

@ -7546,6 +7546,90 @@ static const ParseRuleOption parseRuleOption[CFG_OPTION_TOTAL] =
), // opt/repo-storage-port
), // opt/repo-storage-port
// -----------------------------------------------------------------------------------------------------------------------------
PARSE_RULE_OPTION // opt/repo-storage-upload-chunk-size
( // opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_NAME("repo-storage-upload-chunk-size"), // opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_TYPE(cfgOptTypeSize), // opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_RESET(true), // opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_REQUIRED(false), // opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_SECTION(cfgSectionGlobal), // opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_GROUP_MEMBER(true), // opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_GROUP_ID(cfgOptGrpRepo), // opt/repo-storage-upload-chunk-size
// opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_COMMAND_ROLE_MAIN_VALID_LIST // opt/repo-storage-upload-chunk-size
( // opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_COMMAND(cfgCmdArchiveGet) // opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_COMMAND(cfgCmdArchivePush) // opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_COMMAND(cfgCmdBackup) // opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_COMMAND(cfgCmdCheck) // opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_COMMAND(cfgCmdExpire) // opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_COMMAND(cfgCmdInfo) // opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_COMMAND(cfgCmdRepoCreate) // opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_COMMAND(cfgCmdRepoGet) // opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_COMMAND(cfgCmdRepoLs) // opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_COMMAND(cfgCmdRepoPut) // opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_COMMAND(cfgCmdRepoRm) // opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_COMMAND(cfgCmdRestore) // opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaCreate) // opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaDelete) // opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaUpgrade) // opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_COMMAND(cfgCmdVerify) // opt/repo-storage-upload-chunk-size
), // opt/repo-storage-upload-chunk-size
// opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_COMMAND_ROLE_ASYNC_VALID_LIST // opt/repo-storage-upload-chunk-size
( // opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_COMMAND(cfgCmdArchiveGet) // opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_COMMAND(cfgCmdArchivePush) // opt/repo-storage-upload-chunk-size
), // opt/repo-storage-upload-chunk-size
// opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_COMMAND_ROLE_LOCAL_VALID_LIST // opt/repo-storage-upload-chunk-size
( // opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_COMMAND(cfgCmdArchiveGet) // opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_COMMAND(cfgCmdArchivePush) // opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_COMMAND(cfgCmdBackup) // opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_COMMAND(cfgCmdRestore) // opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_COMMAND(cfgCmdVerify) // opt/repo-storage-upload-chunk-size
), // opt/repo-storage-upload-chunk-size
// opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_COMMAND_ROLE_REMOTE_VALID_LIST // opt/repo-storage-upload-chunk-size
( // opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_COMMAND(cfgCmdArchiveGet) // opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_COMMAND(cfgCmdArchivePush) // opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_COMMAND(cfgCmdCheck) // opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_COMMAND(cfgCmdInfo) // opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_COMMAND(cfgCmdRepoCreate) // opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_COMMAND(cfgCmdRepoGet) // opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_COMMAND(cfgCmdRepoLs) // opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_COMMAND(cfgCmdRepoPut) // opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_COMMAND(cfgCmdRepoRm) // opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_COMMAND(cfgCmdRestore) // opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaCreate) // opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaDelete) // opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaUpgrade) // opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_COMMAND(cfgCmdVerify) // opt/repo-storage-upload-chunk-size
), // opt/repo-storage-upload-chunk-size
// opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTIONAL // opt/repo-storage-upload-chunk-size
( // opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTIONAL_GROUP // opt/repo-storage-upload-chunk-size
( // opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTIONAL_DEPEND // opt/repo-storage-upload-chunk-size
( // opt/repo-storage-upload-chunk-size
PARSE_RULE_VAL_OPT(cfgOptRepoType), // opt/repo-storage-upload-chunk-size
PARSE_RULE_VAL_STRID(parseRuleValStrIdAzure), // opt/repo-storage-upload-chunk-size
PARSE_RULE_VAL_STRID(parseRuleValStrIdGcs), // opt/repo-storage-upload-chunk-size
PARSE_RULE_VAL_STRID(parseRuleValStrIdS3), // opt/repo-storage-upload-chunk-size
), // opt/repo-storage-upload-chunk-size
// opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTIONAL_ALLOW_RANGE // opt/repo-storage-upload-chunk-size
( // opt/repo-storage-upload-chunk-size
PARSE_RULE_VAL_INT(parseRuleValInt65536), // opt/repo-storage-upload-chunk-size
PARSE_RULE_VAL_INT(parseRuleValInt1099511627776), // opt/repo-storage-upload-chunk-size
), // opt/repo-storage-upload-chunk-size
), // opt/repo-storage-upload-chunk-size
), // opt/repo-storage-upload-chunk-size
), // opt/repo-storage-upload-chunk-size
// -----------------------------------------------------------------------------------------------------------------------------
PARSE_RULE_OPTION // opt/repo-storage-verify-tls
( // opt/repo-storage-verify-tls
PARSE_RULE_OPTION_NAME("repo-storage-verify-tls"), // opt/repo-storage-verify-tls
@ -9201,6 +9285,7 @@ static const uint8_t optionResolveOrder[] =
cfgOptRepoStorageCaPath, // opt-resolve-order
cfgOptRepoStorageHost, // opt-resolve-order
cfgOptRepoStoragePort, // opt-resolve-order
cfgOptRepoStorageUploadChunkSize, // opt-resolve-order
cfgOptRepoStorageVerifyTls, // opt-resolve-order
cfgOptTarget, // opt-resolve-order
cfgOptTargetAction, // opt-resolve-order

4
src/storage/azure/helper.c
View File

@ -79,8 +79,8 @@ storageAzureHelper(const unsigned int repoIdx, const bool write, StoragePathExpr
result = storageAzureNew(
cfgOptionIdxStr(cfgOptRepoPath, repoIdx), write, pathExpressionCallback,
cfgOptionIdxStr(cfgOptRepoAzureContainer, repoIdx), cfgOptionIdxStr(cfgOptRepoAzureAccount, repoIdx), keyType, key,
STORAGE_AZURE_BLOCKSIZE_MIN, endpoint, uriStyle, port, ioTimeoutMs(), cfgOptionIdxBool(cfgOptRepoStorageVerifyTls,
repoIdx), cfgOptionIdxStrNull(cfgOptRepoStorageCaFile, repoIdx),
(size_t)cfgOptionIdxUInt64(cfgOptRepoStorageUploadChunkSize, repoIdx), endpoint, uriStyle, port, ioTimeoutMs(),
cfgOptionIdxBool(cfgOptRepoStorageVerifyTls, repoIdx), cfgOptionIdxStrNull(cfgOptRepoStorageCaFile, repoIdx),
cfgOptionIdxStrNull(cfgOptRepoStorageCaPath, repoIdx));
}
MEM_CONTEXT_PRIOR_END();

5
src/storage/azure/storage.h
View File

@ -29,11 +29,6 @@ typedef enum
storageAzureUriStylePath = STRID5("path", 0x450300),
} StorageAzureUriStyle;
/***********************************************************************************************************************************
Defaults
***********************************************************************************************************************************/
#define STORAGE_AZURE_BLOCKSIZE_MIN ((size_t)4 * 1024 * 1024)
/***********************************************************************************************************************************
Constructors
***********************************************************************************************************************************/

4
src/storage/gcs/helper.c
View File

@ -24,8 +24,8 @@ storageGcsHelper(const unsigned int repoIdx, const bool write, StoragePathExpres
Storage *const result = storageGcsNew(
cfgOptionIdxStr(cfgOptRepoPath, repoIdx), write, pathExpressionCallback, cfgOptionIdxStr(cfgOptRepoGcsBucket, repoIdx),
(StorageGcsKeyType)cfgOptionIdxStrId(cfgOptRepoGcsKeyType, repoIdx), cfgOptionIdxStrNull(cfgOptRepoGcsKey, repoIdx),
STORAGE_GCS_CHUNKSIZE_DEFAULT, cfgOptionIdxStr(cfgOptRepoGcsEndpoint, repoIdx), ioTimeoutMs(),
cfgOptionIdxBool(cfgOptRepoStorageVerifyTls, repoIdx), cfgOptionIdxStrNull(cfgOptRepoStorageCaFile, repoIdx),
(size_t)cfgOptionIdxUInt64(cfgOptRepoStorageUploadChunkSize, repoIdx), cfgOptionIdxStr(cfgOptRepoGcsEndpoint, repoIdx),
ioTimeoutMs(), cfgOptionIdxBool(cfgOptRepoStorageVerifyTls, repoIdx), cfgOptionIdxStrNull(cfgOptRepoStorageCaFile, repoIdx),
cfgOptionIdxStrNull(cfgOptRepoStorageCaPath, repoIdx));
FUNCTION_LOG_RETURN(STORAGE, result);

5
src/storage/gcs/storage.h
View File

@ -21,11 +21,6 @@ typedef enum
storageGcsKeyTypeToken = STRID5("token", 0xe2adf40),
} StorageGcsKeyType;
/***********************************************************************************************************************************
Defaults
***********************************************************************************************************************************/
#define STORAGE_GCS_CHUNKSIZE_DEFAULT ((size_t)4 * 1024 * 1024)
/***********************************************************************************************************************************
Constructors
***********************************************************************************************************************************/

7
src/storage/s3/helper.c
View File

@ -85,9 +85,10 @@ storageS3Helper(const unsigned int repoIdx, const bool write, StoragePathExpress
endPoint, (StorageS3UriStyle)cfgOptionIdxStrId(cfgOptRepoS3UriStyle, repoIdx),
cfgOptionIdxStr(cfgOptRepoS3Region, repoIdx), keyType, cfgOptionIdxStrNull(cfgOptRepoS3Key, repoIdx),
cfgOptionIdxStrNull(cfgOptRepoS3KeySecret, repoIdx), cfgOptionIdxStrNull(cfgOptRepoS3Token, repoIdx),
cfgOptionIdxStrNull(cfgOptRepoS3KmsKeyId, repoIdx), role, webIdToken, STORAGE_S3_PARTSIZE_MIN, host, port,
ioTimeoutMs(), cfgOptionIdxBool(cfgOptRepoStorageVerifyTls, repoIdx),
cfgOptionIdxStrNull(cfgOptRepoStorageCaFile, repoIdx), cfgOptionIdxStrNull(cfgOptRepoStorageCaPath, repoIdx));
cfgOptionIdxStrNull(cfgOptRepoS3KmsKeyId, repoIdx), role, webIdToken,
(size_t)cfgOptionIdxUInt64(cfgOptRepoStorageUploadChunkSize, repoIdx), host, port, ioTimeoutMs(),
cfgOptionIdxBool(cfgOptRepoStorageVerifyTls, repoIdx), cfgOptionIdxStrNull(cfgOptRepoStorageCaFile, repoIdx),
cfgOptionIdxStrNull(cfgOptRepoStorageCaPath, repoIdx));
}
MEM_CONTEXT_PRIOR_END();
}

5
src/storage/s3/storage.h
View File

@ -30,11 +30,6 @@ typedef enum
storageS3UriStylePath = STRID5("path", 0x450300),
} StorageS3UriStyle;
/***********************************************************************************************************************************
Defaults
***********************************************************************************************************************************/
#define STORAGE_S3_PARTSIZE_MIN ((size_t)5 * 1024 * 1024)
/***********************************************************************************************************************************
Constructors
***********************************************************************************************************************************/

7
test/src/module/command/helpTest.c
View File

@ -231,7 +231,8 @@ testRun(void)
" files [default=/etc/pgbackrest/conf.d]\n"
" --config-path base path of pgBackRest configuration files\n"
" [default=/etc/pgbackrest]\n"
" --delta restore or backup using checksums [default=n]\n"
" --delta restore or backup using checksums\n"
" [default=n]\n"
" --io-timeout I/O timeout [default=60]\n"
" --lock-path path where lock files are stored\n"
" [default=/tmp/pgbackrest]\n"
@ -285,7 +286,8 @@ testRun(void)
" file\n"
" [default=/etc/pgbackrest/pgbackrest.conf]\n"
" --repo-host-config-include-path pgBackRest repository host configuration\n"
" include path [default=/etc/pgbackrest/conf.d]\n"
" include path\n"
" [default=/etc/pgbackrest/conf.d]\n"
" --repo-host-config-path pgBackRest repository host configuration\n"
" path [default=/etc/pgbackrest]\n"
" --repo-host-key-file repository host key file\n"
@ -309,6 +311,7 @@ testRun(void)
" --repo-storage-ca-path repository storage CA path\n"
" --repo-storage-host repository storage host\n"
" --repo-storage-port repository storage port [default=443]\n"
" --repo-storage-upload-chunk-size repository storage upload chunk size\n"
" --repo-storage-verify-tls repository storage certificate verify\n"
" [default=y]\n"
" --repo-type type of storage used for the repository\n"

55
test/src/module/config/loadTest.c
View File

@ -532,6 +532,61 @@ testRun(void)
TEST_RESULT_LOG(
"P00 WARN: 'compress' and 'compress-type' options should not both be set\n"
" HINT: 'compress-type' is preferred and 'compress' is deprecated.");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("S3 default chunk size");
argList = strLstNew();
hrnCfgArgRawZ(argList, cfgOptStanza, "db");
hrnCfgArgRawZ(argList, cfgOptPgPath, "/path/to/pg");
hrnCfgArgKeyRawZ(argList, cfgOptRepoType, 1, "s3");
hrnCfgArgKeyRawZ(argList, cfgOptRepoS3Bucket, 1, "bucket");
hrnCfgArgKeyRawZ(argList, cfgOptRepoS3Region, 1, "region");
hrnCfgArgKeyRawZ(argList, cfgOptRepoS3Endpoint, 1, "endpoint");
hrnCfgArgKeyRawZ(argList, cfgOptRepoS3KeyType, 1, "auto");
hrnCfgArgKeyRawZ(argList, cfgOptRepoPath, 1, "/repo");
HRN_CFG_LOAD(cfgCmdArchiveGet, argList);
TEST_RESULT_UINT(cfgOptionUInt64(cfgOptRepoStorageUploadChunkSize), 5 * 1024 * 1024, "default chunk size");
TEST_RESULT_UINT(cfgOptionSource(cfgOptRepoStorageUploadChunkSize), cfgSourceDefault, "chunk size source is default");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("S3 custom chunk size");
argList = strLstNew();
hrnCfgArgRawZ(argList, cfgOptStanza, "db");
hrnCfgArgRawZ(argList, cfgOptPgPath, "/path/to/pg");
hrnCfgArgKeyRawZ(argList, cfgOptRepoType, 1, "s3");
hrnCfgArgKeyRawZ(argList, cfgOptRepoS3Bucket, 1, "bucket");
hrnCfgArgKeyRawZ(argList, cfgOptRepoS3Region, 1, "region");
hrnCfgArgKeyRawZ(argList, cfgOptRepoS3Endpoint, 1, "endpoint");
hrnCfgArgKeyRawZ(argList, cfgOptRepoS3KeyType, 1, "auto");
hrnCfgArgKeyRawZ(argList, cfgOptRepoStorageUploadChunkSize, 1, "64KiB");
hrnCfgArgKeyRawZ(argList, cfgOptRepoPath, 1, "/repo");
HRN_CFG_LOAD(cfgCmdArchiveGet, argList);
TEST_RESULT_UINT(cfgOptionUInt64(cfgOptRepoStorageUploadChunkSize), 64 * 1024, "chunk size set");
TEST_RESULT_UINT(cfgOptionSource(cfgOptRepoStorageUploadChunkSize), cfgSourceParam, "chunk size source is param");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("Azure default chunk size");
hrnCfgEnvKeyRawZ(cfgOptRepoAzureAccount, 1, "account");
hrnCfgEnvKeyRawZ(cfgOptRepoAzureKey, 1, "mykey");
argList = strLstNew();
hrnCfgArgRawZ(argList, cfgOptStanza, "db");
hrnCfgArgRawZ(argList, cfgOptPgPath, "/path/to/pg");
hrnCfgArgKeyRawZ(argList, cfgOptRepoType, 1, "azure");
hrnCfgArgKeyRawZ(argList, cfgOptRepoAzureContainer, 1, "container");
hrnCfgArgKeyRawZ(argList, cfgOptRepoPath, 1, "/repo");
HRN_CFG_LOAD(cfgCmdArchiveGet, argList);
TEST_RESULT_UINT(cfgOptionUInt64(cfgOptRepoStorageUploadChunkSize), 4 * 1024 * 1024, "default chunk size");
TEST_RESULT_UINT(cfgOptionSource(cfgOptRepoStorageUploadChunkSize), cfgSourceDefault, "chunk size source is default");
hrnCfgEnvKeyRemoveRaw(cfgOptRepoAzureAccount, 1);
hrnCfgEnvKeyRemoveRaw(cfgOptRepoAzureKey, 1);
}
// *****************************************************************************************************************************

2
test/src/module/storage/azureTest.c
View File

@ -203,7 +203,7 @@ testRun(void)
strNewEncode(encodeBase64, ((StorageAzure *)storageDriver(storage))->sharedKey), TEST_KEY_SHARED_STR, "check key");
TEST_RESULT_STR_Z(((StorageAzure *)storageDriver(storage))->host, TEST_ACCOUNT ".blob.core.windows.net", "check host");
TEST_RESULT_STR_Z(((StorageAzure *)storageDriver(storage))->pathPrefix, "/" TEST_CONTAINER, "check path prefix");
TEST_RESULT_UINT(((StorageAzure *)storageDriver(storage))->blockSize, STORAGE_AZURE_BLOCKSIZE_MIN, "check block size");
TEST_RESULT_UINT(((StorageAzure *)storageDriver(storage))->blockSize, 4 * 1024 * 1024, "check block size");
TEST_RESULT_BOOL(storageFeature(storage, storageFeaturePath), false, "check path feature");
// -------------------------------------------------------------------------------------------------------------------------

2
test/src/module/storage/gcsTest.c
View File

@ -223,7 +223,7 @@ testRun(void)
TEST_RESULT_STR_Z(storage->path, "/repo", "check path");
TEST_RESULT_STR(((StorageGcs *)storageDriver(storage))->bucket, TEST_BUCKET_STR, "check bucket");
TEST_RESULT_STR_Z(((StorageGcs *)storageDriver(storage))->endpoint, "storage.googleapis.com", "check endpoint");
TEST_RESULT_UINT(((StorageGcs *)storageDriver(storage))->chunkSize, STORAGE_GCS_CHUNKSIZE_DEFAULT, "check chunk size");
TEST_RESULT_UINT(((StorageGcs *)storageDriver(storage))->chunkSize, 4 * 1024 * 1024, "check chunk size");
TEST_RESULT_STR(((StorageGcs *)storageDriver(storage))->token, TEST_TOKEN_STR, "check token");
TEST_RESULT_BOOL(storageFeature(storage, storageFeaturePath), false, "check path feature");
}

1
test/src/module/storage/s3Test.c
View File

@ -419,6 +419,7 @@ testRun(void)
TEST_RESULT_STR(s3->path, path, "check path");
TEST_RESULT_BOOL(storageFeature(s3, storageFeaturePath), false, "check path feature");
TEST_RESULT_UINT(driver->partSize, 5 * 1024 * 1024, "check part size");
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("coverage for noop functions");