db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2024-12-12 10:04:14 +02:00
Code Issues Releases Activity

Allow upload chunk size to be configured for object stores.

Browse Source 
There are performance benefits to increasing the upload chunk size as long as the tradeoff with additional memory consumption is acceptable.

Make the chunk size configurable for S3, GCS, and Azure, but don't attempt to do any validation of the chunk size beyond some sane limits. The defaults remain as is for each storage type to avoid any unintentional regressions.
This commit is contained in:
David Steele 2022-08-24 09:33:26 -04:00 committed by GitHub
parent 37b4592e52
commit b05d31f531
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 317 additions and 136 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
src/build/config/config.yaml
View File

@ -2245,3 +2245,17 @@ option:
deprecate: deprecate:
repo?-azure-port: {} repo?-azure-port: {}
repo?-s3-port: {} repo?-s3-port: {}
repo-storage-upload-chunk-size:
section: global
group: repo
type: size
required: false
allow-range: [64KiB, 1TiB]
command: repo-type
depend:
option: repo-type
list:
- azure
- gcs
- s3

22
src/build/help/help.xml
View File

@ -972,6 +972,28 @@
<example>9000</example> <example>9000</example>
</config-key> </config-key>
<config-key id="repo-storage-upload-chunk-size" name="Repository Storage Upload Chunk Size">
<summary>Repository storage upload chunk size.</summary>
<text>
<p>Object stores such as S3 allow files to be uploaded in chunks when the file is too large to be stored in memory. Even if the file can be stored in memory, it is more memory efficient to limit the amount of memory used for uploads.</p>
<p>A larger chunk size will generally lead to better performance because it will minimize upload requests and allow more files to be uploaded in a single request rather than in chunks. The disadvantage is that memory usage will be higher and because the chunk buffer must be allocated per process, larger <br-option>process-max</br-option> values will lead to more memory being consumed overall.</p>
<p>Default chunk sizes by repo type:</p>
<list>
<list-item><id>azure</id> - 4MiB</list-item>
<list-item><id>gcs</id> - 4MiB</list-item>
<list-item><id>s3</id> - 5MiB</list-item>
</list>
<p>Note that valid chunk sizes vary by storage type and by platform. For example, <proper>AWS S3</proper> has a minimum chunk size of 5MiB but <proper>S3</proper> clones may accept lower values. Terminology for chunk size varies by storage type, so when searching min/max values use <quote>part size</quote> for <proper>AWS S3</proper>, <quote>chunk size</quote> for <proper>GCS</proper>, and <quote>block size</quote> for <proper>Azure</proper>. No attempt is made to validate configured chunk sizes so selecting an invalid value will lead to errors from the storage service or undefined behavior.</p>
</text>
<example>16MiB</example>
</config-key>
<config-key id="repo-storage-verify-tls" name="Repository Storage Certificate Verify"> <config-key id="repo-storage-verify-tls" name="Repository Storage Certificate Verify">
<summary>Repository storage certificate verify.</summary> <summary>Repository storage certificate verify.</summary>

3
src/config/config.auto.h
View File

@ -128,7 +128,7 @@ Option constants
#define CFGOPT_TYPE "type" #define CFGOPT_TYPE "type"
#define CFGOPT_VERBOSE "verbose" #define CFGOPT_VERBOSE "verbose"
#define CFG_OPTION_TOTAL 155 #define CFG_OPTION_TOTAL 156
/*********************************************************************************************************************************** /***********************************************************************************************************************************
Option value constants Option value constants
@ -483,6 +483,7 @@ typedef enum
cfgOptRepoStorageCaPath, cfgOptRepoStorageCaPath,
cfgOptRepoStorageHost, cfgOptRepoStorageHost,
cfgOptRepoStoragePort, cfgOptRepoStoragePort,
cfgOptRepoStorageUploadChunkSize,
cfgOptRepoStorageVerifyTls, cfgOptRepoStorageVerifyTls,
cfgOptRepoType, cfgOptRepoType,
cfgOptResume, cfgOptResume,

14
src/config/load.c
View File

@ -288,6 +288,20 @@ cfgLoadUpdateOption(void)
} }
} }
// Set default upload chunk size if not set
if (cfgOptionValid(cfgOptRepoStorageUploadChunkSize))
{
for (unsigned int repoIdx = 0; repoIdx < cfgOptionGroupIdxTotal(cfgOptGrpRepo); repoIdx++)
{
if (!cfgOptionIdxTest(cfgOptRepoStorageUploadChunkSize, repoIdx))
{
cfgOptionIdxSet(
cfgOptRepoStorageUploadChunkSize, repoIdx, cfgSourceDefault,
VARINT64((cfgOptionIdxStrId(cfgOptRepoType, repoIdx) == CFGOPTVAL_REPO_TYPE_S3 ? 5 : 4) * 1024 * 1024));
}
}
}
// Set pg-host-port/repo-host-port default when pg-host-type/repo-host-type is tls. ??? This should be handled in the parser but // Set pg-host-port/repo-host-port default when pg-host-type/repo-host-type is tls. ??? This should be handled in the parser but
// it requires a default that depends on another option value and that is not currently possible. // it requires a default that depends on another option value and that is not currently possible.
#define HOST_PORT_TLS 8432 #define HOST_PORT_TLS 8432

85
src/config/parse.auto.c.inc
View File

@ -7546,6 +7546,90 @@ static const ParseRuleOption parseRuleOption[CFG_OPTION_TOTAL] =
), // opt/repo-storage-port ), // opt/repo-storage-port
), // opt/repo-storage-port ), // opt/repo-storage-port
// ----------------------------------------------------------------------------------------------------------------------------- // -----------------------------------------------------------------------------------------------------------------------------
PARSE_RULE_OPTION // opt/repo-storage-upload-chunk-size
( // opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_NAME("repo-storage-upload-chunk-size"), // opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_TYPE(cfgOptTypeSize), // opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_RESET(true), // opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_REQUIRED(false), // opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_SECTION(cfgSectionGlobal), // opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_GROUP_MEMBER(true), // opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_GROUP_ID(cfgOptGrpRepo), // opt/repo-storage-upload-chunk-size
// opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_COMMAND_ROLE_MAIN_VALID_LIST // opt/repo-storage-upload-chunk-size
( // opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_COMMAND(cfgCmdArchiveGet) // opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_COMMAND(cfgCmdArchivePush) // opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_COMMAND(cfgCmdBackup) // opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_COMMAND(cfgCmdCheck) // opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_COMMAND(cfgCmdExpire) // opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_COMMAND(cfgCmdInfo) // opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_COMMAND(cfgCmdRepoCreate) // opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_COMMAND(cfgCmdRepoGet) // opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_COMMAND(cfgCmdRepoLs) // opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_COMMAND(cfgCmdRepoPut) // opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_COMMAND(cfgCmdRepoRm) // opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_COMMAND(cfgCmdRestore) // opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaCreate) // opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaDelete) // opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaUpgrade) // opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_COMMAND(cfgCmdVerify) // opt/repo-storage-upload-chunk-size
), // opt/repo-storage-upload-chunk-size
// opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_COMMAND_ROLE_ASYNC_VALID_LIST // opt/repo-storage-upload-chunk-size
( // opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_COMMAND(cfgCmdArchiveGet) // opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_COMMAND(cfgCmdArchivePush) // opt/repo-storage-upload-chunk-size
), // opt/repo-storage-upload-chunk-size
// opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_COMMAND_ROLE_LOCAL_VALID_LIST // opt/repo-storage-upload-chunk-size
( // opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_COMMAND(cfgCmdArchiveGet) // opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_COMMAND(cfgCmdArchivePush) // opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_COMMAND(cfgCmdBackup) // opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_COMMAND(cfgCmdRestore) // opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_COMMAND(cfgCmdVerify) // opt/repo-storage-upload-chunk-size
), // opt/repo-storage-upload-chunk-size
// opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_COMMAND_ROLE_REMOTE_VALID_LIST // opt/repo-storage-upload-chunk-size
( // opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_COMMAND(cfgCmdArchiveGet) // opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_COMMAND(cfgCmdArchivePush) // opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_COMMAND(cfgCmdCheck) // opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_COMMAND(cfgCmdInfo) // opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_COMMAND(cfgCmdRepoCreate) // opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_COMMAND(cfgCmdRepoGet) // opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_COMMAND(cfgCmdRepoLs) // opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_COMMAND(cfgCmdRepoPut) // opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_COMMAND(cfgCmdRepoRm) // opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_COMMAND(cfgCmdRestore) // opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaCreate) // opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaDelete) // opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaUpgrade) // opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTION_COMMAND(cfgCmdVerify) // opt/repo-storage-upload-chunk-size
), // opt/repo-storage-upload-chunk-size
// opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTIONAL // opt/repo-storage-upload-chunk-size
( // opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTIONAL_GROUP // opt/repo-storage-upload-chunk-size
( // opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTIONAL_DEPEND // opt/repo-storage-upload-chunk-size
( // opt/repo-storage-upload-chunk-size
PARSE_RULE_VAL_OPT(cfgOptRepoType), // opt/repo-storage-upload-chunk-size
PARSE_RULE_VAL_STRID(parseRuleValStrIdAzure), // opt/repo-storage-upload-chunk-size
PARSE_RULE_VAL_STRID(parseRuleValStrIdGcs), // opt/repo-storage-upload-chunk-size
PARSE_RULE_VAL_STRID(parseRuleValStrIdS3), // opt/repo-storage-upload-chunk-size
), // opt/repo-storage-upload-chunk-size
// opt/repo-storage-upload-chunk-size
PARSE_RULE_OPTIONAL_ALLOW_RANGE // opt/repo-storage-upload-chunk-size
( // opt/repo-storage-upload-chunk-size
PARSE_RULE_VAL_INT(parseRuleValInt65536), // opt/repo-storage-upload-chunk-size
PARSE_RULE_VAL_INT(parseRuleValInt1099511627776), // opt/repo-storage-upload-chunk-size
), // opt/repo-storage-upload-chunk-size
), // opt/repo-storage-upload-chunk-size
), // opt/repo-storage-upload-chunk-size
), // opt/repo-storage-upload-chunk-size
// -----------------------------------------------------------------------------------------------------------------------------
PARSE_RULE_OPTION // opt/repo-storage-verify-tls PARSE_RULE_OPTION // opt/repo-storage-verify-tls
( // opt/repo-storage-verify-tls ( // opt/repo-storage-verify-tls
PARSE_RULE_OPTION_NAME("repo-storage-verify-tls"), // opt/repo-storage-verify-tls PARSE_RULE_OPTION_NAME("repo-storage-verify-tls"), // opt/repo-storage-verify-tls
@ -9201,6 +9285,7 @@ static const uint8_t optionResolveOrder[] =
cfgOptRepoStorageCaPath, // opt-resolve-order cfgOptRepoStorageCaPath, // opt-resolve-order
cfgOptRepoStorageHost, // opt-resolve-order cfgOptRepoStorageHost, // opt-resolve-order
cfgOptRepoStoragePort, // opt-resolve-order cfgOptRepoStoragePort, // opt-resolve-order
cfgOptRepoStorageUploadChunkSize, // opt-resolve-order
cfgOptRepoStorageVerifyTls, // opt-resolve-order cfgOptRepoStorageVerifyTls, // opt-resolve-order
cfgOptTarget, // opt-resolve-order cfgOptTarget, // opt-resolve-order
cfgOptTargetAction, // opt-resolve-order cfgOptTargetAction, // opt-resolve-order

4
src/storage/azure/helper.c
View File

@ -79,8 +79,8 @@ storageAzureHelper(const unsigned int repoIdx, const bool write, StoragePathExpr
result = storageAzureNew( result = storageAzureNew(
cfgOptionIdxStr(cfgOptRepoPath, repoIdx), write, pathExpressionCallback, cfgOptionIdxStr(cfgOptRepoPath, repoIdx), write, pathExpressionCallback,
cfgOptionIdxStr(cfgOptRepoAzureContainer, repoIdx), cfgOptionIdxStr(cfgOptRepoAzureAccount, repoIdx), keyType, key, cfgOptionIdxStr(cfgOptRepoAzureContainer, repoIdx), cfgOptionIdxStr(cfgOptRepoAzureAccount, repoIdx), keyType, key,
STORAGE_AZURE_BLOCKSIZE_MIN, endpoint, uriStyle, port, ioTimeoutMs(), cfgOptionIdxBool(cfgOptRepoStorageVerifyTls, (size_t)cfgOptionIdxUInt64(cfgOptRepoStorageUploadChunkSize, repoIdx), endpoint, uriStyle, port, ioTimeoutMs(),
repoIdx), cfgOptionIdxStrNull(cfgOptRepoStorageCaFile, repoIdx), cfgOptionIdxBool(cfgOptRepoStorageVerifyTls, repoIdx), cfgOptionIdxStrNull(cfgOptRepoStorageCaFile, repoIdx),
cfgOptionIdxStrNull(cfgOptRepoStorageCaPath, repoIdx)); cfgOptionIdxStrNull(cfgOptRepoStorageCaPath, repoIdx));
} }
MEM_CONTEXT_PRIOR_END(); MEM_CONTEXT_PRIOR_END();

5
src/storage/azure/storage.h
View File

@ -29,11 +29,6 @@ typedef enum
storageAzureUriStylePath = STRID5("path", 0x450300), storageAzureUriStylePath = STRID5("path", 0x450300),
} StorageAzureUriStyle; } StorageAzureUriStyle;
/***********************************************************************************************************************************
Defaults
***********************************************************************************************************************************/
#define STORAGE_AZURE_BLOCKSIZE_MIN ((size_t)4 * 1024 * 1024)
/*********************************************************************************************************************************** /***********************************************************************************************************************************
Constructors Constructors
***********************************************************************************************************************************/ ***********************************************************************************************************************************/

4
src/storage/gcs/helper.c
View File

@ -24,8 +24,8 @@ storageGcsHelper(const unsigned int repoIdx, const bool write, StoragePathExpres
Storage *const result = storageGcsNew( Storage *const result = storageGcsNew(
cfgOptionIdxStr(cfgOptRepoPath, repoIdx), write, pathExpressionCallback, cfgOptionIdxStr(cfgOptRepoGcsBucket, repoIdx), cfgOptionIdxStr(cfgOptRepoPath, repoIdx), write, pathExpressionCallback, cfgOptionIdxStr(cfgOptRepoGcsBucket, repoIdx),
(StorageGcsKeyType)cfgOptionIdxStrId(cfgOptRepoGcsKeyType, repoIdx), cfgOptionIdxStrNull(cfgOptRepoGcsKey, repoIdx), (StorageGcsKeyType)cfgOptionIdxStrId(cfgOptRepoGcsKeyType, repoIdx), cfgOptionIdxStrNull(cfgOptRepoGcsKey, repoIdx),
STORAGE_GCS_CHUNKSIZE_DEFAULT, cfgOptionIdxStr(cfgOptRepoGcsEndpoint, repoIdx), ioTimeoutMs(), (size_t)cfgOptionIdxUInt64(cfgOptRepoStorageUploadChunkSize, repoIdx), cfgOptionIdxStr(cfgOptRepoGcsEndpoint, repoIdx),
cfgOptionIdxBool(cfgOptRepoStorageVerifyTls, repoIdx), cfgOptionIdxStrNull(cfgOptRepoStorageCaFile, repoIdx), ioTimeoutMs(), cfgOptionIdxBool(cfgOptRepoStorageVerifyTls, repoIdx), cfgOptionIdxStrNull(cfgOptRepoStorageCaFile, repoIdx),
cfgOptionIdxStrNull(cfgOptRepoStorageCaPath, repoIdx)); cfgOptionIdxStrNull(cfgOptRepoStorageCaPath, repoIdx));
FUNCTION_LOG_RETURN(STORAGE, result); FUNCTION_LOG_RETURN(STORAGE, result);

5
src/storage/gcs/storage.h
View File

@ -21,11 +21,6 @@ typedef enum
storageGcsKeyTypeToken = STRID5("token", 0xe2adf40), storageGcsKeyTypeToken = STRID5("token", 0xe2adf40),
} StorageGcsKeyType; } StorageGcsKeyType;
/***********************************************************************************************************************************
Defaults
***********************************************************************************************************************************/
#define STORAGE_GCS_CHUNKSIZE_DEFAULT ((size_t)4 * 1024 * 1024)
/*********************************************************************************************************************************** /***********************************************************************************************************************************
Constructors Constructors
***********************************************************************************************************************************/ ***********************************************************************************************************************************/

7
src/storage/s3/helper.c
View File

@ -85,9 +85,10 @@ storageS3Helper(const unsigned int repoIdx, const bool write, StoragePathExpress
endPoint, (StorageS3UriStyle)cfgOptionIdxStrId(cfgOptRepoS3UriStyle, repoIdx), endPoint, (StorageS3UriStyle)cfgOptionIdxStrId(cfgOptRepoS3UriStyle, repoIdx),
cfgOptionIdxStr(cfgOptRepoS3Region, repoIdx), keyType, cfgOptionIdxStrNull(cfgOptRepoS3Key, repoIdx), cfgOptionIdxStr(cfgOptRepoS3Region, repoIdx), keyType, cfgOptionIdxStrNull(cfgOptRepoS3Key, repoIdx),
cfgOptionIdxStrNull(cfgOptRepoS3KeySecret, repoIdx), cfgOptionIdxStrNull(cfgOptRepoS3Token, repoIdx), cfgOptionIdxStrNull(cfgOptRepoS3KeySecret, repoIdx), cfgOptionIdxStrNull(cfgOptRepoS3Token, repoIdx),
cfgOptionIdxStrNull(cfgOptRepoS3KmsKeyId, repoIdx), role, webIdToken, STORAGE_S3_PARTSIZE_MIN, host, port, cfgOptionIdxStrNull(cfgOptRepoS3KmsKeyId, repoIdx), role, webIdToken,
ioTimeoutMs(), cfgOptionIdxBool(cfgOptRepoStorageVerifyTls, repoIdx), (size_t)cfgOptionIdxUInt64(cfgOptRepoStorageUploadChunkSize, repoIdx), host, port, ioTimeoutMs(),
cfgOptionIdxStrNull(cfgOptRepoStorageCaFile, repoIdx), cfgOptionIdxStrNull(cfgOptRepoStorageCaPath, repoIdx)); cfgOptionIdxBool(cfgOptRepoStorageVerifyTls, repoIdx), cfgOptionIdxStrNull(cfgOptRepoStorageCaFile, repoIdx),
cfgOptionIdxStrNull(cfgOptRepoStorageCaPath, repoIdx));
} }
MEM_CONTEXT_PRIOR_END(); MEM_CONTEXT_PRIOR_END();
} }

5
src/storage/s3/storage.h
View File

@ -30,11 +30,6 @@ typedef enum
storageS3UriStylePath = STRID5("path", 0x450300), storageS3UriStylePath = STRID5("path", 0x450300),
} StorageS3UriStyle; } StorageS3UriStyle;
/***********************************************************************************************************************************
Defaults
***********************************************************************************************************************************/
#define STORAGE_S3_PARTSIZE_MIN ((size_t)5 * 1024 * 1024)
/*********************************************************************************************************************************** /***********************************************************************************************************************************
Constructors Constructors
***********************************************************************************************************************************/ ***********************************************************************************************************************************/

225
test/src/module/command/helpTest.c
View File

@ -194,129 +194,132 @@ testRun(void)
"\n" "\n"
"Command Options:\n" "Command Options:\n"
"\n" "\n"
" --archive-mode preserve or disable archiving on restored\n" " --archive-mode preserve or disable archiving on restored\n"
" cluster [default=preserve]\n" " cluster [default=preserve]\n"
" --db-exclude restore excluding the specified databases\n" " --db-exclude restore excluding the specified databases\n"
" --db-include restore only specified databases\n" " --db-include restore only specified databases\n"
" [current=db1, db2]\n" " [current=db1, db2]\n"
" --force force a restore [default=n]\n" " --force force a restore [default=n]\n"
" --link-all restore all symlinks [default=n]\n" " --link-all restore all symlinks [default=n]\n"
" --link-map modify the destination of a symlink\n" " --link-map modify the destination of a symlink\n"
" [current=/link1=/dest1, /link2=/dest2]\n" " [current=/link1=/dest1, /link2=/dest2]\n"
" --recovery-option set an option in recovery.conf\n" " --recovery-option set an option in recovery.conf\n"
" --set backup set to restore [default=latest]\n" " --set backup set to restore [default=latest]\n"
" --tablespace-map restore a tablespace into the specified\n" " --tablespace-map restore a tablespace into the specified\n"
" directory\n" " directory\n"
" --tablespace-map-all restore all tablespaces into the specified\n" " --tablespace-map-all restore all tablespaces into the specified\n"
" directory\n" " directory\n"
" --target recovery target\n" " --target recovery target\n"
" --target-action action to take when recovery target is\n" " --target-action action to take when recovery target is\n"
" reached [default=pause]\n" " reached [default=pause]\n"
" --target-exclusive stop just before the recovery target is\n" " --target-exclusive stop just before the recovery target is\n"
" reached [default=n]\n" " reached [default=n]\n"
" --target-timeline recover along a timeline\n" " --target-timeline recover along a timeline\n"
" --type recovery type [default=default]\n" " --type recovery type [default=default]\n"
"\n" "\n"
"General Options:\n" "General Options:\n"
"\n" "\n"
" --buffer-size buffer size for I/O operations\n" " --buffer-size buffer size for I/O operations\n"
" [current=32768, default=1MiB]\n" " [current=32768, default=1MiB]\n"
" --cmd pgBackRest command\n" " --cmd pgBackRest command\n"
" [default=/path/to/pgbackrest]\n" " [default=/path/to/pgbackrest]\n"
" --cmd-ssh SSH client command [default=ssh]\n" " --cmd-ssh SSH client command [default=ssh]\n"
" --compress-level-network network compression level [default=3]\n" " --compress-level-network network compression level [default=3]\n"
" --config pgBackRest configuration file\n" " --config pgBackRest configuration file\n"
" [default=/etc/pgbackrest/pgbackrest.conf]\n" " [default=/etc/pgbackrest/pgbackrest.conf]\n"
" --config-include-path path to additional pgBackRest configuration\n" " --config-include-path path to additional pgBackRest configuration\n"
" files [default=/etc/pgbackrest/conf.d]\n" " files [default=/etc/pgbackrest/conf.d]\n"
" --config-path base path of pgBackRest configuration files\n" " --config-path base path of pgBackRest configuration files\n"
" [default=/etc/pgbackrest]\n" " [default=/etc/pgbackrest]\n"
" --delta restore or backup using checksums [default=n]\n" " --delta restore or backup using checksums\n"
" --io-timeout I/O timeout [default=60]\n" " [default=n]\n"
" --lock-path path where lock files are stored\n" " --io-timeout I/O timeout [default=60]\n"
" [default=/tmp/pgbackrest]\n" " --lock-path path where lock files are stored\n"
" --neutral-umask use a neutral umask [default=y]\n" " [default=/tmp/pgbackrest]\n"
" --process-max max processes to use for compress/transfer\n" " --neutral-umask use a neutral umask [default=y]\n"
" [default=1]\n" " --process-max max processes to use for compress/transfer\n"
" --protocol-timeout protocol timeout [default=1830]\n" " [default=1]\n"
" --sck-keep-alive keep-alive enable [default=y]\n" " --protocol-timeout protocol timeout [default=1830]\n"
" --stanza defines the stanza\n" " --sck-keep-alive keep-alive enable [default=y]\n"
" --tcp-keep-alive-count keep-alive count\n" " --stanza defines the stanza\n"
" --tcp-keep-alive-idle keep-alive idle time\n" " --tcp-keep-alive-count keep-alive count\n"
" --tcp-keep-alive-interval keep-alive interval time\n" " --tcp-keep-alive-idle keep-alive idle time\n"
" --tcp-keep-alive-interval keep-alive interval time\n"
"\n" "\n"
"Log Options:\n" "Log Options:\n"
"\n" "\n"
" --log-level-console level for console logging [default=warn]\n" " --log-level-console level for console logging [default=warn]\n"
" --log-level-file level for file logging [default=info]\n" " --log-level-file level for file logging [default=info]\n"
" --log-level-stderr level for stderr logging [default=warn]\n" " --log-level-stderr level for stderr logging [default=warn]\n"
" --log-path path where log files are stored\n" " --log-path path where log files are stored\n"
" [default=/var/log/pgbackrest]\n" " [default=/var/log/pgbackrest]\n"
" --log-subprocess enable logging in subprocesses [default=n]\n" " --log-subprocess enable logging in subprocesses [default=n]\n"
" --log-timestamp enable timestamp in logging [default=y]\n" " --log-timestamp enable timestamp in logging [default=y]\n"
"\n", "\n",
"Repository Options:\n" "Repository Options:\n"
"\n" "\n"
" --repo set repository\n" " --repo set repository\n"
" --repo-azure-account azure repository account\n" " --repo-azure-account azure repository account\n"
" --repo-azure-container azure repository container\n" " --repo-azure-container azure repository container\n"
" --repo-azure-endpoint azure repository endpoint\n" " --repo-azure-endpoint azure repository endpoint\n"
" [default=blob.core.windows.net]\n" " [default=blob.core.windows.net]\n"
" --repo-azure-key azure repository key\n" " --repo-azure-key azure repository key\n"
" --repo-azure-key-type azure repository key type [default=shared]\n" " --repo-azure-key-type azure repository key type [default=shared]\n"
" --repo-azure-uri-style azure URI Style [default=host]\n" " --repo-azure-uri-style azure URI Style [default=host]\n"
" --repo-cipher-pass repository cipher passphrase\n" " --repo-cipher-pass repository cipher passphrase\n"
" [current=<redacted>]\n" " [current=<redacted>]\n"
" --repo-cipher-type cipher used to encrypt the repository\n" " --repo-cipher-type cipher used to encrypt the repository\n"
" [current=aes-256-cbc, default=none]\n" " [current=aes-256-cbc, default=none]\n"
" --repo-gcs-bucket GCS repository bucket\n" " --repo-gcs-bucket GCS repository bucket\n"
" --repo-gcs-endpoint GCS repository endpoint\n" " --repo-gcs-endpoint GCS repository endpoint\n"
" [default=storage.googleapis.com]\n" " [default=storage.googleapis.com]\n"
" --repo-gcs-key GCS repository key\n" " --repo-gcs-key GCS repository key\n"
" --repo-gcs-key-type GCS repository key type [default=service]\n" " --repo-gcs-key-type GCS repository key type [default=service]\n"
" --repo-host repository host when operating remotely via\n" " --repo-host repository host when operating remotely via\n"
" SSH [current=backup.example.net]\n" " SSH [current=backup.example.net]\n"
" --repo-host-ca-file repository host certificate authority file\n" " --repo-host-ca-file repository host certificate authority file\n"
" --repo-host-ca-path repository host certificate authority path\n" " --repo-host-ca-path repository host certificate authority path\n"
" --repo-host-cert-file repository host certificate file\n" " --repo-host-cert-file repository host certificate file\n"
" --repo-host-cmd repository host pgBackRest command\n" " --repo-host-cmd repository host pgBackRest command\n"
" [default=/path/to/pgbackrest]\n" " [default=/path/to/pgbackrest]\n"
" --repo-host-config pgBackRest repository host configuration\n" " --repo-host-config pgBackRest repository host configuration\n"
" file\n" " file\n"
" [default=/etc/pgbackrest/pgbackrest.conf]\n" " [default=/etc/pgbackrest/pgbackrest.conf]\n"
" --repo-host-config-include-path pgBackRest repository host configuration\n" " --repo-host-config-include-path pgBackRest repository host configuration\n"
" include path [default=/etc/pgbackrest/conf.d]\n" " include path\n"
" --repo-host-config-path pgBackRest repository host configuration\n" " [default=/etc/pgbackrest/conf.d]\n"
" path [default=/etc/pgbackrest]\n" " --repo-host-config-path pgBackRest repository host configuration\n"
" --repo-host-key-file repository host key file\n" " path [default=/etc/pgbackrest]\n"
" --repo-host-port repository host port when repo-host is set\n" " --repo-host-key-file repository host key file\n"
" --repo-host-type repository host protocol type [default=ssh]\n" " --repo-host-port repository host port when repo-host is set\n"
" --repo-host-user repository host user when repo-host is set\n" " --repo-host-type repository host protocol type [default=ssh]\n"
" [default=pgbackrest]\n" " --repo-host-user repository host user when repo-host is set\n"
" --repo-path path where backups and archive are stored\n" " [default=pgbackrest]\n"
" [default=/var/lib/pgbackrest]\n" " --repo-path path where backups and archive are stored\n"
" --repo-s3-bucket S3 repository bucket\n" " [default=/var/lib/pgbackrest]\n"
" --repo-s3-endpoint S3 repository endpoint\n" " --repo-s3-bucket S3 repository bucket\n"
" --repo-s3-key S3 repository access key\n" " --repo-s3-endpoint S3 repository endpoint\n"
" --repo-s3-key-secret S3 repository secret access key\n" " --repo-s3-key S3 repository access key\n"
" --repo-s3-key-type S3 repository key type [default=shared]\n" " --repo-s3-key-secret S3 repository secret access key\n"
" --repo-s3-kms-key-id S3 repository KMS key\n" " --repo-s3-key-type S3 repository key type [default=shared]\n"
" --repo-s3-region S3 repository region\n" " --repo-s3-kms-key-id S3 repository KMS key\n"
" --repo-s3-role S3 repository role\n" " --repo-s3-region S3 repository region\n"
" --repo-s3-token S3 repository security token\n" " --repo-s3-role S3 repository role\n"
" --repo-s3-uri-style S3 URI Style [default=host]\n" " --repo-s3-token S3 repository security token\n"
" --repo-storage-ca-file repository storage CA file\n" " --repo-s3-uri-style S3 URI Style [default=host]\n"
" --repo-storage-ca-path repository storage CA path\n" " --repo-storage-ca-file repository storage CA file\n"
" --repo-storage-host repository storage host\n" " --repo-storage-ca-path repository storage CA path\n"
" --repo-storage-port repository storage port [default=443]\n" " --repo-storage-host repository storage host\n"
" --repo-storage-verify-tls repository storage certificate verify\n" " --repo-storage-port repository storage port [default=443]\n"
" [default=y]\n" " --repo-storage-upload-chunk-size repository storage upload chunk size\n"
" --repo-type type of storage used for the repository\n" " --repo-storage-verify-tls repository storage certificate verify\n"
" [default=posix]\n" " [default=y]\n"
" --repo-type type of storage used for the repository\n"
" [default=posix]\n"
"\n" "\n"
"Stanza Options:\n" "Stanza Options:\n"
"\n" "\n"
" --pg-path postgreSQL data directory\n" " --pg-path postgreSQL data directory\n"
"\n" "\n"
"Use 'pgbackrest help restore [option]' for more information.\n"); "Use 'pgbackrest help restore [option]' for more information.\n");

55
test/src/module/config/loadTest.c
View File

@ -532,6 +532,61 @@ testRun(void)
TEST_RESULT_LOG( TEST_RESULT_LOG(
"P00 WARN: 'compress' and 'compress-type' options should not both be set\n" "P00 WARN: 'compress' and 'compress-type' options should not both be set\n"
" HINT: 'compress-type' is preferred and 'compress' is deprecated."); " HINT: 'compress-type' is preferred and 'compress' is deprecated.");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("S3 default chunk size");
argList = strLstNew();
hrnCfgArgRawZ(argList, cfgOptStanza, "db");
hrnCfgArgRawZ(argList, cfgOptPgPath, "/path/to/pg");
hrnCfgArgKeyRawZ(argList, cfgOptRepoType, 1, "s3");
hrnCfgArgKeyRawZ(argList, cfgOptRepoS3Bucket, 1, "bucket");
hrnCfgArgKeyRawZ(argList, cfgOptRepoS3Region, 1, "region");
hrnCfgArgKeyRawZ(argList, cfgOptRepoS3Endpoint, 1, "endpoint");
hrnCfgArgKeyRawZ(argList, cfgOptRepoS3KeyType, 1, "auto");
hrnCfgArgKeyRawZ(argList, cfgOptRepoPath, 1, "/repo");
HRN_CFG_LOAD(cfgCmdArchiveGet, argList);
TEST_RESULT_UINT(cfgOptionUInt64(cfgOptRepoStorageUploadChunkSize), 5 * 1024 * 1024, "default chunk size");
TEST_RESULT_UINT(cfgOptionSource(cfgOptRepoStorageUploadChunkSize), cfgSourceDefault, "chunk size source is default");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("S3 custom chunk size");
argList = strLstNew();
hrnCfgArgRawZ(argList, cfgOptStanza, "db");
hrnCfgArgRawZ(argList, cfgOptPgPath, "/path/to/pg");
hrnCfgArgKeyRawZ(argList, cfgOptRepoType, 1, "s3");
hrnCfgArgKeyRawZ(argList, cfgOptRepoS3Bucket, 1, "bucket");
hrnCfgArgKeyRawZ(argList, cfgOptRepoS3Region, 1, "region");
hrnCfgArgKeyRawZ(argList, cfgOptRepoS3Endpoint, 1, "endpoint");
hrnCfgArgKeyRawZ(argList, cfgOptRepoS3KeyType, 1, "auto");
hrnCfgArgKeyRawZ(argList, cfgOptRepoStorageUploadChunkSize, 1, "64KiB");
hrnCfgArgKeyRawZ(argList, cfgOptRepoPath, 1, "/repo");
HRN_CFG_LOAD(cfgCmdArchiveGet, argList);
TEST_RESULT_UINT(cfgOptionUInt64(cfgOptRepoStorageUploadChunkSize), 64 * 1024, "chunk size set");
TEST_RESULT_UINT(cfgOptionSource(cfgOptRepoStorageUploadChunkSize), cfgSourceParam, "chunk size source is param");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("Azure default chunk size");
hrnCfgEnvKeyRawZ(cfgOptRepoAzureAccount, 1, "account");
hrnCfgEnvKeyRawZ(cfgOptRepoAzureKey, 1, "mykey");
argList = strLstNew();
hrnCfgArgRawZ(argList, cfgOptStanza, "db");
hrnCfgArgRawZ(argList, cfgOptPgPath, "/path/to/pg");
hrnCfgArgKeyRawZ(argList, cfgOptRepoType, 1, "azure");
hrnCfgArgKeyRawZ(argList, cfgOptRepoAzureContainer, 1, "container");
hrnCfgArgKeyRawZ(argList, cfgOptRepoPath, 1, "/repo");
HRN_CFG_LOAD(cfgCmdArchiveGet, argList);
TEST_RESULT_UINT(cfgOptionUInt64(cfgOptRepoStorageUploadChunkSize), 4 * 1024 * 1024, "default chunk size");
TEST_RESULT_UINT(cfgOptionSource(cfgOptRepoStorageUploadChunkSize), cfgSourceDefault, "chunk size source is default");
hrnCfgEnvKeyRemoveRaw(cfgOptRepoAzureAccount, 1);
hrnCfgEnvKeyRemoveRaw(cfgOptRepoAzureKey, 1);
} }
// ***************************************************************************************************************************** // *****************************************************************************************************************************

2
test/src/module/storage/azureTest.c
View File

@ -203,7 +203,7 @@ testRun(void)
strNewEncode(encodeBase64, ((StorageAzure *)storageDriver(storage))->sharedKey), TEST_KEY_SHARED_STR, "check key"); strNewEncode(encodeBase64, ((StorageAzure *)storageDriver(storage))->sharedKey), TEST_KEY_SHARED_STR, "check key");
TEST_RESULT_STR_Z(((StorageAzure *)storageDriver(storage))->host, TEST_ACCOUNT ".blob.core.windows.net", "check host"); TEST_RESULT_STR_Z(((StorageAzure *)storageDriver(storage))->host, TEST_ACCOUNT ".blob.core.windows.net", "check host");
TEST_RESULT_STR_Z(((StorageAzure *)storageDriver(storage))->pathPrefix, "/" TEST_CONTAINER, "check path prefix"); TEST_RESULT_STR_Z(((StorageAzure *)storageDriver(storage))->pathPrefix, "/" TEST_CONTAINER, "check path prefix");
TEST_RESULT_UINT(((StorageAzure *)storageDriver(storage))->blockSize, STORAGE_AZURE_BLOCKSIZE_MIN, "check block size"); TEST_RESULT_UINT(((StorageAzure *)storageDriver(storage))->blockSize, 4 * 1024 * 1024, "check block size");
TEST_RESULT_BOOL(storageFeature(storage, storageFeaturePath), false, "check path feature"); TEST_RESULT_BOOL(storageFeature(storage, storageFeaturePath), false, "check path feature");
// ------------------------------------------------------------------------------------------------------------------------- // -------------------------------------------------------------------------------------------------------------------------

2
test/src/module/storage/gcsTest.c
View File

@ -223,7 +223,7 @@ testRun(void)
TEST_RESULT_STR_Z(storage->path, "/repo", "check path"); TEST_RESULT_STR_Z(storage->path, "/repo", "check path");
TEST_RESULT_STR(((StorageGcs *)storageDriver(storage))->bucket, TEST_BUCKET_STR, "check bucket"); TEST_RESULT_STR(((StorageGcs *)storageDriver(storage))->bucket, TEST_BUCKET_STR, "check bucket");
TEST_RESULT_STR_Z(((StorageGcs *)storageDriver(storage))->endpoint, "storage.googleapis.com", "check endpoint"); TEST_RESULT_STR_Z(((StorageGcs *)storageDriver(storage))->endpoint, "storage.googleapis.com", "check endpoint");
TEST_RESULT_UINT(((StorageGcs *)storageDriver(storage))->chunkSize, STORAGE_GCS_CHUNKSIZE_DEFAULT, "check chunk size"); TEST_RESULT_UINT(((StorageGcs *)storageDriver(storage))->chunkSize, 4 * 1024 * 1024, "check chunk size");
TEST_RESULT_STR(((StorageGcs *)storageDriver(storage))->token, TEST_TOKEN_STR, "check token"); TEST_RESULT_STR(((StorageGcs *)storageDriver(storage))->token, TEST_TOKEN_STR, "check token");
TEST_RESULT_BOOL(storageFeature(storage, storageFeaturePath), false, "check path feature"); TEST_RESULT_BOOL(storageFeature(storage, storageFeaturePath), false, "check path feature");
} }

1
test/src/module/storage/s3Test.c
View File

@ -419,6 +419,7 @@ testRun(void)
TEST_RESULT_STR(s3->path, path, "check path"); TEST_RESULT_STR(s3->path, path, "check path");
TEST_RESULT_BOOL(storageFeature(s3, storageFeaturePath), false, "check path feature"); TEST_RESULT_BOOL(storageFeature(s3, storageFeaturePath), false, "check path feature");
TEST_RESULT_UINT(driver->partSize, 5 * 1024 * 1024, "check part size");
// ----------------------------------------------------------------------------------------------------------------- // -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("coverage for noop functions"); TEST_TITLE("coverage for noop functions");