mirror of
https://github.com/pgbackrest/pgbackrest.git
synced 2024-12-12 10:04:14 +02:00
Allow upload chunk size to be configured for object stores.
There are performance benefits to increasing the upload chunk size as long as the tradeoff with additional memory consumption is acceptable. Make the chunk size configurable for S3, GCS, and Azure, but don't attempt to do any validation of the chunk size beyond some sane limits. The defaults remain as is for each storage type to avoid any unintentional regressions.
This commit is contained in:
David Steele
GitHub
parent
37b4592e52
commit
b05d31f531
@ -2245,3 +2245,17 @@ option:
|
||||
deprecate:
|
||||
repo?-azure-port: {}
|
||||
repo?-s3-port: {}
|
||||
|
||||
repo-storage-upload-chunk-size:
|
||||
section: global
|
||||
group: repo
|
||||
type: size
|
||||
required: false
|
||||
allow-range: [64KiB, 1TiB]
|
||||
command: repo-type
|
||||
depend:
|
||||
option: repo-type
|
||||
list:
|
||||
- azure
|
||||
- gcs
|
||||
- s3
|
||||
|
||||
@ -972,6 +972,28 @@
|
||||
<example>9000</example>
|
||||
</config-key>
|
||||
|
||||
<config-key id="repo-storage-upload-chunk-size" name="Repository Storage Upload Chunk Size">
|
||||
<summary>Repository storage upload chunk size.</summary>
|
||||
|
||||
<text>
|
||||
<p>Object stores such as S3 allow files to be uploaded in chunks when the file is too large to be stored in memory. Even if the file can be stored in memory, it is more memory efficient to limit the amount of memory used for uploads.</p>
|
||||
|
||||
<p>A larger chunk size will generally lead to better performance because it will minimize upload requests and allow more files to be uploaded in a single request rather than in chunks. The disadvantage is that memory usage will be higher and because the chunk buffer must be allocated per process, larger <br-option>process-max</br-option> values will lead to more memory being consumed overall.</p>
|
||||
|
||||
<p>Default chunk sizes by repo type:</p>
|
||||
|
||||
<list>
|
||||
<list-item><id>azure</id> - 4MiB</list-item>
|
||||
<list-item><id>gcs</id> - 4MiB</list-item>
|
||||
<list-item><id>s3</id> - 5MiB</list-item>
|
||||
</list>
|
||||
|
||||
<p>Note that valid chunk sizes vary by storage type and by platform. For example, <proper>AWS S3</proper> has a minimum chunk size of 5MiB but <proper>S3</proper> clones may accept lower values. Terminology for chunk size varies by storage type, so when searching min/max values use <quote>part size</quote> for <proper>AWS S3</proper>, <quote>chunk size</quote> for <proper>GCS</proper>, and <quote>block size</quote> for <proper>Azure</proper>. No attempt is made to validate configured chunk sizes so selecting an invalid value will lead to errors from the storage service or undefined behavior.</p>
|
||||
</text>
|
||||
|
||||
<example>16MiB</example>
|
||||
</config-key>
|
||||
|
||||
<config-key id="repo-storage-verify-tls" name="Repository Storage Certificate Verify">
|
||||
<summary>Repository storage certificate verify.</summary>
|
||||
|
||||
|
||||
@ -128,7 +128,7 @@ Option constants
|
||||
#define CFGOPT_TYPE "type"
|
||||
#define CFGOPT_VERBOSE "verbose"
|
||||
|
||||
#define CFG_OPTION_TOTAL 155
|
||||
#define CFG_OPTION_TOTAL 156
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Option value constants
|
||||
@ -483,6 +483,7 @@ typedef enum
|
||||
cfgOptRepoStorageCaPath,
|
||||
cfgOptRepoStorageHost,
|
||||
cfgOptRepoStoragePort,
|
||||
cfgOptRepoStorageUploadChunkSize,
|
||||
cfgOptRepoStorageVerifyTls,
|
||||
cfgOptRepoType,
|
||||
cfgOptResume,
|
||||
|
||||
@ -288,6 +288,20 @@ cfgLoadUpdateOption(void)
|
||||
}
|
||||
}
|
||||
|
||||
// Set default upload chunk size if not set
|
||||
if (cfgOptionValid(cfgOptRepoStorageUploadChunkSize))
|
||||
{
|
||||
for (unsigned int repoIdx = 0; repoIdx < cfgOptionGroupIdxTotal(cfgOptGrpRepo); repoIdx++)
|
||||
{
|
||||
if (!cfgOptionIdxTest(cfgOptRepoStorageUploadChunkSize, repoIdx))
|
||||
{
|
||||
cfgOptionIdxSet(
|
||||
cfgOptRepoStorageUploadChunkSize, repoIdx, cfgSourceDefault,
|
||||
VARINT64((cfgOptionIdxStrId(cfgOptRepoType, repoIdx) == CFGOPTVAL_REPO_TYPE_S3 ? 5 : 4) * 1024 * 1024));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Set pg-host-port/repo-host-port default when pg-host-type/repo-host-type is tls. ??? This should be handled in the parser but
|
||||
// it requires a default that depends on another option value and that is not currently possible.
|
||||
#define HOST_PORT_TLS 8432
|
||||
|
||||
@ -7546,6 +7546,90 @@ static const ParseRuleOption parseRuleOption[CFG_OPTION_TOTAL] =
|
||||
), // opt/repo-storage-port
|
||||
), // opt/repo-storage-port
|
||||
// -----------------------------------------------------------------------------------------------------------------------------
|
||||
PARSE_RULE_OPTION // opt/repo-storage-upload-chunk-size
|
||||
( // opt/repo-storage-upload-chunk-size
|
||||
PARSE_RULE_OPTION_NAME("repo-storage-upload-chunk-size"), // opt/repo-storage-upload-chunk-size
|
||||
PARSE_RULE_OPTION_TYPE(cfgOptTypeSize), // opt/repo-storage-upload-chunk-size
|
||||
PARSE_RULE_OPTION_RESET(true), // opt/repo-storage-upload-chunk-size
|
||||
PARSE_RULE_OPTION_REQUIRED(false), // opt/repo-storage-upload-chunk-size
|
||||
PARSE_RULE_OPTION_SECTION(cfgSectionGlobal), // opt/repo-storage-upload-chunk-size
|
||||
PARSE_RULE_OPTION_GROUP_MEMBER(true), // opt/repo-storage-upload-chunk-size
|
||||
PARSE_RULE_OPTION_GROUP_ID(cfgOptGrpRepo), // opt/repo-storage-upload-chunk-size
|
||||
// opt/repo-storage-upload-chunk-size
|
||||
PARSE_RULE_OPTION_COMMAND_ROLE_MAIN_VALID_LIST // opt/repo-storage-upload-chunk-size
|
||||
( // opt/repo-storage-upload-chunk-size
|
||||
PARSE_RULE_OPTION_COMMAND(cfgCmdArchiveGet) // opt/repo-storage-upload-chunk-size
|
||||
PARSE_RULE_OPTION_COMMAND(cfgCmdArchivePush) // opt/repo-storage-upload-chunk-size
|
||||
PARSE_RULE_OPTION_COMMAND(cfgCmdBackup) // opt/repo-storage-upload-chunk-size
|
||||
PARSE_RULE_OPTION_COMMAND(cfgCmdCheck) // opt/repo-storage-upload-chunk-size
|
||||
PARSE_RULE_OPTION_COMMAND(cfgCmdExpire) // opt/repo-storage-upload-chunk-size
|
||||
PARSE_RULE_OPTION_COMMAND(cfgCmdInfo) // opt/repo-storage-upload-chunk-size
|
||||
PARSE_RULE_OPTION_COMMAND(cfgCmdRepoCreate) // opt/repo-storage-upload-chunk-size
|
||||
PARSE_RULE_OPTION_COMMAND(cfgCmdRepoGet) // opt/repo-storage-upload-chunk-size
|
||||
PARSE_RULE_OPTION_COMMAND(cfgCmdRepoLs) // opt/repo-storage-upload-chunk-size
|
||||
PARSE_RULE_OPTION_COMMAND(cfgCmdRepoPut) // opt/repo-storage-upload-chunk-size
|
||||
PARSE_RULE_OPTION_COMMAND(cfgCmdRepoRm) // opt/repo-storage-upload-chunk-size
|
||||
PARSE_RULE_OPTION_COMMAND(cfgCmdRestore) // opt/repo-storage-upload-chunk-size
|
||||
PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaCreate) // opt/repo-storage-upload-chunk-size
|
||||
PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaDelete) // opt/repo-storage-upload-chunk-size
|
||||
PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaUpgrade) // opt/repo-storage-upload-chunk-size
|
||||
PARSE_RULE_OPTION_COMMAND(cfgCmdVerify) // opt/repo-storage-upload-chunk-size
|
||||
), // opt/repo-storage-upload-chunk-size
|
||||
// opt/repo-storage-upload-chunk-size
|
||||
PARSE_RULE_OPTION_COMMAND_ROLE_ASYNC_VALID_LIST // opt/repo-storage-upload-chunk-size
|
||||
( // opt/repo-storage-upload-chunk-size
|
||||
PARSE_RULE_OPTION_COMMAND(cfgCmdArchiveGet) // opt/repo-storage-upload-chunk-size
|
||||
PARSE_RULE_OPTION_COMMAND(cfgCmdArchivePush) // opt/repo-storage-upload-chunk-size
|
||||
), // opt/repo-storage-upload-chunk-size
|
||||
// opt/repo-storage-upload-chunk-size
|
||||
PARSE_RULE_OPTION_COMMAND_ROLE_LOCAL_VALID_LIST // opt/repo-storage-upload-chunk-size
|
||||
( // opt/repo-storage-upload-chunk-size
|
||||
PARSE_RULE_OPTION_COMMAND(cfgCmdArchiveGet) // opt/repo-storage-upload-chunk-size
|
||||
PARSE_RULE_OPTION_COMMAND(cfgCmdArchivePush) // opt/repo-storage-upload-chunk-size
|
||||
PARSE_RULE_OPTION_COMMAND(cfgCmdBackup) // opt/repo-storage-upload-chunk-size
|
||||
PARSE_RULE_OPTION_COMMAND(cfgCmdRestore) // opt/repo-storage-upload-chunk-size
|
||||
PARSE_RULE_OPTION_COMMAND(cfgCmdVerify) // opt/repo-storage-upload-chunk-size
|
||||
), // opt/repo-storage-upload-chunk-size
|
||||
// opt/repo-storage-upload-chunk-size
|
||||
PARSE_RULE_OPTION_COMMAND_ROLE_REMOTE_VALID_LIST // opt/repo-storage-upload-chunk-size
|
||||
( // opt/repo-storage-upload-chunk-size
|
||||
PARSE_RULE_OPTION_COMMAND(cfgCmdArchiveGet) // opt/repo-storage-upload-chunk-size
|
||||
PARSE_RULE_OPTION_COMMAND(cfgCmdArchivePush) // opt/repo-storage-upload-chunk-size
|
||||
PARSE_RULE_OPTION_COMMAND(cfgCmdCheck) // opt/repo-storage-upload-chunk-size
|
||||
PARSE_RULE_OPTION_COMMAND(cfgCmdInfo) // opt/repo-storage-upload-chunk-size
|
||||
PARSE_RULE_OPTION_COMMAND(cfgCmdRepoCreate) // opt/repo-storage-upload-chunk-size
|
||||
PARSE_RULE_OPTION_COMMAND(cfgCmdRepoGet) // opt/repo-storage-upload-chunk-size
|
||||
PARSE_RULE_OPTION_COMMAND(cfgCmdRepoLs) // opt/repo-storage-upload-chunk-size
|
||||
PARSE_RULE_OPTION_COMMAND(cfgCmdRepoPut) // opt/repo-storage-upload-chunk-size
|
||||
PARSE_RULE_OPTION_COMMAND(cfgCmdRepoRm) // opt/repo-storage-upload-chunk-size
|
||||
PARSE_RULE_OPTION_COMMAND(cfgCmdRestore) // opt/repo-storage-upload-chunk-size
|
||||
PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaCreate) // opt/repo-storage-upload-chunk-size
|
||||
PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaDelete) // opt/repo-storage-upload-chunk-size
|
||||
PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaUpgrade) // opt/repo-storage-upload-chunk-size
|
||||
PARSE_RULE_OPTION_COMMAND(cfgCmdVerify) // opt/repo-storage-upload-chunk-size
|
||||
), // opt/repo-storage-upload-chunk-size
|
||||
// opt/repo-storage-upload-chunk-size
|
||||
PARSE_RULE_OPTIONAL // opt/repo-storage-upload-chunk-size
|
||||
( // opt/repo-storage-upload-chunk-size
|
||||
PARSE_RULE_OPTIONAL_GROUP // opt/repo-storage-upload-chunk-size
|
||||
( // opt/repo-storage-upload-chunk-size
|
||||
PARSE_RULE_OPTIONAL_DEPEND // opt/repo-storage-upload-chunk-size
|
||||
( // opt/repo-storage-upload-chunk-size
|
||||
PARSE_RULE_VAL_OPT(cfgOptRepoType), // opt/repo-storage-upload-chunk-size
|
||||
PARSE_RULE_VAL_STRID(parseRuleValStrIdAzure), // opt/repo-storage-upload-chunk-size
|
||||
PARSE_RULE_VAL_STRID(parseRuleValStrIdGcs), // opt/repo-storage-upload-chunk-size
|
||||
PARSE_RULE_VAL_STRID(parseRuleValStrIdS3), // opt/repo-storage-upload-chunk-size
|
||||
), // opt/repo-storage-upload-chunk-size
|
||||
// opt/repo-storage-upload-chunk-size
|
||||
PARSE_RULE_OPTIONAL_ALLOW_RANGE // opt/repo-storage-upload-chunk-size
|
||||
( // opt/repo-storage-upload-chunk-size
|
||||
PARSE_RULE_VAL_INT(parseRuleValInt65536), // opt/repo-storage-upload-chunk-size
|
||||
PARSE_RULE_VAL_INT(parseRuleValInt1099511627776), // opt/repo-storage-upload-chunk-size
|
||||
), // opt/repo-storage-upload-chunk-size
|
||||
), // opt/repo-storage-upload-chunk-size
|
||||
), // opt/repo-storage-upload-chunk-size
|
||||
), // opt/repo-storage-upload-chunk-size
|
||||
// -----------------------------------------------------------------------------------------------------------------------------
|
||||
PARSE_RULE_OPTION // opt/repo-storage-verify-tls
|
||||
( // opt/repo-storage-verify-tls
|
||||
PARSE_RULE_OPTION_NAME("repo-storage-verify-tls"), // opt/repo-storage-verify-tls
|
||||
@ -9201,6 +9285,7 @@ static const uint8_t optionResolveOrder[] =
|
||||
cfgOptRepoStorageCaPath, // opt-resolve-order
|
||||
cfgOptRepoStorageHost, // opt-resolve-order
|
||||
cfgOptRepoStoragePort, // opt-resolve-order
|
||||
cfgOptRepoStorageUploadChunkSize, // opt-resolve-order
|
||||
cfgOptRepoStorageVerifyTls, // opt-resolve-order
|
||||
cfgOptTarget, // opt-resolve-order
|
||||
cfgOptTargetAction, // opt-resolve-order
|
||||
|
||||
@ -79,8 +79,8 @@ storageAzureHelper(const unsigned int repoIdx, const bool write, StoragePathExpr
|
||||
result = storageAzureNew(
|
||||
cfgOptionIdxStr(cfgOptRepoPath, repoIdx), write, pathExpressionCallback,
|
||||
cfgOptionIdxStr(cfgOptRepoAzureContainer, repoIdx), cfgOptionIdxStr(cfgOptRepoAzureAccount, repoIdx), keyType, key,
|
||||
STORAGE_AZURE_BLOCKSIZE_MIN, endpoint, uriStyle, port, ioTimeoutMs(), cfgOptionIdxBool(cfgOptRepoStorageVerifyTls,
|
||||
repoIdx), cfgOptionIdxStrNull(cfgOptRepoStorageCaFile, repoIdx),
|
||||
(size_t)cfgOptionIdxUInt64(cfgOptRepoStorageUploadChunkSize, repoIdx), endpoint, uriStyle, port, ioTimeoutMs(),
|
||||
cfgOptionIdxBool(cfgOptRepoStorageVerifyTls, repoIdx), cfgOptionIdxStrNull(cfgOptRepoStorageCaFile, repoIdx),
|
||||
cfgOptionIdxStrNull(cfgOptRepoStorageCaPath, repoIdx));
|
||||
}
|
||||
MEM_CONTEXT_PRIOR_END();
|
||||
|
||||
@ -29,11 +29,6 @@ typedef enum
|
||||
storageAzureUriStylePath = STRID5("path", 0x450300),
|
||||
} StorageAzureUriStyle;
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Defaults
|
||||
***********************************************************************************************************************************/
|
||||
#define STORAGE_AZURE_BLOCKSIZE_MIN ((size_t)4 * 1024 * 1024)
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Constructors
|
||||
***********************************************************************************************************************************/
|
||||
|
||||
@ -24,8 +24,8 @@ storageGcsHelper(const unsigned int repoIdx, const bool write, StoragePathExpres
|
||||
Storage *const result = storageGcsNew(
|
||||
cfgOptionIdxStr(cfgOptRepoPath, repoIdx), write, pathExpressionCallback, cfgOptionIdxStr(cfgOptRepoGcsBucket, repoIdx),
|
||||
(StorageGcsKeyType)cfgOptionIdxStrId(cfgOptRepoGcsKeyType, repoIdx), cfgOptionIdxStrNull(cfgOptRepoGcsKey, repoIdx),
|
||||
STORAGE_GCS_CHUNKSIZE_DEFAULT, cfgOptionIdxStr(cfgOptRepoGcsEndpoint, repoIdx), ioTimeoutMs(),
|
||||
cfgOptionIdxBool(cfgOptRepoStorageVerifyTls, repoIdx), cfgOptionIdxStrNull(cfgOptRepoStorageCaFile, repoIdx),
|
||||
(size_t)cfgOptionIdxUInt64(cfgOptRepoStorageUploadChunkSize, repoIdx), cfgOptionIdxStr(cfgOptRepoGcsEndpoint, repoIdx),
|
||||
ioTimeoutMs(), cfgOptionIdxBool(cfgOptRepoStorageVerifyTls, repoIdx), cfgOptionIdxStrNull(cfgOptRepoStorageCaFile, repoIdx),
|
||||
cfgOptionIdxStrNull(cfgOptRepoStorageCaPath, repoIdx));
|
||||
|
||||
FUNCTION_LOG_RETURN(STORAGE, result);
|
||||
|
||||
@ -21,11 +21,6 @@ typedef enum
|
||||
storageGcsKeyTypeToken = STRID5("token", 0xe2adf40),
|
||||
} StorageGcsKeyType;
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Defaults
|
||||
***********************************************************************************************************************************/
|
||||
#define STORAGE_GCS_CHUNKSIZE_DEFAULT ((size_t)4 * 1024 * 1024)
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Constructors
|
||||
***********************************************************************************************************************************/
|
||||
|
||||
@ -85,9 +85,10 @@ storageS3Helper(const unsigned int repoIdx, const bool write, StoragePathExpress
|
||||
endPoint, (StorageS3UriStyle)cfgOptionIdxStrId(cfgOptRepoS3UriStyle, repoIdx),
|
||||
cfgOptionIdxStr(cfgOptRepoS3Region, repoIdx), keyType, cfgOptionIdxStrNull(cfgOptRepoS3Key, repoIdx),
|
||||
cfgOptionIdxStrNull(cfgOptRepoS3KeySecret, repoIdx), cfgOptionIdxStrNull(cfgOptRepoS3Token, repoIdx),
|
||||
cfgOptionIdxStrNull(cfgOptRepoS3KmsKeyId, repoIdx), role, webIdToken, STORAGE_S3_PARTSIZE_MIN, host, port,
|
||||
ioTimeoutMs(), cfgOptionIdxBool(cfgOptRepoStorageVerifyTls, repoIdx),
|
||||
cfgOptionIdxStrNull(cfgOptRepoStorageCaFile, repoIdx), cfgOptionIdxStrNull(cfgOptRepoStorageCaPath, repoIdx));
|
||||
cfgOptionIdxStrNull(cfgOptRepoS3KmsKeyId, repoIdx), role, webIdToken,
|
||||
(size_t)cfgOptionIdxUInt64(cfgOptRepoStorageUploadChunkSize, repoIdx), host, port, ioTimeoutMs(),
|
||||
cfgOptionIdxBool(cfgOptRepoStorageVerifyTls, repoIdx), cfgOptionIdxStrNull(cfgOptRepoStorageCaFile, repoIdx),
|
||||
cfgOptionIdxStrNull(cfgOptRepoStorageCaPath, repoIdx));
|
||||
}
|
||||
MEM_CONTEXT_PRIOR_END();
|
||||
}
|
||||
|
||||
@ -30,11 +30,6 @@ typedef enum
|
||||
storageS3UriStylePath = STRID5("path", 0x450300),
|
||||
} StorageS3UriStyle;
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Defaults
|
||||
***********************************************************************************************************************************/
|
||||
#define STORAGE_S3_PARTSIZE_MIN ((size_t)5 * 1024 * 1024)
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Constructors
|
||||
***********************************************************************************************************************************/
|
||||
|
||||
@ -194,129 +194,132 @@ testRun(void)
|
||||
"\n"
|
||||
"Command Options:\n"
|
||||
"\n"
|
||||
" --archive-mode preserve or disable archiving on restored\n"
|
||||
" cluster [default=preserve]\n"
|
||||
" --db-exclude restore excluding the specified databases\n"
|
||||
" --db-include restore only specified databases\n"
|
||||
" [current=db1, db2]\n"
|
||||
" --force force a restore [default=n]\n"
|
||||
" --link-all restore all symlinks [default=n]\n"
|
||||
" --link-map modify the destination of a symlink\n"
|
||||
" [current=/link1=/dest1, /link2=/dest2]\n"
|
||||
" --recovery-option set an option in recovery.conf\n"
|
||||
" --set backup set to restore [default=latest]\n"
|
||||
" --tablespace-map restore a tablespace into the specified\n"
|
||||
" directory\n"
|
||||
" --tablespace-map-all restore all tablespaces into the specified\n"
|
||||
" directory\n"
|
||||
" --target recovery target\n"
|
||||
" --target-action action to take when recovery target is\n"
|
||||
" reached [default=pause]\n"
|
||||
" --target-exclusive stop just before the recovery target is\n"
|
||||
" reached [default=n]\n"
|
||||
" --target-timeline recover along a timeline\n"
|
||||
" --type recovery type [default=default]\n"
|
||||
" --archive-mode preserve or disable archiving on restored\n"
|
||||
" cluster [default=preserve]\n"
|
||||
" --db-exclude restore excluding the specified databases\n"
|
||||
" --db-include restore only specified databases\n"
|
||||
" [current=db1, db2]\n"
|
||||
" --force force a restore [default=n]\n"
|
||||
" --link-all restore all symlinks [default=n]\n"
|
||||
" --link-map modify the destination of a symlink\n"
|
||||
" [current=/link1=/dest1, /link2=/dest2]\n"
|
||||
" --recovery-option set an option in recovery.conf\n"
|
||||
" --set backup set to restore [default=latest]\n"
|
||||
" --tablespace-map restore a tablespace into the specified\n"
|
||||
" directory\n"
|
||||
" --tablespace-map-all restore all tablespaces into the specified\n"
|
||||
" directory\n"
|
||||
" --target recovery target\n"
|
||||
" --target-action action to take when recovery target is\n"
|
||||
" reached [default=pause]\n"
|
||||
" --target-exclusive stop just before the recovery target is\n"
|
||||
" reached [default=n]\n"
|
||||
" --target-timeline recover along a timeline\n"
|
||||
" --type recovery type [default=default]\n"
|
||||
"\n"
|
||||
"General Options:\n"
|
||||
"\n"
|
||||
" --buffer-size buffer size for I/O operations\n"
|
||||
" [current=32768, default=1MiB]\n"
|
||||
" --cmd pgBackRest command\n"
|
||||
" [default=/path/to/pgbackrest]\n"
|
||||
" --cmd-ssh SSH client command [default=ssh]\n"
|
||||
" --compress-level-network network compression level [default=3]\n"
|
||||
" --config pgBackRest configuration file\n"
|
||||
" [default=/etc/pgbackrest/pgbackrest.conf]\n"
|
||||
" --config-include-path path to additional pgBackRest configuration\n"
|
||||
" files [default=/etc/pgbackrest/conf.d]\n"
|
||||
" --config-path base path of pgBackRest configuration files\n"
|
||||
" [default=/etc/pgbackrest]\n"
|
||||
" --delta restore or backup using checksums [default=n]\n"
|
||||
" --io-timeout I/O timeout [default=60]\n"
|
||||
" --lock-path path where lock files are stored\n"
|
||||
" [default=/tmp/pgbackrest]\n"
|
||||
" --neutral-umask use a neutral umask [default=y]\n"
|
||||
" --process-max max processes to use for compress/transfer\n"
|
||||
" [default=1]\n"
|
||||
" --protocol-timeout protocol timeout [default=1830]\n"
|
||||
" --sck-keep-alive keep-alive enable [default=y]\n"
|
||||
" --stanza defines the stanza\n"
|
||||
" --tcp-keep-alive-count keep-alive count\n"
|
||||
" --tcp-keep-alive-idle keep-alive idle time\n"
|
||||
" --tcp-keep-alive-interval keep-alive interval time\n"
|
||||
" --buffer-size buffer size for I/O operations\n"
|
||||
" [current=32768, default=1MiB]\n"
|
||||
" --cmd pgBackRest command\n"
|
||||
" [default=/path/to/pgbackrest]\n"
|
||||
" --cmd-ssh SSH client command [default=ssh]\n"
|
||||
" --compress-level-network network compression level [default=3]\n"
|
||||
" --config pgBackRest configuration file\n"
|
||||
" [default=/etc/pgbackrest/pgbackrest.conf]\n"
|
||||
" --config-include-path path to additional pgBackRest configuration\n"
|
||||
" files [default=/etc/pgbackrest/conf.d]\n"
|
||||
" --config-path base path of pgBackRest configuration files\n"
|
||||
" [default=/etc/pgbackrest]\n"
|
||||
" --delta restore or backup using checksums\n"
|
||||
" [default=n]\n"
|
||||
" --io-timeout I/O timeout [default=60]\n"
|
||||
" --lock-path path where lock files are stored\n"
|
||||
" [default=/tmp/pgbackrest]\n"
|
||||
" --neutral-umask use a neutral umask [default=y]\n"
|
||||
" --process-max max processes to use for compress/transfer\n"
|
||||
" [default=1]\n"
|
||||
" --protocol-timeout protocol timeout [default=1830]\n"
|
||||
" --sck-keep-alive keep-alive enable [default=y]\n"
|
||||
" --stanza defines the stanza\n"
|
||||
" --tcp-keep-alive-count keep-alive count\n"
|
||||
" --tcp-keep-alive-idle keep-alive idle time\n"
|
||||
" --tcp-keep-alive-interval keep-alive interval time\n"
|
||||
"\n"
|
||||
"Log Options:\n"
|
||||
"\n"
|
||||
" --log-level-console level for console logging [default=warn]\n"
|
||||
" --log-level-file level for file logging [default=info]\n"
|
||||
" --log-level-stderr level for stderr logging [default=warn]\n"
|
||||
" --log-path path where log files are stored\n"
|
||||
" [default=/var/log/pgbackrest]\n"
|
||||
" --log-subprocess enable logging in subprocesses [default=n]\n"
|
||||
" --log-timestamp enable timestamp in logging [default=y]\n"
|
||||
" --log-level-console level for console logging [default=warn]\n"
|
||||
" --log-level-file level for file logging [default=info]\n"
|
||||
" --log-level-stderr level for stderr logging [default=warn]\n"
|
||||
" --log-path path where log files are stored\n"
|
||||
" [default=/var/log/pgbackrest]\n"
|
||||
" --log-subprocess enable logging in subprocesses [default=n]\n"
|
||||
" --log-timestamp enable timestamp in logging [default=y]\n"
|
||||
"\n",
|
||||
"Repository Options:\n"
|
||||
"\n"
|
||||
" --repo set repository\n"
|
||||
" --repo-azure-account azure repository account\n"
|
||||
" --repo-azure-container azure repository container\n"
|
||||
" --repo-azure-endpoint azure repository endpoint\n"
|
||||
" [default=blob.core.windows.net]\n"
|
||||
" --repo-azure-key azure repository key\n"
|
||||
" --repo-azure-key-type azure repository key type [default=shared]\n"
|
||||
" --repo-azure-uri-style azure URI Style [default=host]\n"
|
||||
" --repo-cipher-pass repository cipher passphrase\n"
|
||||
" [current=<redacted>]\n"
|
||||
" --repo-cipher-type cipher used to encrypt the repository\n"
|
||||
" [current=aes-256-cbc, default=none]\n"
|
||||
" --repo-gcs-bucket GCS repository bucket\n"
|
||||
" --repo-gcs-endpoint GCS repository endpoint\n"
|
||||
" [default=storage.googleapis.com]\n"
|
||||
" --repo-gcs-key GCS repository key\n"
|
||||
" --repo-gcs-key-type GCS repository key type [default=service]\n"
|
||||
" --repo-host repository host when operating remotely via\n"
|
||||
" SSH [current=backup.example.net]\n"
|
||||
" --repo-host-ca-file repository host certificate authority file\n"
|
||||
" --repo-host-ca-path repository host certificate authority path\n"
|
||||
" --repo-host-cert-file repository host certificate file\n"
|
||||
" --repo-host-cmd repository host pgBackRest command\n"
|
||||
" [default=/path/to/pgbackrest]\n"
|
||||
" --repo-host-config pgBackRest repository host configuration\n"
|
||||
" file\n"
|
||||
" [default=/etc/pgbackrest/pgbackrest.conf]\n"
|
||||
" --repo-host-config-include-path pgBackRest repository host configuration\n"
|
||||
" include path [default=/etc/pgbackrest/conf.d]\n"
|
||||
" --repo-host-config-path pgBackRest repository host configuration\n"
|
||||
" path [default=/etc/pgbackrest]\n"
|
||||
" --repo-host-key-file repository host key file\n"
|
||||
" --repo-host-port repository host port when repo-host is set\n"
|
||||
" --repo-host-type repository host protocol type [default=ssh]\n"
|
||||
" --repo-host-user repository host user when repo-host is set\n"
|
||||
" [default=pgbackrest]\n"
|
||||
" --repo-path path where backups and archive are stored\n"
|
||||
" [default=/var/lib/pgbackrest]\n"
|
||||
" --repo-s3-bucket S3 repository bucket\n"
|
||||
" --repo-s3-endpoint S3 repository endpoint\n"
|
||||
" --repo-s3-key S3 repository access key\n"
|
||||
" --repo-s3-key-secret S3 repository secret access key\n"
|
||||
" --repo-s3-key-type S3 repository key type [default=shared]\n"
|
||||
" --repo-s3-kms-key-id S3 repository KMS key\n"
|
||||
" --repo-s3-region S3 repository region\n"
|
||||
" --repo-s3-role S3 repository role\n"
|
||||
" --repo-s3-token S3 repository security token\n"
|
||||
" --repo-s3-uri-style S3 URI Style [default=host]\n"
|
||||
" --repo-storage-ca-file repository storage CA file\n"
|
||||
" --repo-storage-ca-path repository storage CA path\n"
|
||||
" --repo-storage-host repository storage host\n"
|
||||
" --repo-storage-port repository storage port [default=443]\n"
|
||||
" --repo-storage-verify-tls repository storage certificate verify\n"
|
||||
" [default=y]\n"
|
||||
" --repo-type type of storage used for the repository\n"
|
||||
" [default=posix]\n"
|
||||
" --repo set repository\n"
|
||||
" --repo-azure-account azure repository account\n"
|
||||
" --repo-azure-container azure repository container\n"
|
||||
" --repo-azure-endpoint azure repository endpoint\n"
|
||||
" [default=blob.core.windows.net]\n"
|
||||
" --repo-azure-key azure repository key\n"
|
||||
" --repo-azure-key-type azure repository key type [default=shared]\n"
|
||||
" --repo-azure-uri-style azure URI Style [default=host]\n"
|
||||
" --repo-cipher-pass repository cipher passphrase\n"
|
||||
" [current=<redacted>]\n"
|
||||
" --repo-cipher-type cipher used to encrypt the repository\n"
|
||||
" [current=aes-256-cbc, default=none]\n"
|
||||
" --repo-gcs-bucket GCS repository bucket\n"
|
||||
" --repo-gcs-endpoint GCS repository endpoint\n"
|
||||
" [default=storage.googleapis.com]\n"
|
||||
" --repo-gcs-key GCS repository key\n"
|
||||
" --repo-gcs-key-type GCS repository key type [default=service]\n"
|
||||
" --repo-host repository host when operating remotely via\n"
|
||||
" SSH [current=backup.example.net]\n"
|
||||
" --repo-host-ca-file repository host certificate authority file\n"
|
||||
" --repo-host-ca-path repository host certificate authority path\n"
|
||||
" --repo-host-cert-file repository host certificate file\n"
|
||||
" --repo-host-cmd repository host pgBackRest command\n"
|
||||
" [default=/path/to/pgbackrest]\n"
|
||||
" --repo-host-config pgBackRest repository host configuration\n"
|
||||
" file\n"
|
||||
" [default=/etc/pgbackrest/pgbackrest.conf]\n"
|
||||
" --repo-host-config-include-path pgBackRest repository host configuration\n"
|
||||
" include path\n"
|
||||
" [default=/etc/pgbackrest/conf.d]\n"
|
||||
" --repo-host-config-path pgBackRest repository host configuration\n"
|
||||
" path [default=/etc/pgbackrest]\n"
|
||||
" --repo-host-key-file repository host key file\n"
|
||||
" --repo-host-port repository host port when repo-host is set\n"
|
||||
" --repo-host-type repository host protocol type [default=ssh]\n"
|
||||
" --repo-host-user repository host user when repo-host is set\n"
|
||||
" [default=pgbackrest]\n"
|
||||
" --repo-path path where backups and archive are stored\n"
|
||||
" [default=/var/lib/pgbackrest]\n"
|
||||
" --repo-s3-bucket S3 repository bucket\n"
|
||||
" --repo-s3-endpoint S3 repository endpoint\n"
|
||||
" --repo-s3-key S3 repository access key\n"
|
||||
" --repo-s3-key-secret S3 repository secret access key\n"
|
||||
" --repo-s3-key-type S3 repository key type [default=shared]\n"
|
||||
" --repo-s3-kms-key-id S3 repository KMS key\n"
|
||||
" --repo-s3-region S3 repository region\n"
|
||||
" --repo-s3-role S3 repository role\n"
|
||||
" --repo-s3-token S3 repository security token\n"
|
||||
" --repo-s3-uri-style S3 URI Style [default=host]\n"
|
||||
" --repo-storage-ca-file repository storage CA file\n"
|
||||
" --repo-storage-ca-path repository storage CA path\n"
|
||||
" --repo-storage-host repository storage host\n"
|
||||
" --repo-storage-port repository storage port [default=443]\n"
|
||||
" --repo-storage-upload-chunk-size repository storage upload chunk size\n"
|
||||
" --repo-storage-verify-tls repository storage certificate verify\n"
|
||||
" [default=y]\n"
|
||||
" --repo-type type of storage used for the repository\n"
|
||||
" [default=posix]\n"
|
||||
"\n"
|
||||
"Stanza Options:\n"
|
||||
"\n"
|
||||
" --pg-path postgreSQL data directory\n"
|
||||
" --pg-path postgreSQL data directory\n"
|
||||
"\n"
|
||||
"Use 'pgbackrest help restore [option]' for more information.\n");
|
||||
|
||||
|
||||
@ -532,6 +532,61 @@ testRun(void)
|
||||
TEST_RESULT_LOG(
|
||||
"P00 WARN: 'compress' and 'compress-type' options should not both be set\n"
|
||||
" HINT: 'compress-type' is preferred and 'compress' is deprecated.");
|
||||
|
||||
// -------------------------------------------------------------------------------------------------------------------------
|
||||
TEST_TITLE("S3 default chunk size");
|
||||
|
||||
argList = strLstNew();
|
||||
hrnCfgArgRawZ(argList, cfgOptStanza, "db");
|
||||
hrnCfgArgRawZ(argList, cfgOptPgPath, "/path/to/pg");
|
||||
hrnCfgArgKeyRawZ(argList, cfgOptRepoType, 1, "s3");
|
||||
hrnCfgArgKeyRawZ(argList, cfgOptRepoS3Bucket, 1, "bucket");
|
||||
hrnCfgArgKeyRawZ(argList, cfgOptRepoS3Region, 1, "region");
|
||||
hrnCfgArgKeyRawZ(argList, cfgOptRepoS3Endpoint, 1, "endpoint");
|
||||
hrnCfgArgKeyRawZ(argList, cfgOptRepoS3KeyType, 1, "auto");
|
||||
hrnCfgArgKeyRawZ(argList, cfgOptRepoPath, 1, "/repo");
|
||||
HRN_CFG_LOAD(cfgCmdArchiveGet, argList);
|
||||
|
||||
TEST_RESULT_UINT(cfgOptionUInt64(cfgOptRepoStorageUploadChunkSize), 5 * 1024 * 1024, "default chunk size");
|
||||
TEST_RESULT_UINT(cfgOptionSource(cfgOptRepoStorageUploadChunkSize), cfgSourceDefault, "chunk size source is default");
|
||||
|
||||
// -------------------------------------------------------------------------------------------------------------------------
|
||||
TEST_TITLE("S3 custom chunk size");
|
||||
|
||||
argList = strLstNew();
|
||||
hrnCfgArgRawZ(argList, cfgOptStanza, "db");
|
||||
hrnCfgArgRawZ(argList, cfgOptPgPath, "/path/to/pg");
|
||||
hrnCfgArgKeyRawZ(argList, cfgOptRepoType, 1, "s3");
|
||||
hrnCfgArgKeyRawZ(argList, cfgOptRepoS3Bucket, 1, "bucket");
|
||||
hrnCfgArgKeyRawZ(argList, cfgOptRepoS3Region, 1, "region");
|
||||
hrnCfgArgKeyRawZ(argList, cfgOptRepoS3Endpoint, 1, "endpoint");
|
||||
hrnCfgArgKeyRawZ(argList, cfgOptRepoS3KeyType, 1, "auto");
|
||||
hrnCfgArgKeyRawZ(argList, cfgOptRepoStorageUploadChunkSize, 1, "64KiB");
|
||||
hrnCfgArgKeyRawZ(argList, cfgOptRepoPath, 1, "/repo");
|
||||
HRN_CFG_LOAD(cfgCmdArchiveGet, argList);
|
||||
|
||||
TEST_RESULT_UINT(cfgOptionUInt64(cfgOptRepoStorageUploadChunkSize), 64 * 1024, "chunk size set");
|
||||
TEST_RESULT_UINT(cfgOptionSource(cfgOptRepoStorageUploadChunkSize), cfgSourceParam, "chunk size source is param");
|
||||
|
||||
// -------------------------------------------------------------------------------------------------------------------------
|
||||
TEST_TITLE("Azure default chunk size");
|
||||
|
||||
hrnCfgEnvKeyRawZ(cfgOptRepoAzureAccount, 1, "account");
|
||||
hrnCfgEnvKeyRawZ(cfgOptRepoAzureKey, 1, "mykey");
|
||||
|
||||
argList = strLstNew();
|
||||
hrnCfgArgRawZ(argList, cfgOptStanza, "db");
|
||||
hrnCfgArgRawZ(argList, cfgOptPgPath, "/path/to/pg");
|
||||
hrnCfgArgKeyRawZ(argList, cfgOptRepoType, 1, "azure");
|
||||
hrnCfgArgKeyRawZ(argList, cfgOptRepoAzureContainer, 1, "container");
|
||||
hrnCfgArgKeyRawZ(argList, cfgOptRepoPath, 1, "/repo");
|
||||
HRN_CFG_LOAD(cfgCmdArchiveGet, argList);
|
||||
|
||||
TEST_RESULT_UINT(cfgOptionUInt64(cfgOptRepoStorageUploadChunkSize), 4 * 1024 * 1024, "default chunk size");
|
||||
TEST_RESULT_UINT(cfgOptionSource(cfgOptRepoStorageUploadChunkSize), cfgSourceDefault, "chunk size source is default");
|
||||
|
||||
hrnCfgEnvKeyRemoveRaw(cfgOptRepoAzureAccount, 1);
|
||||
hrnCfgEnvKeyRemoveRaw(cfgOptRepoAzureKey, 1);
|
||||
}
|
||||
|
||||
// *****************************************************************************************************************************
|
||||
|
||||
@ -203,7 +203,7 @@ testRun(void)
|
||||
strNewEncode(encodeBase64, ((StorageAzure *)storageDriver(storage))->sharedKey), TEST_KEY_SHARED_STR, "check key");
|
||||
TEST_RESULT_STR_Z(((StorageAzure *)storageDriver(storage))->host, TEST_ACCOUNT ".blob.core.windows.net", "check host");
|
||||
TEST_RESULT_STR_Z(((StorageAzure *)storageDriver(storage))->pathPrefix, "/" TEST_CONTAINER, "check path prefix");
|
||||
TEST_RESULT_UINT(((StorageAzure *)storageDriver(storage))->blockSize, STORAGE_AZURE_BLOCKSIZE_MIN, "check block size");
|
||||
TEST_RESULT_UINT(((StorageAzure *)storageDriver(storage))->blockSize, 4 * 1024 * 1024, "check block size");
|
||||
TEST_RESULT_BOOL(storageFeature(storage, storageFeaturePath), false, "check path feature");
|
||||
|
||||
// -------------------------------------------------------------------------------------------------------------------------
|
||||
|
||||
@ -223,7 +223,7 @@ testRun(void)
|
||||
TEST_RESULT_STR_Z(storage->path, "/repo", "check path");
|
||||
TEST_RESULT_STR(((StorageGcs *)storageDriver(storage))->bucket, TEST_BUCKET_STR, "check bucket");
|
||||
TEST_RESULT_STR_Z(((StorageGcs *)storageDriver(storage))->endpoint, "storage.googleapis.com", "check endpoint");
|
||||
TEST_RESULT_UINT(((StorageGcs *)storageDriver(storage))->chunkSize, STORAGE_GCS_CHUNKSIZE_DEFAULT, "check chunk size");
|
||||
TEST_RESULT_UINT(((StorageGcs *)storageDriver(storage))->chunkSize, 4 * 1024 * 1024, "check chunk size");
|
||||
TEST_RESULT_STR(((StorageGcs *)storageDriver(storage))->token, TEST_TOKEN_STR, "check token");
|
||||
TEST_RESULT_BOOL(storageFeature(storage, storageFeaturePath), false, "check path feature");
|
||||
}
|
||||
|
||||
@ -419,6 +419,7 @@ testRun(void)
|
||||
|
||||
TEST_RESULT_STR(s3->path, path, "check path");
|
||||
TEST_RESULT_BOOL(storageFeature(s3, storageFeaturePath), false, "check path feature");
|
||||
TEST_RESULT_UINT(driver->partSize, 5 * 1024 * 1024, "check part size");
|
||||
|
||||
// -----------------------------------------------------------------------------------------------------------------
|
||||
TEST_TITLE("coverage for noop functions");
|
||||
|
||||
Loading…
Reference in New Issue
Block a user