From b05d31f531b55b747175cf94c963f95574c54d2b Mon Sep 17 00:00:00 2001 From: David Steele Date: Wed, 24 Aug 2022 09:33:26 -0400 Subject: [PATCH] Allow upload chunk size to be configured for object stores. There are performance benefits to increasing the upload chunk size as long as the tradeoff with additional memory consumption is acceptable. Make the chunk size configurable for S3, GCS, and Azure, but don't attempt to do any validation of the chunk size beyond some sane limits. The defaults remain as is for each storage type to avoid any unintentional regressions. --- src/build/config/config.yaml | 14 ++ src/build/help/help.xml | 22 +++ src/config/config.auto.h | 3 +- src/config/load.c | 14 ++ src/config/parse.auto.c.inc | 85 +++++++++++ src/storage/azure/helper.c | 4 +- src/storage/azure/storage.h | 5 - src/storage/gcs/helper.c | 4 +- src/storage/gcs/storage.h | 5 - src/storage/s3/helper.c | 7 +- src/storage/s3/storage.h | 5 - test/src/module/command/helpTest.c | 225 ++++++++++++++-------------- test/src/module/config/loadTest.c | 55 +++++++ test/src/module/storage/azureTest.c | 2 +- test/src/module/storage/gcsTest.c | 2 +- test/src/module/storage/s3Test.c | 1 + 16 files changed, 317 insertions(+), 136 deletions(-) diff --git a/src/build/config/config.yaml b/src/build/config/config.yaml index b3ab1e265..3643c8a8f 100644 --- a/src/build/config/config.yaml +++ b/src/build/config/config.yaml @@ -2245,3 +2245,17 @@ option: deprecate: repo?-azure-port: {} repo?-s3-port: {} + + repo-storage-upload-chunk-size: + section: global + group: repo + type: size + required: false + allow-range: [64KiB, 1TiB] + command: repo-type + depend: + option: repo-type + list: + - azure + - gcs + - s3 diff --git a/src/build/help/help.xml b/src/build/help/help.xml index 12b3a83b4..352a6a334 100644 --- a/src/build/help/help.xml +++ b/src/build/help/help.xml @@ -972,6 +972,28 @@ 9000 + + Repository storage upload chunk size. + + +

Object stores such as S3 allow files to be uploaded in chunks when the file is too large to be stored in memory. Even if the file can be stored in memory, it is more memory efficient to limit the amount of memory used for uploads.

+ +

A larger chunk size will generally lead to better performance because it will minimize upload requests and allow more files to be uploaded in a single request rather than in chunks. The disadvantage is that memory usage will be higher and because the chunk buffer must be allocated per process, larger process-max values will lead to more memory being consumed overall.

+ +

Default chunk sizes by repo type:

+ + + azure - 4MiB + gcs - 4MiB + s3 - 5MiB + + +

Note that valid chunk sizes vary by storage type and by platform. For example, AWS S3 has a minimum chunk size of 5MiB but S3 clones may accept lower values. Terminology for chunk size varies by storage type, so when searching min/max values use part size for AWS S3, chunk size for GCS, and block size for Azure. No attempt is made to validate configured chunk sizes so selecting an invalid value will lead to errors from the storage service or undefined behavior.

+ + + 16MiB + + Repository storage certificate verify. diff --git a/src/config/config.auto.h b/src/config/config.auto.h index ea03df6b1..b5a306c5a 100644 --- a/src/config/config.auto.h +++ b/src/config/config.auto.h @@ -128,7 +128,7 @@ Option constants #define CFGOPT_TYPE "type" #define CFGOPT_VERBOSE "verbose" -#define CFG_OPTION_TOTAL 155 +#define CFG_OPTION_TOTAL 156 /*********************************************************************************************************************************** Option value constants @@ -483,6 +483,7 @@ typedef enum cfgOptRepoStorageCaPath, cfgOptRepoStorageHost, cfgOptRepoStoragePort, + cfgOptRepoStorageUploadChunkSize, cfgOptRepoStorageVerifyTls, cfgOptRepoType, cfgOptResume, diff --git a/src/config/load.c b/src/config/load.c index 0f6709605..deea2dfee 100644 --- a/src/config/load.c +++ b/src/config/load.c @@ -288,6 +288,20 @@ cfgLoadUpdateOption(void) } } + // Set default upload chunk size if not set + if (cfgOptionValid(cfgOptRepoStorageUploadChunkSize)) + { + for (unsigned int repoIdx = 0; repoIdx < cfgOptionGroupIdxTotal(cfgOptGrpRepo); repoIdx++) + { + if (!cfgOptionIdxTest(cfgOptRepoStorageUploadChunkSize, repoIdx)) + { + cfgOptionIdxSet( + cfgOptRepoStorageUploadChunkSize, repoIdx, cfgSourceDefault, + VARINT64((cfgOptionIdxStrId(cfgOptRepoType, repoIdx) == CFGOPTVAL_REPO_TYPE_S3 ? 5 : 4) * 1024 * 1024)); + } + } + } + // Set pg-host-port/repo-host-port default when pg-host-type/repo-host-type is tls. ??? This should be handled in the parser but // it requires a default that depends on another option value and that is not currently possible. #define HOST_PORT_TLS 8432 diff --git a/src/config/parse.auto.c.inc b/src/config/parse.auto.c.inc index 6563598c0..ebff20ede 100644 --- a/src/config/parse.auto.c.inc +++ b/src/config/parse.auto.c.inc @@ -7546,6 +7546,90 @@ static const ParseRuleOption parseRuleOption[CFG_OPTION_TOTAL] = ), // opt/repo-storage-port ), // opt/repo-storage-port // ----------------------------------------------------------------------------------------------------------------------------- + PARSE_RULE_OPTION // opt/repo-storage-upload-chunk-size + ( // opt/repo-storage-upload-chunk-size + PARSE_RULE_OPTION_NAME("repo-storage-upload-chunk-size"), // opt/repo-storage-upload-chunk-size + PARSE_RULE_OPTION_TYPE(cfgOptTypeSize), // opt/repo-storage-upload-chunk-size + PARSE_RULE_OPTION_RESET(true), // opt/repo-storage-upload-chunk-size + PARSE_RULE_OPTION_REQUIRED(false), // opt/repo-storage-upload-chunk-size + PARSE_RULE_OPTION_SECTION(cfgSectionGlobal), // opt/repo-storage-upload-chunk-size + PARSE_RULE_OPTION_GROUP_MEMBER(true), // opt/repo-storage-upload-chunk-size + PARSE_RULE_OPTION_GROUP_ID(cfgOptGrpRepo), // opt/repo-storage-upload-chunk-size + // opt/repo-storage-upload-chunk-size + PARSE_RULE_OPTION_COMMAND_ROLE_MAIN_VALID_LIST // opt/repo-storage-upload-chunk-size + ( // opt/repo-storage-upload-chunk-size + PARSE_RULE_OPTION_COMMAND(cfgCmdArchiveGet) // opt/repo-storage-upload-chunk-size + PARSE_RULE_OPTION_COMMAND(cfgCmdArchivePush) // opt/repo-storage-upload-chunk-size + PARSE_RULE_OPTION_COMMAND(cfgCmdBackup) // opt/repo-storage-upload-chunk-size + PARSE_RULE_OPTION_COMMAND(cfgCmdCheck) // opt/repo-storage-upload-chunk-size + PARSE_RULE_OPTION_COMMAND(cfgCmdExpire) // opt/repo-storage-upload-chunk-size + PARSE_RULE_OPTION_COMMAND(cfgCmdInfo) // opt/repo-storage-upload-chunk-size + PARSE_RULE_OPTION_COMMAND(cfgCmdRepoCreate) // opt/repo-storage-upload-chunk-size + PARSE_RULE_OPTION_COMMAND(cfgCmdRepoGet) // opt/repo-storage-upload-chunk-size + PARSE_RULE_OPTION_COMMAND(cfgCmdRepoLs) // opt/repo-storage-upload-chunk-size + PARSE_RULE_OPTION_COMMAND(cfgCmdRepoPut) // opt/repo-storage-upload-chunk-size + PARSE_RULE_OPTION_COMMAND(cfgCmdRepoRm) // opt/repo-storage-upload-chunk-size + PARSE_RULE_OPTION_COMMAND(cfgCmdRestore) // opt/repo-storage-upload-chunk-size + PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaCreate) // opt/repo-storage-upload-chunk-size + PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaDelete) // opt/repo-storage-upload-chunk-size + PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaUpgrade) // opt/repo-storage-upload-chunk-size + PARSE_RULE_OPTION_COMMAND(cfgCmdVerify) // opt/repo-storage-upload-chunk-size + ), // opt/repo-storage-upload-chunk-size + // opt/repo-storage-upload-chunk-size + PARSE_RULE_OPTION_COMMAND_ROLE_ASYNC_VALID_LIST // opt/repo-storage-upload-chunk-size + ( // opt/repo-storage-upload-chunk-size + PARSE_RULE_OPTION_COMMAND(cfgCmdArchiveGet) // opt/repo-storage-upload-chunk-size + PARSE_RULE_OPTION_COMMAND(cfgCmdArchivePush) // opt/repo-storage-upload-chunk-size + ), // opt/repo-storage-upload-chunk-size + // opt/repo-storage-upload-chunk-size + PARSE_RULE_OPTION_COMMAND_ROLE_LOCAL_VALID_LIST // opt/repo-storage-upload-chunk-size + ( // opt/repo-storage-upload-chunk-size + PARSE_RULE_OPTION_COMMAND(cfgCmdArchiveGet) // opt/repo-storage-upload-chunk-size + PARSE_RULE_OPTION_COMMAND(cfgCmdArchivePush) // opt/repo-storage-upload-chunk-size + PARSE_RULE_OPTION_COMMAND(cfgCmdBackup) // opt/repo-storage-upload-chunk-size + PARSE_RULE_OPTION_COMMAND(cfgCmdRestore) // opt/repo-storage-upload-chunk-size + PARSE_RULE_OPTION_COMMAND(cfgCmdVerify) // opt/repo-storage-upload-chunk-size + ), // opt/repo-storage-upload-chunk-size + // opt/repo-storage-upload-chunk-size + PARSE_RULE_OPTION_COMMAND_ROLE_REMOTE_VALID_LIST // opt/repo-storage-upload-chunk-size + ( // opt/repo-storage-upload-chunk-size + PARSE_RULE_OPTION_COMMAND(cfgCmdArchiveGet) // opt/repo-storage-upload-chunk-size + PARSE_RULE_OPTION_COMMAND(cfgCmdArchivePush) // opt/repo-storage-upload-chunk-size + PARSE_RULE_OPTION_COMMAND(cfgCmdCheck) // opt/repo-storage-upload-chunk-size + PARSE_RULE_OPTION_COMMAND(cfgCmdInfo) // opt/repo-storage-upload-chunk-size + PARSE_RULE_OPTION_COMMAND(cfgCmdRepoCreate) // opt/repo-storage-upload-chunk-size + PARSE_RULE_OPTION_COMMAND(cfgCmdRepoGet) // opt/repo-storage-upload-chunk-size + PARSE_RULE_OPTION_COMMAND(cfgCmdRepoLs) // opt/repo-storage-upload-chunk-size + PARSE_RULE_OPTION_COMMAND(cfgCmdRepoPut) // opt/repo-storage-upload-chunk-size + PARSE_RULE_OPTION_COMMAND(cfgCmdRepoRm) // opt/repo-storage-upload-chunk-size + PARSE_RULE_OPTION_COMMAND(cfgCmdRestore) // opt/repo-storage-upload-chunk-size + PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaCreate) // opt/repo-storage-upload-chunk-size + PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaDelete) // opt/repo-storage-upload-chunk-size + PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaUpgrade) // opt/repo-storage-upload-chunk-size + PARSE_RULE_OPTION_COMMAND(cfgCmdVerify) // opt/repo-storage-upload-chunk-size + ), // opt/repo-storage-upload-chunk-size + // opt/repo-storage-upload-chunk-size + PARSE_RULE_OPTIONAL // opt/repo-storage-upload-chunk-size + ( // opt/repo-storage-upload-chunk-size + PARSE_RULE_OPTIONAL_GROUP // opt/repo-storage-upload-chunk-size + ( // opt/repo-storage-upload-chunk-size + PARSE_RULE_OPTIONAL_DEPEND // opt/repo-storage-upload-chunk-size + ( // opt/repo-storage-upload-chunk-size + PARSE_RULE_VAL_OPT(cfgOptRepoType), // opt/repo-storage-upload-chunk-size + PARSE_RULE_VAL_STRID(parseRuleValStrIdAzure), // opt/repo-storage-upload-chunk-size + PARSE_RULE_VAL_STRID(parseRuleValStrIdGcs), // opt/repo-storage-upload-chunk-size + PARSE_RULE_VAL_STRID(parseRuleValStrIdS3), // opt/repo-storage-upload-chunk-size + ), // opt/repo-storage-upload-chunk-size + // opt/repo-storage-upload-chunk-size + PARSE_RULE_OPTIONAL_ALLOW_RANGE // opt/repo-storage-upload-chunk-size + ( // opt/repo-storage-upload-chunk-size + PARSE_RULE_VAL_INT(parseRuleValInt65536), // opt/repo-storage-upload-chunk-size + PARSE_RULE_VAL_INT(parseRuleValInt1099511627776), // opt/repo-storage-upload-chunk-size + ), // opt/repo-storage-upload-chunk-size + ), // opt/repo-storage-upload-chunk-size + ), // opt/repo-storage-upload-chunk-size + ), // opt/repo-storage-upload-chunk-size + // ----------------------------------------------------------------------------------------------------------------------------- PARSE_RULE_OPTION // opt/repo-storage-verify-tls ( // opt/repo-storage-verify-tls PARSE_RULE_OPTION_NAME("repo-storage-verify-tls"), // opt/repo-storage-verify-tls @@ -9201,6 +9285,7 @@ static const uint8_t optionResolveOrder[] = cfgOptRepoStorageCaPath, // opt-resolve-order cfgOptRepoStorageHost, // opt-resolve-order cfgOptRepoStoragePort, // opt-resolve-order + cfgOptRepoStorageUploadChunkSize, // opt-resolve-order cfgOptRepoStorageVerifyTls, // opt-resolve-order cfgOptTarget, // opt-resolve-order cfgOptTargetAction, // opt-resolve-order diff --git a/src/storage/azure/helper.c b/src/storage/azure/helper.c index 89ef3a4c0..211f5a97d 100644 --- a/src/storage/azure/helper.c +++ b/src/storage/azure/helper.c @@ -79,8 +79,8 @@ storageAzureHelper(const unsigned int repoIdx, const bool write, StoragePathExpr result = storageAzureNew( cfgOptionIdxStr(cfgOptRepoPath, repoIdx), write, pathExpressionCallback, cfgOptionIdxStr(cfgOptRepoAzureContainer, repoIdx), cfgOptionIdxStr(cfgOptRepoAzureAccount, repoIdx), keyType, key, - STORAGE_AZURE_BLOCKSIZE_MIN, endpoint, uriStyle, port, ioTimeoutMs(), cfgOptionIdxBool(cfgOptRepoStorageVerifyTls, - repoIdx), cfgOptionIdxStrNull(cfgOptRepoStorageCaFile, repoIdx), + (size_t)cfgOptionIdxUInt64(cfgOptRepoStorageUploadChunkSize, repoIdx), endpoint, uriStyle, port, ioTimeoutMs(), + cfgOptionIdxBool(cfgOptRepoStorageVerifyTls, repoIdx), cfgOptionIdxStrNull(cfgOptRepoStorageCaFile, repoIdx), cfgOptionIdxStrNull(cfgOptRepoStorageCaPath, repoIdx)); } MEM_CONTEXT_PRIOR_END(); diff --git a/src/storage/azure/storage.h b/src/storage/azure/storage.h index 471b65106..0c453d086 100644 --- a/src/storage/azure/storage.h +++ b/src/storage/azure/storage.h @@ -29,11 +29,6 @@ typedef enum storageAzureUriStylePath = STRID5("path", 0x450300), } StorageAzureUriStyle; -/*********************************************************************************************************************************** -Defaults -***********************************************************************************************************************************/ -#define STORAGE_AZURE_BLOCKSIZE_MIN ((size_t)4 * 1024 * 1024) - /*********************************************************************************************************************************** Constructors ***********************************************************************************************************************************/ diff --git a/src/storage/gcs/helper.c b/src/storage/gcs/helper.c index 73f5d8e07..90c64e300 100644 --- a/src/storage/gcs/helper.c +++ b/src/storage/gcs/helper.c @@ -24,8 +24,8 @@ storageGcsHelper(const unsigned int repoIdx, const bool write, StoragePathExpres Storage *const result = storageGcsNew( cfgOptionIdxStr(cfgOptRepoPath, repoIdx), write, pathExpressionCallback, cfgOptionIdxStr(cfgOptRepoGcsBucket, repoIdx), (StorageGcsKeyType)cfgOptionIdxStrId(cfgOptRepoGcsKeyType, repoIdx), cfgOptionIdxStrNull(cfgOptRepoGcsKey, repoIdx), - STORAGE_GCS_CHUNKSIZE_DEFAULT, cfgOptionIdxStr(cfgOptRepoGcsEndpoint, repoIdx), ioTimeoutMs(), - cfgOptionIdxBool(cfgOptRepoStorageVerifyTls, repoIdx), cfgOptionIdxStrNull(cfgOptRepoStorageCaFile, repoIdx), + (size_t)cfgOptionIdxUInt64(cfgOptRepoStorageUploadChunkSize, repoIdx), cfgOptionIdxStr(cfgOptRepoGcsEndpoint, repoIdx), + ioTimeoutMs(), cfgOptionIdxBool(cfgOptRepoStorageVerifyTls, repoIdx), cfgOptionIdxStrNull(cfgOptRepoStorageCaFile, repoIdx), cfgOptionIdxStrNull(cfgOptRepoStorageCaPath, repoIdx)); FUNCTION_LOG_RETURN(STORAGE, result); diff --git a/src/storage/gcs/storage.h b/src/storage/gcs/storage.h index 8a7bdc582..6b83dbff9 100644 --- a/src/storage/gcs/storage.h +++ b/src/storage/gcs/storage.h @@ -21,11 +21,6 @@ typedef enum storageGcsKeyTypeToken = STRID5("token", 0xe2adf40), } StorageGcsKeyType; -/*********************************************************************************************************************************** -Defaults -***********************************************************************************************************************************/ -#define STORAGE_GCS_CHUNKSIZE_DEFAULT ((size_t)4 * 1024 * 1024) - /*********************************************************************************************************************************** Constructors ***********************************************************************************************************************************/ diff --git a/src/storage/s3/helper.c b/src/storage/s3/helper.c index 835789e92..65c7b4051 100644 --- a/src/storage/s3/helper.c +++ b/src/storage/s3/helper.c @@ -85,9 +85,10 @@ storageS3Helper(const unsigned int repoIdx, const bool write, StoragePathExpress endPoint, (StorageS3UriStyle)cfgOptionIdxStrId(cfgOptRepoS3UriStyle, repoIdx), cfgOptionIdxStr(cfgOptRepoS3Region, repoIdx), keyType, cfgOptionIdxStrNull(cfgOptRepoS3Key, repoIdx), cfgOptionIdxStrNull(cfgOptRepoS3KeySecret, repoIdx), cfgOptionIdxStrNull(cfgOptRepoS3Token, repoIdx), - cfgOptionIdxStrNull(cfgOptRepoS3KmsKeyId, repoIdx), role, webIdToken, STORAGE_S3_PARTSIZE_MIN, host, port, - ioTimeoutMs(), cfgOptionIdxBool(cfgOptRepoStorageVerifyTls, repoIdx), - cfgOptionIdxStrNull(cfgOptRepoStorageCaFile, repoIdx), cfgOptionIdxStrNull(cfgOptRepoStorageCaPath, repoIdx)); + cfgOptionIdxStrNull(cfgOptRepoS3KmsKeyId, repoIdx), role, webIdToken, + (size_t)cfgOptionIdxUInt64(cfgOptRepoStorageUploadChunkSize, repoIdx), host, port, ioTimeoutMs(), + cfgOptionIdxBool(cfgOptRepoStorageVerifyTls, repoIdx), cfgOptionIdxStrNull(cfgOptRepoStorageCaFile, repoIdx), + cfgOptionIdxStrNull(cfgOptRepoStorageCaPath, repoIdx)); } MEM_CONTEXT_PRIOR_END(); } diff --git a/src/storage/s3/storage.h b/src/storage/s3/storage.h index 4e19933c0..e68ef0f55 100644 --- a/src/storage/s3/storage.h +++ b/src/storage/s3/storage.h @@ -30,11 +30,6 @@ typedef enum storageS3UriStylePath = STRID5("path", 0x450300), } StorageS3UriStyle; -/*********************************************************************************************************************************** -Defaults -***********************************************************************************************************************************/ -#define STORAGE_S3_PARTSIZE_MIN ((size_t)5 * 1024 * 1024) - /*********************************************************************************************************************************** Constructors ***********************************************************************************************************************************/ diff --git a/test/src/module/command/helpTest.c b/test/src/module/command/helpTest.c index 2326dff51..36f510337 100644 --- a/test/src/module/command/helpTest.c +++ b/test/src/module/command/helpTest.c @@ -194,129 +194,132 @@ testRun(void) "\n" "Command Options:\n" "\n" - " --archive-mode preserve or disable archiving on restored\n" - " cluster [default=preserve]\n" - " --db-exclude restore excluding the specified databases\n" - " --db-include restore only specified databases\n" - " [current=db1, db2]\n" - " --force force a restore [default=n]\n" - " --link-all restore all symlinks [default=n]\n" - " --link-map modify the destination of a symlink\n" - " [current=/link1=/dest1, /link2=/dest2]\n" - " --recovery-option set an option in recovery.conf\n" - " --set backup set to restore [default=latest]\n" - " --tablespace-map restore a tablespace into the specified\n" - " directory\n" - " --tablespace-map-all restore all tablespaces into the specified\n" - " directory\n" - " --target recovery target\n" - " --target-action action to take when recovery target is\n" - " reached [default=pause]\n" - " --target-exclusive stop just before the recovery target is\n" - " reached [default=n]\n" - " --target-timeline recover along a timeline\n" - " --type recovery type [default=default]\n" + " --archive-mode preserve or disable archiving on restored\n" + " cluster [default=preserve]\n" + " --db-exclude restore excluding the specified databases\n" + " --db-include restore only specified databases\n" + " [current=db1, db2]\n" + " --force force a restore [default=n]\n" + " --link-all restore all symlinks [default=n]\n" + " --link-map modify the destination of a symlink\n" + " [current=/link1=/dest1, /link2=/dest2]\n" + " --recovery-option set an option in recovery.conf\n" + " --set backup set to restore [default=latest]\n" + " --tablespace-map restore a tablespace into the specified\n" + " directory\n" + " --tablespace-map-all restore all tablespaces into the specified\n" + " directory\n" + " --target recovery target\n" + " --target-action action to take when recovery target is\n" + " reached [default=pause]\n" + " --target-exclusive stop just before the recovery target is\n" + " reached [default=n]\n" + " --target-timeline recover along a timeline\n" + " --type recovery type [default=default]\n" "\n" "General Options:\n" "\n" - " --buffer-size buffer size for I/O operations\n" - " [current=32768, default=1MiB]\n" - " --cmd pgBackRest command\n" - " [default=/path/to/pgbackrest]\n" - " --cmd-ssh SSH client command [default=ssh]\n" - " --compress-level-network network compression level [default=3]\n" - " --config pgBackRest configuration file\n" - " [default=/etc/pgbackrest/pgbackrest.conf]\n" - " --config-include-path path to additional pgBackRest configuration\n" - " files [default=/etc/pgbackrest/conf.d]\n" - " --config-path base path of pgBackRest configuration files\n" - " [default=/etc/pgbackrest]\n" - " --delta restore or backup using checksums [default=n]\n" - " --io-timeout I/O timeout [default=60]\n" - " --lock-path path where lock files are stored\n" - " [default=/tmp/pgbackrest]\n" - " --neutral-umask use a neutral umask [default=y]\n" - " --process-max max processes to use for compress/transfer\n" - " [default=1]\n" - " --protocol-timeout protocol timeout [default=1830]\n" - " --sck-keep-alive keep-alive enable [default=y]\n" - " --stanza defines the stanza\n" - " --tcp-keep-alive-count keep-alive count\n" - " --tcp-keep-alive-idle keep-alive idle time\n" - " --tcp-keep-alive-interval keep-alive interval time\n" + " --buffer-size buffer size for I/O operations\n" + " [current=32768, default=1MiB]\n" + " --cmd pgBackRest command\n" + " [default=/path/to/pgbackrest]\n" + " --cmd-ssh SSH client command [default=ssh]\n" + " --compress-level-network network compression level [default=3]\n" + " --config pgBackRest configuration file\n" + " [default=/etc/pgbackrest/pgbackrest.conf]\n" + " --config-include-path path to additional pgBackRest configuration\n" + " files [default=/etc/pgbackrest/conf.d]\n" + " --config-path base path of pgBackRest configuration files\n" + " [default=/etc/pgbackrest]\n" + " --delta restore or backup using checksums\n" + " [default=n]\n" + " --io-timeout I/O timeout [default=60]\n" + " --lock-path path where lock files are stored\n" + " [default=/tmp/pgbackrest]\n" + " --neutral-umask use a neutral umask [default=y]\n" + " --process-max max processes to use for compress/transfer\n" + " [default=1]\n" + " --protocol-timeout protocol timeout [default=1830]\n" + " --sck-keep-alive keep-alive enable [default=y]\n" + " --stanza defines the stanza\n" + " --tcp-keep-alive-count keep-alive count\n" + " --tcp-keep-alive-idle keep-alive idle time\n" + " --tcp-keep-alive-interval keep-alive interval time\n" "\n" "Log Options:\n" "\n" - " --log-level-console level for console logging [default=warn]\n" - " --log-level-file level for file logging [default=info]\n" - " --log-level-stderr level for stderr logging [default=warn]\n" - " --log-path path where log files are stored\n" - " [default=/var/log/pgbackrest]\n" - " --log-subprocess enable logging in subprocesses [default=n]\n" - " --log-timestamp enable timestamp in logging [default=y]\n" + " --log-level-console level for console logging [default=warn]\n" + " --log-level-file level for file logging [default=info]\n" + " --log-level-stderr level for stderr logging [default=warn]\n" + " --log-path path where log files are stored\n" + " [default=/var/log/pgbackrest]\n" + " --log-subprocess enable logging in subprocesses [default=n]\n" + " --log-timestamp enable timestamp in logging [default=y]\n" "\n", "Repository Options:\n" "\n" - " --repo set repository\n" - " --repo-azure-account azure repository account\n" - " --repo-azure-container azure repository container\n" - " --repo-azure-endpoint azure repository endpoint\n" - " [default=blob.core.windows.net]\n" - " --repo-azure-key azure repository key\n" - " --repo-azure-key-type azure repository key type [default=shared]\n" - " --repo-azure-uri-style azure URI Style [default=host]\n" - " --repo-cipher-pass repository cipher passphrase\n" - " [current=]\n" - " --repo-cipher-type cipher used to encrypt the repository\n" - " [current=aes-256-cbc, default=none]\n" - " --repo-gcs-bucket GCS repository bucket\n" - " --repo-gcs-endpoint GCS repository endpoint\n" - " [default=storage.googleapis.com]\n" - " --repo-gcs-key GCS repository key\n" - " --repo-gcs-key-type GCS repository key type [default=service]\n" - " --repo-host repository host when operating remotely via\n" - " SSH [current=backup.example.net]\n" - " --repo-host-ca-file repository host certificate authority file\n" - " --repo-host-ca-path repository host certificate authority path\n" - " --repo-host-cert-file repository host certificate file\n" - " --repo-host-cmd repository host pgBackRest command\n" - " [default=/path/to/pgbackrest]\n" - " --repo-host-config pgBackRest repository host configuration\n" - " file\n" - " [default=/etc/pgbackrest/pgbackrest.conf]\n" - " --repo-host-config-include-path pgBackRest repository host configuration\n" - " include path [default=/etc/pgbackrest/conf.d]\n" - " --repo-host-config-path pgBackRest repository host configuration\n" - " path [default=/etc/pgbackrest]\n" - " --repo-host-key-file repository host key file\n" - " --repo-host-port repository host port when repo-host is set\n" - " --repo-host-type repository host protocol type [default=ssh]\n" - " --repo-host-user repository host user when repo-host is set\n" - " [default=pgbackrest]\n" - " --repo-path path where backups and archive are stored\n" - " [default=/var/lib/pgbackrest]\n" - " --repo-s3-bucket S3 repository bucket\n" - " --repo-s3-endpoint S3 repository endpoint\n" - " --repo-s3-key S3 repository access key\n" - " --repo-s3-key-secret S3 repository secret access key\n" - " --repo-s3-key-type S3 repository key type [default=shared]\n" - " --repo-s3-kms-key-id S3 repository KMS key\n" - " --repo-s3-region S3 repository region\n" - " --repo-s3-role S3 repository role\n" - " --repo-s3-token S3 repository security token\n" - " --repo-s3-uri-style S3 URI Style [default=host]\n" - " --repo-storage-ca-file repository storage CA file\n" - " --repo-storage-ca-path repository storage CA path\n" - " --repo-storage-host repository storage host\n" - " --repo-storage-port repository storage port [default=443]\n" - " --repo-storage-verify-tls repository storage certificate verify\n" - " [default=y]\n" - " --repo-type type of storage used for the repository\n" - " [default=posix]\n" + " --repo set repository\n" + " --repo-azure-account azure repository account\n" + " --repo-azure-container azure repository container\n" + " --repo-azure-endpoint azure repository endpoint\n" + " [default=blob.core.windows.net]\n" + " --repo-azure-key azure repository key\n" + " --repo-azure-key-type azure repository key type [default=shared]\n" + " --repo-azure-uri-style azure URI Style [default=host]\n" + " --repo-cipher-pass repository cipher passphrase\n" + " [current=]\n" + " --repo-cipher-type cipher used to encrypt the repository\n" + " [current=aes-256-cbc, default=none]\n" + " --repo-gcs-bucket GCS repository bucket\n" + " --repo-gcs-endpoint GCS repository endpoint\n" + " [default=storage.googleapis.com]\n" + " --repo-gcs-key GCS repository key\n" + " --repo-gcs-key-type GCS repository key type [default=service]\n" + " --repo-host repository host when operating remotely via\n" + " SSH [current=backup.example.net]\n" + " --repo-host-ca-file repository host certificate authority file\n" + " --repo-host-ca-path repository host certificate authority path\n" + " --repo-host-cert-file repository host certificate file\n" + " --repo-host-cmd repository host pgBackRest command\n" + " [default=/path/to/pgbackrest]\n" + " --repo-host-config pgBackRest repository host configuration\n" + " file\n" + " [default=/etc/pgbackrest/pgbackrest.conf]\n" + " --repo-host-config-include-path pgBackRest repository host configuration\n" + " include path\n" + " [default=/etc/pgbackrest/conf.d]\n" + " --repo-host-config-path pgBackRest repository host configuration\n" + " path [default=/etc/pgbackrest]\n" + " --repo-host-key-file repository host key file\n" + " --repo-host-port repository host port when repo-host is set\n" + " --repo-host-type repository host protocol type [default=ssh]\n" + " --repo-host-user repository host user when repo-host is set\n" + " [default=pgbackrest]\n" + " --repo-path path where backups and archive are stored\n" + " [default=/var/lib/pgbackrest]\n" + " --repo-s3-bucket S3 repository bucket\n" + " --repo-s3-endpoint S3 repository endpoint\n" + " --repo-s3-key S3 repository access key\n" + " --repo-s3-key-secret S3 repository secret access key\n" + " --repo-s3-key-type S3 repository key type [default=shared]\n" + " --repo-s3-kms-key-id S3 repository KMS key\n" + " --repo-s3-region S3 repository region\n" + " --repo-s3-role S3 repository role\n" + " --repo-s3-token S3 repository security token\n" + " --repo-s3-uri-style S3 URI Style [default=host]\n" + " --repo-storage-ca-file repository storage CA file\n" + " --repo-storage-ca-path repository storage CA path\n" + " --repo-storage-host repository storage host\n" + " --repo-storage-port repository storage port [default=443]\n" + " --repo-storage-upload-chunk-size repository storage upload chunk size\n" + " --repo-storage-verify-tls repository storage certificate verify\n" + " [default=y]\n" + " --repo-type type of storage used for the repository\n" + " [default=posix]\n" "\n" "Stanza Options:\n" "\n" - " --pg-path postgreSQL data directory\n" + " --pg-path postgreSQL data directory\n" "\n" "Use 'pgbackrest help restore [option]' for more information.\n"); diff --git a/test/src/module/config/loadTest.c b/test/src/module/config/loadTest.c index 5f1f4211f..a27ec8ed0 100644 --- a/test/src/module/config/loadTest.c +++ b/test/src/module/config/loadTest.c @@ -532,6 +532,61 @@ testRun(void) TEST_RESULT_LOG( "P00 WARN: 'compress' and 'compress-type' options should not both be set\n" " HINT: 'compress-type' is preferred and 'compress' is deprecated."); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("S3 default chunk size"); + + argList = strLstNew(); + hrnCfgArgRawZ(argList, cfgOptStanza, "db"); + hrnCfgArgRawZ(argList, cfgOptPgPath, "/path/to/pg"); + hrnCfgArgKeyRawZ(argList, cfgOptRepoType, 1, "s3"); + hrnCfgArgKeyRawZ(argList, cfgOptRepoS3Bucket, 1, "bucket"); + hrnCfgArgKeyRawZ(argList, cfgOptRepoS3Region, 1, "region"); + hrnCfgArgKeyRawZ(argList, cfgOptRepoS3Endpoint, 1, "endpoint"); + hrnCfgArgKeyRawZ(argList, cfgOptRepoS3KeyType, 1, "auto"); + hrnCfgArgKeyRawZ(argList, cfgOptRepoPath, 1, "/repo"); + HRN_CFG_LOAD(cfgCmdArchiveGet, argList); + + TEST_RESULT_UINT(cfgOptionUInt64(cfgOptRepoStorageUploadChunkSize), 5 * 1024 * 1024, "default chunk size"); + TEST_RESULT_UINT(cfgOptionSource(cfgOptRepoStorageUploadChunkSize), cfgSourceDefault, "chunk size source is default"); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("S3 custom chunk size"); + + argList = strLstNew(); + hrnCfgArgRawZ(argList, cfgOptStanza, "db"); + hrnCfgArgRawZ(argList, cfgOptPgPath, "/path/to/pg"); + hrnCfgArgKeyRawZ(argList, cfgOptRepoType, 1, "s3"); + hrnCfgArgKeyRawZ(argList, cfgOptRepoS3Bucket, 1, "bucket"); + hrnCfgArgKeyRawZ(argList, cfgOptRepoS3Region, 1, "region"); + hrnCfgArgKeyRawZ(argList, cfgOptRepoS3Endpoint, 1, "endpoint"); + hrnCfgArgKeyRawZ(argList, cfgOptRepoS3KeyType, 1, "auto"); + hrnCfgArgKeyRawZ(argList, cfgOptRepoStorageUploadChunkSize, 1, "64KiB"); + hrnCfgArgKeyRawZ(argList, cfgOptRepoPath, 1, "/repo"); + HRN_CFG_LOAD(cfgCmdArchiveGet, argList); + + TEST_RESULT_UINT(cfgOptionUInt64(cfgOptRepoStorageUploadChunkSize), 64 * 1024, "chunk size set"); + TEST_RESULT_UINT(cfgOptionSource(cfgOptRepoStorageUploadChunkSize), cfgSourceParam, "chunk size source is param"); + + // ------------------------------------------------------------------------------------------------------------------------- + TEST_TITLE("Azure default chunk size"); + + hrnCfgEnvKeyRawZ(cfgOptRepoAzureAccount, 1, "account"); + hrnCfgEnvKeyRawZ(cfgOptRepoAzureKey, 1, "mykey"); + + argList = strLstNew(); + hrnCfgArgRawZ(argList, cfgOptStanza, "db"); + hrnCfgArgRawZ(argList, cfgOptPgPath, "/path/to/pg"); + hrnCfgArgKeyRawZ(argList, cfgOptRepoType, 1, "azure"); + hrnCfgArgKeyRawZ(argList, cfgOptRepoAzureContainer, 1, "container"); + hrnCfgArgKeyRawZ(argList, cfgOptRepoPath, 1, "/repo"); + HRN_CFG_LOAD(cfgCmdArchiveGet, argList); + + TEST_RESULT_UINT(cfgOptionUInt64(cfgOptRepoStorageUploadChunkSize), 4 * 1024 * 1024, "default chunk size"); + TEST_RESULT_UINT(cfgOptionSource(cfgOptRepoStorageUploadChunkSize), cfgSourceDefault, "chunk size source is default"); + + hrnCfgEnvKeyRemoveRaw(cfgOptRepoAzureAccount, 1); + hrnCfgEnvKeyRemoveRaw(cfgOptRepoAzureKey, 1); } // ***************************************************************************************************************************** diff --git a/test/src/module/storage/azureTest.c b/test/src/module/storage/azureTest.c index 562e217b8..3a83ab3b2 100644 --- a/test/src/module/storage/azureTest.c +++ b/test/src/module/storage/azureTest.c @@ -203,7 +203,7 @@ testRun(void) strNewEncode(encodeBase64, ((StorageAzure *)storageDriver(storage))->sharedKey), TEST_KEY_SHARED_STR, "check key"); TEST_RESULT_STR_Z(((StorageAzure *)storageDriver(storage))->host, TEST_ACCOUNT ".blob.core.windows.net", "check host"); TEST_RESULT_STR_Z(((StorageAzure *)storageDriver(storage))->pathPrefix, "/" TEST_CONTAINER, "check path prefix"); - TEST_RESULT_UINT(((StorageAzure *)storageDriver(storage))->blockSize, STORAGE_AZURE_BLOCKSIZE_MIN, "check block size"); + TEST_RESULT_UINT(((StorageAzure *)storageDriver(storage))->blockSize, 4 * 1024 * 1024, "check block size"); TEST_RESULT_BOOL(storageFeature(storage, storageFeaturePath), false, "check path feature"); // ------------------------------------------------------------------------------------------------------------------------- diff --git a/test/src/module/storage/gcsTest.c b/test/src/module/storage/gcsTest.c index d063648cf..466a90f70 100644 --- a/test/src/module/storage/gcsTest.c +++ b/test/src/module/storage/gcsTest.c @@ -223,7 +223,7 @@ testRun(void) TEST_RESULT_STR_Z(storage->path, "/repo", "check path"); TEST_RESULT_STR(((StorageGcs *)storageDriver(storage))->bucket, TEST_BUCKET_STR, "check bucket"); TEST_RESULT_STR_Z(((StorageGcs *)storageDriver(storage))->endpoint, "storage.googleapis.com", "check endpoint"); - TEST_RESULT_UINT(((StorageGcs *)storageDriver(storage))->chunkSize, STORAGE_GCS_CHUNKSIZE_DEFAULT, "check chunk size"); + TEST_RESULT_UINT(((StorageGcs *)storageDriver(storage))->chunkSize, 4 * 1024 * 1024, "check chunk size"); TEST_RESULT_STR(((StorageGcs *)storageDriver(storage))->token, TEST_TOKEN_STR, "check token"); TEST_RESULT_BOOL(storageFeature(storage, storageFeaturePath), false, "check path feature"); } diff --git a/test/src/module/storage/s3Test.c b/test/src/module/storage/s3Test.c index a7bdaaf3c..892e61972 100644 --- a/test/src/module/storage/s3Test.c +++ b/test/src/module/storage/s3Test.c @@ -419,6 +419,7 @@ testRun(void) TEST_RESULT_STR(s3->path, path, "check path"); TEST_RESULT_BOOL(storageFeature(s3, storageFeaturePath), false, "check path feature"); + TEST_RESULT_UINT(driver->partSize, 5 * 1024 * 1024, "check part size"); // ----------------------------------------------------------------------------------------------------------------- TEST_TITLE("coverage for noop functions");