1
0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2025-07-05 00:28:52 +02:00

Add repo-azure-uri-style option.

Azurite introduced a breaking change in 8f63964e to use automatically host-style URIs when the endpoint appears to be a multipart hostname.

This option allows the user to configure which style URI will be used, but changing the endpoint might cause breakage if Azurite decides to use a different style. Future changes to Azurite may also cause breakage.
This commit is contained in:
David Steele
2021-09-27 09:01:53 -04:00
parent c8ea17c68f
commit 096829b3b2
11 changed files with 176 additions and 19 deletions

View File

@ -613,7 +613,6 @@
<backrest-config-option if="'{[azure-key-type]}' ne 'shared'" section="global" key="repo{[azure-setup-repo-id]}-azure-key-type">{[azure-key-type]}</backrest-config-option>
<backrest-config-option section="global" key="repo{[azure-setup-repo-id]}-azure-key">{[azure-key]}</backrest-config-option>
<backrest-config-option section="global" key="repo{[azure-setup-repo-id]}-azure-container">{[azure-container]}</backrest-config-option>
<backrest-config-option if="'{[azure-local]}' eq 'y'" section="global" key="repo{[azure-setup-repo-id]}-storage-host">blob.core.windows.net</backrest-config-option>
<backrest-config-option if="'{[azure-all]}' ne 'y'" section="global" key="repo{[azure-setup-repo-id]}-retention-full">4</backrest-config-option>
<backrest-config-option section="global" key="process-max">4</backrest-config-option>
@ -624,7 +623,7 @@
<!-- Set host entries to redirect to local azure server -->
<execute user="root" user-force="y" show="n">
<exe-cmd>echo "{[host-azure-ip]} blob.core.windows.net" | tee -a /etc/hosts</exe-cmd>
<exe-cmd>echo "{[host-azure-ip]} pgbackrest.blob.core.windows.net" | tee -a /etc/hosts</exe-cmd>
</execute>
<execute user="{[azure-setup-user]}" if="'{[azure-setup-create-container]}' eq 'y'" show='n'>
@ -669,7 +668,6 @@
<backrest-config-option section="global" key="repo{[s3-setup-repo-id]}-s3-endpoint">{[s3-endpoint]}</backrest-config-option>
<backrest-config-option section="global" key="repo{[s3-setup-repo-id]}-s3-region">{[s3-region]}</backrest-config-option>
<backrest-config-option if="'{[s3-all]}' ne 'y'" section="global" key="repo{[s3-setup-repo-id]}-retention-full">4</backrest-config-option>
<backrest-config-option section="global" key="repo{[s3-setup-repo-id]}-storage-host" remove="y"/>
<backrest-config-option section="global" key="process-max">4</backrest-config-option>
</backrest-config>
@ -695,7 +693,7 @@
<title>Introduction</title>
<!-- Create Azure server first to allow it time to boot before being used -->
<host-add if="'{[azure-local]}' eq 'y'" id="{[host-azure-id]}" name="{[host-azure]}" user="root" image="mcr.microsoft.com/azure-storage/azurite:3.14.0" os="{[os-type]}" option="-m 128m -v {[fake-cert-path]}/azure-server.crt:/root/public.crt:ro -v {[fake-cert-path]}/azure-server.key:/root/private.key:ro -e AZURITE_ACCOUNTS='{[azure-account]}:{[azure-key]}'" param="azurite-blob --blobPort 443 --blobHost 0.0.0.0 --cert=/root/public.crt --key=/root/private.key" update-hosts="n"/>
<host-add if="'{[azure-local]}' eq 'y'" id="{[host-azure-id]}" name="{[host-azure]}" user="root" image="mcr.microsoft.com/azure-storage/azurite" os="{[os-type]}" option="-m 128m -v {[fake-cert-path]}/azure-server.crt:/root/public.crt:ro -v {[fake-cert-path]}/azure-server.key:/root/private.key:ro -e AZURITE_ACCOUNTS='{[azure-account]}:{[azure-key]}'" param="azurite-blob --blobPort 443 --blobHost 0.0.0.0 --cert=/root/public.crt --key=/root/private.key" update-hosts="n"/>
<!-- Create S3 server first to allow it time to boot before being used -->
<host-add if="'{[s3-local]}' eq 'y'" id="{[host-s3-id]}" name="{[host-s3]}" user="root" image="minio/minio" os="{[os-type]}" option="-m 128m -v {[fake-cert-path]}/s3-server.crt:/root/.minio/certs/public.crt:ro -v {[fake-cert-path]}/s3-server.key:/root/.minio/certs/private.key:ro -e MINIO_REGION={[s3-region]} -e MINIO_DOMAIN={[s3-endpoint]} -e MINIO_BROWSER=off -e MINIO_ACCESS_KEY={[s3-key]} -e MINIO_SECRET_KEY={[s3-key-secret]}" param="server /data --address :443" update-hosts="n"/>

View File

@ -1611,6 +1611,17 @@ option:
- shared
- sas
repo-azure-uri-style:
section: global
group: repo
type: string
default: host
allow-list:
- host
- path
command: repo-type
depend: repo-azure-account
repo-cipher-pass:
section: global
type: string

View File

@ -482,6 +482,22 @@
<example>sas</example>
</config-key>
<!-- ======================================================================================================= -->
<config-key id="repo-azure-uri-style" name="Azure Repository URI Style">
<summary>Azure URI Style.</summary>
<text>
<p>The following URI styles are supported:</p>
<list>
<list-item><id>host</id> - Connect to <id>account.endpoint</id> host.</list-item>
<list-item><id>path</id> - Connect to <id>endpoint</id> host and prepend account to URIs.</list-item>
</list>
</text>
<example>path</example>
</config-key>
<!-- ======================================================================================================= -->
<config-key id="repo-gcs-bucket" name="GCS Repository Bucket">
<summary>GCS repository bucket.</summary>

View File

@ -117,7 +117,7 @@ Option constants
#define CFGOPT_TCP_KEEP_ALIVE_INTERVAL "tcp-keep-alive-interval"
#define CFGOPT_TYPE "type"
#define CFG_OPTION_TOTAL 131
#define CFG_OPTION_TOTAL 132
/***********************************************************************************************************************************
Option value constants
@ -198,6 +198,11 @@ Option value constants
#define CFGOPTVAL_REPO_AZURE_KEY_TYPE_SHARED STRID5("shared", 0x85905130)
#define CFGOPTVAL_REPO_AZURE_KEY_TYPE_SHARED_Z "shared"
#define CFGOPTVAL_REPO_AZURE_URI_STYLE_HOST STRID5("host", 0xa4de80)
#define CFGOPTVAL_REPO_AZURE_URI_STYLE_HOST_Z "host"
#define CFGOPTVAL_REPO_AZURE_URI_STYLE_PATH STRID5("path", 0x450300)
#define CFGOPTVAL_REPO_AZURE_URI_STYLE_PATH_Z "path"
#define CFGOPTVAL_REPO_CIPHER_TYPE_AES_256_CBC STRID5("aes-256-cbc", 0xc43dfbbcdcca10)
#define CFGOPTVAL_REPO_CIPHER_TYPE_AES_256_CBC_Z "aes-256-cbc"
#define CFGOPTVAL_REPO_CIPHER_TYPE_NONE STRID5("none", 0x2b9ee0)
@ -399,6 +404,7 @@ typedef enum
cfgOptRepoAzureEndpoint,
cfgOptRepoAzureKey,
cfgOptRepoAzureKeyType,
cfgOptRepoAzureUriStyle,
cfgOptRepoCipherPass,
cfgOptRepoCipherType,
cfgOptRepoGcsBucket,

View File

@ -3545,6 +3545,88 @@ static const ParseRuleOption parseRuleOption[CFG_OPTION_TOTAL] =
),
),
// -----------------------------------------------------------------------------------------------------------------------------
PARSE_RULE_OPTION
(
PARSE_RULE_OPTION_NAME("repo-azure-uri-style"),
PARSE_RULE_OPTION_TYPE(cfgOptTypeString),
PARSE_RULE_OPTION_RESET(true),
PARSE_RULE_OPTION_REQUIRED(true),
PARSE_RULE_OPTION_SECTION(cfgSectionGlobal),
PARSE_RULE_OPTION_GROUP_MEMBER(true),
PARSE_RULE_OPTION_GROUP_ID(cfgOptGrpRepo),
PARSE_RULE_OPTION_COMMAND_ROLE_MAIN_VALID_LIST
(
PARSE_RULE_OPTION_COMMAND(cfgCmdArchiveGet)
PARSE_RULE_OPTION_COMMAND(cfgCmdArchivePush)
PARSE_RULE_OPTION_COMMAND(cfgCmdBackup)
PARSE_RULE_OPTION_COMMAND(cfgCmdCheck)
PARSE_RULE_OPTION_COMMAND(cfgCmdExpire)
PARSE_RULE_OPTION_COMMAND(cfgCmdInfo)
PARSE_RULE_OPTION_COMMAND(cfgCmdRepoCreate)
PARSE_RULE_OPTION_COMMAND(cfgCmdRepoGet)
PARSE_RULE_OPTION_COMMAND(cfgCmdRepoLs)
PARSE_RULE_OPTION_COMMAND(cfgCmdRepoPut)
PARSE_RULE_OPTION_COMMAND(cfgCmdRepoRm)
PARSE_RULE_OPTION_COMMAND(cfgCmdRestore)
PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaCreate)
PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaDelete)
PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaUpgrade)
PARSE_RULE_OPTION_COMMAND(cfgCmdVerify)
),
PARSE_RULE_OPTION_COMMAND_ROLE_ASYNC_VALID_LIST
(
PARSE_RULE_OPTION_COMMAND(cfgCmdArchiveGet)
PARSE_RULE_OPTION_COMMAND(cfgCmdArchivePush)
),
PARSE_RULE_OPTION_COMMAND_ROLE_LOCAL_VALID_LIST
(
PARSE_RULE_OPTION_COMMAND(cfgCmdArchiveGet)
PARSE_RULE_OPTION_COMMAND(cfgCmdArchivePush)
PARSE_RULE_OPTION_COMMAND(cfgCmdBackup)
PARSE_RULE_OPTION_COMMAND(cfgCmdRestore)
PARSE_RULE_OPTION_COMMAND(cfgCmdVerify)
),
PARSE_RULE_OPTION_COMMAND_ROLE_REMOTE_VALID_LIST
(
PARSE_RULE_OPTION_COMMAND(cfgCmdArchiveGet)
PARSE_RULE_OPTION_COMMAND(cfgCmdArchivePush)
PARSE_RULE_OPTION_COMMAND(cfgCmdCheck)
PARSE_RULE_OPTION_COMMAND(cfgCmdInfo)
PARSE_RULE_OPTION_COMMAND(cfgCmdRepoCreate)
PARSE_RULE_OPTION_COMMAND(cfgCmdRepoGet)
PARSE_RULE_OPTION_COMMAND(cfgCmdRepoLs)
PARSE_RULE_OPTION_COMMAND(cfgCmdRepoPut)
PARSE_RULE_OPTION_COMMAND(cfgCmdRepoRm)
PARSE_RULE_OPTION_COMMAND(cfgCmdRestore)
PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaCreate)
PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaDelete)
PARSE_RULE_OPTION_COMMAND(cfgCmdStanzaUpgrade)
PARSE_RULE_OPTION_COMMAND(cfgCmdVerify)
),
PARSE_RULE_OPTION_OPTIONAL_LIST
(
PARSE_RULE_OPTION_OPTIONAL_ALLOW_LIST
(
"host",
"path"
),
PARSE_RULE_OPTION_OPTIONAL_DEPEND_LIST
(
cfgOptRepoType,
"azure"
),
PARSE_RULE_OPTION_OPTIONAL_DEFAULT("host"),
),
),
// -----------------------------------------------------------------------------------------------------------------------------
PARSE_RULE_OPTION
(
@ -7108,6 +7190,7 @@ static const ConfigOption optionResolveOrder[] =
cfgOptRepoAzureEndpoint,
cfgOptRepoAzureKey,
cfgOptRepoAzureKeyType,
cfgOptRepoAzureUriStyle,
cfgOptRepoCipherPass,
cfgOptRepoGcsKeyType,
cfgOptRepoHost,

View File

@ -682,9 +682,10 @@ static const StorageInterface storageInterfaceAzure =
Storage *
storageAzureNew(
const String *path, bool write, StoragePathExpressionCallback pathExpressionFunction, const String *container,
const String *account, StorageAzureKeyType keyType, const String *key, size_t blockSize, const String *host,
const String *endpoint, unsigned int port, TimeMSec timeout, bool verifyPeer, const String *caFile, const String *caPath)
const String *const path, const bool write, StoragePathExpressionCallback pathExpressionFunction, const String *const container,
const String *const account, const StorageAzureKeyType keyType, const String *const key, const size_t blockSize,
const String *const endpoint, const StorageAzureUriStyle uriStyle, const unsigned int port, const TimeMSec timeout,
const bool verifyPeer, const String *const caFile, const String *const caPath)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(STRING, path);
@ -695,8 +696,8 @@ storageAzureNew(
FUNCTION_LOG_PARAM(STRING_ID, keyType);
FUNCTION_TEST_PARAM(STRING, key);
FUNCTION_LOG_PARAM(SIZE, blockSize);
FUNCTION_LOG_PARAM(STRING, host);
FUNCTION_LOG_PARAM(STRING, endpoint);
FUNCTION_LOG_PARAM(ENUM, uriStyle);
FUNCTION_LOG_PARAM(UINT, port);
FUNCTION_LOG_PARAM(TIME_MSEC, timeout);
FUNCTION_LOG_PARAM(BOOL, verifyPeer);
@ -707,6 +708,7 @@ storageAzureNew(
ASSERT(path != NULL);
ASSERT(container != NULL);
ASSERT(account != NULL);
ASSERT(endpoint != NULL);
ASSERT(key != NULL);
ASSERT(blockSize != 0);
@ -722,8 +724,9 @@ storageAzureNew(
.container = strDup(container),
.account = strDup(account),
.blockSize = blockSize,
.host = host == NULL ? strNewFmt("%s.%s", strZ(account), strZ(endpoint)) : host,
.pathPrefix = host == NULL ? strNewFmt("/%s", strZ(container)) : strNewFmt("/%s/%s", strZ(account), strZ(container)),
.host = uriStyle == storageAzureUriStyleHost ? strNewFmt("%s.%s", strZ(account), strZ(endpoint)) : endpoint,
.pathPrefix = uriStyle == storageAzureUriStyleHost ?
strNewFmt("/%s", strZ(container)) : strNewFmt("/%s/%s", strZ(account), strZ(container)),
};
// Store shared key or parse sas query

View File

@ -20,6 +20,15 @@ typedef enum
storageAzureKeyTypeSas = STRID5("sas", 0x4c330),
} StorageAzureKeyType;
/***********************************************************************************************************************************
URI style
***********************************************************************************************************************************/
typedef enum
{
storageAzureUriStyleHost = STRID5("host", 0xa4de80),
storageAzureUriStylePath = STRID5("path", 0x450300),
} StorageAzureUriStyle;
/***********************************************************************************************************************************
Defaults
***********************************************************************************************************************************/
@ -30,7 +39,8 @@ Constructors
***********************************************************************************************************************************/
Storage *storageAzureNew(
const String *path, bool write, StoragePathExpressionCallback pathExpressionFunction, const String *container,
const String *account, StorageAzureKeyType keyType, const String *key, size_t blockSize, const String *host,
const String *endpoint, unsigned int port, TimeMSec timeout, bool verifyPeer, const String *caFile, const String *caPath);
const String *account, StorageAzureKeyType keyType, const String *key, size_t blockSize, const String *endpoint,
StorageAzureUriStyle uriStyle, unsigned int port, TimeMSec timeout, bool verifyPeer, const String *caFile,
const String *caPath);
#endif

View File

@ -359,16 +359,33 @@ storageRepoGet(unsigned int repoIdx, bool write)
{
// Use Azure storage
case STORAGE_AZURE_TYPE:
{
const String *endpoint = cfgOptionIdxStr(cfgOptRepoAzureEndpoint, repoIdx);
const String *const host = cfgOptionIdxStrNull(cfgOptRepoStorageHost, repoIdx);
StorageAzureUriStyle uriStyle = (StorageAzureUriStyle)cfgOptionIdxStrId(cfgOptRepoAzureUriStyle, repoIdx);
// If the host is set then set it as the endpoint. The host option is used to set path-style URIs when working with
// Azurite. This was ill-advised, so the uri-style option was added to allow the user to select the URI style used
// by the server. Preserve the old behavior when uri-style is defaulted.
if (host != NULL)
{
endpoint = host;
if (cfgOptionIdxSource(cfgOptRepoAzureUriStyle, repoIdx) == cfgSourceDefault)
uriStyle = storageAzureUriStylePath;
}
result = storageAzureNew(
cfgOptionIdxStr(cfgOptRepoPath, repoIdx), write, storageRepoPathExpression,
cfgOptionIdxStr(cfgOptRepoAzureContainer, repoIdx), cfgOptionIdxStr(cfgOptRepoAzureAccount, repoIdx),
(StorageAzureKeyType)cfgOptionIdxStrId(cfgOptRepoAzureKeyType, repoIdx),
cfgOptionIdxStr(cfgOptRepoAzureKey, repoIdx), STORAGE_AZURE_BLOCKSIZE_MIN,
cfgOptionIdxStrNull(cfgOptRepoStorageHost, repoIdx), cfgOptionIdxStr(cfgOptRepoAzureEndpoint, repoIdx),
cfgOptionIdxUInt(cfgOptRepoStoragePort, repoIdx), ioTimeoutMs(),
endpoint, uriStyle, cfgOptionIdxUInt(cfgOptRepoStoragePort, repoIdx), ioTimeoutMs(),
cfgOptionIdxBool(cfgOptRepoStorageVerifyTls, repoIdx), cfgOptionIdxStrNull(cfgOptRepoStorageCaFile, repoIdx),
cfgOptionIdxStrNull(cfgOptRepoStorageCaPath, repoIdx));
break;
}
// Use CIFS storage
case STORAGE_CIFS_TYPE:

View File

@ -63,7 +63,7 @@ sub new
my $strFakeCertPath = "${strProjectPath}/doc/resource/fake-cert";
my $self = $class->SUPER::new(
HOST_AZURE, 'test-' . testRunGet()->vmId() . '-' . HOST_AZURE, 'mcr.microsoft.com/azure-storage/azurite:3.14.0', 'root',
HOST_AZURE, 'test-' . testRunGet()->vmId() . '-' . HOST_AZURE, 'mcr.microsoft.com/azure-storage/azurite', 'root',
'u18', ["${strFakeCertPath}/s3-server.crt:/root/public.crt:ro", "${strFakeCertPath}/s3-server.key:/root/private.key:ro"],
'-e AZURITE_ACCOUNTS="' . HOST_AZURE_ACCOUNT . ':' . HOST_AZURE_KEY . '"',
'azurite-blob --blobPort 443 --blobHost 0.0.0.0 --cert=/root/public.crt --key=/root/private.key -d debug.log', false);

View File

@ -263,6 +263,7 @@ testRun(void)
" [default=blob.core.windows.net]\n"
" --repo-azure-key azure repository key\n"
" --repo-azure-key-type azure repository key type [default=shared]\n"
" --repo-azure-uri-style azure URI Style [default=host]\n"
" --repo-cipher-pass repository cipher passphrase\n"
" [current=<redacted>]\n"
" --repo-cipher-type cipher used to encrypt the repository\n"

View File

@ -197,6 +197,17 @@ testRun(void)
TEST_RESULT_UINT(((StorageAzure *)storageDriver(storage))->blockSize, STORAGE_AZURE_BLOCKSIZE_MIN, "check block size");
TEST_RESULT_BOOL(storageFeature(storage, storageFeaturePath), false, "check path feature");
TEST_RESULT_BOOL(storageFeature(storage, storageFeatureCompress), false, "check compress feature");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("storage with host but force host-style uri");
hrnCfgArgRawZ(argList, cfgOptRepoStorageHost, "test-host");
hrnCfgArgRawStrId(argList, cfgOptRepoAzureUriStyle, storageAzureUriStyleHost);
HRN_CFG_LOAD(cfgCmdArchivePush, argList);
TEST_ASSIGN(storage, storageRepoGet(0, false), "get repo storage");
TEST_RESULT_STR_Z(((StorageAzure *)storageDriver(storage))->host, TEST_ACCOUNT ".test-host", "check host");
TEST_RESULT_STR_Z(((StorageAzure *)storageDriver(storage))->pathPrefix, "/" TEST_CONTAINER, "check path prefix");
}
// *****************************************************************************************************************************
@ -211,7 +222,8 @@ testRun(void)
(StorageAzure *)storageDriver(
storageAzureNew(
STRDEF("/repo"), false, NULL, TEST_CONTAINER_STR, TEST_ACCOUNT_STR, storageAzureKeyTypeShared,
TEST_KEY_SHARED_STR, 16, NULL, STRDEF("blob.core.windows.net"), 443, 1000, true, NULL, NULL)),
TEST_KEY_SHARED_STR, 16, STRDEF("blob.core.windows.net"), storageAzureUriStyleHost, 443, 1000, true, NULL,
NULL)),
"new azure storage - shared key");
// -------------------------------------------------------------------------------------------------------------------------
@ -250,7 +262,7 @@ testRun(void)
(StorageAzure *)storageDriver(
storageAzureNew(
STRDEF("/repo"), false, NULL, TEST_CONTAINER_STR, TEST_ACCOUNT_STR, storageAzureKeyTypeSas, TEST_KEY_SAS_STR,
16, NULL, STRDEF("blob.core.usgovcloudapi.net"), 443, 1000, true, NULL, NULL)),
16, STRDEF("blob.core.usgovcloudapi.net"), storageAzureUriStyleHost, 443, 1000, true, NULL, NULL)),
"new azure storage - sas key");
query = httpQueryAdd(httpQueryNewP(), STRDEF("a"), STRDEF("b"));
@ -278,7 +290,7 @@ testRun(void)
IoWrite *service = hrnServerScriptBegin(HRN_FORK_PARENT_WRITE(0));
// -----------------------------------------------------------------------------------------------------------------
TEST_TITLE("test against local host");
TEST_TITLE("test against local host with path-style URIs");
StringList *argList = strLstNew();
hrnCfgArgRawZ(argList, cfgOptStanza, "test");