db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2024-12-12 10:04:14 +02:00
Code Issues Releases Activity

Add repo-azure-endpoint option.

Browse Source 
This option allows alternate endpoints (e.g. Azure Government) to be configured.
This commit is contained in:
David Steele 2020-10-06 17:15:48 -04:00 committed by GitHub
parent 3ce6b3e505
commit 9377d05072
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 131 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
build/lib/pgBackRestBuild/Config/Data.pm
View File

@ -218,6 +218,7 @@ use constant CFGOPT_REPO_AZURE_ACCOUNT => CFGDEF_RE
use constant CFGOPT_REPO_AZURE_CA_FILE => CFGDEF_REPO_AZURE . '-ca-file';
use constant CFGOPT_REPO_AZURE_CA_PATH => CFGDEF_REPO_AZURE . '-ca-path';
use constant CFGOPT_REPO_AZURE_CONTAINER => CFGDEF_REPO_AZURE . '-container';
use constant CFGOPT_REPO_AZURE_ENDPOINT => CFGDEF_REPO_AZURE . '-endpoint';
use constant CFGOPT_REPO_AZURE_HOST => CFGDEF_REPO_AZURE . '-host';
use constant CFGOPT_REPO_AZURE_KEY => CFGDEF_REPO_AZURE . '-key';
use constant CFGOPT_REPO_AZURE_KEY_TYPE => CFGDEF_REPO_AZURE . '-key-type';
@ -1826,6 +1827,12 @@ my %hConfigDefine =
&CFGDEF_COMMAND => CFGOPT_REPO_TYPE,
},
&CFGOPT_REPO_AZURE_ENDPOINT =>
{
&CFGDEF_INHERIT => CFGOPT_REPO_AZURE_HOST,
&CFGDEF_DEFAULT => 'blob.core.windows.net',
},
&CFGOPT_REPO_AZURE_HOST =>
{
&CFGDEF_SECTION => CFGDEF_SECTION_GLOBAL,

9
doc/xml/reference.xml
View File

@ -348,6 +348,15 @@
<example>pg-backup</example>
</config-key>
<!-- ======================================================================================================= -->
<config-key id="repo-azure-endpoint" name="Azure Repository Endpoint">
<summary>Azure repository endpoint.</summary>
<text>Endpoint used to connect to the blob service. The default is generally correct unless using Azure Government.</text>
<example>blob.core.usgovcloudapi.net</example>
</config-key>
<!-- ======================================================================================================= -->
<config-key id="repo-azure-host" name="Azure Repository Host">
<summary>Azure repository host.</summary>

20
doc/xml/release.xml
View File

@ -13,6 +13,21 @@
<release-list>
<release date="XXXX-XX-XX" version="2.31dev" title="UNDER DEVELOPMENT">
<release-core-list>
<release-feature-list>
<release-item>
<release-item-contributor-list>
<release-item-ideator id="brian.peterson"/>
<release-item-contributor id="david.steele"/>
<release-item-reviewer id="cynthia.shang"/>
<!-- Actually tester, but we don't have a tag for that yet -->
<release-item-reviewer id="brian.peterson"/>
</release-item-contributor-list>
<p>Add <br-option>repo-azure-endpoint</br-option> option.</p>
</release-item>
</release-feature-list>
</release-core-list>
</release>
<release date="2020-10-05" version="2.30" title="PostgreSQL 13 Support">
@ -8727,6 +8742,11 @@
<contributor-id type="github">scrummyin</contributor-id>
</contributor>
<contributor id="brian.peterson">
<contributor-name-display>Brian Peterson</contributor-name-display>
<contributor-id type="github">brimo2k</contributor-id>
</contributor>
<contributor id="bruce.burdick">
<contributor-name-display>Bruce Burdick</contributor-name-display>
<contributor-id type="github">baburdick</contributor-id>

9
src/config/config.auto.c
View File

@ -447,6 +447,7 @@ STRING_EXTERN(CFGOPT_REPO1_AZURE_ACCOUNT_STR, CFGOPT_REPO1
STRING_EXTERN(CFGOPT_REPO1_AZURE_CA_FILE_STR, CFGOPT_REPO1_AZURE_CA_FILE);
STRING_EXTERN(CFGOPT_REPO1_AZURE_CA_PATH_STR, CFGOPT_REPO1_AZURE_CA_PATH);
STRING_EXTERN(CFGOPT_REPO1_AZURE_CONTAINER_STR, CFGOPT_REPO1_AZURE_CONTAINER);
STRING_EXTERN(CFGOPT_REPO1_AZURE_ENDPOINT_STR, CFGOPT_REPO1_AZURE_ENDPOINT);
STRING_EXTERN(CFGOPT_REPO1_AZURE_HOST_STR, CFGOPT_REPO1_AZURE_HOST);
STRING_EXTERN(CFGOPT_REPO1_AZURE_KEY_STR, CFGOPT_REPO1_AZURE_KEY);
STRING_EXTERN(CFGOPT_REPO1_AZURE_KEY_TYPE_STR, CFGOPT_REPO1_AZURE_KEY_TYPE);
@ -1709,6 +1710,14 @@ static ConfigOptionData configOptionData[CFG_OPTION_TOTAL] = CONFIG_OPTION_LIST
CONFIG_OPTION_DEFINE_ID(cfgDefOptRepoAzureContainer)
)
//------------------------------------------------------------------------------------------------------------------------------
CONFIG_OPTION
(
CONFIG_OPTION_NAME(CFGOPT_REPO1_AZURE_ENDPOINT)
CONFIG_OPTION_INDEX(0)
CONFIG_OPTION_DEFINE_ID(cfgDefOptRepoAzureEndpoint)
)
//------------------------------------------------------------------------------------------------------------------------------
CONFIG_OPTION
(

5
src/config/config.auto.h
View File

@ -355,6 +355,8 @@ Option constants
STRING_DECLARE(CFGOPT_REPO1_AZURE_CA_PATH_STR);
#define CFGOPT_REPO1_AZURE_CONTAINER "repo1-azure-container"
STRING_DECLARE(CFGOPT_REPO1_AZURE_CONTAINER_STR);
#define CFGOPT_REPO1_AZURE_ENDPOINT "repo1-azure-endpoint"
STRING_DECLARE(CFGOPT_REPO1_AZURE_ENDPOINT_STR);
#define CFGOPT_REPO1_AZURE_HOST "repo1-azure-host"
STRING_DECLARE(CFGOPT_REPO1_AZURE_HOST_STR);
#define CFGOPT_REPO1_AZURE_KEY "repo1-azure-key"
@ -468,7 +470,7 @@ Option constants
#define CFGOPT_TYPE "type"
STRING_DECLARE(CFGOPT_TYPE_STR);
#define CFG_OPTION_TOTAL 206
#define CFG_OPTION_TOTAL 207
/***********************************************************************************************************************************
Command enum
@ -653,6 +655,7 @@ typedef enum
cfgOptRepoAzureCaFile,
cfgOptRepoAzureCaPath,
cfgOptRepoAzureContainer,
cfgOptRepoAzureEndpoint,
cfgOptRepoAzureHost,
cfgOptRepoAzureKey,
cfgOptRepoAzureKeyType,

54
src/config/define.auto.c
View File

@ -3240,6 +3240,60 @@ static ConfigDefineOptionData configDefineOptionData[] = CFGDEFDATA_OPTION_LIST
)
)
// -----------------------------------------------------------------------------------------------------------------------------
CFGDEFDATA_OPTION
(
CFGDEFDATA_OPTION_NAME("repo-azure-endpoint")
CFGDEFDATA_OPTION_REQUIRED(false)
CFGDEFDATA_OPTION_SECTION(cfgDefSectionGlobal)
CFGDEFDATA_OPTION_TYPE(cfgDefOptTypeString)
CFGDEFDATA_OPTION_INTERNAL(false)
CFGDEFDATA_OPTION_INDEX_TOTAL(1)
CFGDEFDATA_OPTION_SECURE(false)
CFGDEFDATA_OPTION_HELP_SECTION("repository")
CFGDEFDATA_OPTION_HELP_SUMMARY("Azure repository endpoint.")
CFGDEFDATA_OPTION_HELP_DESCRIPTION
(
"Endpoint used to connect to the blob service. The default is generally correct unless using Azure Government."
)
CFGDEFDATA_OPTION_COMMAND_LIST
(
CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGet)
CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchivePush)
CFGDEFDATA_OPTION_COMMAND(cfgDefCmdBackup)
CFGDEFDATA_OPTION_COMMAND(cfgDefCmdCheck)
CFGDEFDATA_OPTION_COMMAND(cfgDefCmdExpire)
CFGDEFDATA_OPTION_COMMAND(cfgDefCmdInfo)
CFGDEFDATA_OPTION_COMMAND(cfgDefCmdRepoCreate)
CFGDEFDATA_OPTION_COMMAND(cfgDefCmdRepoGet)
CFGDEFDATA_OPTION_COMMAND(cfgDefCmdRepoLs)
CFGDEFDATA_OPTION_COMMAND(cfgDefCmdRepoPut)
CFGDEFDATA_OPTION_COMMAND(cfgDefCmdRepoRm)
CFGDEFDATA_OPTION_COMMAND(cfgDefCmdRestore)
CFGDEFDATA_OPTION_COMMAND(cfgDefCmdStanzaCreate)
CFGDEFDATA_OPTION_COMMAND(cfgDefCmdStanzaDelete)
CFGDEFDATA_OPTION_COMMAND(cfgDefCmdStanzaUpgrade)
CFGDEFDATA_OPTION_COMMAND(cfgDefCmdStart)
CFGDEFDATA_OPTION_COMMAND(cfgDefCmdStop)
CFGDEFDATA_OPTION_COMMAND(cfgDefCmdVerify)
)
CFGDEFDATA_OPTION_OPTIONAL_LIST
(
CFGDEFDATA_OPTION_OPTIONAL_DEPEND_LIST
(
cfgDefOptRepoType,
"azure"
)
CFGDEFDATA_OPTION_OPTIONAL_DEFAULT("blob.core.windows.net")
CFGDEFDATA_OPTION_OPTIONAL_PREFIX("repo")
)
)
// -----------------------------------------------------------------------------------------------------------------------------
CFGDEFDATA_OPTION
(

1
src/config/define.auto.h
View File

@ -119,6 +119,7 @@ typedef enum
cfgDefOptRepoAzureCaFile,
cfgDefOptRepoAzureCaPath,
cfgDefOptRepoAzureContainer,
cfgDefOptRepoAzureEndpoint,
cfgDefOptRepoAzureHost,
cfgDefOptRepoAzureKey,
cfgDefOptRepoAzureKeyType,

13
src/config/parse.auto.c
View File

@ -1928,6 +1928,18 @@ static const struct option optionList[] =
.val = PARSE_OPTION_FLAG | PARSE_RESET_FLAG | cfgOptRepoAzureContainer,
},
// repo-azure-endpoint option
// -----------------------------------------------------------------------------------------------------------------------------
{
.name = CFGOPT_REPO1_AZURE_ENDPOINT,
.has_arg = required_argument,
.val = PARSE_OPTION_FLAG | cfgOptRepoAzureEndpoint,
},
{
.name = "reset-" CFGOPT_REPO1_AZURE_ENDPOINT,
.val = PARSE_OPTION_FLAG | PARSE_RESET_FLAG | cfgOptRepoAzureEndpoint,
},
// repo-azure-host option
// -----------------------------------------------------------------------------------------------------------------------------
{
@ -2908,6 +2920,7 @@ static const ConfigOption optionResolveOrder[] =
cfgOptRepoAzureCaFile,
cfgOptRepoAzureCaPath,
cfgOptRepoAzureContainer,
cfgOptRepoAzureEndpoint,
cfgOptRepoAzureHost,
cfgOptRepoAzureKey,
cfgOptRepoAzureKeyType,

13
src/storage/azure/storage.c
View File

@ -27,12 +27,6 @@ Storage type
***********************************************************************************************************************************/
STRING_EXTERN(STORAGE_AZURE_TYPE_STR, STORAGE_AZURE_TYPE);
/***********************************************************************************************************************************
Azure default hosts
***********************************************************************************************************************************/
#define AZURE_HOST "core.windows.net"
#define AZURE_BLOB_HOST "blob." AZURE_HOST
/***********************************************************************************************************************************
Azure http headers
***********************************************************************************************************************************/
@ -730,8 +724,8 @@ static const StorageInterface storageInterfaceAzure =
Storage *
storageAzureNew(
const String *path, bool write, StoragePathExpressionCallback pathExpressionFunction, const String *container,
const String *account, StorageAzureKeyType keyType, const String *key, size_t blockSize, const String *host, unsigned int port,
TimeMSec timeout, bool verifyPeer, const String *caFile, const String *caPath)
const String *account, StorageAzureKeyType keyType, const String *key, size_t blockSize, const String *host,
const String *endpoint, unsigned int port, TimeMSec timeout, bool verifyPeer, const String *caFile, const String *caPath)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(STRING, path);
@ -743,6 +737,7 @@ storageAzureNew(
FUNCTION_TEST_PARAM(STRING, key);
FUNCTION_LOG_PARAM(SIZE, blockSize);
FUNCTION_LOG_PARAM(STRING, host);
FUNCTION_LOG_PARAM(STRING, endpoint);
FUNCTION_LOG_PARAM(UINT, port);
FUNCTION_LOG_PARAM(TIME_MSEC, timeout);
FUNCTION_LOG_PARAM(BOOL, verifyPeer);
@ -769,7 +764,7 @@ storageAzureNew(
.container = strDup(container),
.account = strDup(account),
.blockSize = blockSize,
.host = host == NULL ? strNewFmt("%s." AZURE_BLOB_HOST, strZ(account)) : host,
.host = host == NULL ? strNewFmt("%s.%s", strZ(account), strZ(endpoint)) : host,
.uriPrefix = host == NULL ? strNewFmt("/%s", strZ(container)) : strNewFmt("/%s/%s", strZ(account), strZ(container)),
};

4
src/storage/azure/storage.h
View File

@ -34,7 +34,7 @@ Constructors
***********************************************************************************************************************************/
Storage *storageAzureNew(
const String *path, bool write, StoragePathExpressionCallback pathExpressionFunction, const String *container,
const String *account, StorageAzureKeyType keyType, const String *key, size_t blockSize, const String *host, unsigned int port,
TimeMSec timeout, bool verifyPeer, const String *caFile, const String *caPath);
const String *account, StorageAzureKeyType keyType, const String *key, size_t blockSize, const String *host,
const String *endpoint, unsigned int port, TimeMSec timeout, bool verifyPeer, const String *caFile, const String *caPath);
#endif

5
src/storage/helper.c
View File

@ -360,8 +360,9 @@ storageRepoGet(const String *type, bool write)
strEqZ(cfgOptionStr(cfgOptRepoAzureKeyType), STORAGE_AZURE_KEY_TYPE_SHARED) ?
storageAzureKeyTypeShared : storageAzureKeyTypeSas,
cfgOptionStr(cfgOptRepoAzureKey), STORAGE_AZURE_BLOCKSIZE_MIN, cfgOptionStrNull(cfgOptRepoAzureHost),
cfgOptionUInt(cfgOptRepoAzurePort), ioTimeoutMs(), cfgOptionBool(cfgOptRepoAzureVerifyTls),
cfgOptionStrNull(cfgOptRepoAzureCaFile), cfgOptionStrNull(cfgOptRepoAzureCaPath));
cfgOptionStr(cfgOptRepoAzureEndpoint), cfgOptionUInt(cfgOptRepoAzurePort), ioTimeoutMs(),
cfgOptionBool(cfgOptRepoAzureVerifyTls), cfgOptionStrNull(cfgOptRepoAzureCaFile),
cfgOptionStrNull(cfgOptRepoAzureCaPath));
}
// Use CIFS storage
else if (strEqZ(type, STORAGE_CIFS_TYPE))

2
test/src/module/command/helpTest.c
View File

@ -188,6 +188,8 @@ testRun(void)
" --repo-azure-ca-file azure repository TLS CA file\n"
" --repo-azure-ca-path azure repository TLS CA path\n"
" --repo-azure-container azure repository container\n"
" --repo-azure-endpoint azure repository endpoint\n"
" [default=blob.core.windows.net]\n"
" --repo-azure-host azure repository host\n"
" --repo-azure-key azure repository key\n"
" --repo-azure-key-type azure repository key type [default=shared]\n"

6
test/src/module/storage/azureTest.c
View File

@ -214,7 +214,7 @@ testRun(void)
(StorageAzure *)storageDriver(
storageAzureNew(
STRDEF("/repo"), false, NULL, TEST_CONTAINER_STR, TEST_ACCOUNT_STR, storageAzureKeyTypeShared,
TEST_KEY_SHARED_STR, 16, NULL, 443, 1000, true, NULL, NULL)),
TEST_KEY_SHARED_STR, 16, NULL, STRDEF("blob.core.windows.net"), 443, 1000, true, NULL, NULL)),
"new azure storage - shared key");
// -------------------------------------------------------------------------------------------------------------------------
@ -253,7 +253,7 @@ testRun(void)
(StorageAzure *)storageDriver(
storageAzureNew(
STRDEF("/repo"), false, NULL, TEST_CONTAINER_STR, TEST_ACCOUNT_STR, storageAzureKeyTypeSas, TEST_KEY_SAS_STR,
16, NULL, 443, 1000, true, NULL, NULL)),
16, NULL, STRDEF("blob.core.usgovcloudapi.net"), 443, 1000, true, NULL, NULL)),
"new azure storage - sas key");
query = httpQueryAdd(httpQueryNewP(), STRDEF("a"), STRDEF("b"));
@ -261,7 +261,7 @@ testRun(void)
TEST_RESULT_VOID(storageAzureAuth(storage, HTTP_VERB_GET_STR, STRDEF("/path/file"), query, dateTime, header), "auth");
TEST_RESULT_STR_Z(
httpHeaderToLog(header), "{content-length: '66', host: 'account.blob.core.windows.net'}", "check headers");
httpHeaderToLog(header), "{content-length: '66', host: 'account.blob.core.usgovcloudapi.net'}", "check headers");
TEST_RESULT_STR_Z(httpQueryRenderP(query), "a=b&sig=key", "check query");
}