Add repo-s3-port option for setting a non-standard S3 service port.

If this option is set then ports appended to repo-s3-endpoint or repo-s3-host will be ignored.

Setting this option explicitly may be the only way to use a bare ipv6 address with S3 (since multiple colons confuse the parser) but we plan to improve this in the future.

build/lib/pgBackRestBuild/Config/Data.pm

@@ -287,6 +287,8 @@ use constant CFGOPT_REPO_S3_ENDPOINT                                => CFGDEF_RE
     push @EXPORT, qw(CFGOPT_REPO_S3_ENDPOINT);
 use constant CFGOPT_REPO_S3_HOST                                    => CFGDEF_REPO_S3 . '-host';
     push @EXPORT, qw(CFGOPT_REPO_S3_HOST);
+use constant CFGOPT_REPO_S3_PORT                                    => CFGDEF_REPO_S3 . '-port';
+    push @EXPORT, qw(CFGOPT_REPO_S3_PORT);
 use constant CFGOPT_REPO_S3_REGION                                  => CFGDEF_REPO_S3 . '-region';
     push @EXPORT, qw(CFGOPT_REPO_S3_REGION);
 use constant CFGOPT_REPO_S3_TOKEN                                   => CFGDEF_REPO_S3 . '-token';
@@ -1862,6 +1864,18 @@ my %hConfigDefine =
         &CFGDEF_COMMAND => CFGOPT_REPO_TYPE,
     },
 
+    &CFGOPT_REPO_S3_PORT =>
+    {
+        &CFGDEF_SECTION => CFGDEF_SECTION_GLOBAL,
+        &CFGDEF_TYPE => CFGDEF_TYPE_INTEGER,
+        &CFGDEF_PREFIX => CFGDEF_PREFIX_REPO,
+        &CFGDEF_INDEX_TOTAL => CFGDEF_INDEX_REPO,
+        &CFGDEF_DEFAULT => 443,
+        &CFGDEF_ALLOW_RANGE => [1, 65535],
+        &CFGDEF_DEPEND => CFGOPT_REPO_S3_BUCKET,
+        &CFGDEF_COMMAND => CFGOPT_REPO_TYPE,
+    },
+
     &CFGOPT_REPO_S3_REGION,
     {
         &CFGDEF_INHERIT => CFGOPT_REPO_S3_BUCKET,

doc/xml/reference.xml

@@ -437,6 +437,15 @@
                         <example>127.0.0.1</example>
                     </config-key>
 
+                    <!-- CONFIG - REPO SECTION - REPO-S3-PORT KEY -->
+                    <config-key id="repo-s3-port" name="S3 Repository Port">
+                        <summary>S3 repository port.</summary>
+
+                        <text>Port to use when connecting to the endpoint (or host if specified).</text>
+
+                        <example>9000</example>
+                    </config-key>
+
                     <!-- CONFIG - REPO SECTION - REPO-S3-REGION KEY -->
                     <config-key id="repo-s3-region" name="S3 Repository Region">
                         <summary>S3 repository region.</summary>

doc/xml/release.xml

@@ -33,6 +33,12 @@
                     </release-item>
                 </release-bug-list>
 
+                <release-feature-list>
+                    <release-item>
+                        <p>Add <br-option>repo-s3-port</br-option> option for setting a non-standard <proper>S3</proper> service port.</p>
+                    </release-item>
+                </release-feature-list>
+
                 <release-improvement-list>
                     <release-item>
                         <release-item-contributor-list>

lib/pgBackRest/LibCAuto.pm

@@ -271,6 +271,7 @@ sub libcAutoExportTag
             'CFGOPT_REPO_S3_HOST',
             'CFGOPT_REPO_S3_KEY',
             'CFGOPT_REPO_S3_KEY_SECRET',
+            'CFGOPT_REPO_S3_PORT',
             'CFGOPT_REPO_S3_REGION',
             'CFGOPT_REPO_S3_TOKEN',
             'CFGOPT_REPO_S3_VERIFY_TLS',

src/config/config.auto.c

@@ -447,6 +447,7 @@ STRING_EXTERN(CFGOPT_REPO1_S3_ENDPOINT_STR,                         CFGOPT_REPO1
 STRING_EXTERN(CFGOPT_REPO1_S3_HOST_STR,                             CFGOPT_REPO1_S3_HOST);
 STRING_EXTERN(CFGOPT_REPO1_S3_KEY_STR,                              CFGOPT_REPO1_S3_KEY);
 STRING_EXTERN(CFGOPT_REPO1_S3_KEY_SECRET_STR,                       CFGOPT_REPO1_S3_KEY_SECRET);
+STRING_EXTERN(CFGOPT_REPO1_S3_PORT_STR,                             CFGOPT_REPO1_S3_PORT);
 STRING_EXTERN(CFGOPT_REPO1_S3_REGION_STR,                           CFGOPT_REPO1_S3_REGION);
 STRING_EXTERN(CFGOPT_REPO1_S3_TOKEN_STR,                            CFGOPT_REPO1_S3_TOKEN);
 STRING_EXTERN(CFGOPT_REPO1_S3_VERIFY_TLS_STR,                       CFGOPT_REPO1_S3_VERIFY_TLS);
@@ -1634,6 +1635,14 @@ static ConfigOptionData configOptionData[CFG_OPTION_TOTAL] = CONFIG_OPTION_LIST
         CONFIG_OPTION_DEFINE_ID(cfgDefOptRepoS3KeySecret)
     )
 
+    //------------------------------------------------------------------------------------------------------------------------------
+    CONFIG_OPTION
+    (
+        CONFIG_OPTION_NAME(CFGOPT_REPO1_S3_PORT)
+        CONFIG_OPTION_INDEX(0)
+        CONFIG_OPTION_DEFINE_ID(cfgDefOptRepoS3Port)
+    )
+
     //------------------------------------------------------------------------------------------------------------------------------
     CONFIG_OPTION
     (

src/config/config.auto.h

@@ -343,6 +343,8 @@ Option constants
     STRING_DECLARE(CFGOPT_REPO1_S3_KEY_STR);
 #define CFGOPT_REPO1_S3_KEY_SECRET                                  "repo1-s3-key-secret"
     STRING_DECLARE(CFGOPT_REPO1_S3_KEY_SECRET_STR);
+#define CFGOPT_REPO1_S3_PORT                                        "repo1-s3-port"
+    STRING_DECLARE(CFGOPT_REPO1_S3_PORT_STR);
 #define CFGOPT_REPO1_S3_REGION                                      "repo1-s3-region"
     STRING_DECLARE(CFGOPT_REPO1_S3_REGION_STR);
 #define CFGOPT_REPO1_S3_TOKEN                                       "repo1-s3-token"
@@ -386,7 +388,7 @@ Option constants
 #define CFGOPT_TYPE                                                 "type"
     STRING_DECLARE(CFGOPT_TYPE_STR);
 
-#define CFG_OPTION_TOTAL                                            166
+#define CFG_OPTION_TOTAL                                            167
 
 /***********************************************************************************************************************************
 Command enum
@@ -565,6 +567,7 @@ typedef enum
     cfgOptRepoS3Host,
     cfgOptRepoS3Key,
     cfgOptRepoS3KeySecret,
+    cfgOptRepoS3Port,
     cfgOptRepoS3Region,
     cfgOptRepoS3Token,
     cfgOptRepoS3VerifyTls,

src/config/define.auto.c

@@ -3720,6 +3720,60 @@ static ConfigDefineOptionData configDefineOptionData[] = CFGDEFDATA_OPTION_LIST
         )
     )
 
+    // -----------------------------------------------------------------------------------------------------------------------------
+    CFGDEFDATA_OPTION
+    (
+        CFGDEFDATA_OPTION_NAME("repo-s3-port")
+        CFGDEFDATA_OPTION_REQUIRED(true)
+        CFGDEFDATA_OPTION_SECTION(cfgDefSectionGlobal)
+        CFGDEFDATA_OPTION_TYPE(cfgDefOptTypeInteger)
+        CFGDEFDATA_OPTION_INTERNAL(false)
+
+        CFGDEFDATA_OPTION_INDEX_TOTAL(1)
+        CFGDEFDATA_OPTION_SECURE(false)
+
+        CFGDEFDATA_OPTION_HELP_SECTION("repository")
+        CFGDEFDATA_OPTION_HELP_SUMMARY("S3 repository port.")
+        CFGDEFDATA_OPTION_HELP_DESCRIPTION
+        (
+            "Port to use when connecting to the endpoint (or host if specified)."
+        )
+
+        CFGDEFDATA_OPTION_COMMAND_LIST
+        (
+            CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGet)
+            CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchiveGetAsync)
+            CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchivePush)
+            CFGDEFDATA_OPTION_COMMAND(cfgDefCmdArchivePushAsync)
+            CFGDEFDATA_OPTION_COMMAND(cfgDefCmdBackup)
+            CFGDEFDATA_OPTION_COMMAND(cfgDefCmdCheck)
+            CFGDEFDATA_OPTION_COMMAND(cfgDefCmdExpire)
+            CFGDEFDATA_OPTION_COMMAND(cfgDefCmdInfo)
+            CFGDEFDATA_OPTION_COMMAND(cfgDefCmdLocal)
+            CFGDEFDATA_OPTION_COMMAND(cfgDefCmdLs)
+            CFGDEFDATA_OPTION_COMMAND(cfgDefCmdRemote)
+            CFGDEFDATA_OPTION_COMMAND(cfgDefCmdRestore)
+            CFGDEFDATA_OPTION_COMMAND(cfgDefCmdStanzaCreate)
+            CFGDEFDATA_OPTION_COMMAND(cfgDefCmdStanzaDelete)
+            CFGDEFDATA_OPTION_COMMAND(cfgDefCmdStanzaUpgrade)
+            CFGDEFDATA_OPTION_COMMAND(cfgDefCmdStart)
+            CFGDEFDATA_OPTION_COMMAND(cfgDefCmdStop)
+        )
+
+        CFGDEFDATA_OPTION_OPTIONAL_LIST
+        (
+            CFGDEFDATA_OPTION_OPTIONAL_ALLOW_RANGE(1, 65535)
+            CFGDEFDATA_OPTION_OPTIONAL_DEPEND_LIST
+            (
+                cfgDefOptRepoType,
+                "s3"
+            )
+
+            CFGDEFDATA_OPTION_OPTIONAL_DEFAULT("443")
+            CFGDEFDATA_OPTION_OPTIONAL_PREFIX("repo")
+        )
+    )
+
     // -----------------------------------------------------------------------------------------------------------------------------
     CFGDEFDATA_OPTION
     (

src/config/define.auto.h

@@ -127,6 +127,7 @@ typedef enum
     cfgDefOptRepoS3Host,
     cfgDefOptRepoS3Key,
     cfgDefOptRepoS3KeySecret,
+    cfgDefOptRepoS3Port,
     cfgDefOptRepoS3Region,
     cfgDefOptRepoS3Token,
     cfgDefOptRepoS3VerifyTls,

src/config/parse.auto.c

@@ -2008,6 +2008,18 @@ static const struct option optionList[] =
         .val = PARSE_OPTION_FLAG | PARSE_DEPRECATE_FLAG | cfgOptRepoS3KeySecret,
     },
 
+    // repo-s3-port option
+    // -----------------------------------------------------------------------------------------------------------------------------
+    {
+        .name = CFGOPT_REPO1_S3_PORT,
+        .has_arg = required_argument,
+        .val = PARSE_OPTION_FLAG | cfgOptRepoS3Port,
+    },
+    {
+        .name = "reset-" CFGOPT_REPO1_S3_PORT,
+        .val = PARSE_OPTION_FLAG | PARSE_RESET_FLAG | cfgOptRepoS3Port,
+    },
+
     // repo-s3-region option and deprecations
     // -----------------------------------------------------------------------------------------------------------------------------
     {
@@ -2421,6 +2433,7 @@ static const ConfigOption optionResolveOrder[] =
     cfgOptRepoS3Host,
     cfgOptRepoS3Key,
     cfgOptRepoS3KeySecret,
+    cfgOptRepoS3Port,
     cfgOptRepoS3Region,
     cfgOptRepoS3Token,
     cfgOptRepoS3VerifyTls,

src/perl/embed.auto.c

@@ -8226,6 +8226,7 @@ static const EmbeddedModule embeddedModule[] =
             "'CFGOPT_REPO_S3_HOST',\n"
             "'CFGOPT_REPO_S3_KEY',\n"
             "'CFGOPT_REPO_S3_KEY_SECRET',\n"
+            "'CFGOPT_REPO_S3_PORT',\n"
             "'CFGOPT_REPO_S3_REGION',\n"
             "'CFGOPT_REPO_S3_TOKEN',\n"
             "'CFGOPT_REPO_S3_VERIFY_TLS',\n"

src/storage/helper.c

@@ -322,12 +322,16 @@ storageRepoGet(const String *type, bool write)
     else if (strEqZ(type, STORAGE_TYPE_S3))
     {
         // Set the default port
-        unsigned int port = STORAGE_S3_PORT_DEFAULT;
+        unsigned int port = cfgOptionUInt(cfgOptRepoS3Port);
 
         // Extract port from the endpoint and host if it is present
         const String *endPoint = cfgOptionHostPort(cfgOptRepoS3Endpoint, &port);
         const String *host = cfgOptionHostPort(cfgOptRepoS3Host, &port);
 
+        // If the port option was set explicitly then use it in preference to appended ports
+        if (cfgOptionSource(cfgOptRepoS3Port) != cfgSourceDefault)
+            port = cfgOptionUInt(cfgOptRepoS3Port);
+
         result = storageS3New(
             cfgOptionStr(cfgOptRepoPath), write, storageRepoPathExpression, cfgOptionStr(cfgOptRepoS3Bucket), endPoint,
             cfgOptionStr(cfgOptRepoS3Region), cfgOptionStr(cfgOptRepoS3Key), cfgOptionStr(cfgOptRepoS3KeySecret),

src/storage/s3/storage.h

@@ -15,7 +15,6 @@ Storage type
 /***********************************************************************************************************************************
 Defaults
 ***********************************************************************************************************************************/
-#define STORAGE_S3_PORT_DEFAULT                                     443
 #define STORAGE_S3_TIMEOUT_DEFAULT                                  60000
 #define STORAGE_S3_PARTSIZE_MIN                                     ((size_t)5 * 1024 * 1024)
 #define STORAGE_S3_DELETE_MAX                                       1000

test/src/module/command/helpTest.c

@@ -208,6 +208,7 @@ testRun(void)
             "  --repo-s3-host                   s3 repository host\n"
             "  --repo-s3-key                    s3 repository access key\n"
             "  --repo-s3-key-secret             s3 repository secret access key\n"
+            "  --repo-s3-port                   s3 repository port [default=443]\n"
             "  --repo-s3-region                 s3 repository region\n"
             "  --repo-s3-token                  s3 repository security token\n"
             "  --repo-s3-verify-tls             verify S3 server certificate [default=y]\n"

test/src/module/storage/s3Test.c

@@ -599,6 +599,39 @@ testRun(void)
             strPtr(((StorageS3 *)storage->driver)->secretAccessKey), strPtr(secretAccessKey), "    check secret access key");
         TEST_RESULT_STR(
             strPtr(((StorageS3 *)storage->driver)->securityToken), strPtr(securityToken), "    check security token");
+
+        // Use the port option to override both
+        // -------------------------------------------------------------------------------------------------------------------------
+        argList = strLstNew();
+        strLstAddZ(argList, "pgbackrest");
+        strLstAddZ(argList, "--stanza=db");
+        strLstAddZ(argList, "--repo1-type=s3");
+        strLstAdd(argList, strNewFmt("--repo1-path=%s", strPtr(path)));
+        strLstAdd(argList, strNewFmt("--repo1-s3-bucket=%s", strPtr(bucket)));
+        strLstAdd(argList, strNewFmt("--repo1-s3-region=%s", strPtr(region)));
+        strLstAdd(argList, strNewFmt("--repo1-s3-endpoint=%s:999", strPtr(endPoint)));
+        strLstAdd(argList, strNewFmt("--repo1-s3-host=%s:7777", strPtr(host)));
+        strLstAddZ(argList, "--repo1-s3-port=9001");
+        strLstAddZ(argList, "--repo1-s3-ca-path=" TLS_CERT_FAKE_PATH);
+        strLstAddZ(argList, "--repo1-s3-ca-file=" TLS_CERT_FAKE_PATH "/pgbackrest-test.crt");
+        setenv("PGBACKREST_REPO1_S3_KEY", strPtr(accessKey), true);
+        setenv("PGBACKREST_REPO1_S3_KEY_SECRET", strPtr(secretAccessKey), true);
+        setenv("PGBACKREST_REPO1_S3_TOKEN", strPtr(securityToken), true);
+        strLstAddZ(argList, "archive-get");
+        harnessCfgLoad(strLstSize(argList), strLstPtr(argList));
+
+        TEST_ASSIGN(storage, storageRepoGet(strNew(STORAGE_TYPE_S3), false), "get S3 repo storage with options");
+        TEST_RESULT_STR(strPtr(((StorageS3 *)storage->driver)->bucket), strPtr(bucket), "    check bucket");
+        TEST_RESULT_STR(strPtr(((StorageS3 *)storage->driver)->region), strPtr(region), "    check region");
+        TEST_RESULT_STR(
+            strPtr(((StorageS3 *)storage->driver)->bucketEndpoint), strPtr(strNewFmt("%s.%s", strPtr(bucket), strPtr(endPoint))),
+            "    check host");
+        TEST_RESULT_UINT(((StorageS3 *)storage->driver)->port, 9001, "    check port");
+        TEST_RESULT_STR(strPtr(((StorageS3 *)storage->driver)->accessKey), strPtr(accessKey), "    check access key");
+        TEST_RESULT_STR(
+            strPtr(((StorageS3 *)storage->driver)->secretAccessKey), strPtr(secretAccessKey), "    check secret access key");
+        TEST_RESULT_STR(
+            strPtr(((StorageS3 *)storage->driver)->securityToken), strPtr(securityToken), "    check security token");
     }
 
     // *****************************************************************************************************************************