fs: allow global variables to be overriden or set on backend creation

This allows backend config to contain

- `override.var` - set var during remote creation only
- `global.var` - set var in the global config permanently

Fixes #8563

docs/content/docs.md

@@ -437,6 +437,103 @@ Do not use single character names on Windows as it creates ambiguity with Window
 drives' names, e.g.: remote called `C` is indistinguishable from `C` drive. Rclone
 will always assume that single letter name refers to a drive.
 
+## Adding global configuration to a remote
+
+It is possible to add global configuration to the remote configuration which
+will be applied just before the remote is created.
+
+This can be done in two ways. The first is to use `override.var = value` in the
+config file or the connection string for a temporary change, and the second is
+to use `global.var = value` in the config file or connection string for a
+permanent change.
+
+This is explained fully below.
+
+### override.var
+
+This is used to override a global variable **just** for the duration of the
+remote creation. It won't affect other remotes even if they are created at the
+same time.
+
+This is very useful for overriding networking config needed for just for that
+remote. For example, say you have a remote which needs `--no-check-certificate`
+as it is running on test infrastructure without a proper certificate. You could
+supply the `--no-check-certificate` flag to rclone, but this will affect **all**
+the remotes. To make it just affect this remote you use an override. You could
+put this in the config file:
+
+```ini
+[remote]
+type = XXX
+...
+override.no_check_certificate = true
+```
+
+or use it in the connection string `remote,override.no_check_certificate=true:`
+(or just `remote,override.no_check_certificate:`).
+
+Note how the global flag name loses its initial `--` and gets `-` replaced with
+`_` and gets an `override.` prefix.
+
+Not all global variables make sense to be overridden like this as the config is
+only applied during the remote creation. Here is a non exhaustive list of ones
+which might be useful:
+
+- `bind_addr`
+- `ca_cert`
+- `client_cert`
+- `client_key`
+- `connect_timeout`
+- `disable_http2`
+- `disable_http_keep_alives`
+- `dump`
+- `expect_continue_timeout`
+- `headers`
+- `http_proxy`
+- `low_level_retries`
+- `max_connections`
+- `no_check_certificate`
+- `no_gzip`
+- `timeout`
+- `traffic_class`
+- `use_cookies`
+- `use_server_modtime`
+- `user_agent`
+
+An `override.var` will override all other config methods, but **just** for the
+duration of the creation of the remote.
+
+### global.var
+
+This is used to set a global variable **for everything**. The global variable is
+set just before the remote is created.
+
+This is useful for parameters (eg sync parameters) which can't be set as an
+`override`. For example, say you have a remote where you would always like to
+use the `--checksum` flag. You could supply the `--checksum` flag to rclone on
+every command line, but instead you could put this in the config file:
+
+```ini
+[remote]
+type = XXX
+...
+global.checksum = true
+```
+
+or use it in the connection string `remote,global.checksum=true:` (or just
+`remote,global.checksum:`). This is equivalent to using the `--checksum` flag.
+
+Note how the global flag name loses its initial `--` and gets `-` replaced with
+`_` and gets a `global.` prefix.
+
+Any global variable can be set like this and it is exactly equivalent to using
+the equivalent flag on the command line. This means it will affect all uses of
+rclone.
+
+If two remotes set the same global variable then the first one instantiated will
+be overridden by the second one. A `global.var` will override all other config
+methods when the remote is created.
+
 ## Quoting and the shell
 
 When you are typing commands to your computer you are using something

fs/fspath/path.go

@@ -20,7 +20,7 @@ const (
 var (
 	errInvalidCharacters = errors.New("config name contains invalid characters - may only contain numbers, letters, `_`, `-`, `.`, `+`, `@` and space, while not start with `-` or space, and not end with space")
 	errCantBeEmpty       = errors.New("can't use empty string as a path")
-	errBadConfigParam    = errors.New("config parameters may only contain `0-9`, `A-Z`, `a-z` and `_`")
+	errBadConfigParam    = errors.New("config parameters may only contain `0-9`, `A-Z`, `a-z`, `_` and `.`")
 	errEmptyConfigParam  = errors.New("config parameters can't be empty")
 	errConfigNameEmpty   = errors.New("config name can't be empty")
 	errConfigName        = errors.New("config name needs a trailing `:`")
@@ -79,7 +79,8 @@ func isConfigParam(c rune) bool {
 	return ((c >= 'a' && c <= 'z') ||
 		(c >= 'A' && c <= 'Z') ||
 		(c >= '0' && c <= '9') ||
-		c == '_')
+		c == '_' ||
+		c == '.')
 }
 
 // Parsed is returned from Parse with the results of the connection string decomposition

fs/newfs.go

@@ -7,12 +7,15 @@ import (
 	"crypto/md5"
 	"encoding/base64"
 	"fmt"
+	"maps"
 	"os"
 	"path/filepath"
+	"slices"
 	"strings"
 	"sync"
 
 	"github.com/rclone/rclone/fs/config/configmap"
+	"github.com/rclone/rclone/fs/config/configstruct"
 	"github.com/rclone/rclone/fs/fspath"
 )
 
@@ -65,6 +68,10 @@ func NewFs(ctx context.Context, path string) (Fs, error) {
 		overriddenConfig[suffix] = extraConfig
 		overriddenConfigMu.Unlock()
 	}
+	ctx, err = addConfigToContext(ctx, configName, config)
+	if err != nil {
+		return nil, err
+	}
 	f, err := fsInfo.NewFs(ctx, configName, fsPath, config)
 	if f != nil && (err == nil || err == ErrorIsFile) {
 		addReverse(f, fsInfo)
@@ -72,6 +79,54 @@ func NewFs(ctx context.Context, path string) (Fs, error) {
 	return f, err
 }
 
+// Add "global" config or "override" to ctx and the global config if required.
+//
+// This looks through keys prefixed with "global." or "override." in
+// config and sets ctx and optionally the global context if "global.".
+func addConfigToContext(ctx context.Context, configName string, config configmap.Getter) (newCtx context.Context, err error) {
+	overrideConfig := make(configmap.Simple)
+	globalConfig := make(configmap.Simple)
+	for i := range ConfigOptionsInfo {
+		opt := &ConfigOptionsInfo[i]
+		globalName := "global." + opt.Name
+		value, isSet := config.Get(globalName)
+		if isSet {
+			// Set both override and global if global
+			overrideConfig[opt.Name] = value
+			globalConfig[opt.Name] = value
+		}
+		overrideName := "override." + opt.Name
+		value, isSet = config.Get(overrideName)
+		if isSet {
+			overrideConfig[opt.Name] = value
+		}
+	}
+	if len(overrideConfig) == 0 && len(globalConfig) == 0 {
+		return ctx, nil
+	}
+	newCtx, ci := AddConfig(ctx)
+	overrideKeys := slices.Collect(maps.Keys(overrideConfig))
+	slices.Sort(overrideKeys)
+	globalKeys := slices.Collect(maps.Keys(globalConfig))
+	slices.Sort(globalKeys)
+	// Set the config in the newCtx
+	err = configstruct.Set(overrideConfig, ci)
+	if err != nil {
+		return ctx, fmt.Errorf("failed to set override config variables %q: %w", overrideKeys, err)
+	}
+	Debugf(configName, "Set overridden config %q for backend startup", overrideKeys)
+	// Set the global context only
+	if len(globalConfig) != 0 {
+		globalCI := GetConfig(context.Background())
+		err = configstruct.Set(globalConfig, globalCI)
+		if err != nil {
+			return ctx, fmt.Errorf("failed to set global config variables %q: %w", globalKeys, err)
+		}
+		Debugf(configName, "Set global config %q at backend startup", overrideKeys)
+	}
+	return newCtx, nil
+}
+
 // ConfigFs makes the config for calling NewFs with.
 //
 // It parses the path which is of the form remote:path

fs/newfs_internal_test.go

@@ -0,0 +1,55 @@
+package fs
+
+import (
+	"context"
+	"testing"
+
+	"github.com/rclone/rclone/fs/config/configmap"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// When no override/global keys exist, ctx must be returned unchanged.
+func TestAddConfigToContext_NoChanges(t *testing.T) {
+	ctx := context.Background()
+	newCtx, err := addConfigToContext(ctx, "unit-test", configmap.Simple{})
+	require.NoError(t, err)
+	assert.Equal(t, newCtx, ctx)
+}
+
+// A single override.key must create a new ctx, but leave the
+// background ctx untouched.
+func TestAddConfigToContext_OverrideOnly(t *testing.T) {
+	override := configmap.Simple{
+		"override.user_agent": "potato",
+	}
+	ctx := context.Background()
+	globalCI := GetConfig(ctx)
+	original := globalCI.UserAgent
+	newCtx, err := addConfigToContext(ctx, "unit-test", override)
+	require.NoError(t, err)
+	assert.NotEqual(t, newCtx, ctx)
+	assert.Equal(t, original, globalCI.UserAgent)
+	ci := GetConfig(newCtx)
+	assert.Equal(t, "potato", ci.UserAgent)
+}
+
+// A single global.key must create a new ctx and update the
+// background/global config.
+func TestAddConfigToContext_GlobalOnly(t *testing.T) {
+	global := configmap.Simple{
+		"global.user_agent": "potato2",
+	}
+	ctx := context.Background()
+	globalCI := GetConfig(ctx)
+	original := globalCI.UserAgent
+	defer func() {
+		globalCI.UserAgent = original
+	}()
+	newCtx, err := addConfigToContext(ctx, "unit-test", global)
+	require.NoError(t, err)
+	assert.NotEqual(t, newCtx, ctx)
+	assert.Equal(t, "potato2", globalCI.UserAgent)
+	ci := GetConfig(newCtx)
+	assert.Equal(t, "potato2", ci.UserAgent)
+}

fs/newfs_test.go

@@ -42,4 +42,21 @@ func TestNewFs(t *testing.T) {
 
 	assert.Equal(t, ":mockfs{S_NHG}:/tmp", fs.ConfigString(f3))
 	assert.Equal(t, ":mockfs,potato='true':/tmp", fs.ConfigStringFull(f3))
+
+	// Check that the overrides work
+	globalCI := fs.GetConfig(ctx)
+	original := globalCI.UserAgent
+	defer func() {
+		globalCI.UserAgent = original
+	}()
+
+	f4, err := fs.NewFs(ctx, ":mockfs,global.user_agent='julian':/tmp")
+	require.NoError(t, err)
+	assert.Equal(t, ":mockfs", f4.Name())
+	assert.Equal(t, "/tmp", f4.Root())
+
+	assert.Equal(t, ":mockfs:/tmp", fs.ConfigString(f4))
+	assert.Equal(t, ":mockfs:/tmp", fs.ConfigStringFull(f4))
+
+	assert.Equal(t, "julian", globalCI.UserAgent)
 }