From 2e21c58e6a6c69be8c2c46cda157d977388687c5 Mon Sep 17 00:00:00 2001
From: Nick Craig-Wood <nick@craig-wood.com>
Date: Thu, 5 Nov 2020 11:33:32 +0000
Subject: [PATCH] fs: deglobalise the config #4685
This is done by making fs.Config private and attaching it to the
context instead.
The Config should be obtained with fs.GetConfig and fs.AddConfig
should be used to get a new mutable config that can be changed.
---
backend/amazonclouddrive/amazonclouddrive.go | 15 +-
backend/azureblob/azureblob.go | 13 +-
backend/b2/b2.go | 17 +-
backend/box/box.go | 15 +-
backend/cache/cache_internal_test.go | 3 +-
backend/drive/drive.go | 23 +-
backend/dropbox/dropbox.go | 2 +-
backend/fichier/fichier.go | 4 +-
backend/fichier/fichier_test.go | 2 -
backend/filefabric/filefabric.go | 4 +-
backend/ftp/ftp.go | 83 +++----
.../googlecloudstorage/googlecloudstorage.go | 4 +-
backend/googlephotos/googlephotos.go | 4 +-
backend/http/http.go | 14 +-
backend/hubic/hubic.go | 7 +-
backend/jottacloud/jottacloud.go | 8 +-
backend/koofr/koofr.go | 2 +-
backend/mailru/mailru.go | 13 +-
backend/mega/mega.go | 7 +-
backend/onedrive/onedrive.go | 14 +-
backend/opendrive/opendrive.go | 4 +-
backend/pcloud/pcloud.go | 2 +-
backend/premiumizeme/premiumizeme.go | 4 +-
backend/putio/fs.go | 4 +-
backend/qingstor/qingstor.go | 6 +-
backend/s3/s3.go | 27 ++-
backend/seafile/pacer.go | 4 +-
backend/seafile/seafile.go | 9 +-
backend/sftp/sftp.go | 96 ++++----
backend/sharefile/sharefile.go | 13 +-
backend/sharefile/upload.go | 2 +-
backend/sugarsync/sugarsync.go | 6 +-
backend/swift/swift.go | 20 +-
backend/webdav/odrvcookie/fetch.go | 2 +-
backend/webdav/webdav.go | 4 +-
backend/yandex/yandex.go | 21 +-
cmd/cmd.go | 23 +-
cmd/config/config.go | 2 +-
cmd/help.go | 4 +-
cmd/info/info.go | 6 +-
cmd/lsd/lsd.go | 3 +-
cmd/ncdu/scan/scan.go | 3 +-
cmd/rc/rc.go | 2 +-
cmd/serve/sftp/server.go | 2 +-
cmd/tree/tree.go | 3 +-
fs/accounting/accounting.go | 20 +-
fs/accounting/accounting_test.go | 24 +-
fs/accounting/inprogress.go | 3 +-
fs/accounting/stats.go | 31 +--
fs/accounting/stats_groups.go | 5 +-
fs/accounting/stats_test.go | 3 +-
fs/accounting/token_bucket.go | 8 +-
fs/accounting/transfer.go | 6 +-
fs/accounting/transfermap.go | 5 +-
fs/asyncreader/asyncreader.go | 32 +--
fs/asyncreader/asyncreader_test.go | 35 ++-
fs/config.go | 33 ++-
fs/config/config.go | 59 ++---
fs/config/config_test.go | 42 ++--
fs/config/configflags/configflags.go | 200 ++++++++--------
fs/config_test.go | 29 +++
fs/filter/filter.go | 10 +-
fs/fs.go | 12 +-
fs/fs_test.go | 14 +-
fs/fshttp/http.go | 10 +-
fs/log.go | 15 +-
fs/log/log.go | 7 +-
fs/march/march.go | 30 ++-
fs/march/march_test.go | 4 +-
fs/operations/check.go | 13 +-
fs/operations/check_test.go | 29 ++-
fs/operations/dedupe.go | 10 +-
fs/operations/dedupe_test.go | 6 +-
fs/operations/lsjson.go | 2 +-
fs/operations/multithread.go | 13 +-
fs/operations/multithread_test.go | 70 +++---
fs/operations/operations.go | 173 ++++++++------
fs/operations/operations_internal_test.go | 11 +-
fs/operations/operations_test.go | 94 ++++----
fs/rc/config_test.go | 4 +-
fs/sync/sync.go | 91 +++----
fs/sync/sync_test.go | 226 ++++++++++--------
fs/walk/walk.go | 21 +-
fstest/fstest.go | 13 +-
fstest/fstests/fstests.go | 7 +-
fstest/test_all/run.go | 4 +-
lib/oauthutil/oauthutil.go | 4 +-
vfs/read.go | 5 +-
vfs/vfscache/cache.go | 2 +-
vfs/vfscache/downloaders/downloaders.go | 4 +-
vfs/vfscache/item.go | 4 +-
vfs/vfscache/writeback/writeback.go | 2 +-
vfs/vfscache/writeback/writeback_test.go | 4 +-
93 files changed, 1128 insertions(+), 847 deletions(-)
create mode 100644 fs/config_test.go
diff --git a/backend/amazonclouddrive/amazonclouddrive.go b/backend/amazonclouddrive/amazonclouddrive.go
index 21f109da7..67dbb7367 100644
--- a/backend/amazonclouddrive/amazonclouddrive.go
+++ b/backend/amazonclouddrive/amazonclouddrive.go
@@ -144,6 +144,7 @@ type Fs struct {
name string // name of this remote
features *fs.Features // optional features
opt Options // options for this Fs
+ ci *fs.ConfigInfo // global config
c *acd.Client // the connection to the acd server
noAuthClient *http.Client // unauthenticated http client
root string // the path we are working on
@@ -247,7 +248,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
return nil, err
}
root = parsePath(root)
- baseClient := fshttp.NewClient(fs.Config)
+ baseClient := fshttp.NewClient(fs.GetConfig(ctx))
if do, ok := baseClient.Transport.(interface {
SetRequestFilter(f func(req *http.Request))
}); ok {
@@ -261,13 +262,15 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
}
c := acd.NewClient(oAuthClient)
+ ci := fs.GetConfig(ctx)
f := &Fs{
name: name,
root: root,
opt: *opt,
+ ci: ci,
c: c,
- pacer: fs.NewPacer(pacer.NewAmazonCloudDrive(pacer.MinSleep(minSleep))),
- noAuthClient: fshttp.NewClient(fs.Config),
+ pacer: fs.NewPacer(ctx, pacer.NewAmazonCloudDrive(pacer.MinSleep(minSleep))),
+ noAuthClient: fshttp.NewClient(ci),
}
f.features = (&fs.Features{
CaseInsensitive: true,
@@ -501,7 +504,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
if err != nil {
return nil, err
}
- maxTries := fs.Config.LowLevelRetries
+ maxTries := f.ci.LowLevelRetries
var iErr error
for tries := 1; tries <= maxTries; tries++ {
entries = nil
@@ -716,7 +719,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
dstObj fs.Object
srcErr, dstErr error
)
- for i := 1; i <= fs.Config.LowLevelRetries; i++ {
+ for i := 1; i <= f.ci.LowLevelRetries; i++ {
_, srcErr = srcObj.fs.NewObject(ctx, srcObj.remote) // try reading the object
if srcErr != nil && srcErr != fs.ErrorObjectNotFound {
// exit if error on source
@@ -731,7 +734,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
// finished if src not found and dst found
break
}
- fs.Debugf(src, "Wait for directory listing to update after move %d/%d", i, fs.Config.LowLevelRetries)
+ fs.Debugf(src, "Wait for directory listing to update after move %d/%d", i, f.ci.LowLevelRetries)
time.Sleep(1 * time.Second)
}
return dstObj, dstErr
diff --git a/backend/azureblob/azureblob.go b/backend/azureblob/azureblob.go
index 096907db9..c65db6cf7 100644
--- a/backend/azureblob/azureblob.go
+++ b/backend/azureblob/azureblob.go
@@ -187,6 +187,7 @@ type Fs struct {
name string // name of this remote
root string // the path we are working on if any
opt Options // parsed config options
+ ci *fs.ConfigInfo // global config
features *fs.Features // optional features
client *http.Client // http client we are using
svcURL *azblob.ServiceURL // reference to serviceURL
@@ -409,18 +410,20 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
string(azblob.AccessTierHot), string(azblob.AccessTierCool), string(azblob.AccessTierArchive))
}
+ ci := fs.GetConfig(ctx)
f := &Fs{
name: name,
opt: *opt,
- pacer: fs.NewPacer(pacer.NewS3(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
- uploadToken: pacer.NewTokenDispenser(fs.Config.Transfers),
- client: fshttp.NewClient(fs.Config),
+ ci: ci,
+ pacer: fs.NewPacer(ctx, pacer.NewS3(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
+ uploadToken: pacer.NewTokenDispenser(ci.Transfers),
+ client: fshttp.NewClient(fs.GetConfig(ctx)),
cache: bucket.NewCache(),
cntURLcache: make(map[string]*azblob.ContainerURL, 1),
pool: pool.New(
time.Duration(opt.MemoryPoolFlushTime),
int(opt.ChunkSize),
- fs.Config.Transfers,
+ ci.Transfers,
opt.MemoryPoolUseMmap,
),
}
@@ -1035,7 +1038,7 @@ func (f *Fs) getMemoryPool(size int64) *pool.Pool {
return pool.New(
time.Duration(f.opt.MemoryPoolFlushTime),
int(size),
- fs.Config.Transfers,
+ f.ci.Transfers,
f.opt.MemoryPoolUseMmap,
)
}
diff --git a/backend/b2/b2.go b/backend/b2/b2.go
index e3dc38d44..bfec9fbfd 100644
--- a/backend/b2/b2.go
+++ b/backend/b2/b2.go
@@ -214,6 +214,7 @@ type Fs struct {
name string // name of this remote
root string // the path we are working on if any
opt Options // parsed config options
+ ci *fs.ConfigInfo // global config
features *fs.Features // optional features
srv *rest.Client // the connection to the b2 server
rootBucket string // bucket part of root (if any)
@@ -415,20 +416,22 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
if opt.Endpoint == "" {
opt.Endpoint = defaultEndpoint
}
+ ci := fs.GetConfig(ctx)
f := &Fs{
name: name,
opt: *opt,
- srv: rest.NewClient(fshttp.NewClient(fs.Config)).SetErrorHandler(errorHandler),
+ ci: ci,
+ srv: rest.NewClient(fshttp.NewClient(fs.GetConfig(ctx))).SetErrorHandler(errorHandler),
cache: bucket.NewCache(),
_bucketID: make(map[string]string, 1),
_bucketType: make(map[string]string, 1),
uploads: make(map[string][]*api.GetUploadURLResponse),
- pacer: fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
- uploadToken: pacer.NewTokenDispenser(fs.Config.Transfers),
+ pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
+ uploadToken: pacer.NewTokenDispenser(ci.Transfers),
pool: pool.New(
time.Duration(opt.MemoryPoolFlushTime),
int(opt.ChunkSize),
- fs.Config.Transfers,
+ ci.Transfers,
opt.MemoryPoolUseMmap,
),
}
@@ -1167,10 +1170,10 @@ func (f *Fs) purge(ctx context.Context, dir string, oldOnly bool) error {
}
// Delete Config.Transfers in parallel
- toBeDeleted := make(chan *api.File, fs.Config.Transfers)
+ toBeDeleted := make(chan *api.File, f.ci.Transfers)
var wg sync.WaitGroup
- wg.Add(fs.Config.Transfers)
- for i := 0; i < fs.Config.Transfers; i++ {
+ wg.Add(f.ci.Transfers)
+ for i := 0; i < f.ci.Transfers; i++ {
go func() {
defer wg.Done()
for object := range toBeDeleted {
diff --git a/backend/box/box.go b/backend/box/box.go
index 9f5eb90b9..dcbdaf40f 100644
--- a/backend/box/box.go
+++ b/backend/box/box.go
@@ -91,7 +91,7 @@ func init() {
var err error
// If using box config.json, use JWT auth
if ok && boxSubTypeOk && jsonFile != "" && boxSubType != "" {
- err = refreshJWTToken(jsonFile, boxSubType, name, m)
+ err = refreshJWTToken(ctx, jsonFile, boxSubType, name, m)
if err != nil {
log.Fatalf("Failed to configure token with jwt authentication: %v", err)
}
@@ -153,7 +153,7 @@ func init() {
})
}
-func refreshJWTToken(jsonFile string, boxSubType string, name string, m configmap.Mapper) error {
+func refreshJWTToken(ctx context.Context, jsonFile string, boxSubType string, name string, m configmap.Mapper) error {
jsonFile = env.ShellExpand(jsonFile)
boxConfig, err := getBoxConfig(jsonFile)
if err != nil {
@@ -169,7 +169,7 @@ func refreshJWTToken(jsonFile string, boxSubType string, name string, m configma
}
signingHeaders := getSigningHeaders(boxConfig)
queryParams := getQueryParams(boxConfig)
- client := fshttp.NewClient(fs.Config)
+ client := fshttp.NewClient(fs.GetConfig(ctx))
err = jwtutil.Config("box", name, claims, signingHeaders, queryParams, privateKey, m, client)
return err
}
@@ -386,7 +386,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
root = parsePath(root)
- client := fshttp.NewClient(fs.Config)
+ client := fshttp.NewClient(fs.GetConfig(ctx))
var ts *oauthutil.TokenSource
// If not using an accessToken, create an oauth client and tokensource
if opt.AccessToken == "" {
@@ -396,13 +396,14 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
}
}
+ ci := fs.GetConfig(ctx)
f := &Fs{
name: name,
root: root,
opt: *opt,
srv: rest.NewClient(client).SetRoot(rootURL),
- pacer: fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
- uploadToken: pacer.NewTokenDispenser(fs.Config.Transfers),
+ pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
+ uploadToken: pacer.NewTokenDispenser(ci.Transfers),
}
f.features = (&fs.Features{
CaseInsensitive: true,
@@ -423,7 +424,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
// should do so whether there are uploads pending or not.
if ok && boxSubTypeOk && jsonFile != "" && boxSubType != "" {
f.tokenRenewer = oauthutil.NewRenew(f.String(), ts, func() error {
- err := refreshJWTToken(jsonFile, boxSubType, name, m)
+ err := refreshJWTToken(ctx, jsonFile, boxSubType, name, m)
return err
})
f.tokenRenewer.Start()
diff --git a/backend/cache/cache_internal_test.go b/backend/cache/cache_internal_test.go
index 4d8e8643b..1cf6603e6 100644
--- a/backend/cache/cache_internal_test.go
+++ b/backend/cache/cache_internal_test.go
@@ -925,7 +925,8 @@ func (r *run) newCacheFs(t *testing.T, remote, id string, needRemote, purge bool
boltDb, err := cache.GetPersistent(runInstance.dbPath, runInstance.chunkPath, &cache.Features{PurgeDb: true})
require.NoError(t, err)
- fs.Config.LowLevelRetries = 1
+ ci := fs.GetConfig(context.Background())
+ ci.LowLevelRetries = 1
// Instantiate root
if purge {
diff --git a/backend/drive/drive.go b/backend/drive/drive.go
index 6128f4269..5f02d7e19 100755
--- a/backend/drive/drive.go
+++ b/backend/drive/drive.go
@@ -564,6 +564,7 @@ type Fs struct {
name string // name of this remote
root string // the path we are working on
opt Options // parsed options
+ ci *fs.ConfigInfo // global config
features *fs.Features // optional features
svc *drive.Service // the connection to the drive server
v2Svc *drive_v2.Service // used to create download links for the v2 api
@@ -940,8 +941,10 @@ func parseExtensions(extensionsIn ...string) (extensions, mimeTypes []string, er
// Figure out if the user wants to use a team drive
func configTeamDrive(ctx context.Context, opt *Options, m configmap.Mapper, name string) error {
+ ci := fs.GetConfig(ctx)
+
// Stop if we are running non-interactive config
- if fs.Config.AutoConfirm {
+ if ci.AutoConfirm {
return nil
}
if opt.TeamDriveID == "" {
@@ -979,8 +982,8 @@ func configTeamDrive(ctx context.Context, opt *Options, m configmap.Mapper, name
}
// getClient makes an http client according to the options
-func getClient(opt *Options) *http.Client {
- t := fshttp.NewTransportCustom(fs.Config, func(t *http.Transport) {
+func getClient(ctx context.Context, opt *Options) *http.Client {
+ t := fshttp.NewTransportCustom(fs.GetConfig(ctx), func(t *http.Transport) {
if opt.DisableHTTP2 {
t.TLSNextProto = map[string]func(string, *tls.Conn) http.RoundTripper{}
}
@@ -999,7 +1002,7 @@ func getServiceAccountClient(ctx context.Context, opt *Options, credentialsData
if opt.Impersonate != "" {
conf.Subject = opt.Impersonate
}
- ctxWithSpecialClient := oauthutil.Context(ctx, getClient(opt))
+ ctxWithSpecialClient := oauthutil.Context(ctx, getClient(ctx, opt))
return oauth2.NewClient(ctxWithSpecialClient, conf.TokenSource(ctxWithSpecialClient)), nil
}
@@ -1021,7 +1024,7 @@ func createOAuthClient(ctx context.Context, opt *Options, name string, m configm
return nil, errors.Wrap(err, "failed to create oauth client from service account")
}
} else {
- oAuthClient, _, err = oauthutil.NewClientWithBaseClient(ctx, name, m, driveConfig, getClient(opt))
+ oAuthClient, _, err = oauthutil.NewClientWithBaseClient(ctx, name, m, driveConfig, getClient(ctx, opt))
if err != nil {
return nil, errors.Wrap(err, "failed to create oauth client")
}
@@ -1090,11 +1093,13 @@ func newFs(ctx context.Context, name, path string, m configmap.Mapper) (*Fs, err
return nil, err
}
+ ci := fs.GetConfig(ctx)
f := &Fs{
name: name,
root: root,
opt: *opt,
- pacer: fs.NewPacer(pacer.NewGoogleDrive(pacer.MinSleep(opt.PacerMinSleep), pacer.Burst(opt.PacerBurst))),
+ ci: ci,
+ pacer: fs.NewPacer(ctx, pacer.NewGoogleDrive(pacer.MinSleep(opt.PacerMinSleep), pacer.Burst(opt.PacerBurst))),
m: m,
grouping: listRGrouping,
listRmu: new(sync.Mutex),
@@ -1803,7 +1808,7 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
mu := sync.Mutex{} // protects in and overflow
wg := sync.WaitGroup{}
in := make(chan listREntry, listRInputBuffer)
- out := make(chan error, fs.Config.Checkers)
+ out := make(chan error, f.ci.Checkers)
list := walk.NewListRHelper(callback)
overflow := []listREntry{}
listed := 0
@@ -1842,7 +1847,7 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
wg.Add(1)
in <- listREntry{directoryID, dir}
- for i := 0; i < fs.Config.Checkers; i++ {
+ for i := 0; i < f.ci.Checkers; i++ {
go f.listRRunner(ctx, &wg, in, out, cb, sendJob)
}
go func() {
@@ -1875,7 +1880,7 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
mu.Unlock()
}()
// wait until the all workers to finish
- for i := 0; i < fs.Config.Checkers; i++ {
+ for i := 0; i < f.ci.Checkers; i++ {
e := <-out
mu.Lock()
// if one worker returns an error early, close the input so all other workers exit
diff --git a/backend/dropbox/dropbox.go b/backend/dropbox/dropbox.go
index f03acf990..faf660712 100755
--- a/backend/dropbox/dropbox.go
+++ b/backend/dropbox/dropbox.go
@@ -324,7 +324,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
f := &Fs{
name: name,
opt: *opt,
- pacer: fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
+ pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
}
config := dropbox.Config{
LogLevel: dropbox.LogOff, // logging in the SDK: LogOff, LogDebug, LogInfo
diff --git a/backend/fichier/fichier.go b/backend/fichier/fichier.go
index 18782257f..b4f407bb5 100644
--- a/backend/fichier/fichier.go
+++ b/backend/fichier/fichier.go
@@ -186,7 +186,7 @@ func NewFs(ctx context.Context, name string, root string, config configmap.Mappe
name: name,
root: root,
opt: *opt,
- pacer: fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant), pacer.AttackConstant(attackConstant))),
+ pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant), pacer.AttackConstant(attackConstant))),
baseClient: &http.Client{},
}
@@ -195,7 +195,7 @@ func NewFs(ctx context.Context, name string, root string, config configmap.Mappe
CanHaveEmptyDirectories: true,
}).Fill(ctx, f)
- client := fshttp.NewClient(fs.Config)
+ client := fshttp.NewClient(fs.GetConfig(ctx))
f.rest = rest.NewClient(client).SetRoot(apiBaseURL)
diff --git a/backend/fichier/fichier_test.go b/backend/fichier/fichier_test.go
index 408eaf9e5..531e6e9d3 100644
--- a/backend/fichier/fichier_test.go
+++ b/backend/fichier/fichier_test.go
@@ -4,13 +4,11 @@ package fichier
import (
"testing"
- "github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fstest/fstests"
)
// TestIntegration runs integration tests against the remote
func TestIntegration(t *testing.T) {
- fs.Config.LogLevel = fs.LogLevelDebug
fstests.Run(t, &fstests.Opt{
RemoteName: "TestFichier:",
})
diff --git a/backend/filefabric/filefabric.go b/backend/filefabric/filefabric.go
index 877b6d07c..5dd97184f 100644
--- a/backend/filefabric/filefabric.go
+++ b/backend/filefabric/filefabric.go
@@ -425,7 +425,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
root = parsePath(root)
- client := fshttp.NewClient(fs.Config)
+ client := fshttp.NewClient(fs.GetConfig(ctx))
f := &Fs{
name: name,
@@ -433,7 +433,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
opt: *opt,
m: m,
srv: rest.NewClient(client).SetRoot(opt.URL),
- pacer: fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
+ pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
token: opt.Token,
}
f.features = (&fs.Features{
diff --git a/backend/ftp/ftp.go b/backend/ftp/ftp.go
index f034d2346..a27fd267b 100644
--- a/backend/ftp/ftp.go
+++ b/backend/ftp/ftp.go
@@ -122,10 +122,11 @@ type Options struct {
// Fs represents a remote FTP server
type Fs struct {
- name string // name of this remote
- root string // the path we are working on if any
- opt Options // parsed options
- features *fs.Features // optional features
+ name string // name of this remote
+ root string // the path we are working on if any
+ opt Options // parsed options
+ ci *fs.ConfigInfo // global config
+ features *fs.Features // optional features
url string
user string
pass string
@@ -210,9 +211,9 @@ func (dl *debugLog) Write(p []byte) (n int, err error) {
}
// Open a new connection to the FTP server.
-func (f *Fs) ftpConnection() (*ftp.ServerConn, error) {
+func (f *Fs) ftpConnection(ctx context.Context) (*ftp.ServerConn, error) {
fs.Debugf(f, "Connecting to FTP server")
- ftpConfig := []ftp.DialOption{ftp.DialWithTimeout(fs.Config.ConnectTimeout)}
+ ftpConfig := []ftp.DialOption{ftp.DialWithTimeout(f.ci.ConnectTimeout)}
if f.opt.TLS && f.opt.ExplicitTLS {
fs.Errorf(f, "Implicit TLS and explicit TLS are mutually incompatible. Please revise your config")
return nil, errors.New("Implicit TLS and explicit TLS are mutually incompatible. Please revise your config")
@@ -235,8 +236,8 @@ func (f *Fs) ftpConnection() (*ftp.ServerConn, error) {
if f.opt.DisableMLSD {
ftpConfig = append(ftpConfig, ftp.DialWithDisabledMLSD(true))
}
- if fs.Config.Dump&(fs.DumpHeaders|fs.DumpBodies|fs.DumpRequests|fs.DumpResponses) != 0 {
- ftpConfig = append(ftpConfig, ftp.DialWithDebugOutput(&debugLog{auth: fs.Config.Dump&fs.DumpAuth != 0}))
+ if f.ci.Dump&(fs.DumpHeaders|fs.DumpBodies|fs.DumpRequests|fs.DumpResponses) != 0 {
+ ftpConfig = append(ftpConfig, ftp.DialWithDebugOutput(&debugLog{auth: f.ci.Dump&fs.DumpAuth != 0}))
}
c, err := ftp.Dial(f.dialAddr, ftpConfig...)
if err != nil {
@@ -253,7 +254,7 @@ func (f *Fs) ftpConnection() (*ftp.ServerConn, error) {
}
// Get an FTP connection from the pool, or open a new one
-func (f *Fs) getFtpConnection() (c *ftp.ServerConn, err error) {
+func (f *Fs) getFtpConnection(ctx context.Context) (c *ftp.ServerConn, err error) {
if f.opt.Concurrency > 0 {
f.tokens.Get()
}
@@ -266,7 +267,7 @@ func (f *Fs) getFtpConnection() (c *ftp.ServerConn, err error) {
if c != nil {
return c, nil
}
- c, err = f.ftpConnection()
+ c, err = f.ftpConnection(ctx)
if err != nil && f.opt.Concurrency > 0 {
f.tokens.Put()
}
@@ -336,10 +337,12 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (ff fs.Fs
protocol = "ftps://"
}
u := protocol + path.Join(dialAddr+"/", root)
+ ci := fs.GetConfig(ctx)
f := &Fs{
name: name,
root: root,
opt: *opt,
+ ci: ci,
url: u,
user: user,
pass: pass,
@@ -350,7 +353,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (ff fs.Fs
CanHaveEmptyDirectories: true,
}).Fill(ctx, f)
// Make a connection and pool it to return errors early
- c, err := f.getFtpConnection()
+ c, err := f.getFtpConnection(ctx)
if err != nil {
return nil, errors.Wrap(err, "NewFs")
}
@@ -421,7 +424,7 @@ func (f *Fs) dirFromStandardPath(dir string) string {
}
// findItem finds a directory entry for the name in its parent directory
-func (f *Fs) findItem(remote string) (entry *ftp.Entry, err error) {
+func (f *Fs) findItem(ctx context.Context, remote string) (entry *ftp.Entry, err error) {
// defer fs.Trace(remote, "")("o=%v, err=%v", &o, &err)
fullPath := path.Join(f.root, remote)
if fullPath == "" || fullPath == "." || fullPath == "/" {
@@ -435,7 +438,7 @@ func (f *Fs) findItem(remote string) (entry *ftp.Entry, err error) {
dir := path.Dir(fullPath)
base := path.Base(fullPath)
- c, err := f.getFtpConnection()
+ c, err := f.getFtpConnection(ctx)
if err != nil {
return nil, errors.Wrap(err, "findItem")
}
@@ -457,7 +460,7 @@ func (f *Fs) findItem(remote string) (entry *ftp.Entry, err error) {
// it returns the error fs.ErrorObjectNotFound.
func (f *Fs) NewObject(ctx context.Context, remote string) (o fs.Object, err error) {
// defer fs.Trace(remote, "")("o=%v, err=%v", &o, &err)
- entry, err := f.findItem(remote)
+ entry, err := f.findItem(ctx, remote)
if err != nil {
return nil, err
}
@@ -479,8 +482,8 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (o fs.Object, err err
}
// dirExists checks the directory pointed to by remote exists or not
-func (f *Fs) dirExists(remote string) (exists bool, err error) {
- entry, err := f.findItem(remote)
+func (f *Fs) dirExists(ctx context.Context, remote string) (exists bool, err error) {
+ entry, err := f.findItem(ctx, remote)
if err != nil {
return false, errors.Wrap(err, "dirExists")
}
@@ -501,7 +504,7 @@ func (f *Fs) dirExists(remote string) (exists bool, err error) {
// found.
func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
// defer log.Trace(dir, "dir=%q", dir)("entries=%v, err=%v", &entries, &err)
- c, err := f.getFtpConnection()
+ c, err := f.getFtpConnection(ctx)
if err != nil {
return nil, errors.Wrap(err, "list")
}
@@ -522,7 +525,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
}()
// Wait for List for up to Timeout seconds
- timer := time.NewTimer(fs.Config.Timeout)
+ timer := time.NewTimer(f.ci.Timeout)
select {
case listErr = <-errchan:
timer.Stop()
@@ -539,7 +542,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
// doesn't exist, so check it really doesn't exist if no
// entries found.
if len(files) == 0 {
- exists, err := f.dirExists(dir)
+ exists, err := f.dirExists(ctx, dir)
if err != nil {
return nil, errors.Wrap(err, "list")
}
@@ -592,7 +595,7 @@ func (f *Fs) Precision() time.Duration {
// nil and the error
func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
// fs.Debugf(f, "Trying to put file %s", src.Remote())
- err := f.mkParentDir(src.Remote())
+ err := f.mkParentDir(ctx, src.Remote())
if err != nil {
return nil, errors.Wrap(err, "Put mkParentDir failed")
}
@@ -610,12 +613,12 @@ func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, opt
}
// getInfo reads the FileInfo for a path
-func (f *Fs) getInfo(remote string) (fi *FileInfo, err error) {
+func (f *Fs) getInfo(ctx context.Context, remote string) (fi *FileInfo, err error) {
// defer fs.Trace(remote, "")("fi=%v, err=%v", &fi, &err)
dir := path.Dir(remote)
base := path.Base(remote)
- c, err := f.getFtpConnection()
+ c, err := f.getFtpConnection(ctx)
if err != nil {
return nil, errors.Wrap(err, "getInfo")
}
@@ -642,12 +645,12 @@ func (f *Fs) getInfo(remote string) (fi *FileInfo, err error) {
}
// mkdir makes the directory and parents using unrooted paths
-func (f *Fs) mkdir(abspath string) error {
+func (f *Fs) mkdir(ctx context.Context, abspath string) error {
abspath = path.Clean(abspath)
if abspath == "." || abspath == "/" {
return nil
}
- fi, err := f.getInfo(abspath)
+ fi, err := f.getInfo(ctx, abspath)
if err == nil {
if fi.IsDir {
return nil
@@ -657,11 +660,11 @@ func (f *Fs) mkdir(abspath string) error {
return errors.Wrapf(err, "mkdir %q failed", abspath)
}
parent := path.Dir(abspath)
- err = f.mkdir(parent)
+ err = f.mkdir(ctx, parent)
if err != nil {
return err
}
- c, connErr := f.getFtpConnection()
+ c, connErr := f.getFtpConnection(ctx)
if connErr != nil {
return errors.Wrap(connErr, "mkdir")
}
@@ -681,23 +684,23 @@ func (f *Fs) mkdir(abspath string) error {
// mkParentDir makes the parent of remote if necessary and any
// directories above that
-func (f *Fs) mkParentDir(remote string) error {
+func (f *Fs) mkParentDir(ctx context.Context, remote string) error {
parent := path.Dir(remote)
- return f.mkdir(path.Join(f.root, parent))
+ return f.mkdir(ctx, path.Join(f.root, parent))
}
// Mkdir creates the directory if it doesn't exist
func (f *Fs) Mkdir(ctx context.Context, dir string) (err error) {
// defer fs.Trace(dir, "")("err=%v", &err)
root := path.Join(f.root, dir)
- return f.mkdir(root)
+ return f.mkdir(ctx, root)
}
// Rmdir removes the directory (container, bucket) if empty
//
// Return an error if it doesn't exist or isn't empty
func (f *Fs) Rmdir(ctx context.Context, dir string) error {
- c, err := f.getFtpConnection()
+ c, err := f.getFtpConnection(ctx)
if err != nil {
return errors.Wrap(translateErrorFile(err), "Rmdir")
}
@@ -713,11 +716,11 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
fs.Debugf(src, "Can't move - not same remote type")
return nil, fs.ErrorCantMove
}
- err := f.mkParentDir(remote)
+ err := f.mkParentDir(ctx, remote)
if err != nil {
return nil, errors.Wrap(err, "Move mkParentDir failed")
}
- c, err := f.getFtpConnection()
+ c, err := f.getFtpConnection(ctx)
if err != nil {
return nil, errors.Wrap(err, "Move")
}
@@ -754,7 +757,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
dstPath := path.Join(f.root, dstRemote)
// Check if destination exists
- fi, err := f.getInfo(dstPath)
+ fi, err := f.getInfo(ctx, dstPath)
if err == nil {
if fi.IsDir {
return fs.ErrorDirExists
@@ -765,13 +768,13 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
}
// Make sure the parent directory exists
- err = f.mkdir(path.Dir(dstPath))
+ err = f.mkdir(ctx, path.Dir(dstPath))
if err != nil {
return errors.Wrap(err, "DirMove mkParentDir dst failed")
}
// Do the move
- c, err := f.getFtpConnection()
+ c, err := f.getFtpConnection(ctx)
if err != nil {
return errors.Wrap(err, "DirMove")
}
@@ -903,7 +906,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (rc io.Read
}
}
}
- c, err := o.fs.getFtpConnection()
+ c, err := o.fs.getFtpConnection(ctx)
if err != nil {
return nil, errors.Wrap(err, "open")
}
@@ -938,7 +941,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
fs.Debugf(o, "Removed after failed upload: %v", err)
}
}
- c, err := o.fs.getFtpConnection()
+ c, err := o.fs.getFtpConnection(ctx)
if err != nil {
return errors.Wrap(err, "Update")
}
@@ -950,7 +953,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
return errors.Wrap(err, "update stor")
}
o.fs.putFtpConnection(&c, nil)
- o.info, err = o.fs.getInfo(path)
+ o.info, err = o.fs.getInfo(ctx, path)
if err != nil {
return errors.Wrap(err, "update getinfo")
}
@@ -962,14 +965,14 @@ func (o *Object) Remove(ctx context.Context) (err error) {
// defer fs.Trace(o, "")("err=%v", &err)
path := path.Join(o.fs.root, o.remote)
// Check if it's a directory or a file
- info, err := o.fs.getInfo(path)
+ info, err := o.fs.getInfo(ctx, path)
if err != nil {
return err
}
if info.IsDir {
err = o.fs.Rmdir(ctx, o.remote)
} else {
- c, err := o.fs.getFtpConnection()
+ c, err := o.fs.getFtpConnection(ctx)
if err != nil {
return errors.Wrap(err, "Remove")
}
diff --git a/backend/googlecloudstorage/googlecloudstorage.go b/backend/googlecloudstorage/googlecloudstorage.go
index 9e91530a7..b80ec6053 100644
--- a/backend/googlecloudstorage/googlecloudstorage.go
+++ b/backend/googlecloudstorage/googlecloudstorage.go
@@ -375,7 +375,7 @@ func getServiceAccountClient(ctx context.Context, credentialsData []byte) (*http
if err != nil {
return nil, errors.Wrap(err, "error processing credentials")
}
- ctxWithSpecialClient := oauthutil.Context(ctx, fshttp.NewClient(fs.Config))
+ ctxWithSpecialClient := oauthutil.Context(ctx, fshttp.NewClient(fs.GetConfig(ctx)))
return oauth2.NewClient(ctxWithSpecialClient, conf.TokenSource(ctxWithSpecialClient)), nil
}
@@ -432,7 +432,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
name: name,
root: root,
opt: *opt,
- pacer: fs.NewPacer(pacer.NewGoogleDrive(pacer.MinSleep(minSleep))),
+ pacer: fs.NewPacer(ctx, pacer.NewGoogleDrive(pacer.MinSleep(minSleep))),
cache: bucket.NewCache(),
}
f.setRoot(root)
diff --git a/backend/googlephotos/googlephotos.go b/backend/googlephotos/googlephotos.go
index 858349f97..286fd8fa2 100644
--- a/backend/googlephotos/googlephotos.go
+++ b/backend/googlephotos/googlephotos.go
@@ -254,7 +254,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
return nil, err
}
- baseClient := fshttp.NewClient(fs.Config)
+ baseClient := fshttp.NewClient(fs.GetConfig(ctx))
oAuthClient, ts, err := oauthutil.NewClientWithBaseClient(ctx, name, m, oauthConfig, baseClient)
if err != nil {
return nil, errors.Wrap(err, "failed to configure Box")
@@ -272,7 +272,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
unAuth: rest.NewClient(baseClient),
srv: rest.NewClient(oAuthClient).SetRoot(rootURL),
ts: ts,
- pacer: fs.NewPacer(pacer.NewGoogleDrive(pacer.MinSleep(minSleep))),
+ pacer: fs.NewPacer(ctx, pacer.NewGoogleDrive(pacer.MinSleep(minSleep))),
startTime: time.Now(),
albums: map[bool]*albums{},
uploaded: dirtree.New(),
diff --git a/backend/http/http.go b/backend/http/http.go
index f7ccb269a..84b34dd43 100644
--- a/backend/http/http.go
+++ b/backend/http/http.go
@@ -115,8 +115,9 @@ type Options struct {
type Fs struct {
name string
root string
- features *fs.Features // optional features
- opt Options // options for this backend
+ features *fs.Features // optional features
+ opt Options // options for this backend
+ ci *fs.ConfigInfo // global config
endpoint *url.URL
endpointURL string // endpoint as a string
httpClient *http.Client
@@ -171,7 +172,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
return nil, err
}
- client := fshttp.NewClient(fs.Config)
+ client := fshttp.NewClient(fs.GetConfig(ctx))
var isFile = false
if !strings.HasSuffix(u.String(), "/") {
@@ -209,10 +210,12 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
return nil, err
}
+ ci := fs.GetConfig(ctx)
f := &Fs{
name: name,
root: root,
opt: *opt,
+ ci: ci,
httpClient: client,
endpoint: u,
endpointURL: u.String(),
@@ -439,14 +442,15 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
var (
entriesMu sync.Mutex // to protect entries
wg sync.WaitGroup
- in = make(chan string, fs.Config.Checkers)
+ checkers = f.ci.Checkers
+ in = make(chan string, checkers)
)
add := func(entry fs.DirEntry) {
entriesMu.Lock()
entries = append(entries, entry)
entriesMu.Unlock()
}
- for i := 0; i < fs.Config.Checkers; i++ {
+ for i := 0; i < checkers; i++ {
wg.Add(1)
go func() {
defer wg.Done()
diff --git a/backend/hubic/hubic.go b/backend/hubic/hubic.go
index b945e90d9..48fcfd331 100644
--- a/backend/hubic/hubic.go
+++ b/backend/hubic/hubic.go
@@ -157,11 +157,12 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
}
// Make the swift Connection
+ ci := fs.GetConfig(ctx)
c := &swiftLib.Connection{
Auth: newAuth(f),
- ConnectTimeout: 10 * fs.Config.ConnectTimeout, // Use the timeouts in the transport
- Timeout: 10 * fs.Config.Timeout, // Use the timeouts in the transport
- Transport: fshttp.NewTransport(fs.Config),
+ ConnectTimeout: 10 * ci.ConnectTimeout, // Use the timeouts in the transport
+ Timeout: 10 * ci.Timeout, // Use the timeouts in the transport
+ Transport: fshttp.NewTransport(fs.GetConfig(ctx)),
}
err = c.Authenticate()
if err != nil {
diff --git a/backend/jottacloud/jottacloud.go b/backend/jottacloud/jottacloud.go
index f44fb6c52..f04445879 100644
--- a/backend/jottacloud/jottacloud.go
+++ b/backend/jottacloud/jottacloud.go
@@ -230,7 +230,7 @@ func shouldRetry(resp *http.Response, err error) (bool, error) {
// v1config configure a jottacloud backend using legacy authentication
func v1config(ctx context.Context, name string, m configmap.Mapper) {
- srv := rest.NewClient(fshttp.NewClient(fs.Config))
+ srv := rest.NewClient(fshttp.NewClient(fs.GetConfig(ctx)))
fmt.Printf("\nDo you want to create a machine specific API key?\n\nRclone has it's own Jottacloud API KEY which works fine as long as one only uses rclone on a single machine. When you want to use rclone with this account on more than one machine it's recommended to create a machine specific API key. These keys can NOT be shared between machines.\n\n")
if config.Confirm(false) {
@@ -365,7 +365,7 @@ func doAuthV1(ctx context.Context, srv *rest.Client, username, password string)
// v2config configure a jottacloud backend using the modern JottaCli token based authentication
func v2config(ctx context.Context, name string, m configmap.Mapper) {
- srv := rest.NewClient(fshttp.NewClient(fs.Config))
+ srv := rest.NewClient(fshttp.NewClient(fs.GetConfig(ctx)))
fmt.Printf("Generate a personal login token here: https://www.jottacloud.com/web/secure\n")
fmt.Printf("Login Token> ")
@@ -661,7 +661,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
return nil, errors.New("Outdated config - please reconfigure this backend")
}
- baseClient := fshttp.NewClient(fs.Config)
+ baseClient := fshttp.NewClient(fs.GetConfig(ctx))
if ver == configVersion {
oauthConfig.ClientID = "jottacli"
@@ -711,7 +711,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
opt: *opt,
srv: rest.NewClient(oAuthClient).SetRoot(rootURL),
apiSrv: rest.NewClient(oAuthClient).SetRoot(apiURL),
- pacer: fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
+ pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
}
f.features = (&fs.Features{
CaseInsensitive: true,
diff --git a/backend/koofr/koofr.go b/backend/koofr/koofr.go
index a15c6b160..9dd91df4b 100644
--- a/backend/koofr/koofr.go
+++ b/backend/koofr/koofr.go
@@ -267,7 +267,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (ff fs.Fs
return nil, err
}
httpClient := httpclient.New()
- httpClient.Client = fshttp.NewClient(fs.Config)
+ httpClient.Client = fshttp.NewClient(fs.GetConfig(ctx))
client := koofrclient.NewKoofrClientWithHTTPClient(opt.Endpoint, httpClient)
basicAuth := fmt.Sprintf("Basic %s",
base64.StdEncoding.EncodeToString([]byte(opt.User+":"+pass)))
diff --git a/backend/mailru/mailru.go b/backend/mailru/mailru.go
index 9141c54f7..48bef60f8 100644
--- a/backend/mailru/mailru.go
+++ b/backend/mailru/mailru.go
@@ -273,6 +273,7 @@ type Fs struct {
name string
root string // root path
opt Options // parsed options
+ ci *fs.ConfigInfo // global config
speedupGlobs []string // list of file name patterns eligible for speedup
speedupAny bool // true if all file names are eligible for speedup
features *fs.Features // optional features
@@ -312,10 +313,12 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
// However the f.root string should not have leading or trailing slashes
root = strings.Trim(root, "/")
+ ci := fs.GetConfig(ctx)
f := &Fs{
name: name,
root: root,
opt: *opt,
+ ci: ci,
m: m,
}
@@ -324,7 +327,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
}
f.quirks.parseQuirks(opt.Quirks)
- f.pacer = fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleepPacer), pacer.MaxSleep(maxSleepPacer), pacer.DecayConstant(decayConstPacer)))
+ f.pacer = fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleepPacer), pacer.MaxSleep(maxSleepPacer), pacer.DecayConstant(decayConstPacer)))
f.features = (&fs.Features{
CaseInsensitive: true,
@@ -335,7 +338,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
}).Fill(ctx, f)
// Override few config settings and create a client
- clientConfig := *fs.Config
+ clientConfig := *fs.GetConfig(ctx)
if opt.UserAgent != "" {
clientConfig.UserAgent = opt.UserAgent
}
@@ -692,7 +695,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
entries, err = f.listM1(ctx, f.absPath(dir), 0, maxInt32)
}
- if err == nil && fs.Config.LogLevel >= fs.LogLevelDebug {
+ if err == nil && f.ci.LogLevel >= fs.LogLevelDebug {
names := []string{}
for _, entry := range entries {
names = append(names, entry.Remote())
@@ -956,7 +959,7 @@ func (t *treeState) NextRecord() (fs.DirEntry, error) {
return nil, r.Error()
}
- if fs.Config.LogLevel >= fs.LogLevelDebug {
+ if t.f.ci.LogLevel >= fs.LogLevelDebug {
ctime, _ := modTime.MarshalJSON()
fs.Debugf(t.f, "binDir %d.%d %q %q (%d) %s", t.level, itemType, t.currDir, name, size, ctime)
}
@@ -2376,7 +2379,7 @@ func (p *serverPool) addServer(url string, now time.Time) {
expiry := now.Add(p.expirySec * time.Second)
expiryStr := []byte("-")
- if fs.Config.LogLevel >= fs.LogLevelInfo {
+ if p.fs.ci.LogLevel >= fs.LogLevelInfo {
expiryStr, _ = expiry.MarshalJSON()
}
diff --git a/backend/mega/mega.go b/backend/mega/mega.go
index 0beeba0eb..187c046e9 100644
--- a/backend/mega/mega.go
+++ b/backend/mega/mega.go
@@ -194,6 +194,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
return nil, errors.Wrap(err, "couldn't decrypt password")
}
}
+ ci := fs.GetConfig(ctx)
// cache *mega.Mega on username so we can re-use and share
// them between remotes. They are expensive to make as they
@@ -204,8 +205,8 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
defer megaCacheMu.Unlock()
srv := megaCache[opt.User]
if srv == nil {
- srv = mega.New().SetClient(fshttp.NewClient(fs.Config))
- srv.SetRetries(fs.Config.LowLevelRetries) // let mega do the low level retries
+ srv = mega.New().SetClient(fshttp.NewClient(fs.GetConfig(ctx)))
+ srv.SetRetries(ci.LowLevelRetries) // let mega do the low level retries
srv.SetLogger(func(format string, v ...interface{}) {
fs.Infof("*go-mega*", format, v...)
})
@@ -228,7 +229,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
root: root,
opt: *opt,
srv: srv,
- pacer: fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
+ pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
}
f.features = (&fs.Features{
DuplicateFiles: true,
diff --git a/backend/onedrive/onedrive.go b/backend/onedrive/onedrive.go
index 94224665a..59c77c701 100755
--- a/backend/onedrive/onedrive.go
+++ b/backend/onedrive/onedrive.go
@@ -81,6 +81,7 @@ func init() {
Description: "Microsoft OneDrive",
NewFs: NewFs,
Config: func(ctx context.Context, name string, m configmap.Mapper) {
+ ci := fs.GetConfig(ctx)
err := oauthutil.Config(ctx, "onedrive", name, m, oauthConfig, nil)
if err != nil {
log.Fatalf("Failed to configure token: %v", err)
@@ -88,7 +89,7 @@ func init() {
}
// Stop if we are running non-interactive config
- if fs.Config.AutoConfirm {
+ if ci.AutoConfirm {
return
}
@@ -363,6 +364,7 @@ type Fs struct {
name string // name of this remote
root string // the path we are working on
opt Options // parsed options
+ ci *fs.ConfigInfo // global config
features *fs.Features // optional features
srv *rest.Client // the connection to the one drive server
dirCache *dircache.DirCache // Map of directory path to directory id
@@ -618,14 +620,16 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
return nil, errors.Wrap(err, "failed to configure OneDrive")
}
+ ci := fs.GetConfig(ctx)
f := &Fs{
name: name,
root: root,
opt: *opt,
+ ci: ci,
driveID: opt.DriveID,
driveType: opt.DriveType,
srv: rest.NewClient(oAuthClient).SetRoot(graphURL + "/drives/" + opt.DriveID),
- pacer: fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
+ pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
}
f.features = (&fs.Features{
CaseInsensitive: true,
@@ -971,7 +975,7 @@ func (f *Fs) Precision() time.Duration {
// waitForJob waits for the job with status in url to complete
func (f *Fs) waitForJob(ctx context.Context, location string, o *Object) error {
- deadline := time.Now().Add(fs.Config.Timeout)
+ deadline := time.Now().Add(f.ci.Timeout)
for time.Now().Before(deadline) {
var resp *http.Response
var err error
@@ -1007,7 +1011,7 @@ func (f *Fs) waitForJob(ctx context.Context, location string, o *Object) error {
time.Sleep(1 * time.Second)
}
- return errors.Errorf("async operation didn't complete after %v", fs.Config.Timeout)
+ return errors.Errorf("async operation didn't complete after %v", f.ci.Timeout)
}
// Copy src to this remote using server-side copy operations.
@@ -1300,7 +1304,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
// CleanUp deletes all the hidden files.
func (f *Fs) CleanUp(ctx context.Context) error {
- token := make(chan struct{}, fs.Config.Checkers)
+ token := make(chan struct{}, f.ci.Checkers)
var wg sync.WaitGroup
err := walk.Walk(ctx, f, "", true, -1, func(path string, entries fs.DirEntries, err error) error {
err = entries.ForObjectError(func(obj fs.Object) error {
diff --git a/backend/opendrive/opendrive.go b/backend/opendrive/opendrive.go
index 252fcb6a9..ae803a617 100644
--- a/backend/opendrive/opendrive.go
+++ b/backend/opendrive/opendrive.go
@@ -187,8 +187,8 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
name: name,
root: root,
opt: *opt,
- srv: rest.NewClient(fshttp.NewClient(fs.Config)).SetErrorHandler(errorHandler),
- pacer: fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
+ srv: rest.NewClient(fshttp.NewClient(fs.GetConfig(ctx))).SetErrorHandler(errorHandler),
+ pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
}
f.dirCache = dircache.New(root, "0", f)
diff --git a/backend/pcloud/pcloud.go b/backend/pcloud/pcloud.go
index 7ffec4853..0e564e038 100644
--- a/backend/pcloud/pcloud.go
+++ b/backend/pcloud/pcloud.go
@@ -299,7 +299,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
root: root,
opt: *opt,
srv: rest.NewClient(oAuthClient).SetRoot("https://" + opt.Hostname),
- pacer: fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
+ pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
}
f.features = (&fs.Features{
CaseInsensitive: false,
diff --git a/backend/premiumizeme/premiumizeme.go b/backend/premiumizeme/premiumizeme.go
index ac41e43f2..c7f352f42 100644
--- a/backend/premiumizeme/premiumizeme.go
+++ b/backend/premiumizeme/premiumizeme.go
@@ -252,7 +252,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
return nil, errors.Wrap(err, "failed to configure premiumize.me")
}
} else {
- client = fshttp.NewClient(fs.Config)
+ client = fshttp.NewClient(fs.GetConfig(ctx))
}
f := &Fs{
@@ -260,7 +260,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
root: root,
opt: *opt,
srv: rest.NewClient(client).SetRoot(rootURL),
- pacer: fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
+ pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
}
f.features = (&fs.Features{
CaseInsensitive: true,
diff --git a/backend/putio/fs.go b/backend/putio/fs.go
index 888e28592..73ee159be 100644
--- a/backend/putio/fs.go
+++ b/backend/putio/fs.go
@@ -77,7 +77,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (f fs.Fs,
return nil, err
}
root = parsePath(root)
- httpClient := fshttp.NewClient(fs.Config)
+ httpClient := fshttp.NewClient(fs.GetConfig(ctx))
oAuthClient, _, err := oauthutil.NewClientWithBaseClient(ctx, name, m, putioConfig, httpClient)
if err != nil {
return nil, errors.Wrap(err, "failed to configure putio")
@@ -86,7 +86,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (f fs.Fs,
name: name,
root: root,
opt: *opt,
- pacer: fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
+ pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
client: putio.NewClient(oAuthClient),
httpClient: httpClient,
oAuthClient: oAuthClient,
diff --git a/backend/qingstor/qingstor.go b/backend/qingstor/qingstor.go
index 5e17f4cbe..43fea2ef6 100644
--- a/backend/qingstor/qingstor.go
+++ b/backend/qingstor/qingstor.go
@@ -228,7 +228,7 @@ func qsParseEndpoint(endpoint string) (protocol, host, port string, err error) {
}
// qsConnection makes a connection to qingstor
-func qsServiceConnection(opt *Options) (*qs.Service, error) {
+func qsServiceConnection(ctx context.Context, opt *Options) (*qs.Service, error) {
accessKeyID := opt.AccessKeyID
secretAccessKey := opt.SecretAccessKey
@@ -277,7 +277,7 @@ func qsServiceConnection(opt *Options) (*qs.Service, error) {
cf.Host = host
cf.Port = port
// unsupported in v3.1: cf.ConnectionRetries = opt.ConnectionRetries
- cf.Connection = fshttp.NewClient(fs.Config)
+ cf.Connection = fshttp.NewClient(fs.GetConfig(ctx))
return qs.Init(cf)
}
@@ -334,7 +334,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
if err != nil {
return nil, errors.Wrap(err, "qingstor: upload cutoff")
}
- svc, err := qsServiceConnection(opt)
+ svc, err := qsServiceConnection(ctx, opt)
if err != nil {
return nil, err
}
diff --git a/backend/s3/s3.go b/backend/s3/s3.go
index f70bcaa34..13debe919 100644
--- a/backend/s3/s3.go
+++ b/backend/s3/s3.go
@@ -1285,6 +1285,8 @@ type Fs struct {
name string // the name of the remote
root string // root of the bucket - ignore all objects above this
opt Options // parsed options
+ ci *fs.ConfigInfo // global config
+ ctx context.Context // global context for reading config
features *fs.Features // optional features
c *s3.S3 // the connection to the s3 server
ses *session.Session // the s3 session
@@ -1401,9 +1403,9 @@ func (o *Object) split() (bucket, bucketPath string) {
}
// getClient makes an http client according to the options
-func getClient(opt *Options) *http.Client {
+func getClient(ctx context.Context, opt *Options) *http.Client {
// TODO: Do we need cookies too?
- t := fshttp.NewTransportCustom(fs.Config, func(t *http.Transport) {
+ t := fshttp.NewTransportCustom(fs.GetConfig(ctx), func(t *http.Transport) {
if opt.DisableHTTP2 {
t.TLSNextProto = map[string]func(string, *tls.Conn) http.RoundTripper{}
}
@@ -1414,7 +1416,7 @@ func getClient(opt *Options) *http.Client {
}
// s3Connection makes a connection to s3
-func s3Connection(opt *Options) (*s3.S3, *session.Session, error) {
+func s3Connection(ctx context.Context, opt *Options) (*s3.S3, *session.Session, error) {
// Make the auth
v := credentials.Value{
AccessKeyID: opt.AccessKeyID,
@@ -1492,7 +1494,7 @@ func s3Connection(opt *Options) (*s3.S3, *session.Session, error) {
awsConfig := aws.NewConfig().
WithMaxRetries(0). // Rely on rclone's retry logic
WithCredentials(cred).
- WithHTTPClient(getClient(opt)).
+ WithHTTPClient(getClient(ctx, opt)).
WithS3ForcePathStyle(opt.ForcePathStyle).
WithS3UseAccelerate(opt.UseAccelerateEndpoint).
WithS3UsEast1RegionalEndpoint(endpoints.RegionalS3UsEast1Endpoint)
@@ -1599,23 +1601,26 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
md5sumBinary := md5.Sum([]byte(opt.SSECustomerKey))
opt.SSECustomerKeyMD5 = base64.StdEncoding.EncodeToString(md5sumBinary[:])
}
- c, ses, err := s3Connection(opt)
+ c, ses, err := s3Connection(ctx, opt)
if err != nil {
return nil, err
}
+ ci := fs.GetConfig(ctx)
f := &Fs{
name: name,
opt: *opt,
+ ci: ci,
+ ctx: ctx,
c: c,
ses: ses,
- pacer: fs.NewPacer(pacer.NewS3(pacer.MinSleep(minSleep))),
+ pacer: fs.NewPacer(ctx, pacer.NewS3(pacer.MinSleep(minSleep))),
cache: bucket.NewCache(),
- srv: getClient(opt),
+ srv: getClient(ctx, opt),
pool: pool.New(
time.Duration(opt.MemoryPoolFlushTime),
int(opt.ChunkSize),
- opt.UploadConcurrency*fs.Config.Transfers,
+ opt.UploadConcurrency*ci.Transfers,
opt.MemoryPoolUseMmap,
),
}
@@ -1728,7 +1733,7 @@ func (f *Fs) updateRegionForBucket(bucket string) error {
// Make a new session with the new region
oldRegion := f.opt.Region
f.opt.Region = region
- c, ses, err := s3Connection(&f.opt)
+ c, ses, err := s3Connection(f.ctx, &f.opt)
if err != nil {
return errors.Wrap(err, "creating new session failed")
}
@@ -2343,7 +2348,7 @@ func (f *Fs) getMemoryPool(size int64) *pool.Pool {
return pool.New(
time.Duration(f.opt.MemoryPoolFlushTime),
int(size),
- f.opt.UploadConcurrency*fs.Config.Transfers,
+ f.opt.UploadConcurrency*f.ci.Transfers,
f.opt.MemoryPoolUseMmap,
)
}
@@ -2810,7 +2815,7 @@ func (o *Object) readMetaData(ctx context.Context) (err error) {
// It attempts to read the objects mtime and if that isn't present the
// LastModified returned in the http headers
func (o *Object) ModTime(ctx context.Context) time.Time {
- if fs.Config.UseServerModTime {
+ if o.fs.ci.UseServerModTime {
return o.lastModified
}
err := o.readMetaData(ctx)
diff --git a/backend/seafile/pacer.go b/backend/seafile/pacer.go
index 3c99a5b85..55680193e 100644
--- a/backend/seafile/pacer.go
+++ b/backend/seafile/pacer.go
@@ -1,6 +1,7 @@
package seafile
import (
+ "context"
"fmt"
"net/url"
"sync"
@@ -27,7 +28,7 @@ func init() {
}
// getPacer returns the unique pacer for that remote URL
-func getPacer(remote string) *fs.Pacer {
+func getPacer(ctx context.Context, remote string) *fs.Pacer {
pacerMutex.Lock()
defer pacerMutex.Unlock()
@@ -37,6 +38,7 @@ func getPacer(remote string) *fs.Pacer {
}
pacers[remote] = fs.NewPacer(
+ ctx,
pacer.NewDefault(
pacer.MinSleep(minSleep),
pacer.MaxSleep(maxSleep),
diff --git a/backend/seafile/seafile.go b/backend/seafile/seafile.go
index 751b0ac16..268f0a2c1 100644
--- a/backend/seafile/seafile.go
+++ b/backend/seafile/seafile.go
@@ -197,8 +197,8 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
opt: *opt,
endpoint: u,
endpointURL: u.String(),
- srv: rest.NewClient(fshttp.NewClient(fs.Config)).SetRoot(u.String()),
- pacer: getPacer(opt.URL),
+ srv: rest.NewClient(fshttp.NewClient(fs.GetConfig(ctx))).SetRoot(u.String()),
+ pacer: getPacer(ctx, opt.URL),
}
f.features = (&fs.Features{
CanHaveEmptyDirectories: true,
@@ -297,6 +297,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
// Config callback for 2FA
func Config(ctx context.Context, name string, m configmap.Mapper) {
+ ci := fs.GetConfig(ctx)
serverURL, ok := m.Get(configURL)
if !ok || serverURL == "" {
// If there's no server URL, it means we're trying an operation at the backend level, like a "rclone authorize seafile"
@@ -305,7 +306,7 @@ func Config(ctx context.Context, name string, m configmap.Mapper) {
}
// Stop if we are running non-interactive config
- if fs.Config.AutoConfirm {
+ if ci.AutoConfirm {
return
}
@@ -342,7 +343,7 @@ func Config(ctx context.Context, name string, m configmap.Mapper) {
if !strings.HasPrefix(url, "/") {
url += "/"
}
- srv := rest.NewClient(fshttp.NewClient(fs.Config)).SetRoot(url)
+ srv := rest.NewClient(fshttp.NewClient(fs.GetConfig(ctx))).SetRoot(url)
// We loop asking for a 2FA code
for {
diff --git a/backend/sftp/sftp.go b/backend/sftp/sftp.go
index 00bb3ab4e..537902abe 100644
--- a/backend/sftp/sftp.go
+++ b/backend/sftp/sftp.go
@@ -226,6 +226,7 @@ type Fs struct {
root string
absRoot string
opt Options // parsed options
+ ci *fs.ConfigInfo // global config
m configmap.Mapper // config
features *fs.Features // optional features
config *ssh.ClientConfig
@@ -252,8 +253,8 @@ type Object struct {
// dial starts a client connection to the given SSH server. It is a
// convenience function that connects to the given network address,
// initiates the SSH handshake, and then sets up a Client.
-func (f *Fs) dial(network, addr string, sshConfig *ssh.ClientConfig) (*ssh.Client, error) {
- dialer := fshttp.NewDialer(fs.Config)
+func (f *Fs) dial(ctx context.Context, network, addr string, sshConfig *ssh.ClientConfig) (*ssh.Client, error) {
+ dialer := fshttp.NewDialer(fs.GetConfig(ctx))
conn, err := dialer.Dial(network, addr)
if err != nil {
return nil, err
@@ -299,12 +300,12 @@ func (c *conn) closed() error {
}
// Open a new connection to the SFTP server.
-func (f *Fs) sftpConnection() (c *conn, err error) {
+func (f *Fs) sftpConnection(ctx context.Context) (c *conn, err error) {
// Rate limit rate of new connections
c = &conn{
err: make(chan error, 1),
}
- c.sshClient, err = f.dial("tcp", f.opt.Host+":"+f.opt.Port, f.config)
+ c.sshClient, err = f.dial(ctx, "tcp", f.opt.Host+":"+f.opt.Port, f.config)
if err != nil {
return nil, errors.Wrap(err, "couldn't connect SSH")
}
@@ -347,7 +348,7 @@ func (f *Fs) newSftpClient(conn *ssh.Client, opts ...sftp.ClientOption) (*sftp.C
}
// Get an SFTP connection from the pool, or open a new one
-func (f *Fs) getSftpConnection() (c *conn, err error) {
+func (f *Fs) getSftpConnection(ctx context.Context) (c *conn, err error) {
f.poolMu.Lock()
for len(f.pool) > 0 {
c = f.pool[0]
@@ -364,7 +365,7 @@ func (f *Fs) getSftpConnection() (c *conn, err error) {
return c, nil
}
err = f.pacer.Call(func() (bool, error) {
- c, err = f.sftpConnection()
+ c, err = f.sftpConnection(ctx)
if err != nil {
return true, err
}
@@ -417,7 +418,9 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
// This will hold the Fs object. We need to create it here
// so we can refer to it in the SSH callback, but it's populated
// in NewFsWithConnection
- f := &Fs{}
+ f := &Fs{
+ ci: fs.GetConfig(ctx),
+ }
// Parse config into Options struct
opt := new(Options)
err := configstruct.Set(m, opt)
@@ -435,8 +438,8 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
User: opt.User,
Auth: []ssh.AuthMethod{},
HostKeyCallback: ssh.InsecureIgnoreHostKey(),
- Timeout: fs.Config.ConnectTimeout,
- ClientVersion: "SSH-2.0-" + fs.Config.UserAgent,
+ Timeout: f.ci.ConnectTimeout,
+ ClientVersion: "SSH-2.0-" + f.ci.UserAgent,
}
if opt.KnownHostsFile != "" {
@@ -603,7 +606,7 @@ func NewFsWithConnection(ctx context.Context, f *Fs, name string, root string, m
f.config = sshConfig
f.url = "sftp://" + opt.User + "@" + opt.Host + ":" + opt.Port + "/" + root
f.mkdirLock = newStringLock()
- f.pacer = fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant)))
+ f.pacer = fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant)))
f.savedpswd = ""
f.features = (&fs.Features{
@@ -611,7 +614,7 @@ func NewFsWithConnection(ctx context.Context, f *Fs, name string, root string, m
SlowHash: true,
}).Fill(ctx, f)
// Make a connection and pool it to return errors early
- c, err := f.getSftpConnection()
+ c, err := f.getSftpConnection(ctx)
if err != nil {
return nil, errors.Wrap(err, "NewFs")
}
@@ -679,7 +682,7 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
fs: f,
remote: remote,
}
- err := o.stat()
+ err := o.stat(ctx)
if err != nil {
return nil, err
}
@@ -688,11 +691,11 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
// dirExists returns true,nil if the directory exists, false, nil if
// it doesn't or false, err
-func (f *Fs) dirExists(dir string) (bool, error) {
+func (f *Fs) dirExists(ctx context.Context, dir string) (bool, error) {
if dir == "" {
dir = "."
}
- c, err := f.getSftpConnection()
+ c, err := f.getSftpConnection(ctx)
if err != nil {
return false, errors.Wrap(err, "dirExists")
}
@@ -721,7 +724,7 @@ func (f *Fs) dirExists(dir string) (bool, error) {
// found.
func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
root := path.Join(f.absRoot, dir)
- ok, err := f.dirExists(root)
+ ok, err := f.dirExists(ctx, root)
if err != nil {
return nil, errors.Wrap(err, "List failed")
}
@@ -732,7 +735,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
if sftpDir == "" {
sftpDir = "."
}
- c, err := f.getSftpConnection()
+ c, err := f.getSftpConnection(ctx)
if err != nil {
return nil, errors.Wrap(err, "List")
}
@@ -751,7 +754,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
continue
}
oldInfo := info
- info, err = f.stat(remote)
+ info, err = f.stat(ctx, remote)
if err != nil {
if !os.IsNotExist(err) {
fs.Errorf(remote, "stat of non-regular file failed: %v", err)
@@ -776,7 +779,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
// Put data from <in> into a new remote sftp file object described by <src.Remote()> and <src.ModTime(ctx)>
func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
- err := f.mkParentDir(src.Remote())
+ err := f.mkParentDir(ctx, src.Remote())
if err != nil {
return nil, errors.Wrap(err, "Put mkParentDir failed")
}
@@ -799,19 +802,19 @@ func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, opt
// mkParentDir makes the parent of remote if necessary and any
// directories above that
-func (f *Fs) mkParentDir(remote string) error {
+func (f *Fs) mkParentDir(ctx context.Context, remote string) error {
parent := path.Dir(remote)
- return f.mkdir(path.Join(f.absRoot, parent))
+ return f.mkdir(ctx, path.Join(f.absRoot, parent))
}
// mkdir makes the directory and parents using native paths
-func (f *Fs) mkdir(dirPath string) error {
+func (f *Fs) mkdir(ctx context.Context, dirPath string) error {
f.mkdirLock.Lock(dirPath)
defer f.mkdirLock.Unlock(dirPath)
if dirPath == "." || dirPath == "/" {
return nil
}
- ok, err := f.dirExists(dirPath)
+ ok, err := f.dirExists(ctx, dirPath)
if err != nil {
return errors.Wrap(err, "mkdir dirExists failed")
}
@@ -819,11 +822,11 @@ func (f *Fs) mkdir(dirPath string) error {
return nil
}
parent := path.Dir(dirPath)
- err = f.mkdir(parent)
+ err = f.mkdir(ctx, parent)
if err != nil {
return err
}
- c, err := f.getSftpConnection()
+ c, err := f.getSftpConnection(ctx)
if err != nil {
return errors.Wrap(err, "mkdir")
}
@@ -838,7 +841,7 @@ func (f *Fs) mkdir(dirPath string) error {
// Mkdir makes the root directory of the Fs object
func (f *Fs) Mkdir(ctx context.Context, dir string) error {
root := path.Join(f.absRoot, dir)
- return f.mkdir(root)
+ return f.mkdir(ctx, root)
}
// Rmdir removes the root directory of the Fs object
@@ -854,7 +857,7 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) error {
}
// Remove the directory
root := path.Join(f.absRoot, dir)
- c, err := f.getSftpConnection()
+ c, err := f.getSftpConnection(ctx)
if err != nil {
return errors.Wrap(err, "Rmdir")
}
@@ -870,11 +873,11 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
fs.Debugf(src, "Can't move - not same remote type")
return nil, fs.ErrorCantMove
}
- err := f.mkParentDir(remote)
+ err := f.mkParentDir(ctx, remote)
if err != nil {
return nil, errors.Wrap(err, "Move mkParentDir failed")
}
- c, err := f.getSftpConnection()
+ c, err := f.getSftpConnection(ctx)
if err != nil {
return nil, errors.Wrap(err, "Move")
}
@@ -911,7 +914,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
dstPath := path.Join(f.absRoot, dstRemote)
// Check if destination exists
- ok, err := f.dirExists(dstPath)
+ ok, err := f.dirExists(ctx, dstPath)
if err != nil {
return errors.Wrap(err, "DirMove dirExists dst failed")
}
@@ -920,13 +923,13 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
}
// Make sure the parent directory exists
- err = f.mkdir(path.Dir(dstPath))
+ err = f.mkdir(ctx, path.Dir(dstPath))
if err != nil {
return errors.Wrap(err, "DirMove mkParentDir dst failed")
}
// Do the move
- c, err := f.getSftpConnection()
+ c, err := f.getSftpConnection(ctx)
if err != nil {
return errors.Wrap(err, "DirMove")
}
@@ -942,8 +945,8 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
}
// run runds cmd on the remote end returning standard output
-func (f *Fs) run(cmd string) ([]byte, error) {
- c, err := f.getSftpConnection()
+func (f *Fs) run(ctx context.Context, cmd string) ([]byte, error) {
+ c, err := f.getSftpConnection(ctx)
if err != nil {
return nil, errors.Wrap(err, "run: get SFTP connection")
}
@@ -971,6 +974,7 @@ func (f *Fs) run(cmd string) ([]byte, error) {
// Hashes returns the supported hash types of the filesystem
func (f *Fs) Hashes() hash.Set {
+ ctx := context.TODO()
if f.opt.DisableHashCheck {
return hash.Set(hash.None)
}
@@ -989,7 +993,7 @@ func (f *Fs) Hashes() hash.Set {
}
*changed = true
for _, command := range commands {
- output, err := f.run(command)
+ output, err := f.run(ctx, command)
if err != nil {
continue
}
@@ -1034,7 +1038,7 @@ func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
if len(escapedPath) == 0 {
escapedPath = "/"
}
- stdout, err := f.run("df -k " + escapedPath)
+ stdout, err := f.run(ctx, "df -k "+escapedPath)
if err != nil {
return nil, errors.Wrap(err, "your remote may not support About")
}
@@ -1097,7 +1101,7 @@ func (o *Object) Hash(ctx context.Context, r hash.Type) (string, error) {
return "", hash.ErrUnsupported
}
- c, err := o.fs.getSftpConnection()
+ c, err := o.fs.getSftpConnection(ctx)
if err != nil {
return "", errors.Wrap(err, "Hash get SFTP connection")
}
@@ -1205,8 +1209,8 @@ func (o *Object) setMetadata(info os.FileInfo) {
}
// statRemote stats the file or directory at the remote given
-func (f *Fs) stat(remote string) (info os.FileInfo, err error) {
- c, err := f.getSftpConnection()
+func (f *Fs) stat(ctx context.Context, remote string) (info os.FileInfo, err error) {
+ c, err := f.getSftpConnection(ctx)
if err != nil {
return nil, errors.Wrap(err, "stat")
}
@@ -1217,8 +1221,8 @@ func (f *Fs) stat(remote string) (info os.FileInfo, err error) {
}
// stat updates the info in the Object
-func (o *Object) stat() error {
- info, err := o.fs.stat(o.remote)
+func (o *Object) stat(ctx context.Context) error {
+ info, err := o.fs.stat(ctx, o.remote)
if err != nil {
if os.IsNotExist(err) {
return fs.ErrorObjectNotFound
@@ -1237,7 +1241,7 @@ func (o *Object) stat() error {
// it also updates the info field
func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
if o.fs.opt.SetModTime {
- c, err := o.fs.getSftpConnection()
+ c, err := o.fs.getSftpConnection(ctx)
if err != nil {
return errors.Wrap(err, "SetModTime")
}
@@ -1247,7 +1251,7 @@ func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
return errors.Wrap(err, "SetModTime failed")
}
}
- err := o.stat()
+ err := o.stat(ctx)
if err != nil {
return errors.Wrap(err, "SetModTime stat failed")
}
@@ -1320,7 +1324,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
}
}
}
- c, err := o.fs.getSftpConnection()
+ c, err := o.fs.getSftpConnection(ctx)
if err != nil {
return nil, errors.Wrap(err, "Open")
}
@@ -1344,7 +1348,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
// Clear the hash cache since we are about to update the object
o.md5sum = nil
o.sha1sum = nil
- c, err := o.fs.getSftpConnection()
+ c, err := o.fs.getSftpConnection(ctx)
if err != nil {
return errors.Wrap(err, "Update")
}
@@ -1355,7 +1359,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
}
// remove the file if upload failed
remove := func() {
- c, removeErr := o.fs.getSftpConnection()
+ c, removeErr := o.fs.getSftpConnection(ctx)
if removeErr != nil {
fs.Debugf(src, "Failed to open new SSH connection for delete: %v", removeErr)
return
@@ -1387,7 +1391,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
// Remove a remote sftp file object
func (o *Object) Remove(ctx context.Context) error {
- c, err := o.fs.getSftpConnection()
+ c, err := o.fs.getSftpConnection(ctx)
if err != nil {
return errors.Wrap(err, "Remove")
}
diff --git a/backend/sharefile/sharefile.go b/backend/sharefile/sharefile.go
index fb56872e7..e7b6e53f1 100644
--- a/backend/sharefile/sharefile.go
+++ b/backend/sharefile/sharefile.go
@@ -237,6 +237,7 @@ type Fs struct {
name string // name of this remote
root string // the path we are working on
opt Options // parsed options
+ ci *fs.ConfigInfo // global config
features *fs.Features // optional features
srv *rest.Client // the connection to the server
dirCache *dircache.DirCache // Map of directory path to directory id
@@ -441,12 +442,14 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
return nil, errors.Wrap(err, "failed to configure sharefile")
}
+ ci := fs.GetConfig(ctx)
f := &Fs{
name: name,
root: root,
opt: *opt,
+ ci: ci,
srv: rest.NewClient(client).SetRoot(opt.Endpoint + apiPath),
- pacer: fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
+ pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
}
f.features = (&fs.Features{
CaseInsensitive: true,
@@ -531,8 +534,8 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
// Fill up (or reset) the buffer tokens
func (f *Fs) fillBufferTokens() {
- f.bufferTokens = make(chan []byte, fs.Config.Transfers)
- for i := 0; i < fs.Config.Transfers; i++ {
+ f.bufferTokens = make(chan []byte, f.ci.Transfers)
+ for i := 0; i < f.ci.Transfers; i++ {
f.bufferTokens <- nil
}
}
@@ -1338,7 +1341,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
Overwrite: true,
CreatedDate: modTime,
ModifiedDate: modTime,
- Tool: fs.Config.UserAgent,
+ Tool: o.fs.ci.UserAgent,
}
if isLargeFile {
@@ -1348,7 +1351,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
} else {
// otherwise use threaded which is more efficient
req.Method = "threaded"
- req.ThreadCount = &fs.Config.Transfers
+ req.ThreadCount = &o.fs.ci.Transfers
req.Filesize = &size
}
}
diff --git a/backend/sharefile/upload.go b/backend/sharefile/upload.go
index 87d760f52..12218ab96 100644
--- a/backend/sharefile/upload.go
+++ b/backend/sharefile/upload.go
@@ -58,7 +58,7 @@ func (f *Fs) newLargeUpload(ctx context.Context, o *Object, in io.Reader, src fs
return nil, errors.Errorf("can't use method %q with newLargeUpload", info.Method)
}
- threads := fs.Config.Transfers
+ threads := f.ci.Transfers
if threads > info.MaxNumberOfThreads {
threads = info.MaxNumberOfThreads
}
diff --git a/backend/sugarsync/sugarsync.go b/backend/sugarsync/sugarsync.go
index 039e29870..01b8489a9 100644
--- a/backend/sugarsync/sugarsync.go
+++ b/backend/sugarsync/sugarsync.go
@@ -106,7 +106,7 @@ func init() {
Method: "POST",
Path: "/app-authorization",
}
- srv := rest.NewClient(fshttp.NewClient(fs.Config)).SetRoot(rootURL) // FIXME
+ srv := rest.NewClient(fshttp.NewClient(fs.GetConfig(ctx))).SetRoot(rootURL) // FIXME
// FIXME
//err = f.pacer.Call(func() (bool, error) {
@@ -403,13 +403,13 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
}
root = parsePath(root)
- client := fshttp.NewClient(fs.Config)
+ client := fshttp.NewClient(fs.GetConfig(ctx))
f := &Fs{
name: name,
root: root,
opt: *opt,
srv: rest.NewClient(client).SetRoot(rootURL),
- pacer: fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
+ pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
m: m,
authExpiry: parseExpiry(opt.AuthorizationExpiry),
}
diff --git a/backend/swift/swift.go b/backend/swift/swift.go
index 5e176dae0..f7c79228f 100644
--- a/backend/swift/swift.go
+++ b/backend/swift/swift.go
@@ -221,6 +221,7 @@ type Fs struct {
root string // the path we are working on if any
features *fs.Features // optional features
opt Options // options for this backend
+ ci *fs.ConfigInfo // global config
c *swift.Connection // the connection to the swift server
rootContainer string // container part of root (if any)
rootDirectory string // directory part of root (if any)
@@ -340,7 +341,8 @@ func (o *Object) split() (container, containerPath string) {
}
// swiftConnection makes a connection to swift
-func swiftConnection(opt *Options, name string) (*swift.Connection, error) {
+func swiftConnection(ctx context.Context, opt *Options, name string) (*swift.Connection, error) {
+ ci := fs.GetConfig(ctx)
c := &swift.Connection{
// Keep these in the same order as the Config for ease of checking
UserName: opt.User,
@@ -359,9 +361,9 @@ func swiftConnection(opt *Options, name string) (*swift.Connection, error) {
ApplicationCredentialName: opt.ApplicationCredentialName,
ApplicationCredentialSecret: opt.ApplicationCredentialSecret,
EndpointType: swift.EndpointType(opt.EndpointType),
- ConnectTimeout: 10 * fs.Config.ConnectTimeout, // Use the timeouts in the transport
- Timeout: 10 * fs.Config.Timeout, // Use the timeouts in the transport
- Transport: fshttp.NewTransport(fs.Config),
+ ConnectTimeout: 10 * ci.ConnectTimeout, // Use the timeouts in the transport
+ Timeout: 10 * ci.Timeout, // Use the timeouts in the transport
+ Transport: fshttp.NewTransport(fs.GetConfig(ctx)),
}
if opt.EnvAuth {
err := c.ApplyEnvironment()
@@ -433,12 +435,14 @@ func (f *Fs) setRoot(root string) {
// if noCheckContainer is set then the Fs won't check the container
// exists before creating it.
func NewFsWithConnection(ctx context.Context, opt *Options, name, root string, c *swift.Connection, noCheckContainer bool) (fs.Fs, error) {
+ ci := fs.GetConfig(ctx)
f := &Fs{
name: name,
opt: *opt,
+ ci: ci,
c: c,
noCheckContainer: noCheckContainer,
- pacer: fs.NewPacer(pacer.NewS3(pacer.MinSleep(minSleep))),
+ pacer: fs.NewPacer(ctx, pacer.NewS3(pacer.MinSleep(minSleep))),
cache: bucket.NewCache(),
}
f.setRoot(root)
@@ -485,7 +489,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
return nil, errors.Wrap(err, "swift: chunk size")
}
- c, err := swiftConnection(opt, name)
+ c, err := swiftConnection(ctx, opt, name)
if err != nil {
return nil, err
}
@@ -849,7 +853,7 @@ func (f *Fs) Purge(ctx context.Context, dir string) error {
return fs.ErrorListBucketRequired
}
// Delete all the files including the directory markers
- toBeDeleted := make(chan fs.Object, fs.Config.Transfers)
+ toBeDeleted := make(chan fs.Object, f.ci.Transfers)
delErr := make(chan error, 1)
go func() {
delErr <- operations.DeleteFiles(ctx, toBeDeleted)
@@ -1040,7 +1044,7 @@ func (o *Object) readMetaData() (err error) {
// It attempts to read the objects mtime and if that isn't present the
// LastModified returned in the http headers
func (o *Object) ModTime(ctx context.Context) time.Time {
- if fs.Config.UseServerModTime {
+ if o.fs.ci.UseServerModTime {
return o.lastModified
}
err := o.readMetaData()
diff --git a/backend/webdav/odrvcookie/fetch.go b/backend/webdav/odrvcookie/fetch.go
index 88f715ac1..3870ec693 100644
--- a/backend/webdav/odrvcookie/fetch.go
+++ b/backend/webdav/odrvcookie/fetch.go
@@ -182,7 +182,7 @@ func (ca *CookieAuth) getSPToken(ctx context.Context) (conf *SharepointSuccessRe
}
req = req.WithContext(ctx) // go1.13 can use NewRequestWithContext
- client := fshttp.NewClient(fs.Config)
+ client := fshttp.NewClient(fs.GetConfig(ctx))
resp, err := client.Do(req)
if err != nil {
return nil, errors.Wrap(err, "Error while logging in to endpoint")
diff --git a/backend/webdav/webdav.go b/backend/webdav/webdav.go
index bd6f2a6ee..bbdca9be9 100644
--- a/backend/webdav/webdav.go
+++ b/backend/webdav/webdav.go
@@ -336,8 +336,8 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
opt: *opt,
endpoint: u,
endpointURL: u.String(),
- srv: rest.NewClient(fshttp.NewClient(fs.Config)).SetRoot(u.String()),
- pacer: fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
+ srv: rest.NewClient(fshttp.NewClient(fs.GetConfig(ctx))).SetRoot(u.String()),
+ pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
precision: fs.ModTimeNotSupported,
}
f.features = (&fs.Features{
diff --git a/backend/yandex/yandex.go b/backend/yandex/yandex.go
index 08a2b6250..fcc4d1650 100644
--- a/backend/yandex/yandex.go
+++ b/backend/yandex/yandex.go
@@ -88,12 +88,13 @@ type Options struct {
// Fs represents a remote yandex
type Fs struct {
name string
- root string // root path
- opt Options // parsed options
- features *fs.Features // optional features
- srv *rest.Client // the connection to the yandex server
- pacer *fs.Pacer // pacer for API calls
- diskRoot string // root path with "disk:/" container name
+ root string // root path
+ opt Options // parsed options
+ ci *fs.ConfigInfo // global config
+ features *fs.Features // optional features
+ srv *rest.Client // the connection to the yandex server
+ pacer *fs.Pacer // pacer for API calls
+ diskRoot string // root path with "disk:/" container name
}
// Object describes a swift object
@@ -265,11 +266,13 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
log.Fatalf("Failed to configure Yandex: %v", err)
}
+ ci := fs.GetConfig(ctx)
f := &Fs{
name: name,
opt: *opt,
+ ci: ci,
srv: rest.NewClient(oAuthClient).SetRoot(rootURL),
- pacer: fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
+ pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
}
f.setRoot(root)
f.features = (&fs.Features{
@@ -534,7 +537,7 @@ func (f *Fs) waitForJob(ctx context.Context, location string) (err error) {
RootURL: location,
Method: "GET",
}
- deadline := time.Now().Add(fs.Config.Timeout)
+ deadline := time.Now().Add(f.ci.Timeout)
for time.Now().Before(deadline) {
var resp *http.Response
var body []byte
@@ -565,7 +568,7 @@ func (f *Fs) waitForJob(ctx context.Context, location string) (err error) {
time.Sleep(1 * time.Second)
}
- return errors.Errorf("async operation didn't complete after %v", fs.Config.Timeout)
+ return errors.Errorf("async operation didn't complete after %v", f.ci.Timeout)
}
func (f *Fs) delete(ctx context.Context, path string, hardDelete bool) (err error) {
diff --git a/cmd/cmd.go b/cmd/cmd.go
index 1fc98ac9e..d9b29fa43 100644
--- a/cmd/cmd.go
+++ b/cmd/cmd.go
@@ -234,12 +234,13 @@ func ShowStats() bool {
// Run the function with stats and retries if required
func Run(Retry bool, showStats bool, cmd *cobra.Command, f func() error) {
+ ci := fs.GetConfig(context.Background())
var cmdErr error
stopStats := func() {}
if !showStats && ShowStats() {
showStats = true
}
- if fs.Config.Progress {
+ if ci.Progress {
stopStats = startProgress()
} else if showStats {
stopStats = StartStats()
@@ -291,13 +292,13 @@ func Run(Retry bool, showStats bool, cmd *cobra.Command, f func() error) {
}
fs.Debugf(nil, "%d go routines active\n", runtime.NumGoroutine())
- if fs.Config.Progress && fs.Config.ProgressTerminalTitle {
+ if ci.Progress && ci.ProgressTerminalTitle {
// Clear terminal title
terminal.WriteTerminalTitle("")
}
// dump all running go-routines
- if fs.Config.Dump&fs.DumpGoRoutines != 0 {
+ if ci.Dump&fs.DumpGoRoutines != 0 {
err := pprof.Lookup("goroutine").WriteTo(os.Stdout, 1)
if err != nil {
fs.Errorf(nil, "Failed to dump goroutines: %v", err)
@@ -305,7 +306,7 @@ func Run(Retry bool, showStats bool, cmd *cobra.Command, f func() error) {
}
// dump open files
- if fs.Config.Dump&fs.DumpOpenFiles != 0 {
+ if ci.Dump&fs.DumpOpenFiles != 0 {
c := exec.Command("lsof", "-p", strconv.Itoa(os.Getpid()))
c.Stdout = os.Stdout
c.Stderr = os.Stderr
@@ -372,17 +373,18 @@ func StartStats() func() {
// initConfig is run by cobra after initialising the flags
func initConfig() {
+ ci := fs.GetConfig(context.Background())
// Activate logger systemd support if systemd invocation ID is detected
_, sysdLaunch := systemd.GetInvocationID()
if sysdLaunch {
- fs.Config.LogSystemdSupport = true // used during fslog.InitLogging()
+ ci.LogSystemdSupport = true // used during fslog.InitLogging()
}
// Start the logger
fslog.InitLogging()
// Finish parsing any command line flags
- configflags.SetFlags()
+ configflags.SetFlags(ci)
// Load filters
err := filterflags.Reload()
@@ -396,7 +398,7 @@ func initConfig() {
// Inform user about systemd log support now that we have a logger
if sysdLaunch {
fs.Debugf("rclone", "systemd logging support automatically activated")
- } else if fs.Config.LogSystemdSupport {
+ } else if ci.LogSystemdSupport {
fs.Debugf("rclone", "systemd logging support manually activated")
}
@@ -448,16 +450,17 @@ func initConfig() {
if m, _ := regexp.MatchString("^(bits|bytes)$", *dataRateUnit); m == false {
fs.Errorf(nil, "Invalid unit passed to --stats-unit. Defaulting to bytes.")
- fs.Config.DataRateUnit = "bytes"
+ ci.DataRateUnit = "bytes"
} else {
- fs.Config.DataRateUnit = *dataRateUnit
+ ci.DataRateUnit = *dataRateUnit
}
}
func resolveExitCode(err error) {
+ ci := fs.GetConfig(context.Background())
atexit.Run()
if err == nil {
- if fs.Config.ErrorOnNoTransfer {
+ if ci.ErrorOnNoTransfer {
if accounting.GlobalStats().GetTransfers() == 0 {
os.Exit(exitCodeNoFilesTransferred)
}
diff --git a/cmd/config/config.go b/cmd/config/config.go
index b94895a6d..88c8e937b 100644
--- a/cmd/config/config.go
+++ b/cmd/config/config.go
@@ -43,7 +43,7 @@ password to protect your configuration.
`,
Run: func(command *cobra.Command, args []string) {
cmd.CheckArgs(0, 0, command, args)
- config.EditConfig()
+ config.EditConfig(context.Background())
},
}
diff --git a/cmd/help.go b/cmd/help.go
index 7cd2d34f0..3668b0c97 100644
--- a/cmd/help.go
+++ b/cmd/help.go
@@ -1,6 +1,7 @@
package cmd
import (
+ "context"
"fmt"
"log"
"os"
@@ -166,8 +167,9 @@ func runRoot(cmd *cobra.Command, args []string) {
//
// Helpful example: http://rtfcode.com/xref/moby-17.03.2-ce/cli/cobra.go
func setupRootCommand(rootCmd *cobra.Command) {
+ ci := fs.GetConfig(context.Background())
// Add global flags
- configflags.AddFlags(pflag.CommandLine)
+ configflags.AddFlags(ci, pflag.CommandLine)
filterflags.AddFlags(pflag.CommandLine)
rcflags.AddFlags(pflag.CommandLine)
logflags.AddFlags(pflag.CommandLine)
diff --git a/cmd/info/info.go b/cmd/info/info.go
index 66642fc70..c2006f20a 100644
--- a/cmd/info/info.go
+++ b/cmd/info/info.go
@@ -249,9 +249,11 @@ func (r *results) checkStringPositions(k, s string) {
// check we can write a file with the control chars
func (r *results) checkControls() {
fs.Infof(r.f, "Trying to create control character file names")
+ ci := fs.GetConfig(context.Background())
+
// Concurrency control
- tokens := make(chan struct{}, fs.Config.Checkers)
- for i := 0; i < fs.Config.Checkers; i++ {
+ tokens := make(chan struct{}, ci.Checkers)
+ for i := 0; i < ci.Checkers; i++ {
tokens <- struct{}{}
}
var wg sync.WaitGroup
diff --git a/cmd/lsd/lsd.go b/cmd/lsd/lsd.go
index de102e102..8bb6fcf2d 100644
--- a/cmd/lsd/lsd.go
+++ b/cmd/lsd/lsd.go
@@ -49,9 +49,10 @@ If you just want the directory names use "rclone lsf --dirs-only".
` + lshelp.Help,
Run: func(command *cobra.Command, args []string) {
+ ci := fs.GetConfig(context.Background())
cmd.CheckArgs(1, 1, command, args)
if recurse {
- fs.Config.MaxDepth = 0
+ ci.MaxDepth = 0
}
fsrc := cmd.NewFsSrc(args)
cmd.Run(false, false, command, func() error {
diff --git a/cmd/ncdu/scan/scan.go b/cmd/ncdu/scan/scan.go
index b5b949a69..5957e6916 100644
--- a/cmd/ncdu/scan/scan.go
+++ b/cmd/ncdu/scan/scan.go
@@ -162,12 +162,13 @@ func (d *Dir) AttrI(i int) (size int64, count int64, isDir bool, readable bool)
// Scan the Fs passed in, returning a root directory channel and an
// error channel
func Scan(ctx context.Context, f fs.Fs) (chan *Dir, chan error, chan struct{}) {
+ ci := fs.GetConfig(ctx)
root := make(chan *Dir, 1)
errChan := make(chan error, 1)
updated := make(chan struct{}, 1)
go func() {
parents := map[string]*Dir{}
- err := walk.Walk(ctx, f, "", false, fs.Config.MaxDepth, func(dirPath string, entries fs.DirEntries, err error) error {
+ err := walk.Walk(ctx, f, "", false, ci.MaxDepth, func(dirPath string, entries fs.DirEntries, err error) error {
if err != nil {
return err // FIXME mark directory as errored instead of aborting
}
diff --git a/cmd/rc/rc.go b/cmd/rc/rc.go
index cc4792143..ae5d57aa0 100644
--- a/cmd/rc/rc.go
+++ b/cmd/rc/rc.go
@@ -177,7 +177,7 @@ func doCall(ctx context.Context, path string, in rc.Params) (out rc.Params, err
}
// Do HTTP request
- client := fshttp.NewClient(fs.Config)
+ client := fshttp.NewClient(fs.GetConfig(ctx))
url += path
data, err := json.Marshal(in)
if err != nil {
diff --git a/cmd/serve/sftp/server.go b/cmd/serve/sftp/server.go
index e76d4652e..a617cf2ad 100644
--- a/cmd/serve/sftp/server.go
+++ b/cmd/serve/sftp/server.go
@@ -145,7 +145,7 @@ func (s *server) serve() (err error) {
// An SSH server is represented by a ServerConfig, which holds
// certificate details and handles authentication of ServerConns.
s.config = &ssh.ServerConfig{
- ServerVersion: "SSH-2.0-" + fs.Config.UserAgent,
+ ServerVersion: "SSH-2.0-" + fs.GetConfig(s.ctx).UserAgent,
PasswordCallback: func(c ssh.ConnMetadata, pass []byte) (*ssh.Permissions, error) {
fs.Debugf(describeConn(c), "Password login attempt for %s", c.User())
if s.proxy != nil {
diff --git a/cmd/tree/tree.go b/cmd/tree/tree.go
index 6ced45ee2..09ce291c3 100644
--- a/cmd/tree/tree.go
+++ b/cmd/tree/tree.go
@@ -108,8 +108,9 @@ short options as they conflict with rclone's short options.
opts.CTimeSort = opts.CTimeSort || sort == "ctime"
opts.NameSort = sort == "name"
opts.SizeSort = sort == "size"
+ ci := fs.GetConfig(context.Background())
if opts.DeepLevel == 0 {
- opts.DeepLevel = fs.Config.MaxDepth
+ opts.DeepLevel = ci.MaxDepth
}
cmd.Run(false, false, command, func() error {
return Tree(fsrc, outFile, &opts)
diff --git a/fs/accounting/accounting.go b/fs/accounting/accounting.go
index 7010411a6..34850b5c5 100644
--- a/fs/accounting/accounting.go
+++ b/fs/accounting/accounting.go
@@ -41,6 +41,7 @@ type Account struct {
mu sync.Mutex // mutex protects these values
in io.Reader
ctx context.Context // current context for transfer - may change
+ ci *fs.ConfigInfo
origIn io.ReadCloser
close io.Closer
size int64
@@ -74,6 +75,7 @@ func newAccountSizeName(ctx context.Context, stats *StatsInfo, in io.ReadCloser,
stats: stats,
in: in,
ctx: ctx,
+ ci: fs.GetConfig(ctx),
close: in,
origIn: in,
size: size,
@@ -85,10 +87,10 @@ func newAccountSizeName(ctx context.Context, stats *StatsInfo, in io.ReadCloser,
max: -1,
},
}
- if fs.Config.CutoffMode == fs.CutoffModeHard {
- acc.values.max = int64((fs.Config.MaxTransfer))
+ if acc.ci.CutoffMode == fs.CutoffModeHard {
+ acc.values.max = int64((acc.ci.MaxTransfer))
}
- currLimit := fs.Config.BwLimitFile.LimitAt(time.Now())
+ currLimit := acc.ci.BwLimitFile.LimitAt(time.Now())
if currLimit.Bandwidth > 0 {
fs.Debugf(acc.name, "Limiting file transfer to %v", currLimit.Bandwidth)
acc.tokenBucket = newTokenBucket(currLimit.Bandwidth)
@@ -107,14 +109,14 @@ func (acc *Account) WithBuffer() *Account {
}
acc.withBuf = true
var buffers int
- if acc.size >= int64(fs.Config.BufferSize) || acc.size == -1 {
- buffers = int(int64(fs.Config.BufferSize) / asyncreader.BufferSize)
+ if acc.size >= int64(acc.ci.BufferSize) || acc.size == -1 {
+ buffers = int(int64(acc.ci.BufferSize) / asyncreader.BufferSize)
} else {
buffers = int(acc.size / asyncreader.BufferSize)
}
// On big files add a buffer
if buffers > 0 {
- rc, err := asyncreader.New(acc.origIn, buffers)
+ rc, err := asyncreader.New(acc.ctx, acc.origIn, buffers)
if err != nil {
fs.Errorf(acc.name, "Failed to make buffer: %v", err)
} else {
@@ -472,7 +474,7 @@ func (acc *Account) String() string {
}
}
- if fs.Config.DataRateUnit == "bits" {
+ if acc.ci.DataRateUnit == "bits" {
cur = cur * 8
}
@@ -482,8 +484,8 @@ func (acc *Account) String() string {
}
return fmt.Sprintf("%*s:%3d%% /%s, %s/s, %s",
- fs.Config.StatsFileNameLength,
- shortenName(acc.name, fs.Config.StatsFileNameLength),
+ acc.ci.StatsFileNameLength,
+ shortenName(acc.name, acc.ci.StatsFileNameLength),
percentageDone,
fs.SizeSuffix(b),
fs.SizeSuffix(cur),
diff --git a/fs/accounting/accounting_test.go b/fs/accounting/accounting_test.go
index f826d7cd0..53cb09a7b 100644
--- a/fs/accounting/accounting_test.go
+++ b/fs/accounting/accounting_test.go
@@ -258,13 +258,14 @@ func TestAccountAccounter(t *testing.T) {
func TestAccountMaxTransfer(t *testing.T) {
ctx := context.Background()
- old := fs.Config.MaxTransfer
- oldMode := fs.Config.CutoffMode
+ ci := fs.GetConfig(ctx)
+ old := ci.MaxTransfer
+ oldMode := ci.CutoffMode
- fs.Config.MaxTransfer = 15
+ ci.MaxTransfer = 15
defer func() {
- fs.Config.MaxTransfer = old
- fs.Config.CutoffMode = oldMode
+ ci.MaxTransfer = old
+ ci.CutoffMode = oldMode
}()
in := ioutil.NopCloser(bytes.NewBuffer(make([]byte, 100)))
@@ -284,7 +285,7 @@ func TestAccountMaxTransfer(t *testing.T) {
assert.Equal(t, ErrorMaxTransferLimitReachedFatal, err)
assert.True(t, fserrors.IsFatalError(err))
- fs.Config.CutoffMode = fs.CutoffModeSoft
+ ci.CutoffMode = fs.CutoffModeSoft
stats = NewStats(ctx)
acc = newAccountSizeName(ctx, stats, in, 1, "test")
@@ -301,13 +302,14 @@ func TestAccountMaxTransfer(t *testing.T) {
func TestAccountMaxTransferWriteTo(t *testing.T) {
ctx := context.Background()
- old := fs.Config.MaxTransfer
- oldMode := fs.Config.CutoffMode
+ ci := fs.GetConfig(ctx)
+ old := ci.MaxTransfer
+ oldMode := ci.CutoffMode
- fs.Config.MaxTransfer = 15
+ ci.MaxTransfer = 15
defer func() {
- fs.Config.MaxTransfer = old
- fs.Config.CutoffMode = oldMode
+ ci.MaxTransfer = old
+ ci.CutoffMode = oldMode
}()
in := ioutil.NopCloser(readers.NewPatternReader(1024))
diff --git a/fs/accounting/inprogress.go b/fs/accounting/inprogress.go
index edd21a616..7fcbad781 100644
--- a/fs/accounting/inprogress.go
+++ b/fs/accounting/inprogress.go
@@ -15,8 +15,9 @@ type inProgress struct {
// newInProgress makes a new inProgress object
func newInProgress(ctx context.Context) *inProgress {
+ ci := fs.GetConfig(ctx)
return &inProgress{
- m: make(map[string]*Account, fs.Config.Transfers),
+ m: make(map[string]*Account, ci.Transfers),
}
}
diff --git a/fs/accounting/stats.go b/fs/accounting/stats.go
index c9502d2aa..6e448caa2 100644
--- a/fs/accounting/stats.go
+++ b/fs/accounting/stats.go
@@ -24,6 +24,7 @@ var startTime = time.Now()
type StatsInfo struct {
mu sync.RWMutex
ctx context.Context
+ ci *fs.ConfigInfo
bytes int64
errors int64
lastError error
@@ -52,10 +53,12 @@ type StatsInfo struct {
// NewStats creates an initialised StatsInfo
func NewStats(ctx context.Context) *StatsInfo {
+ ci := fs.GetConfig(ctx)
return &StatsInfo{
ctx: ctx,
- checking: newTransferMap(fs.Config.Checkers, "checking"),
- transferring: newTransferMap(fs.Config.Transfers, "transferring"),
+ ci: ci,
+ checking: newTransferMap(ci.Checkers, "checking"),
+ transferring: newTransferMap(ci.Transfers, "transferring"),
inProgress: newInProgress(ctx),
}
}
@@ -243,7 +246,7 @@ func (s *StatsInfo) String() string {
}
displaySpeed := speed
- if fs.Config.DataRateUnit == "bits" {
+ if s.ci.DataRateUnit == "bits" {
displaySpeed *= 8
}
@@ -259,7 +262,7 @@ func (s *StatsInfo) String() string {
dateString = ""
)
- if !fs.Config.StatsOneLine {
+ if !s.ci.StatsOneLine {
_, _ = fmt.Fprintf(buf, "\nTransferred: ")
} else {
xfrchk := []string{}
@@ -272,9 +275,9 @@ func (s *StatsInfo) String() string {
if len(xfrchk) > 0 {
xfrchkString = fmt.Sprintf(" (%s)", strings.Join(xfrchk, ", "))
}
- if fs.Config.StatsOneLineDate {
+ if s.ci.StatsOneLineDate {
t := time.Now()
- dateString = t.Format(fs.Config.StatsOneLineDateFormat) // Including the separator so people can customize it
+ dateString = t.Format(s.ci.StatsOneLineDateFormat) // Including the separator so people can customize it
}
}
@@ -283,17 +286,17 @@ func (s *StatsInfo) String() string {
fs.SizeSuffix(s.bytes),
fs.SizeSuffix(totalSize).Unit("Bytes"),
percent(s.bytes, totalSize),
- fs.SizeSuffix(displaySpeed).Unit(strings.Title(fs.Config.DataRateUnit)+"/s"),
+ fs.SizeSuffix(displaySpeed).Unit(strings.Title(s.ci.DataRateUnit)+"/s"),
etaString(currentSize, totalSize, speed),
xfrchkString,
)
- if fs.Config.ProgressTerminalTitle {
+ if s.ci.ProgressTerminalTitle {
// Writes ETA to the terminal title
terminal.WriteTerminalTitle("ETA: " + etaString(currentSize, totalSize, speed))
}
- if !fs.Config.StatsOneLine {
+ if !s.ci.StatsOneLine {
_, _ = buf.WriteRune('\n')
errorDetails := ""
switch {
@@ -333,7 +336,7 @@ func (s *StatsInfo) String() string {
s.mu.RUnlock()
// Add per transfer stats if required
- if !fs.Config.StatsOneLine {
+ if !s.ci.StatsOneLine {
if !s.checking.empty() {
_, _ = fmt.Fprintf(buf, "Checking:\n%s\n", s.checking.String(s.ctx, s.inProgress, s.transferring))
}
@@ -361,11 +364,11 @@ func (s *StatsInfo) Transferred() []TransferSnapshot {
// Log outputs the StatsInfo to the log
func (s *StatsInfo) Log() {
- if fs.Config.UseJSONLog {
+ if s.ci.UseJSONLog {
out, _ := s.RemoteStats()
- fs.LogLevelPrintf(fs.Config.StatsLogLevel, nil, "%v%v\n", s, fs.LogValue("stats", out))
+ fs.LogLevelPrintf(s.ci.StatsLogLevel, nil, "%v%v\n", s, fs.LogValue("stats", out))
} else {
- fs.LogLevelPrintf(fs.Config.StatsLogLevel, nil, "%v\n", s)
+ fs.LogLevelPrintf(s.ci.StatsLogLevel, nil, "%v\n", s)
}
}
@@ -681,7 +684,7 @@ func (s *StatsInfo) PruneTransfers() {
}
s.mu.Lock()
// remove a transfer from the start if we are over quota
- if len(s.startedTransfers) > MaxCompletedTransfers+fs.Config.Transfers {
+ if len(s.startedTransfers) > MaxCompletedTransfers+s.ci.Transfers {
for i, tr := range s.startedTransfers {
if tr.IsDone() {
s.removeTransfer(tr, i)
diff --git a/fs/accounting/stats_groups.go b/fs/accounting/stats_groups.go
index 4767eb5e3..0aace47c3 100644
--- a/fs/accounting/stats_groups.go
+++ b/fs/accounting/stats_groups.go
@@ -308,13 +308,14 @@ func newStatsGroups() *statsGroups {
func (sg *statsGroups) set(ctx context.Context, group string, stats *StatsInfo) {
sg.mu.Lock()
defer sg.mu.Unlock()
+ ci := fs.GetConfig(ctx)
// Limit number of groups kept in memory.
- if len(sg.order) >= fs.Config.MaxStatsGroups {
+ if len(sg.order) >= ci.MaxStatsGroups {
group := sg.order[0]
fs.LogPrintf(fs.LogLevelDebug, nil, "Max number of stats groups reached removing %s", group)
delete(sg.m, group)
- r := (len(sg.order) - fs.Config.MaxStatsGroups) + 1
+ r := (len(sg.order) - ci.MaxStatsGroups) + 1
sg.order = sg.order[r:]
}
diff --git a/fs/accounting/stats_test.go b/fs/accounting/stats_test.go
index 1b1814b2b..3359e3a7b 100644
--- a/fs/accounting/stats_test.go
+++ b/fs/accounting/stats_test.go
@@ -386,6 +386,7 @@ func TestTimeRangeDuration(t *testing.T) {
func TestPruneTransfers(t *testing.T) {
ctx := context.Background()
+ ci := fs.GetConfig(ctx)
for _, test := range []struct {
Name string
Transfers int
@@ -396,7 +397,7 @@ func TestPruneTransfers(t *testing.T) {
Name: "Limited number of StartedTransfers",
Limit: 100,
Transfers: 200,
- ExpectedStartedTransfers: 100 + fs.Config.Transfers,
+ ExpectedStartedTransfers: 100 + ci.Transfers,
},
{
Name: "Unlimited number of StartedTransfers",
diff --git a/fs/accounting/token_bucket.go b/fs/accounting/token_bucket.go
index b4a72f1d6..547bfc6ad 100644
--- a/fs/accounting/token_bucket.go
+++ b/fs/accounting/token_bucket.go
@@ -36,8 +36,9 @@ func newTokenBucket(bandwidth fs.SizeSuffix) *rate.Limiter {
// StartTokenBucket starts the token bucket if necessary
func StartTokenBucket(ctx context.Context) {
+ ci := fs.GetConfig(ctx)
currLimitMu.Lock()
- currLimit := fs.Config.BwLimit.LimitAt(time.Now())
+ currLimit := ci.BwLimit.LimitAt(time.Now())
currLimitMu.Unlock()
if currLimit.Bandwidth > 0 {
@@ -52,16 +53,17 @@ func StartTokenBucket(ctx context.Context) {
// StartTokenTicker creates a ticker to update the bandwidth limiter every minute.
func StartTokenTicker(ctx context.Context) {
+ ci := fs.GetConfig(ctx)
// If the timetable has a single entry or was not specified, we don't need
// a ticker to update the bandwidth.
- if len(fs.Config.BwLimit) <= 1 {
+ if len(ci.BwLimit) <= 1 {
return
}
ticker := time.NewTicker(time.Minute)
go func() {
for range ticker.C {
- limitNow := fs.Config.BwLimit.LimitAt(time.Now())
+ limitNow := ci.BwLimit.LimitAt(time.Now())
currLimitMu.Lock()
if currLimit.Bandwidth != limitNow.Bandwidth {
diff --git a/fs/accounting/transfer.go b/fs/accounting/transfer.go
index 8172bb727..93de4bcbd 100644
--- a/fs/accounting/transfer.go
+++ b/fs/accounting/transfer.go
@@ -99,10 +99,11 @@ func (tr *Transfer) Done(ctx context.Context, err error) {
acc := tr.acc
tr.mu.RUnlock()
+ ci := fs.GetConfig(ctx)
if acc != nil {
// Close the file if it is still open
if err := acc.Close(); err != nil {
- fs.LogLevelPrintf(fs.Config.StatsLogLevel, nil, "can't close account: %+v\n", err)
+ fs.LogLevelPrintf(ci.StatsLogLevel, nil, "can't close account: %+v\n", err)
}
// Signal done with accounting
acc.Done()
@@ -128,10 +129,11 @@ func (tr *Transfer) Reset(ctx context.Context) {
acc := tr.acc
tr.acc = nil
tr.mu.RUnlock()
+ ci := fs.GetConfig(ctx)
if acc != nil {
if err := acc.Close(); err != nil {
- fs.LogLevelPrintf(fs.Config.StatsLogLevel, nil, "can't close account: %+v\n", err)
+ fs.LogLevelPrintf(ci.StatsLogLevel, nil, "can't close account: %+v\n", err)
}
}
}
diff --git a/fs/accounting/transfermap.go b/fs/accounting/transfermap.go
index 22a5f8c1c..ed64bf369 100644
--- a/fs/accounting/transfermap.go
+++ b/fs/accounting/transfermap.go
@@ -92,6 +92,7 @@ func (tm *transferMap) _sortedSlice() []*Transfer {
func (tm *transferMap) String(ctx context.Context, progress *inProgress, exclude *transferMap) string {
tm.mu.RLock()
defer tm.mu.RUnlock()
+ ci := fs.GetConfig(ctx)
stringList := make([]string, 0, len(tm.items))
for _, tr := range tm._sortedSlice() {
if exclude != nil {
@@ -107,8 +108,8 @@ func (tm *transferMap) String(ctx context.Context, progress *inProgress, exclude
out = acc.String()
} else {
out = fmt.Sprintf("%*s: %s",
- fs.Config.StatsFileNameLength,
- shortenName(tr.remote, fs.Config.StatsFileNameLength),
+ ci.StatsFileNameLength,
+ shortenName(tr.remote, ci.StatsFileNameLength),
tm.name,
)
}
diff --git a/fs/asyncreader/asyncreader.go b/fs/asyncreader/asyncreader.go
index 6b97c8f6c..92d389002 100644
--- a/fs/asyncreader/asyncreader.go
+++ b/fs/asyncreader/asyncreader.go
@@ -3,6 +3,7 @@
package asyncreader
import (
+ "context"
"io"
"sync"
"time"
@@ -29,17 +30,18 @@ var ErrorStreamAbandoned = errors.New("stream abandoned")
// This should be fully transparent, except that once an error
// has been returned from the Reader, it will not recover.
type AsyncReader struct {
- in io.ReadCloser // Input reader
- ready chan *buffer // Buffers ready to be handed to the reader
- token chan struct{} // Tokens which allow a buffer to be taken
- exit chan struct{} // Closes when finished
- buffers int // Number of buffers
- err error // If an error has occurred it is here
- cur *buffer // Current buffer being served
- exited chan struct{} // Channel is closed been the async reader shuts down
- size int // size of buffer to use
- closed bool // whether we have closed the underlying stream
- mu sync.Mutex // lock for Read/WriteTo/Abandon/Close
+ in io.ReadCloser // Input reader
+ ready chan *buffer // Buffers ready to be handed to the reader
+ token chan struct{} // Tokens which allow a buffer to be taken
+ exit chan struct{} // Closes when finished
+ buffers int // Number of buffers
+ err error // If an error has occurred it is here
+ cur *buffer // Current buffer being served
+ exited chan struct{} // Channel is closed been the async reader shuts down
+ size int // size of buffer to use
+ closed bool // whether we have closed the underlying stream
+ mu sync.Mutex // lock for Read/WriteTo/Abandon/Close
+ ci *fs.ConfigInfo // for reading config
}
// New returns a reader that will asynchronously read from
@@ -48,14 +50,16 @@ type AsyncReader struct {
// function has returned.
// The input can be read from the returned reader.
// When done use Close to release the buffers and close the supplied input.
-func New(rd io.ReadCloser, buffers int) (*AsyncReader, error) {
+func New(ctx context.Context, rd io.ReadCloser, buffers int) (*AsyncReader, error) {
if buffers <= 0 {
return nil, errors.New("number of buffers too small")
}
if rd == nil {
return nil, errors.New("nil reader supplied")
}
- a := &AsyncReader{}
+ a := &AsyncReader{
+ ci: fs.GetConfig(ctx),
+ }
a.init(rd, buffers)
return a, nil
}
@@ -114,7 +118,7 @@ func (a *AsyncReader) putBuffer(b *buffer) {
func (a *AsyncReader) getBuffer() *buffer {
bufferPoolOnce.Do(func() {
// Initialise the buffer pool when used
- bufferPool = pool.New(bufferCacheFlushTime, BufferSize, bufferCacheSize, fs.Config.UseMmap)
+ bufferPool = pool.New(bufferCacheFlushTime, BufferSize, bufferCacheSize, a.ci.UseMmap)
})
return &buffer{
buf: bufferPool.Get(),
diff --git a/fs/asyncreader/asyncreader_test.go b/fs/asyncreader/asyncreader_test.go
index 4aab06604..342494d06 100644
--- a/fs/asyncreader/asyncreader_test.go
+++ b/fs/asyncreader/asyncreader_test.go
@@ -3,6 +3,7 @@ package asyncreader
import (
"bufio"
"bytes"
+ "context"
"fmt"
"io"
"io/ioutil"
@@ -20,8 +21,10 @@ import (
)
func TestAsyncReader(t *testing.T) {
+ ctx := context.Background()
+
buf := ioutil.NopCloser(bytes.NewBufferString("Testbuffer"))
- ar, err := New(buf, 4)
+ ar, err := New(ctx, buf, 4)
require.NoError(t, err)
var dst = make([]byte, 100)
@@ -46,7 +49,7 @@ func TestAsyncReader(t *testing.T) {
// Test Close without reading everything
buf = ioutil.NopCloser(bytes.NewBuffer(make([]byte, 50000)))
- ar, err = New(buf, 4)
+ ar, err = New(ctx, buf, 4)
require.NoError(t, err)
err = ar.Close()
require.NoError(t, err)
@@ -54,8 +57,10 @@ func TestAsyncReader(t *testing.T) {
}
func TestAsyncWriteTo(t *testing.T) {
+ ctx := context.Background()
+
buf := ioutil.NopCloser(bytes.NewBufferString("Testbuffer"))
- ar, err := New(buf, 4)
+ ar, err := New(ctx, buf, 4)
require.NoError(t, err)
var dst = &bytes.Buffer{}
@@ -73,15 +78,17 @@ func TestAsyncWriteTo(t *testing.T) {
}
func TestAsyncReaderErrors(t *testing.T) {
+ ctx := context.Background()
+
// test nil reader
- _, err := New(nil, 4)
+ _, err := New(ctx, nil, 4)
require.Error(t, err)
// invalid buffer number
buf := ioutil.NopCloser(bytes.NewBufferString("Testbuffer"))
- _, err = New(buf, 0)
+ _, err = New(ctx, buf, 0)
require.Error(t, err)
- _, err = New(buf, -1)
+ _, err = New(ctx, buf, -1)
require.Error(t, err)
}
@@ -140,6 +147,8 @@ var bufsizes = []int{
// Test various input buffer sizes, number of buffers and read sizes.
func TestAsyncReaderSizes(t *testing.T) {
+ ctx := context.Background()
+
var texts [31]string
str := ""
all := ""
@@ -161,7 +170,7 @@ func TestAsyncReaderSizes(t *testing.T) {
bufsize := bufsizes[k]
read := readmaker.fn(strings.NewReader(text))
buf := bufio.NewReaderSize(read, bufsize)
- ar, _ := New(ioutil.NopCloser(buf), l)
+ ar, _ := New(ctx, ioutil.NopCloser(buf), l)
s := bufreader.fn(ar)
// "timeout" expects the Reader to recover, AsyncReader does not.
if s != text && readmaker.name != "timeout" {
@@ -179,6 +188,8 @@ func TestAsyncReaderSizes(t *testing.T) {
// Test various input buffer sizes, number of buffers and read sizes.
func TestAsyncReaderWriteTo(t *testing.T) {
+ ctx := context.Background()
+
var texts [31]string
str := ""
all := ""
@@ -200,7 +211,7 @@ func TestAsyncReaderWriteTo(t *testing.T) {
bufsize := bufsizes[k]
read := readmaker.fn(strings.NewReader(text))
buf := bufio.NewReaderSize(read, bufsize)
- ar, _ := New(ioutil.NopCloser(buf), l)
+ ar, _ := New(ctx, ioutil.NopCloser(buf), l)
dst := &bytes.Buffer{}
_, err := ar.WriteTo(dst)
if err != nil && err != io.EOF && err != iotest.ErrTimeout {
@@ -246,8 +257,10 @@ func (z *zeroReader) Close() error {
// Test closing and abandoning
func testAsyncReaderClose(t *testing.T, writeto bool) {
+ ctx := context.Background()
+
zr := &zeroReader{}
- a, err := New(zr, 16)
+ a, err := New(ctx, zr, 16)
require.NoError(t, err)
var copyN int64
var copyErr error
@@ -287,6 +300,8 @@ func TestAsyncReaderCloseRead(t *testing.T) { testAsyncReaderClose(t, false)
func TestAsyncReaderCloseWriteTo(t *testing.T) { testAsyncReaderClose(t, true) }
func TestAsyncReaderSkipBytes(t *testing.T) {
+ ctx := context.Background()
+
t.Parallel()
data := make([]byte, 15000)
buf := make([]byte, len(data))
@@ -312,7 +327,7 @@ func TestAsyncReaderSkipBytes(t *testing.T) {
t.Run(fmt.Sprintf("%d", initialRead), func(t *testing.T) {
for _, skip := range skips {
t.Run(fmt.Sprintf("%d", skip), func(t *testing.T) {
- ar, err := New(ioutil.NopCloser(bytes.NewReader(data)), buffers)
+ ar, err := New(ctx, ioutil.NopCloser(bytes.NewReader(data)), buffers)
require.NoError(t, err)
wantSkipFalse := false
diff --git a/fs/config.go b/fs/config.go
index d02a0f494..87ffe6fcf 100644
--- a/fs/config.go
+++ b/fs/config.go
@@ -1,6 +1,7 @@
package fs
import (
+ "context"
"net"
"strings"
"time"
@@ -10,8 +11,8 @@ import (
// Global
var (
- // Config is the global config
- Config = NewConfig()
+ // globalConfig for rclone
+ globalConfig = NewConfig()
// Read a value from the config file
//
@@ -162,6 +163,34 @@ func NewConfig() *ConfigInfo {
return c
}
+type configContextKeyType struct{}
+
+// Context key for config
+var configContextKey = configContextKeyType{}
+
+// GetConfig returns the global or context sensitive context
+func GetConfig(ctx context.Context) *ConfigInfo {
+ if ctx == nil {
+ return globalConfig
+ }
+ c := ctx.Value(configContextKey)
+ if c == nil {
+ return globalConfig
+ }
+ return c.(*ConfigInfo)
+}
+
+// AddConfig returns a mutable config structure based on a shallow
+// copy of that found in ctx and returns a new context with that added
+// to it.
+func AddConfig(ctx context.Context) (context.Context, *ConfigInfo) {
+ c := GetConfig(ctx)
+ cCopy := new(ConfigInfo)
+ *cCopy = *c
+ newCtx := context.WithValue(ctx, configContextKey, cCopy)
+ return newCtx, cCopy
+}
+
// ConfigToEnv converts a config section and name, e.g. ("myremote",
// "ignore-size") into an environment name
// "RCLONE_CONFIG_MYREMOTE_IGNORE_SIZE"
diff --git a/fs/config/config.go b/fs/config/config.go
index b4e669fc4..c8fe6c51f 100644
--- a/fs/config/config.go
+++ b/fs/config/config.go
@@ -236,7 +236,7 @@ func LoadConfig(ctx context.Context) {
accounting.StartTokenTicker(ctx)
// Start the transactions per second limiter
- fshttp.StartHTTPTokenBucket()
+ fshttp.StartHTTPTokenBucket(ctx)
}
var errorConfigFileNotFound = errors.New("config file not found")
@@ -244,6 +244,8 @@ var errorConfigFileNotFound = errors.New("config file not found")
// loadConfigFile will load a config file, and
// automatically decrypt it.
func loadConfigFile() (*goconfig.ConfigFile, error) {
+ ctx := context.Background()
+ ci := fs.GetConfig(ctx)
var usingPasswordCommand bool
b, err := ioutil.ReadFile(ConfigPath)
@@ -278,11 +280,11 @@ func loadConfigFile() (*goconfig.ConfigFile, error) {
}
if len(configKey) == 0 {
- if len(fs.Config.PasswordCommand) != 0 {
+ if len(ci.PasswordCommand) != 0 {
var stdout bytes.Buffer
var stderr bytes.Buffer
- cmd := exec.Command(fs.Config.PasswordCommand[0], fs.Config.PasswordCommand[1:]...)
+ cmd := exec.Command(ci.PasswordCommand[0], ci.PasswordCommand[1:]...)
cmd.Stdout = &stdout
cmd.Stderr = &stderr
@@ -358,7 +360,7 @@ func loadConfigFile() (*goconfig.ConfigFile, error) {
if usingPasswordCommand {
return nil, errors.New("using --password-command derived password, unable to decrypt configuration")
}
- if !fs.Config.AskPassword {
+ if !ci.AskPassword {
return nil, errors.New("unable to decrypt configuration and not allowed to ask for password - set RCLONE_CONFIG_PASS to your configuration password")
}
getConfigPassword("Enter configuration password:")
@@ -600,15 +602,17 @@ func saveConfig() error {
// SaveConfig calling function which saves configuration file.
// if saveConfig returns error trying again after sleep.
func SaveConfig() {
+ ctx := context.Background()
+ ci := fs.GetConfig(ctx)
var err error
- for i := 0; i < fs.Config.LowLevelRetries+1; i++ {
+ for i := 0; i < ci.LowLevelRetries+1; i++ {
if err = saveConfig(); err == nil {
return
}
waitingTimeMs := mathrand.Intn(1000)
time.Sleep(time.Duration(waitingTimeMs) * time.Millisecond)
}
- log.Fatalf("Failed to save config after %d tries: %v", fs.Config.LowLevelRetries, err)
+ log.Fatalf("Failed to save config after %d tries: %v", ci.LowLevelRetries, err)
return
}
@@ -746,7 +750,8 @@ func Confirm(Default bool) bool {
// that, but if it isn't set then it will return the Default value
// passed in
func ConfirmWithConfig(ctx context.Context, m configmap.Getter, configName string, Default bool) bool {
- if fs.Config.AutoConfirm {
+ ci := fs.GetConfig(ctx)
+ if ci.AutoConfirm {
configString, ok := m.Get(configName)
if ok {
configValue, err := strconv.ParseBool(configString)
@@ -897,12 +902,12 @@ func MustFindByName(name string) *fs.RegInfo {
}
// RemoteConfig runs the config helper for the remote if needed
-func RemoteConfig(name string) {
+func RemoteConfig(ctx context.Context, name string) {
fmt.Printf("Remote config\n")
f := MustFindByName(name)
if f.Config != nil {
m := fs.ConfigMap(f, name)
- f.Config(context.Background(), name, m)
+ f.Config(ctx, name, m)
}
}
@@ -1023,13 +1028,11 @@ func ChooseOption(o *fs.Option, name string) string {
return in
}
-// Suppress the confirm prompts and return a function to undo that
-func suppressConfirm(ctx context.Context) func() {
- old := fs.Config.AutoConfirm
- fs.Config.AutoConfirm = true
- return func() {
- fs.Config.AutoConfirm = old
- }
+// Suppress the confirm prompts by altering the context config
+func suppressConfirm(ctx context.Context) context.Context {
+ newCtx, ci := fs.AddConfig(ctx)
+ ci.AutoConfirm = true
+ return newCtx
}
// UpdateRemote adds the keyValues passed in to the remote of name.
@@ -1042,7 +1045,7 @@ func UpdateRemote(ctx context.Context, name string, keyValues rc.Params, doObscu
if err != nil {
return err
}
- defer suppressConfirm(ctx)()
+ ctx = suppressConfirm(ctx)
// Work out which options need to be obscured
needsObscure := map[string]struct{}{}
@@ -1079,7 +1082,7 @@ func UpdateRemote(ctx context.Context, name string, keyValues rc.Params, doObscu
}
getConfigData().SetValue(name, k, vStr)
}
- RemoteConfig(name)
+ RemoteConfig(ctx, name)
SaveConfig()
return nil
}
@@ -1103,11 +1106,11 @@ func CreateRemote(ctx context.Context, name string, provider string, keyValues r
// PasswordRemote adds the keyValues passed in to the remote of name.
// keyValues should be key, value pairs.
func PasswordRemote(ctx context.Context, name string, keyValues rc.Params) error {
+ ctx = suppressConfirm(ctx)
err := fspath.CheckConfigName(name)
if err != nil {
return err
}
- defer suppressConfirm(ctx)()
for k, v := range keyValues {
keyValues[k] = obscure.MustObscure(fmt.Sprint(v))
}
@@ -1206,7 +1209,7 @@ func editOptions(ri *fs.RegInfo, name string, isNew bool) {
}
// NewRemote make a new remote from its name
-func NewRemote(name string) {
+func NewRemote(ctx context.Context, name string) {
var (
newType string
ri *fs.RegInfo
@@ -1226,16 +1229,16 @@ func NewRemote(name string) {
getConfigData().SetValue(name, "type", newType)
editOptions(ri, name, true)
- RemoteConfig(name)
+ RemoteConfig(ctx, name)
if OkRemote(name) {
SaveConfig()
return
}
- EditRemote(ri, name)
+ EditRemote(ctx, ri, name)
}
// EditRemote gets the user to edit a remote
-func EditRemote(ri *fs.RegInfo, name string) {
+func EditRemote(ctx context.Context, ri *fs.RegInfo, name string) {
ShowRemote(name)
fmt.Printf("Edit remote\n")
for {
@@ -1245,7 +1248,7 @@ func EditRemote(ri *fs.RegInfo, name string) {
}
}
SaveConfig()
- RemoteConfig(name)
+ RemoteConfig(ctx, name)
}
// DeleteRemote gets the user to delete a remote
@@ -1307,7 +1310,7 @@ func ShowConfig() {
}
// EditConfig edits the config file interactively
-func EditConfig() {
+func EditConfig(ctx context.Context) {
for {
haveRemotes := len(getConfigData().GetSectionList()) != 0
what := []string{"eEdit existing remote", "nNew remote", "dDelete remote", "rRename remote", "cCopy remote", "sSet configuration password", "qQuit config"}
@@ -1324,9 +1327,9 @@ func EditConfig() {
case 'e':
name := ChooseRemote()
fs := MustFindByName(name)
- EditRemote(fs, name)
+ EditRemote(ctx, fs, name)
case 'n':
- NewRemote(NewRemoteName())
+ NewRemote(ctx, NewRemoteName())
case 'd':
name := ChooseRemote()
DeleteRemote(name)
@@ -1388,7 +1391,7 @@ func SetPassword() {
// rclone authorize "fs name"
// rclone authorize "fs name" "client id" "client secret"
func Authorize(ctx context.Context, args []string, noAutoBrowser bool) {
- defer suppressConfirm(ctx)()
+ ctx = suppressConfirm(ctx)
switch len(args) {
case 1, 3:
default:
diff --git a/fs/config/config_test.go b/fs/config/config_test.go
index dd4cdbd1d..6713e9d17 100644
--- a/fs/config/config_test.go
+++ b/fs/config/config_test.go
@@ -17,6 +17,7 @@ import (
func testConfigFile(t *testing.T, configFileName string) func() {
ctx := context.Background()
+ ci := fs.GetConfig(ctx)
configKey = nil // reset password
_ = os.Unsetenv("_RCLONE_CONFIG_KEY_FILE")
_ = os.Unsetenv("RCLONE_CONFIG_PASS")
@@ -29,13 +30,13 @@ func testConfigFile(t *testing.T, configFileName string) func() {
// temporarily adapt configuration
oldOsStdout := os.Stdout
oldConfigPath := ConfigPath
- oldConfig := fs.Config
+ oldConfig := *ci
oldConfigFile := configFile
oldReadLine := ReadLine
oldPassword := Password
os.Stdout = nil
ConfigPath = path
- fs.Config = &fs.ConfigInfo{}
+ ci = &fs.ConfigInfo{}
configFile = nil
LoadConfig(ctx)
@@ -67,7 +68,7 @@ func testConfigFile(t *testing.T, configFileName string) func() {
ConfigPath = oldConfigPath
ReadLine = oldReadLine
Password = oldPassword
- fs.Config = oldConfig
+ *ci = oldConfig
configFile = oldConfigFile
_ = os.Unsetenv("_RCLONE_CONFIG_KEY_FILE")
@@ -87,6 +88,7 @@ func makeReadLine(answers []string) func() string {
func TestCRUD(t *testing.T) {
defer testConfigFile(t, "crud.conf")()
+ ctx := context.Background()
// script for creating remote
ReadLine = makeReadLine([]string{
@@ -97,7 +99,7 @@ func TestCRUD(t *testing.T) {
"secret", // repeat
"y", // looks good, save
})
- NewRemote("test")
+ NewRemote(ctx, "test")
assert.Equal(t, []string{"test"}, configFile.GetSectionList())
assert.Equal(t, "config_test_remote", FileGet("test", "type"))
@@ -124,6 +126,7 @@ func TestCRUD(t *testing.T) {
func TestChooseOption(t *testing.T) {
defer testConfigFile(t, "crud.conf")()
+ ctx := context.Background()
// script for creating remote
ReadLine = makeReadLine([]string{
@@ -139,7 +142,7 @@ func TestChooseOption(t *testing.T) {
assert.Equal(t, 1024, bits)
return "not very random password", nil
}
- NewRemote("test")
+ NewRemote(ctx, "test")
assert.Equal(t, "false", FileGet("test", "bool"))
assert.Equal(t, "not very random password", obscure.MustReveal(FileGet("test", "pass")))
@@ -151,7 +154,7 @@ func TestChooseOption(t *testing.T) {
"n", // not required
"y", // looks good, save
})
- NewRemote("test")
+ NewRemote(ctx, "test")
assert.Equal(t, "true", FileGet("test", "bool"))
assert.Equal(t, "", FileGet("test", "pass"))
@@ -159,6 +162,7 @@ func TestChooseOption(t *testing.T) {
func TestNewRemoteName(t *testing.T) {
defer testConfigFile(t, "crud.conf")()
+ ctx := context.Background()
// script for creating remote
ReadLine = makeReadLine([]string{
@@ -167,7 +171,7 @@ func TestNewRemoteName(t *testing.T) {
"n", // not required
"y", // looks good, save
})
- NewRemote("test")
+ NewRemote(ctx, "test")
ReadLine = makeReadLine([]string{
"test", // already exists
@@ -293,16 +297,18 @@ func TestConfigLoadEncrypted(t *testing.T) {
}
func TestConfigLoadEncryptedWithValidPassCommand(t *testing.T) {
+ ctx := context.Background()
+ ci := fs.GetConfig(ctx)
oldConfigPath := ConfigPath
- oldConfig := fs.Config
+ oldConfig := *ci
ConfigPath = "./testdata/encrypted.conf"
- // using fs.Config.PasswordCommand, correct password
- fs.Config.PasswordCommand = fs.SpaceSepList{"echo", "asdf"}
+ // using ci.PasswordCommand, correct password
+ ci.PasswordCommand = fs.SpaceSepList{"echo", "asdf"}
defer func() {
ConfigPath = oldConfigPath
configKey = nil // reset password
- fs.Config = oldConfig
- fs.Config.PasswordCommand = nil
+ *ci = oldConfig
+ ci.PasswordCommand = nil
}()
configKey = nil // reset password
@@ -320,16 +326,18 @@ func TestConfigLoadEncryptedWithValidPassCommand(t *testing.T) {
}
func TestConfigLoadEncryptedWithInvalidPassCommand(t *testing.T) {
+ ctx := context.Background()
+ ci := fs.GetConfig(ctx)
oldConfigPath := ConfigPath
- oldConfig := fs.Config
+ oldConfig := *ci
ConfigPath = "./testdata/encrypted.conf"
- // using fs.Config.PasswordCommand, incorrect password
- fs.Config.PasswordCommand = fs.SpaceSepList{"echo", "asdf-blurfl"}
+ // using ci.PasswordCommand, incorrect password
+ ci.PasswordCommand = fs.SpaceSepList{"echo", "asdf-blurfl"}
defer func() {
ConfigPath = oldConfigPath
configKey = nil // reset password
- fs.Config = oldConfig
- fs.Config.PasswordCommand = nil
+ *ci = oldConfig
+ ci.PasswordCommand = nil
}()
configKey = nil // reset password
diff --git a/fs/config/configflags/configflags.go b/fs/config/configflags/configflags.go
index f3bdf116e..4d1ea3665 100644
--- a/fs/config/configflags/configflags.go
+++ b/fs/config/configflags/configflags.go
@@ -35,96 +35,96 @@ var (
)
// AddFlags adds the non filing system specific flags to the command
-func AddFlags(flagSet *pflag.FlagSet) {
- rc.AddOption("main", fs.Config)
+func AddFlags(ci *fs.ConfigInfo, flagSet *pflag.FlagSet) {
+ rc.AddOption("main", ci)
// NB defaults which aren't the zero for the type should be set in fs/config.go NewConfig
flags.CountVarP(flagSet, &verbose, "verbose", "v", "Print lots more stuff (repeat for more)")
flags.BoolVarP(flagSet, &quiet, "quiet", "q", false, "Print as little stuff as possible")
- flags.DurationVarP(flagSet, &fs.Config.ModifyWindow, "modify-window", "", fs.Config.ModifyWindow, "Max time diff to be considered the same")
- flags.IntVarP(flagSet, &fs.Config.Checkers, "checkers", "", fs.Config.Checkers, "Number of checkers to run in parallel.")
- flags.IntVarP(flagSet, &fs.Config.Transfers, "transfers", "", fs.Config.Transfers, "Number of file transfers to run in parallel.")
+ flags.DurationVarP(flagSet, &ci.ModifyWindow, "modify-window", "", ci.ModifyWindow, "Max time diff to be considered the same")
+ flags.IntVarP(flagSet, &ci.Checkers, "checkers", "", ci.Checkers, "Number of checkers to run in parallel.")
+ flags.IntVarP(flagSet, &ci.Transfers, "transfers", "", ci.Transfers, "Number of file transfers to run in parallel.")
flags.StringVarP(flagSet, &config.ConfigPath, "config", "", config.ConfigPath, "Config file.")
flags.StringVarP(flagSet, &config.CacheDir, "cache-dir", "", config.CacheDir, "Directory rclone will use for caching.")
- flags.BoolVarP(flagSet, &fs.Config.CheckSum, "checksum", "c", fs.Config.CheckSum, "Skip based on checksum (if available) & size, not mod-time & size")
- flags.BoolVarP(flagSet, &fs.Config.SizeOnly, "size-only", "", fs.Config.SizeOnly, "Skip based on size only, not mod-time or checksum")
- flags.BoolVarP(flagSet, &fs.Config.IgnoreTimes, "ignore-times", "I", fs.Config.IgnoreTimes, "Don't skip files that match size and time - transfer all files")
- flags.BoolVarP(flagSet, &fs.Config.IgnoreExisting, "ignore-existing", "", fs.Config.IgnoreExisting, "Skip all files that exist on destination")
- flags.BoolVarP(flagSet, &fs.Config.IgnoreErrors, "ignore-errors", "", fs.Config.IgnoreErrors, "delete even if there are I/O errors")
- flags.BoolVarP(flagSet, &fs.Config.DryRun, "dry-run", "n", fs.Config.DryRun, "Do a trial run with no permanent changes")
- flags.BoolVarP(flagSet, &fs.Config.Interactive, "interactive", "i", fs.Config.Interactive, "Enable interactive mode")
- flags.DurationVarP(flagSet, &fs.Config.ConnectTimeout, "contimeout", "", fs.Config.ConnectTimeout, "Connect timeout")
- flags.DurationVarP(flagSet, &fs.Config.Timeout, "timeout", "", fs.Config.Timeout, "IO idle timeout")
- flags.DurationVarP(flagSet, &fs.Config.ExpectContinueTimeout, "expect-continue-timeout", "", fs.Config.ExpectContinueTimeout, "Timeout when using expect / 100-continue in HTTP")
+ flags.BoolVarP(flagSet, &ci.CheckSum, "checksum", "c", ci.CheckSum, "Skip based on checksum (if available) & size, not mod-time & size")
+ flags.BoolVarP(flagSet, &ci.SizeOnly, "size-only", "", ci.SizeOnly, "Skip based on size only, not mod-time or checksum")
+ flags.BoolVarP(flagSet, &ci.IgnoreTimes, "ignore-times", "I", ci.IgnoreTimes, "Don't skip files that match size and time - transfer all files")
+ flags.BoolVarP(flagSet, &ci.IgnoreExisting, "ignore-existing", "", ci.IgnoreExisting, "Skip all files that exist on destination")
+ flags.BoolVarP(flagSet, &ci.IgnoreErrors, "ignore-errors", "", ci.IgnoreErrors, "delete even if there are I/O errors")
+ flags.BoolVarP(flagSet, &ci.DryRun, "dry-run", "n", ci.DryRun, "Do a trial run with no permanent changes")
+ flags.BoolVarP(flagSet, &ci.Interactive, "interactive", "i", ci.Interactive, "Enable interactive mode")
+ flags.DurationVarP(flagSet, &ci.ConnectTimeout, "contimeout", "", ci.ConnectTimeout, "Connect timeout")
+ flags.DurationVarP(flagSet, &ci.Timeout, "timeout", "", ci.Timeout, "IO idle timeout")
+ flags.DurationVarP(flagSet, &ci.ExpectContinueTimeout, "expect-continue-timeout", "", ci.ExpectContinueTimeout, "Timeout when using expect / 100-continue in HTTP")
flags.BoolVarP(flagSet, &dumpHeaders, "dump-headers", "", false, "Dump HTTP headers - may contain sensitive info")
flags.BoolVarP(flagSet, &dumpBodies, "dump-bodies", "", false, "Dump HTTP headers and bodies - may contain sensitive info")
- flags.BoolVarP(flagSet, &fs.Config.InsecureSkipVerify, "no-check-certificate", "", fs.Config.InsecureSkipVerify, "Do not verify the server SSL certificate. Insecure.")
- flags.BoolVarP(flagSet, &fs.Config.AskPassword, "ask-password", "", fs.Config.AskPassword, "Allow prompt for password for encrypted configuration.")
- flags.FVarP(flagSet, &fs.Config.PasswordCommand, "password-command", "", "Command for supplying password for encrypted configuration.")
+ flags.BoolVarP(flagSet, &ci.InsecureSkipVerify, "no-check-certificate", "", ci.InsecureSkipVerify, "Do not verify the server SSL certificate. Insecure.")
+ flags.BoolVarP(flagSet, &ci.AskPassword, "ask-password", "", ci.AskPassword, "Allow prompt for password for encrypted configuration.")
+ flags.FVarP(flagSet, &ci.PasswordCommand, "password-command", "", "Command for supplying password for encrypted configuration.")
flags.BoolVarP(flagSet, &deleteBefore, "delete-before", "", false, "When synchronizing, delete files on destination before transferring")
flags.BoolVarP(flagSet, &deleteDuring, "delete-during", "", false, "When synchronizing, delete files during transfer")
flags.BoolVarP(flagSet, &deleteAfter, "delete-after", "", false, "When synchronizing, delete files on destination after transferring (default)")
- flags.Int64VarP(flagSet, &fs.Config.MaxDelete, "max-delete", "", -1, "When synchronizing, limit the number of deletes")
- flags.BoolVarP(flagSet, &fs.Config.TrackRenames, "track-renames", "", fs.Config.TrackRenames, "When synchronizing, track file renames and do a server-side move if possible")
- flags.StringVarP(flagSet, &fs.Config.TrackRenamesStrategy, "track-renames-strategy", "", fs.Config.TrackRenamesStrategy, "Strategies to use when synchronizing using track-renames hash|modtime|leaf")
- flags.IntVarP(flagSet, &fs.Config.LowLevelRetries, "low-level-retries", "", fs.Config.LowLevelRetries, "Number of low level retries to do.")
- flags.BoolVarP(flagSet, &fs.Config.UpdateOlder, "update", "u", fs.Config.UpdateOlder, "Skip files that are newer on the destination.")
- flags.BoolVarP(flagSet, &fs.Config.UseServerModTime, "use-server-modtime", "", fs.Config.UseServerModTime, "Use server modified time instead of object metadata")
- flags.BoolVarP(flagSet, &fs.Config.NoGzip, "no-gzip-encoding", "", fs.Config.NoGzip, "Don't set Accept-Encoding: gzip.")
- flags.IntVarP(flagSet, &fs.Config.MaxDepth, "max-depth", "", fs.Config.MaxDepth, "If set limits the recursion depth to this.")
- flags.BoolVarP(flagSet, &fs.Config.IgnoreSize, "ignore-size", "", false, "Ignore size when skipping use mod-time or checksum.")
- flags.BoolVarP(flagSet, &fs.Config.IgnoreChecksum, "ignore-checksum", "", fs.Config.IgnoreChecksum, "Skip post copy check of checksums.")
- flags.BoolVarP(flagSet, &fs.Config.IgnoreCaseSync, "ignore-case-sync", "", fs.Config.IgnoreCaseSync, "Ignore case when synchronizing")
- flags.BoolVarP(flagSet, &fs.Config.NoTraverse, "no-traverse", "", fs.Config.NoTraverse, "Don't traverse destination file system on copy.")
- flags.BoolVarP(flagSet, &fs.Config.CheckFirst, "check-first", "", fs.Config.CheckFirst, "Do all the checks before starting transfers.")
- flags.BoolVarP(flagSet, &fs.Config.NoCheckDest, "no-check-dest", "", fs.Config.NoCheckDest, "Don't check the destination, copy regardless.")
- flags.BoolVarP(flagSet, &fs.Config.NoUnicodeNormalization, "no-unicode-normalization", "", fs.Config.NoUnicodeNormalization, "Don't normalize unicode characters in filenames.")
- flags.BoolVarP(flagSet, &fs.Config.NoUpdateModTime, "no-update-modtime", "", fs.Config.NoUpdateModTime, "Don't update destination mod-time if files identical.")
- flags.StringVarP(flagSet, &fs.Config.CompareDest, "compare-dest", "", fs.Config.CompareDest, "Include additional server-side path during comparison.")
- flags.StringVarP(flagSet, &fs.Config.CopyDest, "copy-dest", "", fs.Config.CopyDest, "Implies --compare-dest but also copies files from path into destination.")
- flags.StringVarP(flagSet, &fs.Config.BackupDir, "backup-dir", "", fs.Config.BackupDir, "Make backups into hierarchy based in DIR.")
- flags.StringVarP(flagSet, &fs.Config.Suffix, "suffix", "", fs.Config.Suffix, "Suffix to add to changed files.")
- flags.BoolVarP(flagSet, &fs.Config.SuffixKeepExtension, "suffix-keep-extension", "", fs.Config.SuffixKeepExtension, "Preserve the extension when using --suffix.")
- flags.BoolVarP(flagSet, &fs.Config.UseListR, "fast-list", "", fs.Config.UseListR, "Use recursive list if available. Uses more memory but fewer transactions.")
- flags.Float64VarP(flagSet, &fs.Config.TPSLimit, "tpslimit", "", fs.Config.TPSLimit, "Limit HTTP transactions per second to this.")
- flags.IntVarP(flagSet, &fs.Config.TPSLimitBurst, "tpslimit-burst", "", fs.Config.TPSLimitBurst, "Max burst of transactions for --tpslimit.")
+ flags.Int64VarP(flagSet, &ci.MaxDelete, "max-delete", "", -1, "When synchronizing, limit the number of deletes")
+ flags.BoolVarP(flagSet, &ci.TrackRenames, "track-renames", "", ci.TrackRenames, "When synchronizing, track file renames and do a server-side move if possible")
+ flags.StringVarP(flagSet, &ci.TrackRenamesStrategy, "track-renames-strategy", "", ci.TrackRenamesStrategy, "Strategies to use when synchronizing using track-renames hash|modtime|leaf")
+ flags.IntVarP(flagSet, &ci.LowLevelRetries, "low-level-retries", "", ci.LowLevelRetries, "Number of low level retries to do.")
+ flags.BoolVarP(flagSet, &ci.UpdateOlder, "update", "u", ci.UpdateOlder, "Skip files that are newer on the destination.")
+ flags.BoolVarP(flagSet, &ci.UseServerModTime, "use-server-modtime", "", ci.UseServerModTime, "Use server modified time instead of object metadata")
+ flags.BoolVarP(flagSet, &ci.NoGzip, "no-gzip-encoding", "", ci.NoGzip, "Don't set Accept-Encoding: gzip.")
+ flags.IntVarP(flagSet, &ci.MaxDepth, "max-depth", "", ci.MaxDepth, "If set limits the recursion depth to this.")
+ flags.BoolVarP(flagSet, &ci.IgnoreSize, "ignore-size", "", false, "Ignore size when skipping use mod-time or checksum.")
+ flags.BoolVarP(flagSet, &ci.IgnoreChecksum, "ignore-checksum", "", ci.IgnoreChecksum, "Skip post copy check of checksums.")
+ flags.BoolVarP(flagSet, &ci.IgnoreCaseSync, "ignore-case-sync", "", ci.IgnoreCaseSync, "Ignore case when synchronizing")
+ flags.BoolVarP(flagSet, &ci.NoTraverse, "no-traverse", "", ci.NoTraverse, "Don't traverse destination file system on copy.")
+ flags.BoolVarP(flagSet, &ci.CheckFirst, "check-first", "", ci.CheckFirst, "Do all the checks before starting transfers.")
+ flags.BoolVarP(flagSet, &ci.NoCheckDest, "no-check-dest", "", ci.NoCheckDest, "Don't check the destination, copy regardless.")
+ flags.BoolVarP(flagSet, &ci.NoUnicodeNormalization, "no-unicode-normalization", "", ci.NoUnicodeNormalization, "Don't normalize unicode characters in filenames.")
+ flags.BoolVarP(flagSet, &ci.NoUpdateModTime, "no-update-modtime", "", ci.NoUpdateModTime, "Don't update destination mod-time if files identical.")
+ flags.StringVarP(flagSet, &ci.CompareDest, "compare-dest", "", ci.CompareDest, "Include additional server-side path during comparison.")
+ flags.StringVarP(flagSet, &ci.CopyDest, "copy-dest", "", ci.CopyDest, "Implies --compare-dest but also copies files from path into destination.")
+ flags.StringVarP(flagSet, &ci.BackupDir, "backup-dir", "", ci.BackupDir, "Make backups into hierarchy based in DIR.")
+ flags.StringVarP(flagSet, &ci.Suffix, "suffix", "", ci.Suffix, "Suffix to add to changed files.")
+ flags.BoolVarP(flagSet, &ci.SuffixKeepExtension, "suffix-keep-extension", "", ci.SuffixKeepExtension, "Preserve the extension when using --suffix.")
+ flags.BoolVarP(flagSet, &ci.UseListR, "fast-list", "", ci.UseListR, "Use recursive list if available. Uses more memory but fewer transactions.")
+ flags.Float64VarP(flagSet, &ci.TPSLimit, "tpslimit", "", ci.TPSLimit, "Limit HTTP transactions per second to this.")
+ flags.IntVarP(flagSet, &ci.TPSLimitBurst, "tpslimit-burst", "", ci.TPSLimitBurst, "Max burst of transactions for --tpslimit.")
flags.StringVarP(flagSet, &bindAddr, "bind", "", "", "Local address to bind to for outgoing connections, IPv4, IPv6 or name.")
flags.StringVarP(flagSet, &disableFeatures, "disable", "", "", "Disable a comma separated list of features. Use help to see a list.")
- flags.StringVarP(flagSet, &fs.Config.UserAgent, "user-agent", "", fs.Config.UserAgent, "Set the user-agent to a specified string. The default is rclone/ version")
- flags.BoolVarP(flagSet, &fs.Config.Immutable, "immutable", "", fs.Config.Immutable, "Do not modify files. Fail if existing files have been modified.")
- flags.BoolVarP(flagSet, &fs.Config.AutoConfirm, "auto-confirm", "", fs.Config.AutoConfirm, "If enabled, do not request console confirmation.")
- flags.IntVarP(flagSet, &fs.Config.StatsFileNameLength, "stats-file-name-length", "", fs.Config.StatsFileNameLength, "Max file name length in stats. 0 for no limit")
- flags.FVarP(flagSet, &fs.Config.LogLevel, "log-level", "", "Log level DEBUG|INFO|NOTICE|ERROR")
- flags.FVarP(flagSet, &fs.Config.StatsLogLevel, "stats-log-level", "", "Log level to show --stats output DEBUG|INFO|NOTICE|ERROR")
- flags.FVarP(flagSet, &fs.Config.BwLimit, "bwlimit", "", "Bandwidth limit in kBytes/s, or use suffix b|k|M|G or a full timetable.")
- flags.FVarP(flagSet, &fs.Config.BwLimitFile, "bwlimit-file", "", "Bandwidth limit per file in kBytes/s, or use suffix b|k|M|G or a full timetable.")
- flags.FVarP(flagSet, &fs.Config.BufferSize, "buffer-size", "", "In memory buffer size when reading files for each --transfer.")
- flags.FVarP(flagSet, &fs.Config.StreamingUploadCutoff, "streaming-upload-cutoff", "", "Cutoff for switching to chunked upload if file size is unknown. Upload starts after reaching cutoff or when file ends.")
- flags.FVarP(flagSet, &fs.Config.Dump, "dump", "", "List of items to dump from: "+fs.DumpFlagsList)
- flags.FVarP(flagSet, &fs.Config.MaxTransfer, "max-transfer", "", "Maximum size of data to transfer.")
- flags.DurationVarP(flagSet, &fs.Config.MaxDuration, "max-duration", "", 0, "Maximum duration rclone will transfer data for.")
- flags.FVarP(flagSet, &fs.Config.CutoffMode, "cutoff-mode", "", "Mode to stop transfers when reaching the max transfer limit HARD|SOFT|CAUTIOUS")
- flags.IntVarP(flagSet, &fs.Config.MaxBacklog, "max-backlog", "", fs.Config.MaxBacklog, "Maximum number of objects in sync or check backlog.")
- flags.IntVarP(flagSet, &fs.Config.MaxStatsGroups, "max-stats-groups", "", fs.Config.MaxStatsGroups, "Maximum number of stats groups to keep in memory. On max oldest is discarded.")
- flags.BoolVarP(flagSet, &fs.Config.StatsOneLine, "stats-one-line", "", fs.Config.StatsOneLine, "Make the stats fit on one line.")
- flags.BoolVarP(flagSet, &fs.Config.StatsOneLineDate, "stats-one-line-date", "", fs.Config.StatsOneLineDate, "Enables --stats-one-line and add current date/time prefix.")
- flags.StringVarP(flagSet, &fs.Config.StatsOneLineDateFormat, "stats-one-line-date-format", "", fs.Config.StatsOneLineDateFormat, "Enables --stats-one-line-date and uses custom formatted date. Enclose date string in double quotes (\"). See https://golang.org/pkg/time/#Time.Format")
- flags.BoolVarP(flagSet, &fs.Config.ErrorOnNoTransfer, "error-on-no-transfer", "", fs.Config.ErrorOnNoTransfer, "Sets exit code 9 if no files are transferred, useful in scripts")
- flags.BoolVarP(flagSet, &fs.Config.Progress, "progress", "P", fs.Config.Progress, "Show progress during transfer.")
- flags.BoolVarP(flagSet, &fs.Config.ProgressTerminalTitle, "progress-terminal-title", "", fs.Config.ProgressTerminalTitle, "Show progress on the terminal title. Requires -P/--progress.")
- flags.BoolVarP(flagSet, &fs.Config.Cookie, "use-cookies", "", fs.Config.Cookie, "Enable session cookiejar.")
- flags.BoolVarP(flagSet, &fs.Config.UseMmap, "use-mmap", "", fs.Config.UseMmap, "Use mmap allocator (see docs).")
- flags.StringVarP(flagSet, &fs.Config.CaCert, "ca-cert", "", fs.Config.CaCert, "CA certificate used to verify servers")
- flags.StringVarP(flagSet, &fs.Config.ClientCert, "client-cert", "", fs.Config.ClientCert, "Client SSL certificate (PEM) for mutual TLS auth")
- flags.StringVarP(flagSet, &fs.Config.ClientKey, "client-key", "", fs.Config.ClientKey, "Client SSL private key (PEM) for mutual TLS auth")
- flags.FVarP(flagSet, &fs.Config.MultiThreadCutoff, "multi-thread-cutoff", "", "Use multi-thread downloads for files above this size.")
- flags.IntVarP(flagSet, &fs.Config.MultiThreadStreams, "multi-thread-streams", "", fs.Config.MultiThreadStreams, "Max number of streams to use for multi-thread downloads.")
- flags.BoolVarP(flagSet, &fs.Config.UseJSONLog, "use-json-log", "", fs.Config.UseJSONLog, "Use json log format.")
- flags.StringVarP(flagSet, &fs.Config.OrderBy, "order-by", "", fs.Config.OrderBy, "Instructions on how to order the transfers, e.g. 'size,descending'")
+ flags.StringVarP(flagSet, &ci.UserAgent, "user-agent", "", ci.UserAgent, "Set the user-agent to a specified string. The default is rclone/ version")
+ flags.BoolVarP(flagSet, &ci.Immutable, "immutable", "", ci.Immutable, "Do not modify files. Fail if existing files have been modified.")
+ flags.BoolVarP(flagSet, &ci.AutoConfirm, "auto-confirm", "", ci.AutoConfirm, "If enabled, do not request console confirmation.")
+ flags.IntVarP(flagSet, &ci.StatsFileNameLength, "stats-file-name-length", "", ci.StatsFileNameLength, "Max file name length in stats. 0 for no limit")
+ flags.FVarP(flagSet, &ci.LogLevel, "log-level", "", "Log level DEBUG|INFO|NOTICE|ERROR")
+ flags.FVarP(flagSet, &ci.StatsLogLevel, "stats-log-level", "", "Log level to show --stats output DEBUG|INFO|NOTICE|ERROR")
+ flags.FVarP(flagSet, &ci.BwLimit, "bwlimit", "", "Bandwidth limit in kBytes/s, or use suffix b|k|M|G or a full timetable.")
+ flags.FVarP(flagSet, &ci.BwLimitFile, "bwlimit-file", "", "Bandwidth limit per file in kBytes/s, or use suffix b|k|M|G or a full timetable.")
+ flags.FVarP(flagSet, &ci.BufferSize, "buffer-size", "", "In memory buffer size when reading files for each --transfer.")
+ flags.FVarP(flagSet, &ci.StreamingUploadCutoff, "streaming-upload-cutoff", "", "Cutoff for switching to chunked upload if file size is unknown. Upload starts after reaching cutoff or when file ends.")
+ flags.FVarP(flagSet, &ci.Dump, "dump", "", "List of items to dump from: "+fs.DumpFlagsList)
+ flags.FVarP(flagSet, &ci.MaxTransfer, "max-transfer", "", "Maximum size of data to transfer.")
+ flags.DurationVarP(flagSet, &ci.MaxDuration, "max-duration", "", 0, "Maximum duration rclone will transfer data for.")
+ flags.FVarP(flagSet, &ci.CutoffMode, "cutoff-mode", "", "Mode to stop transfers when reaching the max transfer limit HARD|SOFT|CAUTIOUS")
+ flags.IntVarP(flagSet, &ci.MaxBacklog, "max-backlog", "", ci.MaxBacklog, "Maximum number of objects in sync or check backlog.")
+ flags.IntVarP(flagSet, &ci.MaxStatsGroups, "max-stats-groups", "", ci.MaxStatsGroups, "Maximum number of stats groups to keep in memory. On max oldest is discarded.")
+ flags.BoolVarP(flagSet, &ci.StatsOneLine, "stats-one-line", "", ci.StatsOneLine, "Make the stats fit on one line.")
+ flags.BoolVarP(flagSet, &ci.StatsOneLineDate, "stats-one-line-date", "", ci.StatsOneLineDate, "Enables --stats-one-line and add current date/time prefix.")
+ flags.StringVarP(flagSet, &ci.StatsOneLineDateFormat, "stats-one-line-date-format", "", ci.StatsOneLineDateFormat, "Enables --stats-one-line-date and uses custom formatted date. Enclose date string in double quotes (\"). See https://golang.org/pkg/time/#Time.Format")
+ flags.BoolVarP(flagSet, &ci.ErrorOnNoTransfer, "error-on-no-transfer", "", ci.ErrorOnNoTransfer, "Sets exit code 9 if no files are transferred, useful in scripts")
+ flags.BoolVarP(flagSet, &ci.Progress, "progress", "P", ci.Progress, "Show progress during transfer.")
+ flags.BoolVarP(flagSet, &ci.ProgressTerminalTitle, "progress-terminal-title", "", ci.ProgressTerminalTitle, "Show progress on the terminal title. Requires -P/--progress.")
+ flags.BoolVarP(flagSet, &ci.Cookie, "use-cookies", "", ci.Cookie, "Enable session cookiejar.")
+ flags.BoolVarP(flagSet, &ci.UseMmap, "use-mmap", "", ci.UseMmap, "Use mmap allocator (see docs).")
+ flags.StringVarP(flagSet, &ci.CaCert, "ca-cert", "", ci.CaCert, "CA certificate used to verify servers")
+ flags.StringVarP(flagSet, &ci.ClientCert, "client-cert", "", ci.ClientCert, "Client SSL certificate (PEM) for mutual TLS auth")
+ flags.StringVarP(flagSet, &ci.ClientKey, "client-key", "", ci.ClientKey, "Client SSL private key (PEM) for mutual TLS auth")
+ flags.FVarP(flagSet, &ci.MultiThreadCutoff, "multi-thread-cutoff", "", "Use multi-thread downloads for files above this size.")
+ flags.IntVarP(flagSet, &ci.MultiThreadStreams, "multi-thread-streams", "", ci.MultiThreadStreams, "Max number of streams to use for multi-thread downloads.")
+ flags.BoolVarP(flagSet, &ci.UseJSONLog, "use-json-log", "", ci.UseJSONLog, "Use json log format.")
+ flags.StringVarP(flagSet, &ci.OrderBy, "order-by", "", ci.OrderBy, "Instructions on how to order the transfers, e.g. 'size,descending'")
flags.StringArrayVarP(flagSet, &uploadHeaders, "header-upload", "", nil, "Set HTTP header for upload transactions")
flags.StringArrayVarP(flagSet, &downloadHeaders, "header-download", "", nil, "Set HTTP header for download transactions")
flags.StringArrayVarP(flagSet, &headers, "header", "", nil, "Set HTTP header for all transactions")
- flags.BoolVarP(flagSet, &fs.Config.RefreshTimes, "refresh-times", "", fs.Config.RefreshTimes, "Refresh the modtime of remote files.")
- flags.BoolVarP(flagSet, &fs.Config.LogSystemdSupport, "log-systemd", "", fs.Config.LogSystemdSupport, "Activate systemd integration for the logger.")
+ flags.BoolVarP(flagSet, &ci.RefreshTimes, "refresh-times", "", ci.RefreshTimes, "Refresh the modtime of remote files.")
+ flags.BoolVarP(flagSet, &ci.LogSystemdSupport, "log-systemd", "", ci.LogSystemdSupport, "Activate systemd integration for the logger.")
}
// ParseHeaders converts the strings passed in via the header flags into HTTPOptions
@@ -145,17 +145,17 @@ func ParseHeaders(headers []string) []*fs.HTTPOption {
}
// SetFlags converts any flags into config which weren't straight forward
-func SetFlags() {
+func SetFlags(ci *fs.ConfigInfo) {
if verbose >= 2 {
- fs.Config.LogLevel = fs.LogLevelDebug
+ ci.LogLevel = fs.LogLevelDebug
} else if verbose >= 1 {
- fs.Config.LogLevel = fs.LogLevelInfo
+ ci.LogLevel = fs.LogLevelInfo
}
if quiet {
if verbose > 0 {
log.Fatalf("Can't set -v and -q")
}
- fs.Config.LogLevel = fs.LogLevelError
+ ci.LogLevel = fs.LogLevelError
}
logLevelFlag := pflag.Lookup("log-level")
if logLevelFlag != nil && logLevelFlag.Changed {
@@ -166,13 +166,13 @@ func SetFlags() {
log.Fatalf("Can't set -q and --log-level")
}
}
- if fs.Config.UseJSONLog {
+ if ci.UseJSONLog {
logrus.AddHook(fsLog.NewCallerHook())
logrus.SetFormatter(&logrus.JSONFormatter{
TimestampFormat: "2006-01-02T15:04:05.999999-07:00",
})
logrus.SetLevel(logrus.DebugLevel)
- switch fs.Config.LogLevel {
+ switch ci.LogLevel {
case fs.LogLevelEmergency, fs.LogLevelAlert:
logrus.SetLevel(logrus.PanicLevel)
case fs.LogLevelCritical:
@@ -189,11 +189,11 @@ func SetFlags() {
}
if dumpHeaders {
- fs.Config.Dump |= fs.DumpHeaders
+ ci.Dump |= fs.DumpHeaders
fs.Logf(nil, "--dump-headers is obsolete - please use --dump headers instead")
}
if dumpBodies {
- fs.Config.Dump |= fs.DumpBodies
+ ci.Dump |= fs.DumpBodies
fs.Logf(nil, "--dump-bodies is obsolete - please use --dump bodies instead")
}
@@ -202,26 +202,26 @@ func SetFlags() {
deleteDuring && deleteAfter:
log.Fatalf(`Only one of --delete-before, --delete-during or --delete-after can be used.`)
case deleteBefore:
- fs.Config.DeleteMode = fs.DeleteModeBefore
+ ci.DeleteMode = fs.DeleteModeBefore
case deleteDuring:
- fs.Config.DeleteMode = fs.DeleteModeDuring
+ ci.DeleteMode = fs.DeleteModeDuring
case deleteAfter:
- fs.Config.DeleteMode = fs.DeleteModeAfter
+ ci.DeleteMode = fs.DeleteModeAfter
default:
- fs.Config.DeleteMode = fs.DeleteModeDefault
+ ci.DeleteMode = fs.DeleteModeDefault
}
- if fs.Config.CompareDest != "" && fs.Config.CopyDest != "" {
+ if ci.CompareDest != "" && ci.CopyDest != "" {
log.Fatalf(`Can't use --compare-dest with --copy-dest.`)
}
switch {
- case len(fs.Config.StatsOneLineDateFormat) > 0:
- fs.Config.StatsOneLineDate = true
- fs.Config.StatsOneLine = true
- case fs.Config.StatsOneLineDate:
- fs.Config.StatsOneLineDateFormat = "2006/01/02 15:04:05 - "
- fs.Config.StatsOneLine = true
+ case len(ci.StatsOneLineDateFormat) > 0:
+ ci.StatsOneLineDate = true
+ ci.StatsOneLine = true
+ case ci.StatsOneLineDate:
+ ci.StatsOneLineDateFormat = "2006/01/02 15:04:05 - "
+ ci.StatsOneLine = true
}
if bindAddr != "" {
@@ -232,24 +232,24 @@ func SetFlags() {
if len(addrs) != 1 {
log.Fatalf("--bind: Expecting 1 IP address for %q but got %d", bindAddr, len(addrs))
}
- fs.Config.BindAddr = addrs[0]
+ ci.BindAddr = addrs[0]
}
if disableFeatures != "" {
if disableFeatures == "help" {
log.Fatalf("Possible backend features are: %s\n", strings.Join(new(fs.Features).List(), ", "))
}
- fs.Config.DisableFeatures = strings.Split(disableFeatures, ",")
+ ci.DisableFeatures = strings.Split(disableFeatures, ",")
}
if len(uploadHeaders) != 0 {
- fs.Config.UploadHeaders = ParseHeaders(uploadHeaders)
+ ci.UploadHeaders = ParseHeaders(uploadHeaders)
}
if len(downloadHeaders) != 0 {
- fs.Config.DownloadHeaders = ParseHeaders(downloadHeaders)
+ ci.DownloadHeaders = ParseHeaders(downloadHeaders)
}
if len(headers) != 0 {
- fs.Config.Headers = ParseHeaders(headers)
+ ci.Headers = ParseHeaders(headers)
}
// Make the config file absolute
@@ -260,6 +260,6 @@ func SetFlags() {
// Set whether multi-thread-streams was set
multiThreadStreamsFlag := pflag.Lookup("multi-thread-streams")
- fs.Config.MultiThreadSet = multiThreadStreamsFlag != nil && multiThreadStreamsFlag.Changed
+ ci.MultiThreadSet = multiThreadStreamsFlag != nil && multiThreadStreamsFlag.Changed
}
diff --git a/fs/config_test.go b/fs/config_test.go
new file mode 100644
index 000000000..9813dc218
--- /dev/null
+++ b/fs/config_test.go
@@ -0,0 +1,29 @@
+package fs
+
+import (
+ "context"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+)
+
+func TestGetConfig(t *testing.T) {
+ ctx := context.Background()
+
+ // Check nil
+ config := GetConfig(nil)
+ assert.Equal(t, globalConfig, config)
+
+ // Check empty config
+ config = GetConfig(ctx)
+ assert.Equal(t, globalConfig, config)
+
+ // Check adding a config
+ ctx2, config2 := AddConfig(ctx)
+ config2.Transfers++
+ assert.NotEqual(t, config2, config)
+
+ // Check can get config back
+ config2ctx := GetConfig(ctx2)
+ assert.Equal(t, config2, config2ctx)
+}
diff --git a/fs/filter/filter.go b/fs/filter/filter.go
index fb84c496c..f39c1f71b 100644
--- a/fs/filter/filter.go
+++ b/fs/filter/filter.go
@@ -229,7 +229,7 @@ func NewFilter(opt *Opt) (f *Filter, err error) {
return nil, err
}
}
- if fs.Config.Dump&fs.DumpFilters != 0 {
+ if fs.GetConfig(context.Background()).Dump&fs.DumpFilters != 0 {
fmt.Println("--- start filters ---")
fmt.Println(f.DumpFilters())
fmt.Println("--- end filters ---")
@@ -540,14 +540,16 @@ var errFilesFromNotSet = errors.New("--files-from not set so can't use Filter.Li
// MakeListR makes function to return all the files set using --files-from
func (f *Filter) MakeListR(ctx context.Context, NewObject func(ctx context.Context, remote string) (fs.Object, error)) fs.ListRFn {
return func(ctx context.Context, dir string, callback fs.ListRCallback) error {
+ ci := fs.GetConfig(ctx)
if !f.HaveFilesFrom() {
return errFilesFromNotSet
}
var (
- remotes = make(chan string, fs.Config.Checkers)
- g errgroup.Group
+ checkers = ci.Checkers
+ remotes = make(chan string, checkers)
+ g errgroup.Group
)
- for i := 0; i < fs.Config.Checkers; i++ {
+ for i := 0; i < checkers; i++ {
g.Go(func() (err error) {
var entries = make(fs.DirEntries, 1)
for remote := range remotes {
diff --git a/fs/fs.go b/fs/fs.go
index 31ff742b9..13ccde44e 100644
--- a/fs/fs.go
+++ b/fs/fs.go
@@ -774,7 +774,7 @@ func (ft *Features) Fill(ctx context.Context, f Fs) *Features {
if do, ok := f.(Commander); ok {
ft.Command = do.Command
}
- return ft.DisableList(Config.DisableFeatures)
+ return ft.DisableList(GetConfig(ctx).DisableFeatures)
}
// Mask the Features with the Fs passed in
@@ -854,7 +854,7 @@ func (ft *Features) Mask(ctx context.Context, f Fs) *Features {
ft.Disconnect = nil
}
// Command is always local so we don't mask it
- return ft.DisableList(Config.DisableFeatures)
+ return ft.DisableList(GetConfig(ctx).DisableFeatures)
}
// Wrap makes a Copy of the features passed in, overriding the UnWrap/Wrap
@@ -1399,7 +1399,7 @@ func FileExists(ctx context.Context, fs Fs, remote string) (bool, error) {
// GetModifyWindow calculates the maximum modify window between the given Fses
// and the Config.ModifyWindow parameter.
func GetModifyWindow(ctx context.Context, fss ...Info) time.Duration {
- window := Config.ModifyWindow
+ window := GetConfig(ctx).ModifyWindow
for _, f := range fss {
if f != nil {
precision := f.Precision()
@@ -1424,12 +1424,12 @@ type logCalculator struct {
}
// NewPacer creates a Pacer for the given Fs and Calculator.
-func NewPacer(c pacer.Calculator) *Pacer {
+func NewPacer(ctx context.Context, c pacer.Calculator) *Pacer {
p := &Pacer{
Pacer: pacer.New(
pacer.InvokerOption(pacerInvoker),
- pacer.MaxConnectionsOption(Config.Checkers+Config.Transfers),
- pacer.RetriesOption(Config.LowLevelRetries),
+ pacer.MaxConnectionsOption(GetConfig(ctx).Checkers+GetConfig(ctx).Transfers),
+ pacer.RetriesOption(GetConfig(ctx).LowLevelRetries),
pacer.CalculatorOption(c),
),
}
diff --git a/fs/fs_test.go b/fs/fs_test.go
index 981b74d0c..8e2086411 100644
--- a/fs/fs_test.go
+++ b/fs/fs_test.go
@@ -127,15 +127,15 @@ func (dp *dummyPaced) fn() (bool, error) {
}
func TestPacerCall(t *testing.T) {
- expectedCalled := Config.LowLevelRetries
+ ctx := context.Background()
+ config := GetConfig(ctx)
+ expectedCalled := config.LowLevelRetries
if expectedCalled == 0 {
+ ctx, config = AddConfig(ctx)
expectedCalled = 20
- Config.LowLevelRetries = expectedCalled
- defer func() {
- Config.LowLevelRetries = 0
- }()
+ config.LowLevelRetries = expectedCalled
}
- p := NewPacer(pacer.NewDefault(pacer.MinSleep(1*time.Millisecond), pacer.MaxSleep(2*time.Millisecond)))
+ p := NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(1*time.Millisecond), pacer.MaxSleep(2*time.Millisecond)))
dp := &dummyPaced{retry: true}
err := p.Call(dp.fn)
@@ -144,7 +144,7 @@ func TestPacerCall(t *testing.T) {
}
func TestPacerCallNoRetry(t *testing.T) {
- p := NewPacer(pacer.NewDefault(pacer.MinSleep(1*time.Millisecond), pacer.MaxSleep(2*time.Millisecond)))
+ p := NewPacer(context.Background(), pacer.NewDefault(pacer.MinSleep(1*time.Millisecond), pacer.MaxSleep(2*time.Millisecond)))
dp := &dummyPaced{retry: true}
err := p.CallNoRetry(dp.fn)
diff --git a/fs/fshttp/http.go b/fs/fshttp/http.go
index d384cee8f..ccce08be7 100644
--- a/fs/fshttp/http.go
+++ b/fs/fshttp/http.go
@@ -34,14 +34,14 @@ var (
)
// StartHTTPTokenBucket starts the token bucket if necessary
-func StartHTTPTokenBucket() {
- if fs.Config.TPSLimit > 0 {
- tpsBurst := fs.Config.TPSLimitBurst
+func StartHTTPTokenBucket(ctx context.Context) {
+ if fs.GetConfig(ctx).TPSLimit > 0 {
+ tpsBurst := fs.GetConfig(ctx).TPSLimitBurst
if tpsBurst < 1 {
tpsBurst = 1
}
- tpsBucket = rate.NewLimiter(rate.Limit(fs.Config.TPSLimit), tpsBurst)
- fs.Infof(nil, "Starting HTTP transaction limiter: max %g transactions/s with burst %d", fs.Config.TPSLimit, tpsBurst)
+ tpsBucket = rate.NewLimiter(rate.Limit(fs.GetConfig(ctx).TPSLimit), tpsBurst)
+ fs.Infof(nil, "Starting HTTP transaction limiter: max %g transactions/s with burst %d", fs.GetConfig(ctx).TPSLimit, tpsBurst)
}
}
diff --git a/fs/log.go b/fs/log.go
index 4bba8a922..76c0965aa 100644
--- a/fs/log.go
+++ b/fs/log.go
@@ -1,6 +1,7 @@
package fs
import (
+ "context"
"fmt"
"log"
@@ -72,7 +73,7 @@ func (l *LogLevel) Type() string {
// LogPrint sends the text to the logger of level
var LogPrint = func(level LogLevel, text string) {
var prefix string
- if Config.LogSystemdSupport {
+ if GetConfig(context.TODO()).LogSystemdSupport {
switch level {
case LogLevelDebug:
prefix = sysdjournald.DebugPrefix
@@ -121,7 +122,7 @@ func (j LogValueItem) String() string {
func LogPrintf(level LogLevel, o interface{}, text string, args ...interface{}) {
out := fmt.Sprintf(text, args...)
- if Config.UseJSONLog {
+ if GetConfig(context.TODO()).UseJSONLog {
fields := logrus.Fields{}
if o != nil {
fields = logrus.Fields{
@@ -158,7 +159,7 @@ func LogPrintf(level LogLevel, o interface{}, text string, args ...interface{})
// LogLevelPrintf writes logs at the given level
func LogLevelPrintf(level LogLevel, o interface{}, text string, args ...interface{}) {
- if Config.LogLevel >= level {
+ if GetConfig(context.TODO()).LogLevel >= level {
LogPrintf(level, o, text, args...)
}
}
@@ -166,7 +167,7 @@ func LogLevelPrintf(level LogLevel, o interface{}, text string, args ...interfac
// Errorf writes error log output for this Object or Fs. It
// should always be seen by the user.
func Errorf(o interface{}, text string, args ...interface{}) {
- if Config.LogLevel >= LogLevelError {
+ if GetConfig(context.TODO()).LogLevel >= LogLevelError {
LogPrintf(LogLevelError, o, text, args...)
}
}
@@ -177,7 +178,7 @@ func Errorf(o interface{}, text string, args ...interface{}) {
// important things the user should see. The user can filter these
// out with the -q flag.
func Logf(o interface{}, text string, args ...interface{}) {
- if Config.LogLevel >= LogLevelNotice {
+ if GetConfig(context.TODO()).LogLevel >= LogLevelNotice {
LogPrintf(LogLevelNotice, o, text, args...)
}
}
@@ -186,7 +187,7 @@ func Logf(o interface{}, text string, args ...interface{}) {
// level for logging transfers, deletions and things which should
// appear with the -v flag.
func Infof(o interface{}, text string, args ...interface{}) {
- if Config.LogLevel >= LogLevelInfo {
+ if GetConfig(context.TODO()).LogLevel >= LogLevelInfo {
LogPrintf(LogLevelInfo, o, text, args...)
}
}
@@ -194,7 +195,7 @@ func Infof(o interface{}, text string, args ...interface{}) {
// Debugf writes debugging output for this Object or Fs. Use this for
// debug only. The user must have to specify -vv to see this.
func Debugf(o interface{}, text string, args ...interface{}) {
- if Config.LogLevel >= LogLevelDebug {
+ if GetConfig(context.TODO()).LogLevel >= LogLevelDebug {
LogPrintf(LogLevelDebug, o, text, args...)
}
}
diff --git a/fs/log/log.go b/fs/log/log.go
index fbd1f8118..d87fd970e 100644
--- a/fs/log/log.go
+++ b/fs/log/log.go
@@ -2,6 +2,7 @@
package log
import (
+ "context"
"io"
"log"
"os"
@@ -51,7 +52,7 @@ func fnName() string {
//
// Any pointers in the exit function will be dereferenced
func Trace(o interface{}, format string, a ...interface{}) func(string, ...interface{}) {
- if fs.Config.LogLevel < fs.LogLevelDebug {
+ if fs.GetConfig(context.Background()).LogLevel < fs.LogLevelDebug {
return func(format string, a ...interface{}) {}
}
name := fnName()
@@ -76,7 +77,7 @@ func Trace(o interface{}, format string, a ...interface{}) func(string, ...inter
// Stack logs a stack trace of callers with the o and info passed in
func Stack(o interface{}, info string) {
- if fs.Config.LogLevel < fs.LogLevelDebug {
+ if fs.GetConfig(context.Background()).LogLevel < fs.LogLevelDebug {
return
}
arr := [16 * 1024]byte{}
@@ -90,7 +91,7 @@ func Stack(o interface{}, info string) {
func InitLogging() {
flagsStr := "," + Opt.Format + ","
var flags int
- if !fs.Config.LogSystemdSupport {
+ if !fs.GetConfig(context.Background()).LogSystemdSupport {
if strings.Contains(flagsStr, ",date,") {
flags |= log.Ldate
}
diff --git a/fs/march/march.go b/fs/march/march.go
index 0b8de9a75..7307cc8f6 100644
--- a/fs/march/march.go
+++ b/fs/march/march.go
@@ -49,10 +49,11 @@ type Marcher interface {
}
// init sets up a march over opt.Fsrc, and opt.Fdst calling back callback for each match
-func (m *March) init() {
- m.srcListDir = m.makeListDir(m.Fsrc, m.SrcIncludeAll)
+func (m *March) init(ctx context.Context) {
+ ci := fs.GetConfig(ctx)
+ m.srcListDir = m.makeListDir(ctx, m.Fsrc, m.SrcIncludeAll)
if !m.NoTraverse {
- m.dstListDir = m.makeListDir(m.Fdst, m.DstIncludeAll)
+ m.dstListDir = m.makeListDir(ctx, m.Fdst, m.DstIncludeAll)
}
// Now create the matching transform
// ..normalise the UTF8 first
@@ -65,7 +66,7 @@ func (m *March) init() {
// | Yes | No | No |
// | No | Yes | Yes |
// | Yes | Yes | Yes |
- if m.Fdst.Features().CaseInsensitive || fs.Config.IgnoreCaseSync {
+ if m.Fdst.Features().CaseInsensitive || ci.IgnoreCaseSync {
m.transforms = append(m.transforms, strings.ToLower)
}
}
@@ -75,9 +76,10 @@ type listDirFn func(dir string) (entries fs.DirEntries, err error)
// makeListDir makes constructs a listing function for the given fs
// and includeAll flags for marching through the file system.
-func (m *March) makeListDir(f fs.Fs, includeAll bool) listDirFn {
- if !(fs.Config.UseListR && f.Features().ListR != nil) && // !--fast-list active and
- !(fs.Config.NoTraverse && filter.Active.HaveFilesFrom()) { // !(--files-from and --no-traverse)
+func (m *March) makeListDir(ctx context.Context, f fs.Fs, includeAll bool) listDirFn {
+ ci := fs.GetConfig(ctx)
+ if !(ci.UseListR && f.Features().ListR != nil) && // !--fast-list active and
+ !(ci.NoTraverse && filter.Active.HaveFilesFrom()) { // !(--files-from and --no-traverse)
return func(dir string) (entries fs.DirEntries, err error) {
return list.DirSorted(m.Ctx, f, includeAll, dir)
}
@@ -95,7 +97,7 @@ func (m *March) makeListDir(f fs.Fs, includeAll bool) listDirFn {
mu.Lock()
defer mu.Unlock()
if !started {
- dirs, dirsErr = walk.NewDirTree(m.Ctx, f, m.Dir, includeAll, fs.Config.MaxDepth)
+ dirs, dirsErr = walk.NewDirTree(m.Ctx, f, m.Dir, includeAll, ci.MaxDepth)
started = true
}
if dirsErr != nil {
@@ -122,10 +124,11 @@ type listDirJob struct {
}
// Run starts the matching process off
-func (m *March) Run() error {
- m.init()
+func (m *March) Run(ctx context.Context) error {
+ ci := fs.GetConfig(ctx)
+ m.init(ctx)
- srcDepth := fs.Config.MaxDepth
+ srcDepth := ci.MaxDepth
if srcDepth < 0 {
srcDepth = fs.MaxLevel
}
@@ -141,8 +144,9 @@ func (m *March) Run() error {
// Start some directory listing go routines
var wg sync.WaitGroup // sync closing of go routines
var traversing sync.WaitGroup // running directory traversals
- in := make(chan listDirJob, fs.Config.Checkers)
- for i := 0; i < fs.Config.Checkers; i++ {
+ checkers := ci.Checkers
+ in := make(chan listDirJob, checkers)
+ for i := 0; i < checkers; i++ {
wg.Add(1)
go func() {
defer wg.Done()
diff --git a/fs/march/march_test.go b/fs/march/march_test.go
index f2331ece5..3d1a58040 100644
--- a/fs/march/march_test.go
+++ b/fs/march/march_test.go
@@ -203,7 +203,7 @@ func TestMarch(t *testing.T) {
DstIncludeAll: filter.Active.Opt.DeleteExcluded,
}
- mt.processError(m.Run())
+ mt.processError(m.Run(ctx))
mt.cancel()
err := mt.currentError()
require.NoError(t, err)
@@ -270,7 +270,7 @@ func TestMarchNoTraverse(t *testing.T) {
DstIncludeAll: filter.Active.Opt.DeleteExcluded,
}
- mt.processError(m.Run())
+ mt.processError(m.Run(ctx))
mt.cancel()
err := mt.currentError()
require.NoError(t, err)
diff --git a/fs/operations/check.go b/fs/operations/check.go
index 619b73249..4f095e587 100644
--- a/fs/operations/check.go
+++ b/fs/operations/check.go
@@ -114,16 +114,17 @@ func (c *checkMarch) SrcOnly(src fs.DirEntry) (recurse bool) {
// check to see if two objects are identical using the check function
func (c *checkMarch) checkIdentical(ctx context.Context, dst, src fs.Object) (differ bool, noHash bool, err error) {
+ ci := fs.GetConfig(ctx)
tr := accounting.Stats(ctx).NewCheckingTransfer(src)
defer func() {
tr.Done(ctx, err)
}()
- if sizeDiffers(src, dst) {
+ if sizeDiffers(ctx, src, dst) {
err = errors.Errorf("Sizes differ")
fs.Errorf(src, "%v", err)
return true, false, nil
}
- if fs.Config.SizeOnly {
+ if ci.SizeOnly {
return false, false, nil
}
return c.opt.Check(ctx, dst, src)
@@ -202,11 +203,12 @@ func (c *checkMarch) Match(ctx context.Context, dst, src fs.DirEntry) (recurse b
// it returns true if differences were found
// it also returns whether it couldn't be hashed
func CheckFn(ctx context.Context, opt *CheckOpt) error {
+ ci := fs.GetConfig(ctx)
if opt.Check == nil {
return errors.New("internal error: nil check function")
}
c := &checkMarch{
- tokens: make(chan struct{}, fs.Config.Checkers),
+ tokens: make(chan struct{}, ci.Checkers),
opt: *opt,
}
@@ -219,7 +221,7 @@ func CheckFn(ctx context.Context, opt *CheckOpt) error {
Callback: c,
}
fs.Debugf(c.opt.Fdst, "Waiting for checks to finish")
- err := m.Run()
+ err := m.Run(ctx)
c.wg.Wait() // wait for background go-routines
if c.dstFilesMissing > 0 {
@@ -308,7 +310,8 @@ func CheckEqualReaders(in1, in2 io.Reader) (differ bool, err error) {
//
// it returns true if differences were found
func CheckIdenticalDownload(ctx context.Context, dst, src fs.Object) (differ bool, err error) {
- err = Retry(src, fs.Config.LowLevelRetries, func() error {
+ ci := fs.GetConfig(ctx)
+ err = Retry(src, ci.LowLevelRetries, func() error {
differ, err = checkIdenticalDownload(ctx, dst, src)
return err
})
diff --git a/fs/operations/check_test.go b/fs/operations/check_test.go
index 8dda7b172..df6421243 100644
--- a/fs/operations/check_test.go
+++ b/fs/operations/check_test.go
@@ -24,6 +24,8 @@ import (
func testCheck(t *testing.T, checkFunction func(ctx context.Context, opt *operations.CheckOpt) error) {
r := fstest.NewRun(t)
defer r.Finalise()
+ ctx := context.Background()
+ ci := fs.GetConfig(ctx)
addBuffers := func(opt *operations.CheckOpt) {
opt.Combined = new(bytes.Buffer)
@@ -73,7 +75,7 @@ func testCheck(t *testing.T, checkFunction func(ctx context.Context, opt *operat
OneWay: oneway,
}
addBuffers(&opt)
- err := checkFunction(context.Background(), &opt)
+ err := checkFunction(ctx, &opt)
gotErrors := accounting.GlobalStats().GetErrors()
gotChecks := accounting.GlobalStats().GetChecks()
if wantErrors == 0 && err != nil {
@@ -95,7 +97,7 @@ func testCheck(t *testing.T, checkFunction func(ctx context.Context, opt *operat
})
}
- file1 := r.WriteBoth(context.Background(), "rutabaga", "is tasty", t3)
+ file1 := r.WriteBoth(ctx, "rutabaga", "is tasty", t3)
fstest.CheckItems(t, r.Fremote, file1)
fstest.CheckItems(t, r.Flocal, file1)
check(1, 0, 1, false, map[string]string{
@@ -118,7 +120,7 @@ func testCheck(t *testing.T, checkFunction func(ctx context.Context, opt *operat
"error": "",
})
- file3 := r.WriteObject(context.Background(), "empty space", "-", t2)
+ file3 := r.WriteObject(ctx, "empty space", "-", t2)
fstest.CheckItems(t, r.Fremote, file1, file3)
check(3, 2, 1, false, map[string]string{
"combined": "- empty space\n+ potato2\n= rutabaga\n",
@@ -130,10 +132,10 @@ func testCheck(t *testing.T, checkFunction func(ctx context.Context, opt *operat
})
file2r := file2
- if fs.Config.SizeOnly {
- file2r = r.WriteObject(context.Background(), "potato2", "--Some-Differences-But-Size-Only-Is-Enabled-----------------", t1)
+ if ci.SizeOnly {
+ file2r = r.WriteObject(ctx, "potato2", "--Some-Differences-But-Size-Only-Is-Enabled-----------------", t1)
} else {
- r.WriteObject(context.Background(), "potato2", "------------------------------------------------------------", t1)
+ r.WriteObject(ctx, "potato2", "------------------------------------------------------------", t1)
}
fstest.CheckItems(t, r.Fremote, file1, file2r, file3)
check(4, 1, 2, false, map[string]string{
@@ -157,7 +159,7 @@ func testCheck(t *testing.T, checkFunction func(ctx context.Context, opt *operat
"error": "",
})
- file4 := r.WriteObject(context.Background(), "remotepotato", "------------------------------------------------------------", t1)
+ file4 := r.WriteObject(ctx, "remotepotato", "------------------------------------------------------------", t1)
fstest.CheckItems(t, r.Fremote, file1, file2r, file3r, file4)
check(6, 2, 3, false, map[string]string{
"combined": "* empty space\n= potato2\n= rutabaga\n- remotepotato\n",
@@ -182,11 +184,12 @@ func TestCheck(t *testing.T) {
}
func TestCheckFsError(t *testing.T) {
- dstFs, err := fs.NewFs(context.Background(), "non-existent")
+ ctx := context.Background()
+ dstFs, err := fs.NewFs(ctx, "non-existent")
if err != nil {
t.Fatal(err)
}
- srcFs, err := fs.NewFs(context.Background(), "non-existent")
+ srcFs, err := fs.NewFs(ctx, "non-existent")
if err != nil {
t.Fatal(err)
}
@@ -195,7 +198,7 @@ func TestCheckFsError(t *testing.T) {
Fsrc: srcFs,
OneWay: false,
}
- err = operations.Check(context.Background(), &opt)
+ err = operations.Check(ctx, &opt)
require.Error(t, err)
}
@@ -204,8 +207,10 @@ func TestCheckDownload(t *testing.T) {
}
func TestCheckSizeOnly(t *testing.T) {
- fs.Config.SizeOnly = true
- defer func() { fs.Config.SizeOnly = false }()
+ ctx := context.Background()
+ ci := fs.GetConfig(ctx)
+ ci.SizeOnly = true
+ defer func() { ci.SizeOnly = false }()
TestCheck(t)
}
diff --git a/fs/operations/dedupe.go b/fs/operations/dedupe.go
index 0c7c1e2af..56de6dd3d 100644
--- a/fs/operations/dedupe.go
+++ b/fs/operations/dedupe.go
@@ -75,6 +75,8 @@ func dedupeDeleteAllButOne(ctx context.Context, keep int, remote string, objs []
// dedupeDeleteIdentical deletes all but one of identical (by hash) copies
func dedupeDeleteIdentical(ctx context.Context, ht hash.Type, remote string, objs []fs.Object) (remainingObjs []fs.Object) {
+ ci := fs.GetConfig(ctx)
+
// Make map of IDs
IDs := make(map[string]int, len(objs))
for _, o := range objs {
@@ -104,7 +106,7 @@ func dedupeDeleteIdentical(ctx context.Context, ht hash.Type, remote string, obj
dupesByID := make(map[string][]fs.Object, len(objs))
for _, o := range objs {
ID := ""
- if fs.Config.SizeOnly && o.Size() >= 0 {
+ if ci.SizeOnly && o.Size() >= 0 {
ID = fmt.Sprintf("size %d", o.Size())
} else if ht != hash.None {
hashValue, err := o.Hash(ctx, ht)
@@ -229,8 +231,9 @@ func (x *DeduplicateMode) Type() string {
// dedupeFindDuplicateDirs scans f for duplicate directories
func dedupeFindDuplicateDirs(ctx context.Context, f fs.Fs) ([][]fs.Directory, error) {
+ ci := fs.GetConfig(ctx)
dirs := map[string][]fs.Directory{}
- err := walk.ListR(ctx, f, "", true, fs.Config.MaxDepth, walk.ListDirs, func(entries fs.DirEntries) error {
+ err := walk.ListR(ctx, f, "", true, ci.MaxDepth, walk.ListDirs, func(entries fs.DirEntries) error {
entries.ForDir(func(d fs.Directory) {
dirs[d.Remote()] = append(dirs[d.Remote()], d)
})
@@ -297,6 +300,7 @@ func sortSmallestFirst(objs []fs.Object) {
// Google Drive which can have duplicate file names.
func Deduplicate(ctx context.Context, f fs.Fs, mode DeduplicateMode) error {
fs.Infof(f, "Looking for duplicates using %v mode.", mode)
+ ci := fs.GetConfig(ctx)
// Find duplicate directories first and fix them
duplicateDirs, err := dedupeFindDuplicateDirs(ctx, f)
@@ -315,7 +319,7 @@ func Deduplicate(ctx context.Context, f fs.Fs, mode DeduplicateMode) error {
// Now find duplicate files
files := map[string][]fs.Object{}
- err = walk.ListR(ctx, f, "", true, fs.Config.MaxDepth, walk.ListObjects, func(entries fs.DirEntries) error {
+ err = walk.ListR(ctx, f, "", true, ci.MaxDepth, walk.ListObjects, func(entries fs.DirEntries) error {
entries.ForObject(func(o fs.Object) {
remote := o.Remote()
files[remote] = append(files[remote], o)
diff --git a/fs/operations/dedupe_test.go b/fs/operations/dedupe_test.go
index 9bee4fcf0..985642399 100644
--- a/fs/operations/dedupe_test.go
+++ b/fs/operations/dedupe_test.go
@@ -79,15 +79,17 @@ func TestDeduplicateSizeOnly(t *testing.T) {
r := fstest.NewRun(t)
defer r.Finalise()
skipIfCantDedupe(t, r.Fremote)
+ ctx := context.Background()
+ ci := fs.GetConfig(ctx)
file1 := r.WriteUncheckedObject(context.Background(), "one", "This is one", t1)
file2 := r.WriteUncheckedObject(context.Background(), "one", "THIS IS ONE", t1)
file3 := r.WriteUncheckedObject(context.Background(), "one", "This is another one", t1)
r.CheckWithDuplicates(t, file1, file2, file3)
- fs.Config.SizeOnly = true
+ ci.SizeOnly = true
defer func() {
- fs.Config.SizeOnly = false
+ ci.SizeOnly = false
}()
err := operations.Deduplicate(context.Background(), r.Fremote, operations.DeduplicateSkip)
diff --git a/fs/operations/lsjson.go b/fs/operations/lsjson.go
index 0a0066950..50ad050e1 100644
--- a/fs/operations/lsjson.go
+++ b/fs/operations/lsjson.go
@@ -115,7 +115,7 @@ func ListJSON(ctx context.Context, fsrc fs.Fs, remote string, opt *ListJSONOpt,
hashTypes = append(hashTypes, ht)
}
}
- err := walk.ListR(ctx, fsrc, remote, false, ConfigMaxDepth(opt.Recurse), walk.ListAll, func(entries fs.DirEntries) (err error) {
+ err := walk.ListR(ctx, fsrc, remote, false, ConfigMaxDepth(ctx, opt.Recurse), walk.ListAll, func(entries fs.DirEntries) (err error) {
for _, entry := range entries {
switch entry.(type) {
case fs.Directory:
diff --git a/fs/operations/multithread.go b/fs/operations/multithread.go
index 83e77765c..db246b82a 100644
--- a/fs/operations/multithread.go
+++ b/fs/operations/multithread.go
@@ -18,15 +18,17 @@ const (
// Return a boolean as to whether we should use multi thread copy for
// this transfer
-func doMultiThreadCopy(f fs.Fs, src fs.Object) bool {
+func doMultiThreadCopy(ctx context.Context, f fs.Fs, src fs.Object) bool {
+ ci := fs.GetConfig(ctx)
+
// Disable multi thread if...
// ...it isn't configured
- if fs.Config.MultiThreadStreams <= 1 {
+ if ci.MultiThreadStreams <= 1 {
return false
}
// ...size of object is less than cutoff
- if src.Size() < int64(fs.Config.MultiThreadCutoff) {
+ if src.Size() < int64(ci.MultiThreadCutoff) {
return false
}
// ...source doesn't support it
@@ -36,7 +38,7 @@ func doMultiThreadCopy(f fs.Fs, src fs.Object) bool {
}
// ...if --multi-thread-streams not in use and source and
// destination are both local
- if !fs.Config.MultiThreadSet && dstFeatures.IsLocal && src.Fs().Features().IsLocal {
+ if !ci.MultiThreadSet && dstFeatures.IsLocal && src.Fs().Features().IsLocal {
return false
}
return true
@@ -55,6 +57,7 @@ type multiThreadCopyState struct {
// Copy a single stream into place
func (mc *multiThreadCopyState) copyStream(ctx context.Context, stream int) (err error) {
+ ci := fs.GetConfig(ctx)
defer func() {
if err != nil {
fs.Debugf(mc.src, "multi-thread copy: stream %d/%d failed: %v", stream+1, mc.streams, err)
@@ -71,7 +74,7 @@ func (mc *multiThreadCopyState) copyStream(ctx context.Context, stream int) (err
fs.Debugf(mc.src, "multi-thread copy: stream %d/%d (%d-%d) size %v starting", stream+1, mc.streams, start, end, fs.SizeSuffix(end-start))
- rc, err := NewReOpen(ctx, mc.src, fs.Config.LowLevelRetries, &fs.RangeOption{Start: start, End: end - 1})
+ rc, err := NewReOpen(ctx, mc.src, ci.LowLevelRetries, &fs.RangeOption{Start: start, End: end - 1})
if err != nil {
return errors.Wrap(err, "multipart copy: failed to open source")
}
diff --git a/fs/operations/multithread_test.go b/fs/operations/multithread_test.go
index 849af6bd7..e17c551a7 100644
--- a/fs/operations/multithread_test.go
+++ b/fs/operations/multithread_test.go
@@ -17,64 +17,66 @@ import (
)
func TestDoMultiThreadCopy(t *testing.T) {
- f := mockfs.NewFs(context.Background(), "potato", "")
+ ctx := context.Background()
+ ci := fs.GetConfig(ctx)
+ f := mockfs.NewFs(ctx, "potato", "")
src := mockobject.New("file.txt").WithContent([]byte(random.String(100)), mockobject.SeekModeNone)
- srcFs := mockfs.NewFs(context.Background(), "sausage", "")
+ srcFs := mockfs.NewFs(ctx, "sausage", "")
src.SetFs(srcFs)
- oldStreams := fs.Config.MultiThreadStreams
- oldCutoff := fs.Config.MultiThreadCutoff
- oldIsSet := fs.Config.MultiThreadSet
+ oldStreams := ci.MultiThreadStreams
+ oldCutoff := ci.MultiThreadCutoff
+ oldIsSet := ci.MultiThreadSet
defer func() {
- fs.Config.MultiThreadStreams = oldStreams
- fs.Config.MultiThreadCutoff = oldCutoff
- fs.Config.MultiThreadSet = oldIsSet
+ ci.MultiThreadStreams = oldStreams
+ ci.MultiThreadCutoff = oldCutoff
+ ci.MultiThreadSet = oldIsSet
}()
- fs.Config.MultiThreadStreams, fs.Config.MultiThreadCutoff = 4, 50
- fs.Config.MultiThreadSet = false
+ ci.MultiThreadStreams, ci.MultiThreadCutoff = 4, 50
+ ci.MultiThreadSet = false
nullWriterAt := func(ctx context.Context, remote string, size int64) (fs.WriterAtCloser, error) {
panic("don't call me")
}
f.Features().OpenWriterAt = nullWriterAt
- assert.True(t, doMultiThreadCopy(f, src))
+ assert.True(t, doMultiThreadCopy(ctx, f, src))
- fs.Config.MultiThreadStreams = 0
- assert.False(t, doMultiThreadCopy(f, src))
- fs.Config.MultiThreadStreams = 1
- assert.False(t, doMultiThreadCopy(f, src))
- fs.Config.MultiThreadStreams = 2
- assert.True(t, doMultiThreadCopy(f, src))
+ ci.MultiThreadStreams = 0
+ assert.False(t, doMultiThreadCopy(ctx, f, src))
+ ci.MultiThreadStreams = 1
+ assert.False(t, doMultiThreadCopy(ctx, f, src))
+ ci.MultiThreadStreams = 2
+ assert.True(t, doMultiThreadCopy(ctx, f, src))
- fs.Config.MultiThreadCutoff = 200
- assert.False(t, doMultiThreadCopy(f, src))
- fs.Config.MultiThreadCutoff = 101
- assert.False(t, doMultiThreadCopy(f, src))
- fs.Config.MultiThreadCutoff = 100
- assert.True(t, doMultiThreadCopy(f, src))
+ ci.MultiThreadCutoff = 200
+ assert.False(t, doMultiThreadCopy(ctx, f, src))
+ ci.MultiThreadCutoff = 101
+ assert.False(t, doMultiThreadCopy(ctx, f, src))
+ ci.MultiThreadCutoff = 100
+ assert.True(t, doMultiThreadCopy(ctx, f, src))
f.Features().OpenWriterAt = nil
- assert.False(t, doMultiThreadCopy(f, src))
+ assert.False(t, doMultiThreadCopy(ctx, f, src))
f.Features().OpenWriterAt = nullWriterAt
- assert.True(t, doMultiThreadCopy(f, src))
+ assert.True(t, doMultiThreadCopy(ctx, f, src))
f.Features().IsLocal = true
srcFs.Features().IsLocal = true
- assert.False(t, doMultiThreadCopy(f, src))
- fs.Config.MultiThreadSet = true
- assert.True(t, doMultiThreadCopy(f, src))
- fs.Config.MultiThreadSet = false
- assert.False(t, doMultiThreadCopy(f, src))
+ assert.False(t, doMultiThreadCopy(ctx, f, src))
+ ci.MultiThreadSet = true
+ assert.True(t, doMultiThreadCopy(ctx, f, src))
+ ci.MultiThreadSet = false
+ assert.False(t, doMultiThreadCopy(ctx, f, src))
srcFs.Features().IsLocal = false
- assert.True(t, doMultiThreadCopy(f, src))
+ assert.True(t, doMultiThreadCopy(ctx, f, src))
srcFs.Features().IsLocal = true
- assert.False(t, doMultiThreadCopy(f, src))
+ assert.False(t, doMultiThreadCopy(ctx, f, src))
f.Features().IsLocal = false
- assert.True(t, doMultiThreadCopy(f, src))
+ assert.True(t, doMultiThreadCopy(ctx, f, src))
srcFs.Features().IsLocal = false
- assert.True(t, doMultiThreadCopy(f, src))
+ assert.True(t, doMultiThreadCopy(ctx, f, src))
}
func TestMultithreadCalculateChunks(t *testing.T) {
diff --git a/fs/operations/operations.go b/fs/operations/operations.go
index d967d11de..cfb8d0698 100644
--- a/fs/operations/operations.go
+++ b/fs/operations/operations.go
@@ -119,13 +119,14 @@ func checkHashes(ctx context.Context, src fs.ObjectInfo, dst fs.Object, ht hash.
// Otherwise the file is considered to be not equal including if there
// were errors reading info.
func Equal(ctx context.Context, src fs.ObjectInfo, dst fs.Object) bool {
- return equal(ctx, src, dst, defaultEqualOpt())
+ return equal(ctx, src, dst, defaultEqualOpt(ctx))
}
// sizeDiffers compare the size of src and dst taking into account the
// various ways of ignoring sizes
-func sizeDiffers(src, dst fs.ObjectInfo) bool {
- if fs.Config.IgnoreSize || src.Size() < 0 || dst.Size() < 0 {
+func sizeDiffers(ctx context.Context, src, dst fs.ObjectInfo) bool {
+ ci := fs.GetConfig(ctx)
+ if ci.IgnoreSize || src.Size() < 0 || dst.Size() < 0 {
return false
}
return src.Size() != dst.Size()
@@ -142,11 +143,12 @@ type equalOpt struct {
}
// default set of options for equal()
-func defaultEqualOpt() equalOpt {
+func defaultEqualOpt(ctx context.Context) equalOpt {
+ ci := fs.GetConfig(ctx)
return equalOpt{
- sizeOnly: fs.Config.SizeOnly,
- checkSum: fs.Config.CheckSum,
- updateModTime: !fs.Config.NoUpdateModTime,
+ sizeOnly: ci.SizeOnly,
+ checkSum: ci.CheckSum,
+ updateModTime: !ci.NoUpdateModTime,
forceModTimeMatch: false,
}
}
@@ -161,7 +163,8 @@ func logModTimeUpload(dst fs.Object) {
}
func equal(ctx context.Context, src fs.ObjectInfo, dst fs.Object, opt equalOpt) bool {
- if sizeDiffers(src, dst) {
+ ci := fs.GetConfig(ctx)
+ if sizeDiffers(ctx, src, dst) {
fs.Debugf(src, "Sizes differ (src %d vs dst %d)", src.Size(), dst.Size())
return false
}
@@ -218,7 +221,7 @@ func equal(ctx context.Context, src fs.ObjectInfo, dst fs.Object, opt equalOpt)
fs.Debugf(src, "%v differ", ht)
return false
}
- if ht == hash.None && !fs.Config.RefreshTimes {
+ if ht == hash.None && !ci.RefreshTimes {
// if couldn't check hash, return that they differ
return false
}
@@ -228,7 +231,7 @@ func equal(ctx context.Context, src fs.ObjectInfo, dst fs.Object, opt equalOpt)
if !SkipDestructive(ctx, src, "update modification time") {
// Size and hash the same but mtime different
// Error if objects are treated as immutable
- if fs.Config.Immutable {
+ if ci.Immutable {
fs.Errorf(dst, "StartedAt mismatch between immutable objects")
return false
}
@@ -243,7 +246,7 @@ func equal(ctx context.Context, src fs.ObjectInfo, dst fs.Object, opt equalOpt)
fs.Infof(dst, "src and dst identical but can't set mod time without deleting and re-uploading")
// Remove the file if BackupDir isn't set. If BackupDir is set we would rather have the old file
// put in the BackupDir than deleted which is what will happen if we don't delete it.
- if fs.Config.BackupDir == "" {
+ if ci.BackupDir == "" {
err = dst.Remove(ctx)
if err != nil {
fs.Errorf(dst, "failed to delete before re-upload: %v", err)
@@ -337,11 +340,12 @@ var _ fs.FullObjectInfo = (*OverrideRemote)(nil)
// CommonHash returns a single hash.Type and a HashOption with that
// type which is in common between the two fs.Fs.
-func CommonHash(fa, fb fs.Info) (hash.Type, *fs.HashesOption) {
+func CommonHash(ctx context.Context, fa, fb fs.Info) (hash.Type, *fs.HashesOption) {
+ ci := fs.GetConfig(ctx)
// work out which hash to use - limit to 1 hash in common
var common hash.Set
hashType := hash.None
- if !fs.Config.IgnoreChecksum {
+ if !ci.IgnoreChecksum {
common = fb.Hashes().Overlap(fa.Hashes())
if common.Count() > 0 {
hashType = common.GetOne()
@@ -357,6 +361,7 @@ func CommonHash(fa, fb fs.Info) (hash.Type, *fs.HashesOption) {
// It returns the destination object if possible. Note that this may
// be nil.
func Copy(ctx context.Context, f fs.Fs, dst fs.Object, remote string, src fs.Object) (newDst fs.Object, err error) {
+ ci := fs.GetConfig(ctx)
tr := accounting.Stats(ctx).NewTransfer(src)
defer func() {
tr.Done(ctx, err)
@@ -365,25 +370,25 @@ func Copy(ctx context.Context, f fs.Fs, dst fs.Object, remote string, src fs.Obj
if SkipDestructive(ctx, src, "copy") {
return newDst, nil
}
- maxTries := fs.Config.LowLevelRetries
+ maxTries := ci.LowLevelRetries
tries := 0
doUpdate := dst != nil
- hashType, hashOption := CommonHash(f, src.Fs())
+ hashType, hashOption := CommonHash(ctx, f, src.Fs())
var actionTaken string
for {
// Try server-side copy first - if has optional interface and
// is same underlying remote
actionTaken = "Copied (server-side copy)"
- if fs.Config.MaxTransfer >= 0 {
+ if ci.MaxTransfer >= 0 {
var bytesSoFar int64
- if fs.Config.CutoffMode == fs.CutoffModeCautious {
+ if ci.CutoffMode == fs.CutoffModeCautious {
bytesSoFar = accounting.Stats(ctx).GetBytesWithPending() + src.Size()
} else {
bytesSoFar = accounting.Stats(ctx).GetBytes()
}
- if bytesSoFar >= int64(fs.Config.MaxTransfer) {
- if fs.Config.CutoffMode == fs.CutoffModeHard {
+ if bytesSoFar >= int64(ci.MaxTransfer) {
+ if ci.CutoffMode == fs.CutoffModeHard {
return nil, accounting.ErrorMaxTransferLimitReachedFatal
}
return nil, accounting.ErrorMaxTransferLimitReachedGraceful
@@ -408,12 +413,12 @@ func Copy(ctx context.Context, f fs.Fs, dst fs.Object, remote string, src fs.Obj
}
// If can't server-side copy, do it manually
if err == fs.ErrorCantCopy {
- if doMultiThreadCopy(f, src) {
+ if doMultiThreadCopy(ctx, f, src) {
// Number of streams proportional to size
- streams := src.Size() / int64(fs.Config.MultiThreadCutoff)
+ streams := src.Size() / int64(ci.MultiThreadCutoff)
// With maximum
- if streams > int64(fs.Config.MultiThreadStreams) {
- streams = int64(fs.Config.MultiThreadStreams)
+ if streams > int64(ci.MultiThreadStreams) {
+ streams = int64(ci.MultiThreadStreams)
}
if streams < 2 {
streams = 2
@@ -427,10 +432,10 @@ func Copy(ctx context.Context, f fs.Fs, dst fs.Object, remote string, src fs.Obj
} else {
var in0 io.ReadCloser
options := []fs.OpenOption{hashOption}
- for _, option := range fs.Config.DownloadHeaders {
+ for _, option := range ci.DownloadHeaders {
options = append(options, option)
}
- in0, err = NewReOpen(ctx, src, fs.Config.LowLevelRetries, options...)
+ in0, err = NewReOpen(ctx, src, ci.LowLevelRetries, options...)
if err != nil {
err = errors.Wrap(err, "failed to open source object")
} else {
@@ -452,7 +457,7 @@ func Copy(ctx context.Context, f fs.Fs, dst fs.Object, remote string, src fs.Obj
wrappedSrc = NewOverrideRemote(src, remote)
}
options := []fs.OpenOption{hashOption}
- for _, option := range fs.Config.UploadHeaders {
+ for _, option := range ci.UploadHeaders {
options = append(options, option)
}
if doUpdate {
@@ -491,7 +496,7 @@ func Copy(ctx context.Context, f fs.Fs, dst fs.Object, remote string, src fs.Obj
}
// Verify sizes are the same after transfer
- if sizeDiffers(src, dst) {
+ if sizeDiffers(ctx, src, dst) {
err = errors.Errorf("corrupted on transfer: sizes differ %d vs %d", src.Size(), dst.Size())
fs.Errorf(dst, "%v", err)
err = fs.CountError(err)
@@ -607,16 +612,17 @@ func CanServerSideMove(fdst fs.Fs) bool {
// SuffixName adds the current --suffix to the remote, obeying
// --suffix-keep-extension if set
-func SuffixName(remote string) string {
- if fs.Config.Suffix == "" {
+func SuffixName(ctx context.Context, remote string) string {
+ ci := fs.GetConfig(ctx)
+ if ci.Suffix == "" {
return remote
}
- if fs.Config.SuffixKeepExtension {
+ if ci.SuffixKeepExtension {
ext := path.Ext(remote)
base := remote[:len(remote)-len(ext)]
- return base + fs.Config.Suffix + ext
+ return base + ci.Suffix + ext
}
- return remote + fs.Config.Suffix
+ return remote + ci.Suffix
}
// DeleteFileWithBackupDir deletes a single file respecting --dry-run
@@ -625,12 +631,13 @@ func SuffixName(remote string) string {
// If backupDir is set then it moves the file to there instead of
// deleting
func DeleteFileWithBackupDir(ctx context.Context, dst fs.Object, backupDir fs.Fs) (err error) {
+ ci := fs.GetConfig(ctx)
tr := accounting.Stats(ctx).NewCheckingTransfer(dst)
defer func() {
tr.Done(ctx, err)
}()
numDeletes := accounting.Stats(ctx).Deletes(1)
- if fs.Config.MaxDelete != -1 && numDeletes > fs.Config.MaxDelete {
+ if ci.MaxDelete != -1 && numDeletes > ci.MaxDelete {
return fserrors.FatalError(errors.New("--max-delete threshold reached"))
}
action, actioned := "delete", "Deleted"
@@ -669,11 +676,12 @@ func DeleteFile(ctx context.Context, dst fs.Object) (err error) {
// instead of being deleted.
func DeleteFilesWithBackupDir(ctx context.Context, toBeDeleted fs.ObjectsChan, backupDir fs.Fs) error {
var wg sync.WaitGroup
- wg.Add(fs.Config.Transfers)
+ ci := fs.GetConfig(ctx)
+ wg.Add(ci.Transfers)
var errorCount int32
var fatalErrorCount int32
- for i := 0; i < fs.Config.Transfers; i++ {
+ for i := 0; i < ci.Transfers; i++ {
go func() {
defer wg.Done()
for dst := range toBeDeleted {
@@ -779,7 +787,8 @@ func Retry(o interface{}, maxTries int, fn func() error) (err error) {
//
// Lists in parallel which may get them out of order
func ListFn(ctx context.Context, f fs.Fs, fn func(fs.Object)) error {
- return walk.ListR(ctx, f, "", false, fs.Config.MaxDepth, walk.ListObjects, func(entries fs.DirEntries) error {
+ ci := fs.GetConfig(ctx)
+ return walk.ListR(ctx, f, "", false, ci.MaxDepth, walk.ListObjects, func(entries fs.DirEntries) error {
entries.ForObject(fn)
return nil
})
@@ -898,8 +907,9 @@ func Count(ctx context.Context, f fs.Fs) (objects int64, size int64, err error)
}
// ConfigMaxDepth returns the depth to use for a recursive or non recursive listing.
-func ConfigMaxDepth(recursive bool) int {
- depth := fs.Config.MaxDepth
+func ConfigMaxDepth(ctx context.Context, recursive bool) int {
+ ci := fs.GetConfig(ctx)
+ depth := ci.MaxDepth
if !recursive && depth < 0 {
depth = 1
}
@@ -908,7 +918,7 @@ func ConfigMaxDepth(recursive bool) int {
// ListDir lists the directories/buckets/containers in the Fs to the supplied writer
func ListDir(ctx context.Context, f fs.Fs, w io.Writer) error {
- return walk.ListR(ctx, f, "", false, ConfigMaxDepth(false), walk.ListDirs, func(entries fs.DirEntries) error {
+ return walk.ListR(ctx, f, "", false, ConfigMaxDepth(ctx, false), walk.ListDirs, func(entries fs.DirEntries) error {
entries.ForDir(func(dir fs.Directory) {
if dir != nil {
syncFprintf(w, "%12d %13s %9d %s\n", dir.Size(), dir.ModTime(ctx).Local().Format("2006-01-02 15:04:05"), dir.Items(), dir.Remote())
@@ -985,7 +995,8 @@ func Purge(ctx context.Context, f fs.Fs, dir string) (err error) {
// Delete removes all the contents of a container. Unlike Purge, it
// obeys includes and excludes.
func Delete(ctx context.Context, f fs.Fs) error {
- delChan := make(fs.ObjectsChan, fs.Config.Transfers)
+ ci := fs.GetConfig(ctx)
+ delChan := make(fs.ObjectsChan, ci.Transfers)
delErr := make(chan error, 1)
go func() {
delErr <- DeleteFiles(ctx, delChan)
@@ -1008,10 +1019,11 @@ func Delete(ctx context.Context, f fs.Fs) error {
//
// If the error was ErrorDirNotFound then it will be ignored
func listToChan(ctx context.Context, f fs.Fs, dir string) fs.ObjectsChan {
- o := make(fs.ObjectsChan, fs.Config.Checkers)
+ ci := fs.GetConfig(ctx)
+ o := make(fs.ObjectsChan, ci.Checkers)
go func() {
defer close(o)
- err := walk.ListR(ctx, f, dir, true, fs.Config.MaxDepth, walk.ListObjects, func(entries fs.DirEntries) error {
+ err := walk.ListR(ctx, f, dir, true, ci.MaxDepth, walk.ListObjects, func(entries fs.DirEntries) error {
entries.ForObject(func(obj fs.Object) {
o <- obj
})
@@ -1054,6 +1066,7 @@ type readCloser struct {
// if count >= 0 then only that many characters will be output
func Cat(ctx context.Context, f fs.Fs, w io.Writer, offset, count int64) error {
var mu sync.Mutex
+ ci := fs.GetConfig(ctx)
return ListFn(ctx, f, func(o fs.Object) {
var err error
tr := accounting.Stats(ctx).NewTransfer(o)
@@ -1072,7 +1085,7 @@ func Cat(ctx context.Context, f fs.Fs, w io.Writer, offset, count int64) error {
if opt.Start > 0 || opt.End >= 0 {
options = append(options, &opt)
}
- for _, option := range fs.Config.DownloadHeaders {
+ for _, option := range ci.DownloadHeaders {
options = append(options, option)
}
in, err := o.Open(ctx, options...)
@@ -1098,6 +1111,7 @@ func Cat(ctx context.Context, f fs.Fs, w io.Writer, offset, count int64) error {
// Rcat reads data from the Reader until EOF and uploads it to a file on remote
func Rcat(ctx context.Context, fdst fs.Fs, dstFileName string, in io.ReadCloser, modTime time.Time) (dst fs.Object, err error) {
+ ci := fs.GetConfig(ctx)
tr := accounting.Stats(ctx).NewTransferRemoteSize(dstFileName, -1)
defer func() {
tr.Done(ctx, err)
@@ -1108,7 +1122,7 @@ func Rcat(ctx context.Context, fdst fs.Fs, dstFileName string, in io.ReadCloser,
var trackingIn io.Reader
var hasher *hash.MultiHasher
var options []fs.OpenOption
- if !fs.Config.IgnoreChecksum {
+ if !ci.IgnoreChecksum {
hashes := hash.NewHashSet(fdst.Hashes().GetOne()) // just pick one hash
hashOption := &fs.HashesOption{Hashes: hashes}
options = append(options, hashOption)
@@ -1120,7 +1134,7 @@ func Rcat(ctx context.Context, fdst fs.Fs, dstFileName string, in io.ReadCloser,
} else {
trackingIn = readCounter
}
- for _, option := range fs.Config.UploadHeaders {
+ for _, option := range ci.UploadHeaders {
options = append(options, option)
}
@@ -1140,7 +1154,7 @@ func Rcat(ctx context.Context, fdst fs.Fs, dstFileName string, in io.ReadCloser,
}
// check if file small enough for direct upload
- buf := make([]byte, fs.Config.StreamingUploadCutoff)
+ buf := make([]byte, ci.StreamingUploadCutoff)
if n, err := io.ReadFull(trackingIn, buf); err == io.EOF || err == io.ErrUnexpectedEOF {
fs.Debugf(fdst, "File to upload is small (%d bytes), uploading instead of streaming", n)
src := object.NewMemoryObject(dstFileName, modTime, buf[:n])
@@ -1202,9 +1216,10 @@ func PublicLink(ctx context.Context, f fs.Fs, remote string, expire fs.Duration,
// Rmdirs removes any empty directories (or directories only
// containing empty directories) under f, including f.
func Rmdirs(ctx context.Context, f fs.Fs, dir string, leaveRoot bool) error {
+ ci := fs.GetConfig(ctx)
dirEmpty := make(map[string]bool)
dirEmpty[dir] = !leaveRoot
- err := walk.Walk(ctx, f, dir, true, fs.Config.MaxDepth, func(dirPath string, entries fs.DirEntries, err error) error {
+ err := walk.Walk(ctx, f, dir, true, ci.MaxDepth, func(dirPath string, entries fs.DirEntries, err error) error {
if err != nil {
err = fs.CountError(err)
fs.Errorf(f, "Failed to list %q: %v", dirPath, err)
@@ -1263,9 +1278,10 @@ func Rmdirs(ctx context.Context, f fs.Fs, dir string, leaveRoot bool) error {
// GetCompareDest sets up --compare-dest
func GetCompareDest(ctx context.Context) (CompareDest fs.Fs, err error) {
- CompareDest, err = cache.Get(ctx, fs.Config.CompareDest)
+ ci := fs.GetConfig(ctx)
+ CompareDest, err = cache.Get(ctx, ci.CompareDest)
if err != nil {
- return nil, fserrors.FatalError(errors.Errorf("Failed to make fs for --compare-dest %q: %v", fs.Config.CompareDest, err))
+ return nil, fserrors.FatalError(errors.Errorf("Failed to make fs for --compare-dest %q: %v", ci.CompareDest, err))
}
return CompareDest, nil
}
@@ -1299,9 +1315,10 @@ func compareDest(ctx context.Context, dst, src fs.Object, CompareDest fs.Fs) (No
// GetCopyDest sets up --copy-dest
func GetCopyDest(ctx context.Context, fdst fs.Fs) (CopyDest fs.Fs, err error) {
- CopyDest, err = cache.Get(ctx, fs.Config.CopyDest)
+ ci := fs.GetConfig(ctx)
+ CopyDest, err = cache.Get(ctx, ci.CopyDest)
if err != nil {
- return nil, fserrors.FatalError(errors.Errorf("Failed to make fs for --copy-dest %q: %v", fs.Config.CopyDest, err))
+ return nil, fserrors.FatalError(errors.Errorf("Failed to make fs for --copy-dest %q: %v", ci.CopyDest, err))
}
if !SameConfig(fdst, CopyDest) {
return nil, fserrors.FatalError(errors.New("parameter to --copy-dest has to be on the same remote as destination"))
@@ -1332,7 +1349,7 @@ func copyDest(ctx context.Context, fdst fs.Fs, dst, src fs.Object, CopyDest, bac
default:
return false, err
}
- opt := defaultEqualOpt()
+ opt := defaultEqualOpt(ctx)
opt.updateModTime = false
if equal(ctx, src, CopyDestFile, opt) {
if dst == nil || !Equal(ctx, src, dst) {
@@ -1364,9 +1381,10 @@ func copyDest(ctx context.Context, fdst fs.Fs, dst, src fs.Object, CopyDest, bac
//
// Returns True if src does not need to be copied
func CompareOrCopyDest(ctx context.Context, fdst fs.Fs, dst, src fs.Object, CompareOrCopyDest, backupDir fs.Fs) (NoNeedTransfer bool, err error) {
- if fs.Config.CompareDest != "" {
+ ci := fs.GetConfig(ctx)
+ if ci.CompareDest != "" {
return compareDest(ctx, dst, src, CompareOrCopyDest)
- } else if fs.Config.CopyDest != "" {
+ } else if ci.CopyDest != "" {
return copyDest(ctx, fdst, dst, src, CompareOrCopyDest, backupDir)
}
return false, nil
@@ -1378,22 +1396,23 @@ func CompareOrCopyDest(ctx context.Context, fdst fs.Fs, dst, src fs.Object, Comp
// Returns a flag which indicates whether the file needs to be
// transferred or not.
func NeedTransfer(ctx context.Context, dst, src fs.Object) bool {
+ ci := fs.GetConfig(ctx)
if dst == nil {
fs.Debugf(src, "Need to transfer - File not found at Destination")
return true
}
// If we should ignore existing files, don't transfer
- if fs.Config.IgnoreExisting {
+ if ci.IgnoreExisting {
fs.Debugf(src, "Destination exists, skipping")
return false
}
// If we should upload unconditionally
- if fs.Config.IgnoreTimes {
+ if ci.IgnoreTimes {
fs.Debugf(src, "Transferring unconditionally as --ignore-times is in use")
return true
}
// If UpdateOlder is in effect, skip if dst is newer than src
- if fs.Config.UpdateOlder {
+ if ci.UpdateOlder {
srcModTime := src.ModTime(ctx)
dstModTime := dst.ModTime(ctx)
dt := dstModTime.Sub(srcModTime)
@@ -1411,7 +1430,7 @@ func NeedTransfer(ctx context.Context, dst, src fs.Object) bool {
return false
case dt <= -modifyWindow:
// force --checksum on for the check and do update modtimes by default
- opt := defaultEqualOpt()
+ opt := defaultEqualOpt(ctx)
opt.forceModTimeMatch = true
if equal(ctx, src, dst, opt) {
fs.Debugf(src, "Unchanged skipping")
@@ -1419,8 +1438,8 @@ func NeedTransfer(ctx context.Context, dst, src fs.Object) bool {
}
default:
// Do a size only compare unless --checksum is set
- opt := defaultEqualOpt()
- opt.sizeOnly = !fs.Config.CheckSum
+ opt := defaultEqualOpt(ctx)
+ opt.sizeOnly = !ci.CheckSum
if equal(ctx, src, dst, opt) {
fs.Debugf(src, "Destination mod time is within %v of source and files identical, skipping", modifyWindow)
return false
@@ -1483,7 +1502,7 @@ type copyURLFunc func(ctx context.Context, dstFileName string, in io.ReadCloser,
// copyURLFn copies the data from the url to the function supplied
func copyURLFn(ctx context.Context, dstFileName string, url string, dstFileNameFromURL bool, fn copyURLFunc) (err error) {
- client := fshttp.NewClient(fs.Config)
+ client := fshttp.NewClient(fs.GetConfig(ctx))
resp, err := client.Get(url)
if err != nil {
return err
@@ -1531,10 +1550,11 @@ func CopyURLToWriter(ctx context.Context, url string, out io.Writer) (err error)
// BackupDir returns the correctly configured --backup-dir
func BackupDir(ctx context.Context, fdst fs.Fs, fsrc fs.Fs, srcFileName string) (backupDir fs.Fs, err error) {
- if fs.Config.BackupDir != "" {
- backupDir, err = cache.Get(ctx, fs.Config.BackupDir)
+ ci := fs.GetConfig(ctx)
+ if ci.BackupDir != "" {
+ backupDir, err = cache.Get(ctx, ci.BackupDir)
if err != nil {
- return nil, fserrors.FatalError(errors.Errorf("Failed to make fs for --backup-dir %q: %v", fs.Config.BackupDir, err))
+ return nil, fserrors.FatalError(errors.Errorf("Failed to make fs for --backup-dir %q: %v", ci.BackupDir, err))
}
if !SameConfig(fdst, backupDir) {
return nil, fserrors.FatalError(errors.New("parameter to --backup-dir has to be on the same remote as destination"))
@@ -1547,7 +1567,7 @@ func BackupDir(ctx context.Context, fdst fs.Fs, fsrc fs.Fs, srcFileName string)
return nil, fserrors.FatalError(errors.New("source and parameter to --backup-dir mustn't overlap"))
}
} else {
- if fs.Config.Suffix == "" {
+ if ci.Suffix == "" {
if SameDir(fdst, backupDir) {
return nil, fserrors.FatalError(errors.New("destination and parameter to --backup-dir mustn't be the same"))
}
@@ -1556,7 +1576,7 @@ func BackupDir(ctx context.Context, fdst fs.Fs, fsrc fs.Fs, srcFileName string)
}
}
}
- } else if fs.Config.Suffix != "" {
+ } else if ci.Suffix != "" {
// --backup-dir is not set but --suffix is - use the destination as the backupDir
backupDir = fdst
} else {
@@ -1570,7 +1590,7 @@ func BackupDir(ctx context.Context, fdst fs.Fs, fsrc fs.Fs, srcFileName string)
// MoveBackupDir moves a file to the backup dir
func MoveBackupDir(ctx context.Context, backupDir fs.Fs, dst fs.Object) (err error) {
- remoteWithSuffix := SuffixName(dst.Remote())
+ remoteWithSuffix := SuffixName(ctx, dst.Remote())
overwritten, _ := backupDir.NewObject(ctx, remoteWithSuffix)
_, err = Move(ctx, backupDir, overwritten, remoteWithSuffix, dst)
return err
@@ -1578,6 +1598,7 @@ func MoveBackupDir(ctx context.Context, backupDir fs.Fs, dst fs.Object) (err err
// moveOrCopyFile moves or copies a single file possibly to a new name
func moveOrCopyFile(ctx context.Context, fdst fs.Fs, fsrc fs.Fs, dstFileName string, srcFileName string, cp bool) (err error) {
+ ci := fs.GetConfig(ctx)
dstFilePath := path.Join(fdst.Root(), dstFileName)
srcFilePath := path.Join(fsrc.Root(), srcFileName)
if fdst.Name() == fsrc.Name() && dstFilePath == srcFilePath {
@@ -1599,7 +1620,7 @@ func moveOrCopyFile(ctx context.Context, fdst fs.Fs, fsrc fs.Fs, dstFileName str
// Find dst object if it exists
var dstObj fs.Object
- if !fs.Config.NoCheckDest {
+ if !ci.NoCheckDest {
dstObj, err = fdst.NewObject(ctx, dstFileName)
if err == fs.ErrorObjectNotFound {
dstObj = nil
@@ -1635,18 +1656,18 @@ func moveOrCopyFile(ctx context.Context, fdst fs.Fs, fsrc fs.Fs, dstFileName str
}
var backupDir, copyDestDir fs.Fs
- if fs.Config.BackupDir != "" || fs.Config.Suffix != "" {
+ if ci.BackupDir != "" || ci.Suffix != "" {
backupDir, err = BackupDir(ctx, fdst, fsrc, srcFileName)
if err != nil {
return errors.Wrap(err, "creating Fs for --backup-dir failed")
}
}
- if fs.Config.CompareDest != "" {
+ if ci.CompareDest != "" {
copyDestDir, err = GetCompareDest(ctx)
if err != nil {
return err
}
- } else if fs.Config.CopyDest != "" {
+ } else if ci.CopyDest != "" {
copyDestDir, err = GetCopyDest(ctx, fdst)
if err != nil {
return err
@@ -1853,6 +1874,7 @@ func (l *ListFormat) Format(entry *ListJSONItem) (result string) {
// It does this by loading the directory tree into memory (using ListR
// if available) and doing renames in parallel.
func DirMove(ctx context.Context, f fs.Fs, srcRemote, dstRemote string) (err error) {
+ ci := fs.GetConfig(ctx)
// Use DirMove if possible
if doDirMove := f.Features().DirMove; doDirMove != nil {
err = doDirMove(ctx, f, srcRemote, dstRemote)
@@ -1885,9 +1907,9 @@ func DirMove(ctx context.Context, f fs.Fs, srcRemote, dstRemote string) (err err
o fs.Object
newPath string
}
- renames := make(chan rename, fs.Config.Transfers)
+ renames := make(chan rename, ci.Transfers)
g, gCtx := errgroup.WithContext(context.Background())
- for i := 0; i < fs.Config.Transfers; i++ {
+ for i := 0; i < ci.Transfers; i++ {
g.Go(func() error {
for job := range renames {
dstOverwritten, _ := f.NewObject(gCtx, job.newPath)
@@ -2019,11 +2041,12 @@ func skipDestructiveChoose(ctx context.Context, subject interface{}, action stri
// to action subject".
func SkipDestructive(ctx context.Context, subject interface{}, action string) (skip bool) {
var flag string
+ ci := fs.GetConfig(ctx)
switch {
- case fs.Config.DryRun:
+ case ci.DryRun:
flag = "--dry-run"
skip = true
- case fs.Config.Interactive:
+ case ci.Interactive:
flag = "--interactive"
interactiveMu.Lock()
defer interactiveMu.Unlock()
diff --git a/fs/operations/operations_internal_test.go b/fs/operations/operations_internal_test.go
index cb7f6ccd5..fd85a9819 100644
--- a/fs/operations/operations_internal_test.go
+++ b/fs/operations/operations_internal_test.go
@@ -3,6 +3,7 @@
package operations
import (
+ "context"
"fmt"
"testing"
"time"
@@ -13,6 +14,8 @@ import (
)
func TestSizeDiffers(t *testing.T) {
+ ctx := context.Background()
+ ci := fs.GetConfig(ctx)
when := time.Now()
for _, test := range []struct {
ignoreSize bool
@@ -31,10 +34,10 @@ func TestSizeDiffers(t *testing.T) {
} {
src := object.NewStaticObjectInfo("a", when, test.srcSize, true, nil, nil)
dst := object.NewStaticObjectInfo("a", when, test.dstSize, true, nil, nil)
- oldIgnoreSize := fs.Config.IgnoreSize
- fs.Config.IgnoreSize = test.ignoreSize
- got := sizeDiffers(src, dst)
- fs.Config.IgnoreSize = oldIgnoreSize
+ oldIgnoreSize := ci.IgnoreSize
+ ci.IgnoreSize = test.ignoreSize
+ got := sizeDiffers(ctx, src, dst)
+ ci.IgnoreSize = oldIgnoreSize
assert.Equal(t, test.want, got, fmt.Sprintf("ignoreSize=%v, srcSize=%v, dstSize=%v", test.ignoreSize, test.srcSize, test.dstSize))
}
}
diff --git a/fs/operations/operations_test.go b/fs/operations/operations_test.go
index d9601c527..113e7a198 100644
--- a/fs/operations/operations_test.go
+++ b/fs/operations/operations_test.go
@@ -106,6 +106,7 @@ func TestLs(t *testing.T) {
func TestLsWithFilesFrom(t *testing.T) {
ctx := context.Background()
+ ci := fs.GetConfig(ctx)
r := fstest.NewRun(t)
defer r.Finalise()
file1 := r.WriteBoth(ctx, "potato2", "------------------------------------------------------------", t1)
@@ -132,10 +133,10 @@ func TestLsWithFilesFrom(t *testing.T) {
assert.Equal(t, " 60 potato2\n", buf.String())
// Now try with --no-traverse
- oldNoTraverse := fs.Config.NoTraverse
- fs.Config.NoTraverse = true
+ oldNoTraverse := ci.NoTraverse
+ ci.NoTraverse = true
defer func() {
- fs.Config.NoTraverse = oldNoTraverse
+ ci.NoTraverse = oldNoTraverse
}()
buf.Reset()
@@ -269,9 +270,11 @@ func TestHashSums(t *testing.T) {
}
func TestSuffixName(t *testing.T) {
- origSuffix, origKeepExt := fs.Config.Suffix, fs.Config.SuffixKeepExtension
+ ctx := context.Background()
+ ci := fs.GetConfig(ctx)
+ origSuffix, origKeepExt := ci.Suffix, ci.SuffixKeepExtension
defer func() {
- fs.Config.Suffix, fs.Config.SuffixKeepExtension = origSuffix, origKeepExt
+ ci.Suffix, ci.SuffixKeepExtension = origSuffix, origKeepExt
}()
for _, test := range []struct {
remote string
@@ -288,15 +291,16 @@ func TestSuffixName(t *testing.T) {
{"test", "-suffix", false, "test-suffix"},
{"test", "-suffix", true, "test-suffix"},
} {
- fs.Config.Suffix = test.suffix
- fs.Config.SuffixKeepExtension = test.keepExt
- got := operations.SuffixName(test.remote)
+ ci.Suffix = test.suffix
+ ci.SuffixKeepExtension = test.keepExt
+ got := operations.SuffixName(ctx, test.remote)
assert.Equal(t, test.want, got, fmt.Sprintf("%+v", test))
}
}
func TestCount(t *testing.T) {
ctx := context.Background()
+ ci := fs.GetConfig(ctx)
r := fstest.NewRun(t)
defer r.Finalise()
file1 := r.WriteBoth(ctx, "potato2", "------------------------------------------------------------", t1)
@@ -306,8 +310,8 @@ func TestCount(t *testing.T) {
fstest.CheckItems(t, r.Fremote, file1, file2, file3)
// Check the MaxDepth too
- fs.Config.MaxDepth = 1
- defer func() { fs.Config.MaxDepth = -1 }()
+ ci.MaxDepth = 1
+ defer func() { ci.MaxDepth = -1 }()
objects, size, err := operations.Count(ctx, r.Fremote)
require.NoError(t, err)
@@ -583,6 +587,7 @@ func TestRmdirsLeaveRoot(t *testing.T) {
func TestCopyURL(t *testing.T) {
ctx := context.Background()
+ ci := fs.GetConfig(ctx)
r := fstest.NewRun(t)
defer r.Finalise()
@@ -635,10 +640,10 @@ func TestCopyURL(t *testing.T) {
status = 0
// check when reading from unverified HTTPS server
- fs.Config.InsecureSkipVerify = true
+ ci.InsecureSkipVerify = true
fshttp.ResetTransport()
defer func() {
- fs.Config.InsecureSkipVerify = false
+ ci.InsecureSkipVerify = false
fshttp.ResetTransport()
}()
tss := httptest.NewTLSServer(handler)
@@ -750,16 +755,17 @@ func TestCaseInsensitiveMoveFile(t *testing.T) {
func TestMoveFileBackupDir(t *testing.T) {
ctx := context.Background()
+ ci := fs.GetConfig(ctx)
r := fstest.NewRun(t)
defer r.Finalise()
if !operations.CanServerSideMove(r.Fremote) {
t.Skip("Skipping test as remote does not support server-side move or copy")
}
- oldBackupDir := fs.Config.BackupDir
- fs.Config.BackupDir = r.FremoteName + "/backup"
+ oldBackupDir := ci.BackupDir
+ ci.BackupDir = r.FremoteName + "/backup"
defer func() {
- fs.Config.BackupDir = oldBackupDir
+ ci.BackupDir = oldBackupDir
}()
file1 := r.WriteFile("dst/file1", "file1 contents", t1)
@@ -804,16 +810,17 @@ func TestCopyFile(t *testing.T) {
func TestCopyFileBackupDir(t *testing.T) {
ctx := context.Background()
+ ci := fs.GetConfig(ctx)
r := fstest.NewRun(t)
defer r.Finalise()
if !operations.CanServerSideMove(r.Fremote) {
t.Skip("Skipping test as remote does not support server-side move or copy")
}
- oldBackupDir := fs.Config.BackupDir
- fs.Config.BackupDir = r.FremoteName + "/backup"
+ oldBackupDir := ci.BackupDir
+ ci.BackupDir = r.FremoteName + "/backup"
defer func() {
- fs.Config.BackupDir = oldBackupDir
+ ci.BackupDir = oldBackupDir
}()
file1 := r.WriteFile("dst/file1", "file1 contents", t1)
@@ -832,12 +839,13 @@ func TestCopyFileBackupDir(t *testing.T) {
// Test with CompareDest set
func TestCopyFileCompareDest(t *testing.T) {
ctx := context.Background()
+ ci := fs.GetConfig(ctx)
r := fstest.NewRun(t)
defer r.Finalise()
- fs.Config.CompareDest = r.FremoteName + "/CompareDest"
+ ci.CompareDest = r.FremoteName + "/CompareDest"
defer func() {
- fs.Config.CompareDest = ""
+ ci.CompareDest = ""
}()
fdst, err := fs.NewFs(ctx, r.FremoteName+"/dst")
require.NoError(t, err)
@@ -913,6 +921,7 @@ func TestCopyFileCompareDest(t *testing.T) {
// Test with CopyDest set
func TestCopyFileCopyDest(t *testing.T) {
ctx := context.Background()
+ ci := fs.GetConfig(ctx)
r := fstest.NewRun(t)
defer r.Finalise()
@@ -920,9 +929,9 @@ func TestCopyFileCopyDest(t *testing.T) {
t.Skip("Skipping test as remote does not support server-side copy")
}
- fs.Config.CopyDest = r.FremoteName + "/CopyDest"
+ ci.CopyDest = r.FremoteName + "/CopyDest"
defer func() {
- fs.Config.CopyDest = ""
+ ci.CopyDest = ""
}()
fdst, err := fs.NewFs(ctx, r.FremoteName+"/dst")
@@ -955,7 +964,7 @@ func TestCopyFileCopyDest(t *testing.T) {
// check old dest, new copy, backup-dir
- fs.Config.BackupDir = r.FremoteName + "/BackupDir"
+ ci.BackupDir = r.FremoteName + "/BackupDir"
file3 := r.WriteObject(ctx, "dst/one", "one", t1)
file2 := r.WriteObject(ctx, "CopyDest/one", "onet2", t2)
@@ -971,7 +980,7 @@ func TestCopyFileCopyDest(t *testing.T) {
file3.Path = "BackupDir/one"
fstest.CheckItems(t, r.Fremote, file2, file2dst, file3)
- fs.Config.BackupDir = ""
+ ci.BackupDir = ""
// check empty dest, new copy
file4 := r.WriteObject(ctx, "CopyDest/two", "two", t2)
@@ -1329,11 +1338,13 @@ func TestGetFsInfo(t *testing.T) {
}
func TestRcat(t *testing.T) {
+ ctx := context.Background()
+ ci := fs.GetConfig(ctx)
check := func(withChecksum, ignoreChecksum bool) {
- checksumBefore, ignoreChecksumBefore := fs.Config.CheckSum, fs.Config.IgnoreChecksum
- fs.Config.CheckSum, fs.Config.IgnoreChecksum = withChecksum, ignoreChecksum
+ checksumBefore, ignoreChecksumBefore := ci.CheckSum, ci.IgnoreChecksum
+ ci.CheckSum, ci.IgnoreChecksum = withChecksum, ignoreChecksum
defer func() {
- fs.Config.CheckSum, fs.Config.IgnoreChecksum = checksumBefore, ignoreChecksumBefore
+ ci.CheckSum, ci.IgnoreChecksum = checksumBefore, ignoreChecksumBefore
}()
var prefix string
@@ -1350,13 +1361,13 @@ func TestRcat(t *testing.T) {
r := fstest.NewRun(t)
defer r.Finalise()
- if *fstest.SizeLimit > 0 && int64(fs.Config.StreamingUploadCutoff) > *fstest.SizeLimit {
- savedCutoff := fs.Config.StreamingUploadCutoff
+ if *fstest.SizeLimit > 0 && int64(ci.StreamingUploadCutoff) > *fstest.SizeLimit {
+ savedCutoff := ci.StreamingUploadCutoff
defer func() {
- fs.Config.StreamingUploadCutoff = savedCutoff
+ ci.StreamingUploadCutoff = savedCutoff
}()
- fs.Config.StreamingUploadCutoff = fs.SizeSuffix(*fstest.SizeLimit)
- t.Logf("Adjust StreamingUploadCutoff to size limit %s (was %s)", fs.Config.StreamingUploadCutoff, savedCutoff)
+ ci.StreamingUploadCutoff = fs.SizeSuffix(*fstest.SizeLimit)
+ t.Logf("Adjust StreamingUploadCutoff to size limit %s (was %s)", ci.StreamingUploadCutoff, savedCutoff)
}
fstest.CheckListing(t, r.Fremote, []fstest.Item{})
@@ -1364,7 +1375,7 @@ func TestRcat(t *testing.T) {
data1 := "this is some really nice test data"
path1 := prefix + "small_file_from_pipe"
- data2 := string(make([]byte, fs.Config.StreamingUploadCutoff+1))
+ data2 := string(make([]byte, ci.StreamingUploadCutoff+1))
path2 := prefix + "big_file_from_pipe"
in := ioutil.NopCloser(strings.NewReader(data1))
@@ -1418,14 +1429,15 @@ func TestRcatSize(t *testing.T) {
func TestCopyFileMaxTransfer(t *testing.T) {
ctx := context.Background()
+ ci := fs.GetConfig(ctx)
r := fstest.NewRun(t)
defer r.Finalise()
- old := fs.Config.MaxTransfer
- oldMode := fs.Config.CutoffMode
+ old := ci.MaxTransfer
+ oldMode := ci.CutoffMode
defer func() {
- fs.Config.MaxTransfer = old
- fs.Config.CutoffMode = oldMode
+ ci.MaxTransfer = old
+ ci.CutoffMode = oldMode
accounting.Stats(ctx).ResetCounters()
}()
@@ -1436,8 +1448,8 @@ func TestCopyFileMaxTransfer(t *testing.T) {
file4 := r.WriteFile("TestCopyFileMaxTransfer/file4", "file4 contents"+random.String(sizeCutoff), t2)
// Cutoff mode: Hard
- fs.Config.MaxTransfer = sizeCutoff
- fs.Config.CutoffMode = fs.CutoffModeHard
+ ci.MaxTransfer = sizeCutoff
+ ci.CutoffMode = fs.CutoffModeHard
// file1: Show a small file gets transferred OK
accounting.Stats(ctx).ResetCounters()
@@ -1456,7 +1468,7 @@ func TestCopyFileMaxTransfer(t *testing.T) {
fstest.CheckItems(t, r.Fremote, file1)
// Cutoff mode: Cautious
- fs.Config.CutoffMode = fs.CutoffModeCautious
+ ci.CutoffMode = fs.CutoffModeCautious
// file3: show a large file does not get transferred
accounting.Stats(ctx).ResetCounters()
@@ -1473,7 +1485,7 @@ func TestCopyFileMaxTransfer(t *testing.T) {
}
// Cutoff mode: Soft
- fs.Config.CutoffMode = fs.CutoffModeSoft
+ ci.CutoffMode = fs.CutoffModeSoft
// file4: show a large file does get transferred this time
accounting.Stats(ctx).ResetCounters()
diff --git a/fs/rc/config_test.go b/fs/rc/config_test.go
index 3ee4c27bc..eb46698e8 100644
--- a/fs/rc/config_test.go
+++ b/fs/rc/config_test.go
@@ -75,10 +75,12 @@ func TestOptionsGet(t *testing.T) {
func TestOptionsGetMarshal(t *testing.T) {
defer clearOptionBlock()()
+ ctx := context.Background()
+ ci := fs.GetConfig(ctx)
// Add some real options
AddOption("http", &httplib.DefaultOpt)
- AddOption("main", fs.Config)
+ AddOption("main", ci)
AddOption("rc", &DefaultOpt)
// get them
diff --git a/fs/sync/sync.go b/fs/sync/sync.go
index 165527dae..1eaaafadb 100644
--- a/fs/sync/sync.go
+++ b/fs/sync/sync.go
@@ -30,6 +30,7 @@ type syncCopyMove struct {
deleteEmptySrcDirs bool
dir string
// internal state
+ ci *fs.ConfigInfo // global config
ctx context.Context // internal context for controlling go-routines
cancel func() // cancel the context
inCtx context.Context // internal context for controlling march
@@ -97,7 +98,9 @@ func newSyncCopyMove(ctx context.Context, fdst, fsrc fs.Fs, deleteMode fs.Delete
if (deleteMode != fs.DeleteModeOff || DoMove) && operations.Overlapping(fdst, fsrc) {
return nil, fserrors.FatalError(fs.ErrorOverlapping)
}
+ ci := fs.GetConfig(ctx)
s := &syncCopyMove{
+ ci: ci,
fdst: fdst,
fsrc: fsrc,
deleteMode: deleteMode,
@@ -105,42 +108,42 @@ func newSyncCopyMove(ctx context.Context, fdst, fsrc fs.Fs, deleteMode fs.Delete
copyEmptySrcDirs: copyEmptySrcDirs,
deleteEmptySrcDirs: deleteEmptySrcDirs,
dir: "",
- srcFilesChan: make(chan fs.Object, fs.Config.Checkers+fs.Config.Transfers),
+ srcFilesChan: make(chan fs.Object, ci.Checkers+ci.Transfers),
srcFilesResult: make(chan error, 1),
dstFilesResult: make(chan error, 1),
dstEmptyDirs: make(map[string]fs.DirEntry),
srcEmptyDirs: make(map[string]fs.DirEntry),
- noTraverse: fs.Config.NoTraverse,
- noCheckDest: fs.Config.NoCheckDest,
- noUnicodeNormalization: fs.Config.NoUnicodeNormalization,
- deleteFilesCh: make(chan fs.Object, fs.Config.Checkers),
- trackRenames: fs.Config.TrackRenames,
+ noTraverse: ci.NoTraverse,
+ noCheckDest: ci.NoCheckDest,
+ noUnicodeNormalization: ci.NoUnicodeNormalization,
+ deleteFilesCh: make(chan fs.Object, ci.Checkers),
+ trackRenames: ci.TrackRenames,
commonHash: fsrc.Hashes().Overlap(fdst.Hashes()).GetOne(),
modifyWindow: fs.GetModifyWindow(ctx, fsrc, fdst),
- trackRenamesCh: make(chan fs.Object, fs.Config.Checkers),
- checkFirst: fs.Config.CheckFirst,
+ trackRenamesCh: make(chan fs.Object, ci.Checkers),
+ checkFirst: ci.CheckFirst,
}
- backlog := fs.Config.MaxBacklog
+ backlog := ci.MaxBacklog
if s.checkFirst {
fs.Infof(s.fdst, "Running all checks before starting transfers")
backlog = -1
}
var err error
- s.toBeChecked, err = newPipe(fs.Config.OrderBy, accounting.Stats(ctx).SetCheckQueue, backlog)
+ s.toBeChecked, err = newPipe(ci.OrderBy, accounting.Stats(ctx).SetCheckQueue, backlog)
if err != nil {
return nil, err
}
- s.toBeUploaded, err = newPipe(fs.Config.OrderBy, accounting.Stats(ctx).SetTransferQueue, backlog)
+ s.toBeUploaded, err = newPipe(ci.OrderBy, accounting.Stats(ctx).SetTransferQueue, backlog)
if err != nil {
return nil, err
}
- s.toBeRenamed, err = newPipe(fs.Config.OrderBy, accounting.Stats(ctx).SetRenameQueue, backlog)
+ s.toBeRenamed, err = newPipe(ci.OrderBy, accounting.Stats(ctx).SetRenameQueue, backlog)
if err != nil {
return nil, err
}
// If a max session duration has been defined add a deadline to the context
- if fs.Config.MaxDuration > 0 {
- endTime := time.Now().Add(fs.Config.MaxDuration)
+ if ci.MaxDuration > 0 {
+ endTime := time.Now().Add(ci.MaxDuration)
fs.Infof(s.fdst, "Transfer session deadline: %s", endTime.Format("2006/01/02 15:04:05"))
s.ctx, s.cancel = context.WithDeadline(ctx, endTime)
} else {
@@ -152,7 +155,7 @@ func newSyncCopyMove(ctx context.Context, fdst, fsrc fs.Fs, deleteMode fs.Delete
fs.Errorf(nil, "Ignoring --no-traverse with sync")
s.noTraverse = false
}
- s.trackRenamesStrategy, err = parseTrackRenamesStrategy(fs.Config.TrackRenamesStrategy)
+ s.trackRenamesStrategy, err = parseTrackRenamesStrategy(ci.TrackRenamesStrategy)
if err != nil {
return nil, err
}
@@ -160,7 +163,7 @@ func newSyncCopyMove(ctx context.Context, fdst, fsrc fs.Fs, deleteMode fs.Delete
if s.deleteMode != fs.DeleteModeOff {
return nil, errors.New("can't use --no-check-dest with sync: use copy instead")
}
- if fs.Config.Immutable {
+ if ci.Immutable {
return nil, errors.New("can't use --no-check-dest with --immutable")
}
if s.backupDir != nil {
@@ -199,20 +202,20 @@ func newSyncCopyMove(ctx context.Context, fdst, fsrc fs.Fs, deleteMode fs.Delete
}
}
// Make Fs for --backup-dir if required
- if fs.Config.BackupDir != "" || fs.Config.Suffix != "" {
+ if ci.BackupDir != "" || ci.Suffix != "" {
var err error
s.backupDir, err = operations.BackupDir(ctx, fdst, fsrc, "")
if err != nil {
return nil, err
}
}
- if fs.Config.CompareDest != "" {
+ if ci.CompareDest != "" {
var err error
s.compareCopyDest, err = operations.GetCompareDest(ctx)
if err != nil {
return nil, err
}
- } else if fs.Config.CopyDest != "" {
+ } else if ci.CopyDest != "" {
var err error
s.compareCopyDest, err = operations.GetCopyDest(ctx, fdst)
if err != nil {
@@ -312,7 +315,7 @@ func (s *syncCopyMove) pairChecker(in *pipe, out *pipe, fraction int, wg *sync.W
}
if !NoNeedTransfer && operations.NeedTransfer(s.ctx, pair.Dst, pair.Src) {
// If files are treated as immutable, fail if destination exists and does not match
- if fs.Config.Immutable && pair.Dst != nil {
+ if s.ci.Immutable && pair.Dst != nil {
fs.Errorf(pair.Dst, "Source and destination exist but do not match: immutable file modified")
s.processError(fs.ErrorImmutableModified)
} else {
@@ -389,9 +392,9 @@ func (s *syncCopyMove) pairCopyOrMove(ctx context.Context, in *pipe, fdst fs.Fs,
// This starts the background checkers.
func (s *syncCopyMove) startCheckers() {
- s.checkerWg.Add(fs.Config.Checkers)
- for i := 0; i < fs.Config.Checkers; i++ {
- fraction := (100 * i) / fs.Config.Checkers
+ s.checkerWg.Add(s.ci.Checkers)
+ for i := 0; i < s.ci.Checkers; i++ {
+ fraction := (100 * i) / s.ci.Checkers
go s.pairChecker(s.toBeChecked, s.toBeUploaded, fraction, &s.checkerWg)
}
}
@@ -405,9 +408,9 @@ func (s *syncCopyMove) stopCheckers() {
// This starts the background transfers
func (s *syncCopyMove) startTransfers() {
- s.transfersWg.Add(fs.Config.Transfers)
- for i := 0; i < fs.Config.Transfers; i++ {
- fraction := (100 * i) / fs.Config.Transfers
+ s.transfersWg.Add(s.ci.Transfers)
+ for i := 0; i < s.ci.Transfers; i++ {
+ fraction := (100 * i) / s.ci.Transfers
go s.pairCopyOrMove(s.ctx, s.toBeUploaded, s.fdst, fraction, &s.transfersWg)
}
}
@@ -424,9 +427,9 @@ func (s *syncCopyMove) startRenamers() {
if !s.trackRenames {
return
}
- s.renamerWg.Add(fs.Config.Checkers)
- for i := 0; i < fs.Config.Checkers; i++ {
- fraction := (100 * i) / fs.Config.Checkers
+ s.renamerWg.Add(s.ci.Checkers)
+ for i := 0; i < s.ci.Checkers; i++ {
+ fraction := (100 * i) / s.ci.Checkers
go s.pairRenamer(s.toBeRenamed, s.toBeUploaded, fraction, &s.renamerWg)
}
}
@@ -492,13 +495,13 @@ func (s *syncCopyMove) stopDeleters() {
// checkSrcMap is clear then it assumes that the any source files that
// have been found have been removed from dstFiles already.
func (s *syncCopyMove) deleteFiles(checkSrcMap bool) error {
- if accounting.Stats(s.ctx).Errored() && !fs.Config.IgnoreErrors {
+ if accounting.Stats(s.ctx).Errored() && !s.ci.IgnoreErrors {
fs.Errorf(s.fdst, "%v", fs.ErrorNotDeleting)
return fs.ErrorNotDeleting
}
// Delete the spare files
- toDelete := make(fs.ObjectsChan, fs.Config.Transfers)
+ toDelete := make(fs.ObjectsChan, s.ci.Transfers)
go func() {
outer:
for remote, o := range s.dstFiles {
@@ -524,11 +527,11 @@ func (s *syncCopyMove) deleteFiles(checkSrcMap bool) error {
// This deletes the empty directories in the slice passed in. It
// ignores any errors deleting directories
-func deleteEmptyDirectories(ctx context.Context, f fs.Fs, entriesMap map[string]fs.DirEntry) error {
+func (s *syncCopyMove) deleteEmptyDirectories(ctx context.Context, f fs.Fs, entriesMap map[string]fs.DirEntry) error {
if len(entriesMap) == 0 {
return nil
}
- if accounting.Stats(ctx).Errored() && !fs.Config.IgnoreErrors {
+ if accounting.Stats(ctx).Errored() && !s.ci.IgnoreErrors {
fs.Errorf(f, "%v", fs.ErrorNotDeletingDirs)
return fs.ErrorNotDeletingDirs
}
@@ -729,14 +732,14 @@ func (s *syncCopyMove) makeRenameMap() {
}
// pump all the dstFiles into in
- in := make(chan fs.Object, fs.Config.Checkers)
+ in := make(chan fs.Object, s.ci.Checkers)
go s.pumpMapToChan(s.dstFiles, in)
// now make a map of size,hash for all dstFiles
s.renameMap = make(map[string][]fs.Object)
var wg sync.WaitGroup
- wg.Add(fs.Config.Transfers)
- for i := 0; i < fs.Config.Transfers; i++ {
+ wg.Add(s.ci.Transfers)
+ for i := 0; i < s.ci.Transfers; i++ {
go func() {
defer wg.Done()
for obj := range in {
@@ -829,7 +832,7 @@ func (s *syncCopyMove) run() error {
NoCheckDest: s.noCheckDest,
NoUnicodeNormalization: s.noUnicodeNormalization,
}
- s.processError(m.Run())
+ s.processError(m.Run(s.ctx))
s.stopTrackRenames()
if s.trackRenames {
@@ -860,7 +863,7 @@ func (s *syncCopyMove) run() error {
// Delete files after
if s.deleteMode == fs.DeleteModeAfter {
- if s.currentError() != nil && !fs.Config.IgnoreErrors {
+ if s.currentError() != nil && !s.ci.IgnoreErrors {
fs.Errorf(s.fdst, "%v", fs.ErrorNotDeleting)
} else {
s.processError(s.deleteFiles(false))
@@ -869,10 +872,10 @@ func (s *syncCopyMove) run() error {
// Prune empty directories
if s.deleteMode != fs.DeleteModeOff {
- if s.currentError() != nil && !fs.Config.IgnoreErrors {
+ if s.currentError() != nil && !s.ci.IgnoreErrors {
fs.Errorf(s.fdst, "%v", fs.ErrorNotDeletingDirs)
} else {
- s.processError(deleteEmptyDirectories(s.ctx, s.fdst, s.dstEmptyDirs))
+ s.processError(s.deleteEmptyDirectories(s.ctx, s.fdst, s.dstEmptyDirs))
}
}
@@ -880,7 +883,7 @@ func (s *syncCopyMove) run() error {
// if DoMove and --delete-empty-src-dirs flag is set
if s.DoMove && s.deleteEmptySrcDirs {
//delete empty subdirectories that were part of the move
- s.processError(deleteEmptyDirectories(s.ctx, s.fsrc, s.srcEmptyDirs))
+ s.processError(s.deleteEmptyDirectories(s.ctx, s.fsrc, s.srcEmptyDirs))
}
// Read the error out of the context if there is one
@@ -1038,12 +1041,13 @@ func (s *syncCopyMove) Match(ctx context.Context, dst, src fs.DirEntry) (recurse
//
// dir is the start directory, "" for root
func runSyncCopyMove(ctx context.Context, fdst, fsrc fs.Fs, deleteMode fs.DeleteMode, DoMove bool, deleteEmptySrcDirs bool, copyEmptySrcDirs bool) error {
+ ci := fs.GetConfig(ctx)
if deleteMode != fs.DeleteModeOff && DoMove {
return fserrors.FatalError(errors.New("can't delete and move at the same time"))
}
// Run an extra pass to delete only
if deleteMode == fs.DeleteModeBefore {
- if fs.Config.TrackRenames {
+ if ci.TrackRenames {
return fserrors.FatalError(errors.New("can't use --delete-before with --track-renames"))
}
// only delete stuff during in this pass
@@ -1067,7 +1071,8 @@ func runSyncCopyMove(ctx context.Context, fdst, fsrc fs.Fs, deleteMode fs.Delete
// Sync fsrc into fdst
func Sync(ctx context.Context, fdst, fsrc fs.Fs, copyEmptySrcDirs bool) error {
- return runSyncCopyMove(ctx, fdst, fsrc, fs.Config.DeleteMode, false, false, copyEmptySrcDirs)
+ ci := fs.GetConfig(ctx)
+ return runSyncCopyMove(ctx, fdst, fsrc, ci.DeleteMode, false, false, copyEmptySrcDirs)
}
// CopyDir copies fsrc into fdst
diff --git a/fs/sync/sync_test.go b/fs/sync/sync_test.go
index 235debcfa..bee41dd22 100644
--- a/fs/sync/sync_test.go
+++ b/fs/sync/sync_test.go
@@ -39,14 +39,15 @@ func TestMain(m *testing.M) {
// Check dry run is working
func TestCopyWithDryRun(t *testing.T) {
ctx := context.Background()
+ ci := fs.GetConfig(ctx)
r := fstest.NewRun(t)
defer r.Finalise()
file1 := r.WriteFile("sub dir/hello world", "hello world", t1)
r.Mkdir(ctx, r.Fremote)
- fs.Config.DryRun = true
+ ci.DryRun = true
err := CopyDir(ctx, r.Fremote, r.Flocal, false)
- fs.Config.DryRun = false
+ ci.DryRun = false
require.NoError(t, err)
fstest.CheckItems(t, r.Flocal, file1)
@@ -86,11 +87,12 @@ func TestCopyMissingDirectory(t *testing.T) {
// Now with --no-traverse
func TestCopyNoTraverse(t *testing.T) {
ctx := context.Background()
+ ci := fs.GetConfig(ctx)
r := fstest.NewRun(t)
defer r.Finalise()
- fs.Config.NoTraverse = true
- defer func() { fs.Config.NoTraverse = false }()
+ ci.NoTraverse = true
+ defer func() { ci.NoTraverse = false }()
file1 := r.WriteFile("sub dir/hello world", "hello world", t1)
@@ -104,11 +106,12 @@ func TestCopyNoTraverse(t *testing.T) {
// Now with --check-first
func TestCopyCheckFirst(t *testing.T) {
ctx := context.Background()
+ ci := fs.GetConfig(ctx)
r := fstest.NewRun(t)
defer r.Finalise()
- fs.Config.CheckFirst = true
- defer func() { fs.Config.CheckFirst = false }()
+ ci.CheckFirst = true
+ defer func() { ci.CheckFirst = false }()
file1 := r.WriteFile("sub dir/hello world", "hello world", t1)
@@ -122,11 +125,12 @@ func TestCopyCheckFirst(t *testing.T) {
// Now with --no-traverse
func TestSyncNoTraverse(t *testing.T) {
ctx := context.Background()
+ ci := fs.GetConfig(ctx)
r := fstest.NewRun(t)
defer r.Finalise()
- fs.Config.NoTraverse = true
- defer func() { fs.Config.NoTraverse = false }()
+ ci.NoTraverse = true
+ defer func() { ci.NoTraverse = false }()
file1 := r.WriteFile("sub dir/hello world", "hello world", t1)
@@ -141,14 +145,15 @@ func TestSyncNoTraverse(t *testing.T) {
// Test copy with depth
func TestCopyWithDepth(t *testing.T) {
ctx := context.Background()
+ ci := fs.GetConfig(ctx)
r := fstest.NewRun(t)
defer r.Finalise()
file1 := r.WriteFile("sub dir/hello world", "hello world", t1)
file2 := r.WriteFile("hello world2", "hello world2", t2)
// Check the MaxDepth too
- fs.Config.MaxDepth = 1
- defer func() { fs.Config.MaxDepth = -1 }()
+ ci.MaxDepth = 1
+ defer func() { ci.MaxDepth = -1 }()
err := CopyDir(ctx, r.Fremote, r.Flocal, false)
require.NoError(t, err)
@@ -160,6 +165,7 @@ func TestCopyWithDepth(t *testing.T) {
// Test copy with files from
func testCopyWithFilesFrom(t *testing.T, noTraverse bool) {
ctx := context.Background()
+ ci := fs.GetConfig(ctx)
r := fstest.NewRun(t)
defer r.Finalise()
file1 := r.WriteFile("potato2", "hello world", t1)
@@ -173,12 +179,12 @@ func testCopyWithFilesFrom(t *testing.T, noTraverse bool) {
// Monkey patch the active filter
oldFilter := filter.Active
- oldNoTraverse := fs.Config.NoTraverse
+ oldNoTraverse := ci.NoTraverse
filter.Active = f
- fs.Config.NoTraverse = noTraverse
+ ci.NoTraverse = noTraverse
unpatch := func() {
filter.Active = oldFilter
- fs.Config.NoTraverse = oldNoTraverse
+ ci.NoTraverse = oldNoTraverse
}
defer unpatch()
@@ -332,10 +338,11 @@ func TestCopyRedownload(t *testing.T) {
// to be transferred on the second sync.
func TestSyncBasedOnCheckSum(t *testing.T) {
ctx := context.Background()
+ ci := fs.GetConfig(ctx)
r := fstest.NewRun(t)
defer r.Finalise()
- fs.Config.CheckSum = true
- defer func() { fs.Config.CheckSum = false }()
+ ci.CheckSum = true
+ defer func() { ci.CheckSum = false }()
file1 := r.WriteFile("check sum", "-", t1)
fstest.CheckItems(t, r.Flocal, file1)
@@ -367,10 +374,11 @@ func TestSyncBasedOnCheckSum(t *testing.T) {
// only, we expect nothing to to be transferred on the second sync.
func TestSyncSizeOnly(t *testing.T) {
ctx := context.Background()
+ ci := fs.GetConfig(ctx)
r := fstest.NewRun(t)
defer r.Finalise()
- fs.Config.SizeOnly = true
- defer func() { fs.Config.SizeOnly = false }()
+ ci.SizeOnly = true
+ defer func() { ci.SizeOnly = false }()
file1 := r.WriteFile("sizeonly", "potato", t1)
fstest.CheckItems(t, r.Flocal, file1)
@@ -402,10 +410,11 @@ func TestSyncSizeOnly(t *testing.T) {
// transferred on the second sync.
func TestSyncIgnoreSize(t *testing.T) {
ctx := context.Background()
+ ci := fs.GetConfig(ctx)
r := fstest.NewRun(t)
defer r.Finalise()
- fs.Config.IgnoreSize = true
- defer func() { fs.Config.IgnoreSize = false }()
+ ci.IgnoreSize = true
+ defer func() { ci.IgnoreSize = false }()
file1 := r.WriteFile("ignore-size", "contents", t1)
fstest.CheckItems(t, r.Flocal, file1)
@@ -434,6 +443,7 @@ func TestSyncIgnoreSize(t *testing.T) {
func TestSyncIgnoreTimes(t *testing.T) {
ctx := context.Background()
+ ci := fs.GetConfig(ctx)
r := fstest.NewRun(t)
defer r.Finalise()
file1 := r.WriteBoth(ctx, "existing", "potato", t1)
@@ -447,8 +457,8 @@ func TestSyncIgnoreTimes(t *testing.T) {
// files were identical.
assert.Equal(t, int64(0), accounting.GlobalStats().GetTransfers())
- fs.Config.IgnoreTimes = true
- defer func() { fs.Config.IgnoreTimes = false }()
+ ci.IgnoreTimes = true
+ defer func() { ci.IgnoreTimes = false }()
accounting.GlobalStats().ResetCounters()
err = Sync(ctx, r.Fremote, r.Flocal, false)
@@ -464,12 +474,13 @@ func TestSyncIgnoreTimes(t *testing.T) {
func TestSyncIgnoreExisting(t *testing.T) {
ctx := context.Background()
+ ci := fs.GetConfig(ctx)
r := fstest.NewRun(t)
defer r.Finalise()
file1 := r.WriteFile("existing", "potato", t1)
- fs.Config.IgnoreExisting = true
- defer func() { fs.Config.IgnoreExisting = false }()
+ ci.IgnoreExisting = true
+ defer func() { ci.IgnoreExisting = false }()
accounting.GlobalStats().ResetCounters()
err := Sync(ctx, r.Fremote, r.Flocal, false)
@@ -488,10 +499,11 @@ func TestSyncIgnoreExisting(t *testing.T) {
func TestSyncIgnoreErrors(t *testing.T) {
ctx := context.Background()
+ ci := fs.GetConfig(ctx)
r := fstest.NewRun(t)
- fs.Config.IgnoreErrors = true
+ ci.IgnoreErrors = true
defer func() {
- fs.Config.IgnoreErrors = false
+ ci.IgnoreErrors = false
r.Finalise()
}()
file1 := r.WriteFile("a/potato2", "------------------------------------------------------------", t1)
@@ -561,6 +573,7 @@ func TestSyncIgnoreErrors(t *testing.T) {
func TestSyncAfterChangingModtimeOnly(t *testing.T) {
ctx := context.Background()
+ ci := fs.GetConfig(ctx)
r := fstest.NewRun(t)
defer r.Finalise()
file1 := r.WriteFile("empty space", "-", t2)
@@ -569,8 +582,8 @@ func TestSyncAfterChangingModtimeOnly(t *testing.T) {
fstest.CheckItems(t, r.Flocal, file1)
fstest.CheckItems(t, r.Fremote, file2)
- fs.Config.DryRun = true
- defer func() { fs.Config.DryRun = false }()
+ ci.DryRun = true
+ defer func() { ci.DryRun = false }()
accounting.GlobalStats().ResetCounters()
err := Sync(ctx, r.Fremote, r.Flocal, false)
@@ -579,7 +592,7 @@ func TestSyncAfterChangingModtimeOnly(t *testing.T) {
fstest.CheckItems(t, r.Flocal, file1)
fstest.CheckItems(t, r.Fremote, file2)
- fs.Config.DryRun = false
+ ci.DryRun = false
accounting.GlobalStats().ResetCounters()
err = Sync(ctx, r.Fremote, r.Flocal, false)
@@ -591,6 +604,7 @@ func TestSyncAfterChangingModtimeOnly(t *testing.T) {
func TestSyncAfterChangingModtimeOnlyWithNoUpdateModTime(t *testing.T) {
ctx := context.Background()
+ ci := fs.GetConfig(ctx)
r := fstest.NewRun(t)
defer r.Finalise()
@@ -599,9 +613,9 @@ func TestSyncAfterChangingModtimeOnlyWithNoUpdateModTime(t *testing.T) {
return
}
- fs.Config.NoUpdateModTime = true
+ ci.NoUpdateModTime = true
defer func() {
- fs.Config.NoUpdateModTime = false
+ ci.NoUpdateModTime = false
}()
file1 := r.WriteFile("empty space", "-", t2)
@@ -703,16 +717,17 @@ func TestSyncAfterChangingContentsOnly(t *testing.T) {
// Sync after removing a file and adding a file --dry-run
func TestSyncAfterRemovingAFileAndAddingAFileDryRun(t *testing.T) {
ctx := context.Background()
+ ci := fs.GetConfig(ctx)
r := fstest.NewRun(t)
defer r.Finalise()
file1 := r.WriteFile("potato2", "------------------------------------------------------------", t1)
file2 := r.WriteObject(ctx, "potato", "SMALLER BUT SAME DATE", t2)
file3 := r.WriteBoth(ctx, "empty space", "-", t2)
- fs.Config.DryRun = true
+ ci.DryRun = true
accounting.GlobalStats().ResetCounters()
err := Sync(ctx, r.Fremote, r.Flocal, false)
- fs.Config.DryRun = false
+ ci.DryRun = false
require.NoError(t, err)
fstest.CheckItems(t, r.Flocal, file3, file1)
@@ -885,16 +900,20 @@ func TestSyncAfterRemovingAFileAndAddingAFileSubDirWithErrors(t *testing.T) {
// Sync test delete after
func TestSyncDeleteAfter(t *testing.T) {
+ ctx := context.Background()
+ ci := fs.GetConfig(ctx)
// This is the default so we've checked this already
// check it is the default
- require.Equal(t, fs.Config.DeleteMode, fs.DeleteModeAfter, "Didn't default to --delete-after")
+ require.Equal(t, ci.DeleteMode, fs.DeleteModeAfter, "Didn't default to --delete-after")
}
// Sync test delete during
func TestSyncDeleteDuring(t *testing.T) {
- fs.Config.DeleteMode = fs.DeleteModeDuring
+ ctx := context.Background()
+ ci := fs.GetConfig(ctx)
+ ci.DeleteMode = fs.DeleteModeDuring
defer func() {
- fs.Config.DeleteMode = fs.DeleteModeDefault
+ ci.DeleteMode = fs.DeleteModeDefault
}()
TestSyncAfterRemovingAFileAndAddingAFile(t)
@@ -902,9 +921,11 @@ func TestSyncDeleteDuring(t *testing.T) {
// Sync test delete before
func TestSyncDeleteBefore(t *testing.T) {
- fs.Config.DeleteMode = fs.DeleteModeBefore
+ ctx := context.Background()
+ ci := fs.GetConfig(ctx)
+ ci.DeleteMode = fs.DeleteModeBefore
defer func() {
- fs.Config.DeleteMode = fs.DeleteModeDefault
+ ci.DeleteMode = fs.DeleteModeDefault
}()
TestSyncAfterRemovingAFileAndAddingAFile(t)
@@ -913,12 +934,13 @@ func TestSyncDeleteBefore(t *testing.T) {
// Copy test delete before - shouldn't delete anything
func TestCopyDeleteBefore(t *testing.T) {
ctx := context.Background()
+ ci := fs.GetConfig(ctx)
r := fstest.NewRun(t)
defer r.Finalise()
- fs.Config.DeleteMode = fs.DeleteModeBefore
+ ci.DeleteMode = fs.DeleteModeBefore
defer func() {
- fs.Config.DeleteMode = fs.DeleteModeDefault
+ ci.DeleteMode = fs.DeleteModeDefault
}()
file1 := r.WriteObject(ctx, "potato", "hopefully not deleted", t1)
@@ -997,6 +1019,7 @@ func TestSyncWithExcludeAndDeleteExcluded(t *testing.T) {
// Test with UpdateOlder set
func TestSyncWithUpdateOlder(t *testing.T) {
ctx := context.Background()
+ ci := fs.GetConfig(ctx)
r := fstest.NewRun(t)
defer r.Finalise()
if fs.GetModifyWindow(ctx, r.Fremote) == fs.ModTimeNotSupported {
@@ -1016,12 +1039,12 @@ func TestSyncWithUpdateOlder(t *testing.T) {
fourO := r.WriteObject(ctx, "four", "FOURFOUR", t2minus)
fstest.CheckItems(t, r.Fremote, oneO, twoO, threeO, fourO)
- fs.Config.UpdateOlder = true
- oldModifyWindow := fs.Config.ModifyWindow
- fs.Config.ModifyWindow = fs.ModTimeNotSupported
+ ci.UpdateOlder = true
+ oldModifyWindow := ci.ModifyWindow
+ ci.ModifyWindow = fs.ModTimeNotSupported
defer func() {
- fs.Config.UpdateOlder = false
- fs.Config.ModifyWindow = oldModifyWindow
+ ci.UpdateOlder = false
+ ci.ModifyWindow = oldModifyWindow
}()
err := Sync(ctx, r.Fremote, r.Flocal, false)
@@ -1034,8 +1057,8 @@ func TestSyncWithUpdateOlder(t *testing.T) {
}
// now enable checksum
- fs.Config.CheckSum = true
- defer func() { fs.Config.CheckSum = false }()
+ ci.CheckSum = true
+ defer func() { ci.CheckSum = false }()
err = Sync(ctx, r.Fremote, r.Flocal, false)
require.NoError(t, err)
@@ -1045,6 +1068,7 @@ func TestSyncWithUpdateOlder(t *testing.T) {
// Test with a max transfer duration
func TestSyncWithMaxDuration(t *testing.T) {
ctx := context.Background()
+ ci := fs.GetConfig(ctx)
if *fstest.RemoteName != "" {
t.Skip("Skipping test on non local remote")
}
@@ -1052,14 +1076,14 @@ func TestSyncWithMaxDuration(t *testing.T) {
defer r.Finalise()
maxDuration := 250 * time.Millisecond
- fs.Config.MaxDuration = maxDuration
+ ci.MaxDuration = maxDuration
bytesPerSecond := 300
accounting.SetBwLimit(fs.SizeSuffix(bytesPerSecond))
- oldTransfers := fs.Config.Transfers
- fs.Config.Transfers = 1
+ oldTransfers := ci.Transfers
+ ci.Transfers = 1
defer func() {
- fs.Config.MaxDuration = 0 // reset back to default
- fs.Config.Transfers = oldTransfers
+ ci.MaxDuration = 0 // reset back to default
+ ci.Transfers = oldTransfers
accounting.SetBwLimit(fs.SizeSuffix(0))
}()
@@ -1089,12 +1113,13 @@ func TestSyncWithMaxDuration(t *testing.T) {
// Test with TrackRenames set
func TestSyncWithTrackRenames(t *testing.T) {
ctx := context.Background()
+ ci := fs.GetConfig(ctx)
r := fstest.NewRun(t)
defer r.Finalise()
- fs.Config.TrackRenames = true
+ ci.TrackRenames = true
defer func() {
- fs.Config.TrackRenames = false
+ ci.TrackRenames = false
}()
haveHash := r.Fremote.Hashes().Overlap(r.Flocal.Hashes()).GetOne() != hash.None
@@ -1160,14 +1185,15 @@ func TestRenamesStrategyModtime(t *testing.T) {
func TestSyncWithTrackRenamesStrategyModtime(t *testing.T) {
ctx := context.Background()
+ ci := fs.GetConfig(ctx)
r := fstest.NewRun(t)
defer r.Finalise()
- fs.Config.TrackRenames = true
- fs.Config.TrackRenamesStrategy = "modtime"
+ ci.TrackRenames = true
+ ci.TrackRenamesStrategy = "modtime"
defer func() {
- fs.Config.TrackRenames = false
- fs.Config.TrackRenamesStrategy = "hash"
+ ci.TrackRenames = false
+ ci.TrackRenamesStrategy = "hash"
}()
canTrackRenames := operations.CanServerSideMove(r.Fremote) && r.Fremote.Precision() != fs.ModTimeNotSupported
@@ -1199,14 +1225,15 @@ func TestSyncWithTrackRenamesStrategyModtime(t *testing.T) {
func TestSyncWithTrackRenamesStrategyLeaf(t *testing.T) {
ctx := context.Background()
+ ci := fs.GetConfig(ctx)
r := fstest.NewRun(t)
defer r.Finalise()
- fs.Config.TrackRenames = true
- fs.Config.TrackRenamesStrategy = "leaf"
+ ci.TrackRenames = true
+ ci.TrackRenamesStrategy = "leaf"
defer func() {
- fs.Config.TrackRenames = false
- fs.Config.TrackRenamesStrategy = "hash"
+ ci.TrackRenames = false
+ ci.TrackRenamesStrategy = "hash"
}()
canTrackRenames := operations.CanServerSideMove(r.Fremote) && r.Fremote.Precision() != fs.ModTimeNotSupported
@@ -1445,12 +1472,13 @@ func TestSyncOverlap(t *testing.T) {
// Test with CompareDest set
func TestSyncCompareDest(t *testing.T) {
ctx := context.Background()
+ ci := fs.GetConfig(ctx)
r := fstest.NewRun(t)
defer r.Finalise()
- fs.Config.CompareDest = r.FremoteName + "/CompareDest"
+ ci.CompareDest = r.FremoteName + "/CompareDest"
defer func() {
- fs.Config.CompareDest = ""
+ ci.CompareDest = ""
}()
fdst, err := fs.NewFs(ctx, r.FremoteName+"/dst")
@@ -1533,6 +1561,7 @@ func TestSyncCompareDest(t *testing.T) {
// Test with CopyDest set
func TestSyncCopyDest(t *testing.T) {
ctx := context.Background()
+ ci := fs.GetConfig(ctx)
r := fstest.NewRun(t)
defer r.Finalise()
@@ -1540,9 +1569,9 @@ func TestSyncCopyDest(t *testing.T) {
t.Skip("Skipping test as remote does not support server-side copy")
}
- fs.Config.CopyDest = r.FremoteName + "/CopyDest"
+ ci.CopyDest = r.FremoteName + "/CopyDest"
defer func() {
- fs.Config.CopyDest = ""
+ ci.CopyDest = ""
}()
fdst, err := fs.NewFs(ctx, r.FremoteName+"/dst")
@@ -1577,7 +1606,7 @@ func TestSyncCopyDest(t *testing.T) {
// check old dest, new copy, backup-dir
- fs.Config.BackupDir = r.FremoteName + "/BackupDir"
+ ci.BackupDir = r.FremoteName + "/BackupDir"
file3 := r.WriteObject(ctx, "dst/one", "one", t1)
file2 := r.WriteObject(ctx, "CopyDest/one", "onet2", t2)
@@ -1594,7 +1623,7 @@ func TestSyncCopyDest(t *testing.T) {
file3.Path = "BackupDir/one"
fstest.CheckItems(t, r.Fremote, file2, file2dst, file3)
- fs.Config.BackupDir = ""
+ ci.BackupDir = ""
// check empty dest, new copy
file4 := r.WriteObject(ctx, "CopyDest/two", "two", t2)
@@ -1637,6 +1666,7 @@ func TestSyncCopyDest(t *testing.T) {
// Test with BackupDir set
func testSyncBackupDir(t *testing.T, backupDir string, suffix string, suffixKeepExtension bool) {
ctx := context.Background()
+ ci := fs.GetConfig(ctx)
r := fstest.NewRun(t)
defer r.Finalise()
@@ -1646,10 +1676,10 @@ func testSyncBackupDir(t *testing.T, backupDir string, suffix string, suffixKeep
r.Mkdir(ctx, r.Fremote)
if backupDir != "" {
- fs.Config.BackupDir = r.FremoteName + "/" + backupDir
+ ci.BackupDir = r.FremoteName + "/" + backupDir
backupDir += "/"
} else {
- fs.Config.BackupDir = ""
+ ci.BackupDir = ""
backupDir = "dst/"
// Exclude the suffix from the sync otherwise the sync
// deletes the old backup files
@@ -1662,12 +1692,12 @@ func testSyncBackupDir(t *testing.T, backupDir string, suffix string, suffixKeep
filter.Active = oldFlt
}()
}
- fs.Config.Suffix = suffix
- fs.Config.SuffixKeepExtension = suffixKeepExtension
+ ci.Suffix = suffix
+ ci.SuffixKeepExtension = suffixKeepExtension
defer func() {
- fs.Config.BackupDir = ""
- fs.Config.Suffix = ""
- fs.Config.SuffixKeepExtension = false
+ ci.BackupDir = ""
+ ci.Suffix = ""
+ ci.SuffixKeepExtension = false
}()
// Make the setup so we have one, two, three in the dest
@@ -1742,6 +1772,7 @@ func TestSyncBackupDirSuffixOnly(t *testing.T) {
// Test with Suffix set
func testSyncSuffix(t *testing.T, suffix string, suffixKeepExtension bool) {
ctx := context.Background()
+ ci := fs.GetConfig(ctx)
r := fstest.NewRun(t)
defer r.Finalise()
@@ -1750,12 +1781,12 @@ func testSyncSuffix(t *testing.T, suffix string, suffixKeepExtension bool) {
}
r.Mkdir(ctx, r.Fremote)
- fs.Config.Suffix = suffix
- fs.Config.SuffixKeepExtension = suffixKeepExtension
+ ci.Suffix = suffix
+ ci.SuffixKeepExtension = suffixKeepExtension
defer func() {
- fs.Config.BackupDir = ""
- fs.Config.Suffix = ""
- fs.Config.SuffixKeepExtension = false
+ ci.BackupDir = ""
+ ci.Suffix = ""
+ ci.SuffixKeepExtension = false
}()
// Make the setup so we have one, two, three in the dest
@@ -1865,11 +1896,12 @@ func TestSyncUTFNorm(t *testing.T) {
// Test --immutable
func TestSyncImmutable(t *testing.T) {
ctx := context.Background()
+ ci := fs.GetConfig(ctx)
r := fstest.NewRun(t)
defer r.Finalise()
- fs.Config.Immutable = true
- defer func() { fs.Config.Immutable = false }()
+ ci.Immutable = true
+ defer func() { ci.Immutable = false }()
// Create file on source
file1 := r.WriteFile("existing", "potato", t1)
@@ -1899,6 +1931,7 @@ func TestSyncImmutable(t *testing.T) {
// Test --ignore-case-sync
func TestSyncIgnoreCase(t *testing.T) {
ctx := context.Background()
+ ci := fs.GetConfig(ctx)
r := fstest.NewRun(t)
defer r.Finalise()
@@ -1907,8 +1940,8 @@ func TestSyncIgnoreCase(t *testing.T) {
t.Skip("Skipping test as local or remote are case-insensitive")
}
- fs.Config.IgnoreCaseSync = true
- defer func() { fs.Config.IgnoreCaseSync = false }()
+ ci.IgnoreCaseSync = true
+ defer func() { ci.IgnoreCaseSync = false }()
// Create files with different filename casing
file1 := r.WriteFile("existing", "potato", t1)
@@ -1927,25 +1960,26 @@ func TestSyncIgnoreCase(t *testing.T) {
// Test that aborting on --max-transfer works
func TestMaxTransfer(t *testing.T) {
ctx := context.Background()
- oldMaxTransfer := fs.Config.MaxTransfer
- oldTransfers := fs.Config.Transfers
- oldCheckers := fs.Config.Checkers
- oldCutoff := fs.Config.CutoffMode
- fs.Config.MaxTransfer = 3 * 1024
- fs.Config.Transfers = 1
- fs.Config.Checkers = 1
- fs.Config.CutoffMode = fs.CutoffModeHard
+ ci := fs.GetConfig(ctx)
+ oldMaxTransfer := ci.MaxTransfer
+ oldTransfers := ci.Transfers
+ oldCheckers := ci.Checkers
+ oldCutoff := ci.CutoffMode
+ ci.MaxTransfer = 3 * 1024
+ ci.Transfers = 1
+ ci.Checkers = 1
+ ci.CutoffMode = fs.CutoffModeHard
defer func() {
- fs.Config.MaxTransfer = oldMaxTransfer
- fs.Config.Transfers = oldTransfers
- fs.Config.Checkers = oldCheckers
- fs.Config.CutoffMode = oldCutoff
+ ci.MaxTransfer = oldMaxTransfer
+ ci.Transfers = oldTransfers
+ ci.Checkers = oldCheckers
+ ci.CutoffMode = oldCutoff
}()
test := func(t *testing.T, cutoff fs.CutoffMode) {
r := fstest.NewRun(t)
defer r.Finalise()
- fs.Config.CutoffMode = cutoff
+ ci.CutoffMode = cutoff
if r.Fremote.Name() != "local" {
t.Skip("This test only runs on local")
diff --git a/fs/walk/walk.go b/fs/walk/walk.go
index 79fcbcd47..04510e053 100644
--- a/fs/walk/walk.go
+++ b/fs/walk/walk.go
@@ -59,11 +59,12 @@ type Func func(path string, entries fs.DirEntries, err error) error
//
// NB (f, path) to be replaced by fs.Dir at some point
func Walk(ctx context.Context, f fs.Fs, path string, includeAll bool, maxLevel int, fn Func) error {
- if fs.Config.NoTraverse && filter.Active.HaveFilesFrom() {
+ ci := fs.GetConfig(ctx)
+ if ci.NoTraverse && filter.Active.HaveFilesFrom() {
return walkR(ctx, f, path, includeAll, maxLevel, fn, filter.Active.MakeListR(ctx, f.NewObject))
}
// FIXME should this just be maxLevel < 0 - why the maxLevel > 1
- if (maxLevel < 0 || maxLevel > 1) && fs.Config.UseListR && f.Features().ListR != nil {
+ if (maxLevel < 0 || maxLevel > 1) && ci.UseListR && f.Features().ListR != nil {
return walkListR(ctx, f, path, includeAll, maxLevel, fn)
}
return walkListDirSorted(ctx, f, path, includeAll, maxLevel, fn)
@@ -353,10 +354,11 @@ type listDirFunc func(ctx context.Context, fs fs.Fs, includeAll bool, dir string
func walk(ctx context.Context, f fs.Fs, path string, includeAll bool, maxLevel int, fn Func, listDir listDirFunc) error {
var (
- wg sync.WaitGroup // sync closing of go routines
- traversing sync.WaitGroup // running directory traversals
- doClose sync.Once // close the channel once
- mu sync.Mutex // stop fn being called concurrently
+ wg sync.WaitGroup // sync closing of go routines
+ traversing sync.WaitGroup // running directory traversals
+ doClose sync.Once // close the channel once
+ mu sync.Mutex // stop fn being called concurrently
+ ci = fs.GetConfig(ctx) // current config
)
// listJob describe a directory listing that needs to be done
type listJob struct {
@@ -364,7 +366,7 @@ func walk(ctx context.Context, f fs.Fs, path string, includeAll bool, maxLevel i
depth int
}
- in := make(chan listJob, fs.Config.Checkers)
+ in := make(chan listJob, ci.Checkers)
errs := make(chan error, 1)
quit := make(chan struct{})
closeQuit := func() {
@@ -377,7 +379,7 @@ func walk(ctx context.Context, f fs.Fs, path string, includeAll bool, maxLevel i
}()
})
}
- for i := 0; i < fs.Config.Checkers; i++ {
+ for i := 0; i < ci.Checkers; i++ {
wg.Add(1)
go func() {
defer wg.Done()
@@ -553,8 +555,9 @@ func walkNDirTree(ctx context.Context, f fs.Fs, path string, includeAll bool, ma
//
// NB (f, path) to be replaced by fs.Dir at some point
func NewDirTree(ctx context.Context, f fs.Fs, path string, includeAll bool, maxLevel int) (dirtree.DirTree, error) {
+ ci := fs.GetConfig(ctx)
// if --no-traverse and --files-from build DirTree just from files
- if fs.Config.NoTraverse && filter.Active.HaveFilesFrom() {
+ if ci.NoTraverse && filter.Active.HaveFilesFrom() {
return walkRDirTree(ctx, f, path, includeAll, maxLevel, filter.Active.MakeListR(ctx, f.NewObject))
}
// if have ListR; and recursing; and not using --files-from; then build a DirTree with ListR
diff --git a/fstest/fstest.go b/fstest/fstest.go
index 82a22cc53..b996d2259 100644
--- a/fstest/fstest.go
+++ b/fstest/fstest.go
@@ -59,10 +59,11 @@ func init() {
// Initialise rclone for testing
func Initialise() {
ctx := context.Background()
+ ci := fs.GetConfig(ctx)
// Never ask for passwords, fail instead.
// If your local config is encrypted set environment variable
// "RCLONE_CONFIG_PASS=hunter2" (or your password)
- fs.Config.AskPassword = false
+ ci.AskPassword = false
// Override the config file from the environment - we don't
// parse the flags any more so this doesn't happen
// automatically
@@ -71,16 +72,16 @@ func Initialise() {
}
config.LoadConfig(ctx)
if *Verbose {
- fs.Config.LogLevel = fs.LogLevelDebug
+ ci.LogLevel = fs.LogLevelDebug
}
if *DumpHeaders {
- fs.Config.Dump |= fs.DumpHeaders
+ ci.Dump |= fs.DumpHeaders
}
if *DumpBodies {
- fs.Config.Dump |= fs.DumpBodies
+ ci.Dump |= fs.DumpBodies
}
- fs.Config.LowLevelRetries = *LowLevelRetries
- fs.Config.UseListR = *UseListR
+ ci.LowLevelRetries = *LowLevelRetries
+ ci.UseListR = *UseListR
}
// Item represents an item for checking
diff --git a/fstest/fstests/fstests.go b/fstest/fstests/fstests.go
index b3ea1294f..9a13c43fa 100644
--- a/fstest/fstests/fstests.go
+++ b/fstest/fstests/fstests.go
@@ -295,6 +295,7 @@ func Run(t *testing.T, opt *Opt) {
isLocalRemote bool
purged bool // whether the dir has been purged or not
ctx = context.Background()
+ ci = fs.GetConfig(ctx)
unwrappableFsMethods = []string{"Command"} // these Fs methods don't need to be wrapped ever
)
@@ -316,10 +317,10 @@ func Run(t *testing.T, opt *Opt) {
if remote.Features().ListR == nil {
t.Skip("FS has no ListR interface")
}
- previous := fs.Config.UseListR
- fs.Config.UseListR = true
+ previous := ci.UseListR
+ ci.UseListR = true
return func() {
- fs.Config.UseListR = previous
+ ci.UseListR = previous
}
}
diff --git a/fstest/test_all/run.go b/fstest/test_all/run.go
index 89cb96070..56d6dde2d 100644
--- a/fstest/test_all/run.go
+++ b/fstest/test_all/run.go
@@ -4,6 +4,7 @@ package main
import (
"bytes"
+ "context"
"fmt"
"go/build"
"io"
@@ -345,9 +346,10 @@ func (r *Run) Init() {
r.CmdLine = append(r.CmdLine, "-list-retries", fmt.Sprint(listRetries))
}
r.Try = 1
+ ci := fs.GetConfig(context.Background())
if *verbose {
r.CmdLine = append(r.CmdLine, "-verbose")
- fs.Config.LogLevel = fs.LogLevelDebug
+ ci.LogLevel = fs.LogLevelDebug
}
if *runOnly != "" {
r.CmdLine = append(r.CmdLine, prefix+"run", *runOnly)
diff --git a/lib/oauthutil/oauthutil.go b/lib/oauthutil/oauthutil.go
index 8d54d71ce..5c255284c 100644
--- a/lib/oauthutil/oauthutil.go
+++ b/lib/oauthutil/oauthutil.go
@@ -353,7 +353,7 @@ func NewClientWithBaseClient(ctx context.Context, name string, m configmap.Mappe
// NewClient gets a token from the config file and configures a Client
// with it. It returns the client and a TokenSource which Invalidate may need to be called on
func NewClient(ctx context.Context, name string, m configmap.Mapper, oauthConfig *oauth2.Config) (*http.Client, *TokenSource, error) {
- return NewClientWithBaseClient(ctx, name, m, oauthConfig, fshttp.NewClient(fs.Config))
+ return NewClientWithBaseClient(ctx, name, m, oauthConfig, fshttp.NewClient(fs.GetConfig(ctx)))
}
// AuthResult is returned from the web server after authorization
@@ -526,7 +526,7 @@ version recommended):
}
// Exchange the code for a token
- ctx = Context(ctx, fshttp.NewClient(fs.Config))
+ ctx = Context(ctx, fshttp.NewClient(fs.GetConfig(ctx)))
token, err := oauthConfig.Exchange(ctx, auth.Code)
if err != nil {
return errors.Wrap(err, "failed to get token")
diff --git a/vfs/read.go b/vfs/read.go
index 865b922c9..1dae5a8d8 100644
--- a/vfs/read.go
+++ b/vfs/read.go
@@ -276,6 +276,7 @@ func (fh *ReadFileHandle) readAt(p []byte, off int64) (n int, err error) {
retries := 0
reqSize := len(p)
doReopen := false
+ lowLevelRetries := fs.GetConfig(context.TODO()).LowLevelRetries
for {
if doSeek {
// Are we attempting to seek beyond the end of the
@@ -312,11 +313,11 @@ func (fh *ReadFileHandle) readAt(p []byte, off int64) (n int, err error) {
break
}
}
- if retries >= fs.Config.LowLevelRetries {
+ if retries >= lowLevelRetries {
break
}
retries++
- fs.Errorf(fh.remote, "ReadFileHandle.Read error: low level retry %d/%d: %v", retries, fs.Config.LowLevelRetries, err)
+ fs.Errorf(fh.remote, "ReadFileHandle.Read error: low level retry %d/%d: %v", retries, lowLevelRetries, err)
doSeek = true
doReopen = true
}
diff --git a/vfs/vfscache/cache.go b/vfs/vfscache/cache.go
index 895c8b93a..525fe6f83 100644
--- a/vfs/vfscache/cache.go
+++ b/vfs/vfscache/cache.go
@@ -95,7 +95,7 @@ func New(ctx context.Context, fremote fs.Fs, opt *vfscommon.Options, avFn AddVir
return nil, errors.Wrap(err, "failed to create cache meta remote")
}
- hashType, hashOption := operations.CommonHash(fcache, fremote)
+ hashType, hashOption := operations.CommonHash(ctx, fcache, fremote)
c := &Cache{
fremote: fremote,
diff --git a/vfs/vfscache/downloaders/downloaders.go b/vfs/vfscache/downloaders/downloaders.go
index a82f2a364..190d38117 100644
--- a/vfs/vfscache/downloaders/downloaders.go
+++ b/vfs/vfscache/downloaders/downloaders.go
@@ -283,7 +283,7 @@ func (dls *Downloaders) _ensureDownloader(r ranges.Range) (err error) {
// defer log.Trace(dls.src, "r=%v", r)("err=%v", &err)
// The window includes potentially unread data in the buffer
- window := int64(fs.Config.BufferSize)
+ window := int64(fs.GetConfig(context.TODO()).BufferSize)
// Increase the read range by the read ahead if set
if dls.opt.ReadAhead > 0 {
@@ -521,7 +521,7 @@ func (dl *downloader) open(offset int64) (err error) {
// if offset > 0 {
// rangeOption = &fs.RangeOption{Start: offset, End: size - 1}
// }
- // in0, err := operations.NewReOpen(dl.dls.ctx, dl.dls.src, fs.Config.LowLevelRetries, dl.dls.item.c.hashOption, rangeOption)
+ // in0, err := operations.NewReOpen(dl.dls.ctx, dl.dls.src, ci.LowLevelRetries, dl.dls.item.c.hashOption, rangeOption)
in0 := chunkedreader.New(context.TODO(), dl.dls.src, int64(dl.dls.opt.ChunkSize), int64(dl.dls.opt.ChunkSizeLimit))
_, err = in0.Seek(offset, 0)
diff --git a/vfs/vfscache/item.go b/vfs/vfscache/item.go
index 158e583f2..baecaa8f3 100644
--- a/vfs/vfscache/item.go
+++ b/vfs/vfscache/item.go
@@ -491,7 +491,7 @@ func (item *Item) _createFile(osPath string) (err error) {
// Open the local file from the object passed in. Wraps open()
// to provide recovery from out of space error.
func (item *Item) Open(o fs.Object) (err error) {
- for retries := 0; retries < fs.Config.LowLevelRetries; retries++ {
+ for retries := 0; retries < fs.GetConfig(context.TODO()).LowLevelRetries; retries++ {
item.preAccess()
err = item.open(o)
item.postAccess()
@@ -1190,7 +1190,7 @@ func (item *Item) setModTime(modTime time.Time) {
func (item *Item) ReadAt(b []byte, off int64) (n int, err error) {
n = 0
var expBackOff int
- for retries := 0; retries < fs.Config.LowLevelRetries; retries++ {
+ for retries := 0; retries < fs.GetConfig(context.TODO()).LowLevelRetries; retries++ {
item.preAccess()
n, err = item.readAt(b, off)
item.postAccess()
diff --git a/vfs/vfscache/writeback/writeback.go b/vfs/vfscache/writeback/writeback.go
index f4c367a58..54d664baa 100644
--- a/vfs/vfscache/writeback/writeback.go
+++ b/vfs/vfscache/writeback/writeback.go
@@ -416,7 +416,7 @@ func (wb *WriteBack) processItems(ctx context.Context) {
resetTimer := true
for wbItem := wb._peekItem(); wbItem != nil && time.Until(wbItem.expiry) <= 0; wbItem = wb._peekItem() {
// If reached transfer limit don't restart the timer
- if wb.uploads >= fs.Config.Transfers {
+ if wb.uploads >= fs.GetConfig(context.TODO()).Transfers {
fs.Debugf(wbItem.name, "vfs cache: delaying writeback as --transfers exceeded")
resetTimer = false
break
diff --git a/vfs/vfscache/writeback/writeback_test.go b/vfs/vfscache/writeback/writeback_test.go
index 7f23d8559..25a41d9e6 100644
--- a/vfs/vfscache/writeback/writeback_test.go
+++ b/vfs/vfscache/writeback/writeback_test.go
@@ -493,10 +493,12 @@ func TestWriteBackGetStats(t *testing.T) {
// Test queuing more than fs.Config.Transfers
func TestWriteBackMaxQueue(t *testing.T) {
+ ctx := context.Background()
+ ci := fs.GetConfig(ctx)
wb, cancel := newTestWriteBack(t)
defer cancel()
- maxTransfers := fs.Config.Transfers
+ maxTransfers := ci.Transfers
toTransfer := maxTransfers + 2
// put toTransfer things in the queue