From 8c8ee9905c6435050873f7724a080d8f49e100c5 Mon Sep 17 00:00:00 2001
From: Manoj Ghosh <manoj.ghosh@oracle.com>
Date: Wed, 25 Jan 2023 02:48:16 -0800
Subject: [PATCH] oracleobjectstorage: speed up operations by using S3 pacer
and setting minsleep to 10ms
Uploading 100 files of each 1 MB took 20 seconds before. With above fix it takes around 2 seconds now.
10x time improvement in line with pacer's sleep reduction from 100ms to 10ms
---
backend/oracleobjectstorage/options.go | 4 +---
backend/oracleobjectstorage/oracleobjectstorage.go | 8 ++++++--
2 files changed, 7 insertions(+), 5 deletions(-)
diff --git a/backend/oracleobjectstorage/options.go b/backend/oracleobjectstorage/options.go
index 2916157be..004465093 100644
--- a/backend/oracleobjectstorage/options.go
+++ b/backend/oracleobjectstorage/options.go
@@ -17,9 +17,7 @@ const (
defaultUploadCutoff = fs.SizeSuffix(200 * 1024 * 1024)
defaultUploadConcurrency = 10
maxUploadCutoff = fs.SizeSuffix(5 * 1024 * 1024 * 1024)
- minSleep = 100 * time.Millisecond
- maxSleep = 5 * time.Minute
- decayConstant = 1 // bigger for slower decay, exponential
+ minSleep = 10 * time.Millisecond
defaultCopyTimeoutDuration = fs.Duration(time.Minute)
)
diff --git a/backend/oracleobjectstorage/oracleobjectstorage.go b/backend/oracleobjectstorage/oracleobjectstorage.go
index 6f730f258..c5f64744d 100644
--- a/backend/oracleobjectstorage/oracleobjectstorage.go
+++ b/backend/oracleobjectstorage/oracleobjectstorage.go
@@ -64,14 +64,18 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
if err != nil {
return nil, err
}
- p := pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))
+ pc := fs.NewPacer(ctx, pacer.NewS3(pacer.MinSleep(minSleep)))
+ // Set pacer retries to 2 (1 try and 1 retry) because we are
+ // relying on SDK retry mechanism, but we allow 2 attempts to
+ // retry directory listings after XMLSyntaxError
+ pc.SetRetries(2)
f := &Fs{
name: name,
opt: *opt,
ci: ci,
srv: objectStorageClient,
cache: bucket.NewCache(),
- pacer: fs.NewPacer(ctx, p),
+ pacer: pc,
}
f.setRoot(root)
f.features = (&fs.Features{