diff --git a/amazonclouddrive/amazonclouddrive.go b/amazonclouddrive/amazonclouddrive.go
index 2b1e80688..065a93ba3 100644
--- a/amazonclouddrive/amazonclouddrive.go
+++ b/amazonclouddrive/amazonclouddrive.go
@@ -48,7 +48,9 @@ const (
var (
// Flags
tempLinkThreshold = fs.SizeSuffix(9 << 30) // Download files bigger than this via the tempLink
+ uploadWaitLimit = pflag.DurationP("acd-upload-wait-limit", "", 60*time.Second, "Don't wait for completed uploads to appear if they took less than this time.")
uploadWaitTime = pflag.DurationP("acd-upload-wait-time", "", 2*60*time.Second, "Time to wait after a failed complete upload to see if it appears.")
+ uploadWaitPerGB = pflag.DurationP("acd-upload-wait-per-gb", "", 30*time.Second, "Additional time per GB to wait after a failed complete upload to see if it appears.")
// Description of how to auth for this app
acdConfig = &oauth2.Config{
Scopes: []string{"clouddrive:read_all", "clouddrive:write"},
@@ -478,26 +480,36 @@ func (f *Fs) List(out fs.ListOpts, dir string) {
// At the end of large uploads. The speculation is that the timeout
// is waiting for the sha1 hashing to complete and the file may well
// be properly uploaded.
-func (f *Fs) checkUpload(resp *http.Response, in io.Reader, src fs.ObjectInfo, inInfo *acd.File, inErr error) (fixedError bool, info *acd.File, err error) {
+func (f *Fs) checkUpload(resp *http.Response, in io.Reader, src fs.ObjectInfo, inInfo *acd.File, inErr error, uploadTime time.Duration) (fixedError bool, info *acd.File, err error) {
// Return if no error - all is well
if inErr == nil {
return false, inInfo, inErr
}
// If not one of the errors we can fix return
- if resp == nil || resp.StatusCode != 408 && resp.StatusCode != 500 && resp.StatusCode != 504 {
- return false, inInfo, inErr
- }
- const sleepTime = 5 * time.Second // sleep between tries
- retries := int(*uploadWaitTime / sleepTime) // number of retries
- if retries <= 0 {
- retries = 1
- }
+ // if resp == nil || resp.StatusCode != 408 && resp.StatusCode != 500 && resp.StatusCode != 504 {
+ // return false, inInfo, inErr
+ // }
+ // check to see if we read to the end
buf := make([]byte, 1)
n, err := in.Read(buf)
if !(n == 0 && err == io.EOF) {
- fs.Debug(src, "Upload error detected but didn't finish upload (n=%d, err=%v): %v", n, err, inErr)
+ fs.Debug(src, "Upload error detected but didn't finish upload: %v", inErr)
return false, inInfo, inErr
}
+
+ // Only wait for items which have been in transit for > uploadWaitLimit
+ if uploadTime < *uploadWaitLimit {
+ fs.Debug(src, "Upload error detected but not waiting since it only took %v to upload: %v", uploadTime, inErr)
+ return false, inInfo, inErr
+ }
+
+ // Time we should wait for the upload
+ uploadWaitPerByte := float64(*uploadWaitPerGB) / 1024 / 1024 / 1024
+ timeToWait := time.Duration(uploadWaitPerByte*float64(src.Size())) + *uploadWaitTime
+
+ const sleepTime = 5 * time.Second // sleep between tries
+ retries := int((timeToWait + sleepTime - 1) / sleepTime) // number of retries, rounded up
+
fs.Debug(src, "Error detected after finished upload - waiting to see if object was uploaded correctly: %v", inErr)
remote := src.Remote()
for i := 1; i <= retries; i++ {
@@ -508,7 +520,7 @@ func (f *Fs) checkUpload(resp *http.Response, in io.Reader, src fs.ObjectInfo, i
fs.Debug(src, "Object returned error - waiting (%d/%d): %v", i, retries, err)
} else {
if src.Size() == o.Size() {
- fs.Debug(src, "Object found with correct size - returning with no error")
+ fs.Debug(src, "Object found with correct size %d after waiting (%d/%d) - %v - returning with no error", src.Size(), i, retries, sleepTime*time.Duration(i-1))
info = &acd.File{
Node: o.(*Object).info,
}
@@ -518,7 +530,7 @@ func (f *Fs) checkUpload(resp *http.Response, in io.Reader, src fs.ObjectInfo, i
}
time.Sleep(sleepTime)
}
- fs.Debug(src, "Finished waiting for object - returning original error: %v", inErr)
+ fs.Debug(src, "Giving up waiting for object - returning original error: %v", inErr)
return false, inInfo, inErr
}
@@ -557,6 +569,7 @@ func (f *Fs) Put(in io.Reader, src fs.ObjectInfo) (fs.Object, error) {
var info *acd.File
var resp *http.Response
err = f.pacer.CallNoRetry(func() (bool, error) {
+ start := time.Now()
f.startUpload()
if src.Size() != 0 {
info, resp, err = folder.Put(in, leaf)
@@ -565,7 +578,7 @@ func (f *Fs) Put(in io.Reader, src fs.ObjectInfo) (fs.Object, error) {
}
f.stopUpload()
var ok bool
- ok, info, err = f.checkUpload(resp, in, src, info, err)
+ ok, info, err = f.checkUpload(resp, in, src, info, err, time.Since(start))
if ok {
return false, nil
}
@@ -815,6 +828,7 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo) error {
var resp *http.Response
var err error
err = o.fs.pacer.CallNoRetry(func() (bool, error) {
+ start := time.Now()
o.fs.startUpload()
if size != 0 {
info, resp, err = file.OverwriteSized(in, size)
@@ -823,7 +837,7 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo) error {
}
o.fs.stopUpload()
var ok bool
- ok, info, err = o.fs.checkUpload(resp, in, src, info, err)
+ ok, info, err = o.fs.checkUpload(resp, in, src, info, err, time.Since(start))
if ok {
return false, nil
}
diff --git a/docs/content/amazonclouddrive.md b/docs/content/amazonclouddrive.md
index a4a22c068..1febc535f 100644
--- a/docs/content/amazonclouddrive.md
+++ b/docs/content/amazonclouddrive.md
@@ -133,13 +133,26 @@ To download files above this threshold, rclone requests a `tempLink`
which downloads the file through a temporary URL directly from the
underlying S3 storage.
-#### --acd-upload-wait-time=TIME ####
+#### --acd-upload-wait-time=TIME, --acd-upload-wait-per-gb=TIME, --acd-upload-wait-limit=TIME ####
Sometimes Amazon Drive gives an error when a file has been fully
uploaded but the file appears anyway after a little while. This
-controls the time rclone waits - 2 minutes by default. You might want
-to increase the time if you are having problems with very big files.
-Upload with the `-v` flag for more info.
+happens sometimes for files over 1GB in size and nearly every time for
+files bigger than 10GB. These parameters control the time rclone waits
+for the file to appear.
+
+If the upload took less than `--acd-upload-wait-limit` (default 60s),
+then we go ahead an upload it again as that will be quicker.
+
+We wait `--acd-upload-wait-time` (default 2m) for the file to appear,
+with an additional `--acd-upload-wait-per-gb` (default 30s) per GB of
+the uploaded file.
+
+These values were determined empirically by observing lots of uploads
+of big files for a range of file sizes.
+
+Upload with the `-v` flag to see more info about what rclone is doing
+in this situation.
### Limitations ###