From 8ffde402f624690cd367d48ba43b61b90dfdaeb5 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Thu, 24 Apr 2025 19:36:59 +0100 Subject: [PATCH] operations: fix too many connections open when using --max-memory Before this change we opened the connection before allocating memory. This meant a long wait sometimes for memory and too many connections open. Now we allocate the memory first before opening the connection. --- fs/operations/multithread.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/fs/operations/multithread.go b/fs/operations/multithread.go index 8eb9a1ad7..9abd5f743 100644 --- a/fs/operations/multithread.go +++ b/fs/operations/multithread.go @@ -12,6 +12,7 @@ import ( "github.com/rclone/rclone/fs/accounting" "github.com/rclone/rclone/lib/atexit" "github.com/rclone/rclone/lib/multipart" + "github.com/rclone/rclone/lib/pool" "golang.org/x/sync/errgroup" ) @@ -76,6 +77,13 @@ func (mc *multiThreadCopyState) copyChunk(ctx context.Context, chunk int, writer end := min(start+mc.partSize, mc.size) size := end - start + // Reserve the memory first so we don't open the source and wait for memory buffers for ages + var rw *pool.RW + if !mc.noBuffering { + rw = multipart.NewRW().Reserve(size) + defer fs.CheckClose(rw, &err) + } + fs.Debugf(mc.src, "multi-thread copy: chunk %d/%d (%d-%d) size %v starting", chunk+1, mc.numChunks, start, end, fs.SizeSuffix(size)) rc, err := Open(ctx, mc.src, &fs.RangeOption{Start: start, End: end - 1}) @@ -92,8 +100,6 @@ func (mc *multiThreadCopyState) copyChunk(ctx context.Context, chunk int, writer rs = rc } else { // Read the chunk into buffered reader - rw := multipart.NewRW().Reserve(size) - defer fs.CheckClose(rw, &err) _, err = io.CopyN(rw, rc, size) if err != nil { return fmt.Errorf("multi-thread copy: failed to read chunk: %w", err)