go/rclone
1
0
Fork 0
mirror of https://github.com/rclone/rclone.git synced 2025-08-10 06:09:44 +02:00
Code Issues Releases Activity

operations: fix too many connections open when using --max-memory

Browse Source 
Before this change we opened the connection before allocating memory.
This meant a long wait sometimes for memory and too many connections
open.

Now we allocate the memory first before opening the connection.
This commit is contained in:
Nick Craig-Wood
2025-04-24 19:36:59 +01:00
parent 117d8d9fdb
commit 8ffde402f6
1 changed files with 8 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
fs/operations/multithread.go
View File

@@ -12,6 +12,7 @@ import (
"github.com/rclone/rclone/fs/accounting" "github.com/rclone/rclone/fs/accounting"
"github.com/rclone/rclone/lib/atexit" "github.com/rclone/rclone/lib/atexit"
"github.com/rclone/rclone/lib/multipart" "github.com/rclone/rclone/lib/multipart"
"github.com/rclone/rclone/lib/pool"
"golang.org/x/sync/errgroup" "golang.org/x/sync/errgroup"
) )
@@ -76,6 +77,13 @@ func (mc *multiThreadCopyState) copyChunk(ctx context.Context, chunk int, writer
end := min(start+mc.partSize, mc.size) end := min(start+mc.partSize, mc.size)
size := end - start size := end - start
// Reserve the memory first so we don't open the source and wait for memory buffers for ages
var rw *pool.RW
if !mc.noBuffering {
rw = multipart.NewRW().Reserve(size)
defer fs.CheckClose(rw, &err)
}
fs.Debugf(mc.src, "multi-thread copy: chunk %d/%d (%d-%d) size %v starting", chunk+1, mc.numChunks, start, end, fs.SizeSuffix(size)) fs.Debugf(mc.src, "multi-thread copy: chunk %d/%d (%d-%d) size %v starting", chunk+1, mc.numChunks, start, end, fs.SizeSuffix(size))
rc, err := Open(ctx, mc.src, &fs.RangeOption{Start: start, End: end - 1}) rc, err := Open(ctx, mc.src, &fs.RangeOption{Start: start, End: end - 1})
@@ -92,8 +100,6 @@ func (mc *multiThreadCopyState) copyChunk(ctx context.Context, chunk int, writer
rs = rc rs = rc
} else { } else {
// Read the chunk into buffered reader // Read the chunk into buffered reader
rw := multipart.NewRW().Reserve(size)
defer fs.CheckClose(rw, &err)
_, err = io.CopyN(rw, rc, size) _, err = io.CopyN(rw, rc, size)
if err != nil { if err != nil {
return fmt.Errorf("multi-thread copy: failed to read chunk: %w", err) return fmt.Errorf("multi-thread copy: failed to read chunk: %w", err)