go/rclone
1
0
Fork 0
mirror of https://github.com/rclone/rclone.git synced 2025-02-20 07:48:33 +02:00
Code Issues Releases Activity

Make async buffering start slowly to improve seek performance

Browse Source
This commit is contained in:
Nick Craig-Wood 2017-02-16 23:55:47 +00:00
parent 8c2b50c7ed
commit 6f75290678
1 changed files with 12 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
fs/buffer.go
View File

@ -7,7 +7,10 @@ import (
"github.com/pkg/errors" "github.com/pkg/errors"
) )
const asyncBufferSize = 128 * 1024 const (
asyncBufferSize = 1024 * 1024
softStartInitial = 4 * 1024
)
var asyncBufferPool = sync.Pool{ var asyncBufferPool = sync.Pool{
New: func() interface{} { return newBuffer() }, New: func() interface{} { return newBuffer() },
@ -26,6 +29,7 @@ type asyncReader struct {
err error // If an error has occurred it is here err error // If an error has occurred it is here
cur *buffer // Current buffer being served cur *buffer // Current buffer being served
exited chan struct{} // Channel is closed been the async reader shuts down exited chan struct{} // Channel is closed been the async reader shuts down
size int // size of buffer to use
} }
// newAsyncReader returns a reader that will asynchronously read from // newAsyncReader returns a reader that will asynchronously read from
@ -54,6 +58,7 @@ func (a *asyncReader) init(rd io.ReadCloser, buffers int) {
a.exited = make(chan struct{}, 0) a.exited = make(chan struct{}, 0)
a.buffers = buffers a.buffers = buffers
a.cur = nil a.cur = nil
a.size = softStartInitial
// Create tokens // Create tokens
for i := 0; i < buffers; i++ { for i := 0; i < buffers; i++ {
@ -69,6 +74,10 @@ func (a *asyncReader) init(rd io.ReadCloser, buffers int) {
select { select {
case <-a.token: case <-a.token:
b := a.getBuffer() b := a.getBuffer()
if a.size < asyncBufferSize {
b.buf = b.buf[:a.size]
a.size <<= 1
}
err := b.read(a.in) err := b.read(a.in)
a.ready <- b a.ready <- b
if err != nil { if err != nil {
@ -89,7 +98,8 @@ func (a *asyncReader) putBuffer(b *buffer) {
// get a buffer from the pool // get a buffer from the pool
func (a *asyncReader) getBuffer() *buffer { func (a *asyncReader) getBuffer() *buffer {
return asyncBufferPool.Get().(*buffer) b := asyncBufferPool.Get().(*buffer)
return b
} }
// Read will return the next available data. // Read will return the next available data.