diff --git a/fs/operations/operations.go b/fs/operations/operations.go index a10ad41b2..d207e3c8a 100644 --- a/fs/operations/operations.go +++ b/fs/operations/operations.go @@ -1281,6 +1281,48 @@ type readCloser struct { io.Closer } +// CatFile outputs the file to the io.Writer +// +// if offset == 0 it will be ignored +// if offset > 0 then the file will be seeked to that offset +// if offset < 0 then the file will be seeked that far from the end +// +// if count < 0 then it will be ignored +// if count >= 0 then only that many characters will be output +func CatFile(ctx context.Context, o fs.Object, offset, count int64, w io.Writer) (err error) { + ci := fs.GetConfig(ctx) + tr := accounting.Stats(ctx).NewTransfer(o, nil) + defer func() { + tr.Done(ctx, err) + }() + opt := fs.RangeOption{Start: offset, End: -1} + size := o.Size() + if opt.Start < 0 { + opt.Start += size + } + if count >= 0 { + opt.End = opt.Start + count - 1 + } + var options []fs.OpenOption + if opt.Start > 0 || opt.End >= 0 { + options = append(options, &opt) + } + for _, option := range ci.DownloadHeaders { + options = append(options, option) + } + var in io.ReadCloser + in, err = Open(ctx, o, options...) + if err != nil { + return err + } + if count >= 0 { + in = &readCloser{Reader: &io.LimitedReader{R: in, N: count}, Closer: in} + } + in = tr.Account(ctx, in).WithBuffer() // account and buffer the transfer + _, err = io.Copy(w, in) + return err +} + // Cat any files to the io.Writer // // if offset == 0 it will be ignored @@ -1291,46 +1333,14 @@ type readCloser struct { // if count >= 0 then only that many characters will be output func Cat(ctx context.Context, f fs.Fs, w io.Writer, offset, count int64, sep []byte) error { var mu sync.Mutex - ci := fs.GetConfig(ctx) return ListFn(ctx, f, func(o fs.Object) { - var err error - tr := accounting.Stats(ctx).NewTransfer(o, nil) - defer func() { - tr.Done(ctx, err) - }() - opt := fs.RangeOption{Start: offset, End: -1} - size := o.Size() - if opt.Start < 0 { - opt.Start += size - } - if count >= 0 { - opt.End = opt.Start + count - 1 - } - var options []fs.OpenOption - if opt.Start > 0 || opt.End >= 0 { - options = append(options, &opt) - } - for _, option := range ci.DownloadHeaders { - options = append(options, option) - } - var in io.ReadCloser - in, err = Open(ctx, o, options...) - if err != nil { - err = fs.CountError(ctx, err) - fs.Errorf(o, "Failed to open: %v", err) - return - } - if count >= 0 { - in = &readCloser{Reader: &io.LimitedReader{R: in, N: count}, Closer: in} - } - in = tr.Account(ctx, in).WithBuffer() // account and buffer the transfer - // take the lock just before we output stuff, so at the last possible moment mu.Lock() defer mu.Unlock() - _, err = io.Copy(w, in) + err := CatFile(ctx, o, offset, count, w) if err != nil { err = fs.CountError(ctx, err) fs.Errorf(o, "Failed to send to output: %v", err) + return } if len(sep) > 0 { _, err = w.Write(sep) diff --git a/fs/operations/rc.go b/fs/operations/rc.go index 644a947e9..413e6c556 100644 --- a/fs/operations/rc.go +++ b/fs/operations/rc.go @@ -948,3 +948,52 @@ func rcHashsum(ctx context.Context, in rc.Params) (out rc.Params, err error) { } return out, err } + +func init() { + rc.Add(rc.Call{ + Path: "operations/discard", + AuthRequired: true, + Fn: rcDiscard, + Title: "Read and discard bytes from a file", + Help: `This takes the following parameters: + +- fs - a remote name string e.g. "drive:" +- remote - a file within that remote e.g. "file.txt" +- offset - offset to start reading from, start if unset, from end if -ve +- count - bytes to read, all if unset + +This is similar to the [cat](/commands/rclone_cat/) with the --discard flag. + +It can be used for reading files into the VFS cache. +`, + }) +} + +// Cat a file with --discard +func rcDiscard(ctx context.Context, in rc.Params) (out rc.Params, err error) { + f, remote, err := rc.GetFsAndRemote(ctx, in) + if err != nil { + return nil, err + } + o, err := f.NewObject(ctx, remote) + if err != nil { + return nil, err + } + offset, err := in.GetInt64("offset") + if rc.IsErrParamNotFound(err) { + offset = 0 + } else if err != nil { + return nil, err + } + count, err := in.GetInt64("count") + if rc.IsErrParamNotFound(err) { + count = -1 + } else if err != nil { + return nil, err + } + err = CatFile(ctx, o, offset, count, io.Discard) + if err != nil { + return nil, err + } + return nil, nil +} diff --git a/fs/operations/rc_test.go b/fs/operations/rc_test.go index 2d8d0f2af..1bb0a7e07 100644 --- a/fs/operations/rc_test.go +++ b/fs/operations/rc_test.go @@ -14,12 +14,14 @@ import ( "time" "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/accounting" "github.com/rclone/rclone/fs/cache" "github.com/rclone/rclone/fs/hash" "github.com/rclone/rclone/fs/operations" "github.com/rclone/rclone/fs/rc" "github.com/rclone/rclone/fstest" "github.com/rclone/rclone/lib/diskusage" + "github.com/rclone/rclone/lib/random" "github.com/rclone/rclone/lib/rest" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -866,3 +868,66 @@ func TestRcHashsumFile(t *testing.T) { assert.Equal(t, "md5", out["hashType"]) assert.Equal(t, []string{"0ef726ce9b1a7692357ff70dd321d595 hashsum-file1"}, out["hashsum"]) } + +// operations/discard: read and discard the contents of a file +func TestRcDiscard(t *testing.T) { + ctx := context.Background() + r, call := rcNewRun(t, "operations/discard") + r.Mkdir(ctx, r.Fremote) + + fileContents := "file contents to be discarded" + file := r.WriteBoth(ctx, "discard-file", fileContents, t1) + r.CheckLocalItems(t, file) + r.CheckRemoteItems(t, file) + + for _, tt := range []struct { + name string + in rc.Params + want int64 + }{{ + name: "full read", + in: rc.Params{ + "fs": r.FremoteName, + "remote": file.Path, + }, + want: int64(len(fileContents)), + }, { + name: "start", + in: rc.Params{ + "fs": r.FremoteName, + "remote": file.Path, + "count": 2, + }, + want: 2, + }, { + name: "offset", + in: rc.Params{ + "fs": r.FremoteName, + "remote": file.Path, + "offset": 1, + "count": 3, + }, + want: 3, + }, { + name: "end", + in: rc.Params{ + "fs": r.FremoteName, + "remote": file.Path, + "offset": -1, + "count": 4, + }, + want: 1, + }} { + t.Run(tt.name, func(t *testing.T) { + group := random.String(8) + ctx := accounting.WithStatsGroup(ctx, group) + + out, err := call.Fn(ctx, tt.in) + require.NoError(t, err) + assert.Equal(t, rc.Params(nil), out) + + stats := accounting.Stats(ctx) + assert.Equal(t, tt.want, stats.GetBytes()) + }) + } +}