rc: implement operations/discard to read and discard a file

fs/operations/operations.go

@@ -1281,6 +1281,48 @@ type readCloser struct {
 	io.Closer
 }
 
+// CatFile outputs the file to the io.Writer
+//
+// if offset == 0 it will be ignored
+// if offset > 0 then the file will be seeked to that offset
+// if offset < 0 then the file will be seeked that far from the end
+//
+// if count < 0 then it will be ignored
+// if count >= 0 then only that many characters will be output
+func CatFile(ctx context.Context, o fs.Object, offset, count int64, w io.Writer) (err error) {
+	ci := fs.GetConfig(ctx)
+	tr := accounting.Stats(ctx).NewTransfer(o, nil)
+	defer func() {
+		tr.Done(ctx, err)
+	}()
+	opt := fs.RangeOption{Start: offset, End: -1}
+	size := o.Size()
+	if opt.Start < 0 {
+		opt.Start += size
+	}
+	if count >= 0 {
+		opt.End = opt.Start + count - 1
+	}
+	var options []fs.OpenOption
+	if opt.Start > 0 || opt.End >= 0 {
+		options = append(options, &opt)
+	}
+	for _, option := range ci.DownloadHeaders {
+		options = append(options, option)
+	}
+	var in io.ReadCloser
+	in, err = Open(ctx, o, options...)
+	if err != nil {
+		return err
+	}
+	if count >= 0 {
+		in = &readCloser{Reader: &io.LimitedReader{R: in, N: count}, Closer: in}
+	}
+	in = tr.Account(ctx, in).WithBuffer() // account and buffer the transfer
+	_, err = io.Copy(w, in)
+	return err
+}
+
 // Cat any files to the io.Writer
 //
 // if offset == 0 it will be ignored
@@ -1291,46 +1333,14 @@ type readCloser struct {
 // if count >= 0 then only that many characters will be output
 func Cat(ctx context.Context, f fs.Fs, w io.Writer, offset, count int64, sep []byte) error {
 	var mu sync.Mutex
-	ci := fs.GetConfig(ctx)
 	return ListFn(ctx, f, func(o fs.Object) {
-		var err error
-		tr := accounting.Stats(ctx).NewTransfer(o, nil)
-		defer func() {
-			tr.Done(ctx, err)
-		}()
-		opt := fs.RangeOption{Start: offset, End: -1}
-		size := o.Size()
-		if opt.Start < 0 {
-			opt.Start += size
-		}
-		if count >= 0 {
-			opt.End = opt.Start + count - 1
-		}
-		var options []fs.OpenOption
-		if opt.Start > 0 || opt.End >= 0 {
-			options = append(options, &opt)
-		}
-		for _, option := range ci.DownloadHeaders {
-			options = append(options, option)
-		}
-		var in io.ReadCloser
-		in, err = Open(ctx, o, options...)
-		if err != nil {
-			err = fs.CountError(ctx, err)
-			fs.Errorf(o, "Failed to open: %v", err)
-			return
-		}
-		if count >= 0 {
-			in = &readCloser{Reader: &io.LimitedReader{R: in, N: count}, Closer: in}
-		}
-		in = tr.Account(ctx, in).WithBuffer() // account and buffer the transfer
-		// take the lock just before we output stuff, so at the last possible moment
 		mu.Lock()
 		defer mu.Unlock()
-		_, err = io.Copy(w, in)
+		err := CatFile(ctx, o, offset, count, w)
 		if err != nil {
 			err = fs.CountError(ctx, err)
 			fs.Errorf(o, "Failed to send to output: %v", err)
+			return
 		}
 		if len(sep) > 0 {
 			_, err = w.Write(sep)

fs/operations/rc.go

@@ -948,3 +948,52 @@ func rcHashsum(ctx context.Context, in rc.Params) (out rc.Params, err error) {
 	}
 	return out, err
 }
+
+func init() {
+	rc.Add(rc.Call{
+		Path:         "operations/discard",
+		AuthRequired: true,
+		Fn:           rcDiscard,
+		Title:        "Read and discard bytes from a file",
+		Help: `This takes the following parameters:
+
+- fs - a remote name string e.g. "drive:"
+- remote - a file within that remote e.g. "file.txt"
+- offset - offset to start reading from, start if unset, from end if -ve
+- count - bytes to read, all if unset
+
+This is similar to the [cat](/commands/rclone_cat/) with the --discard flag.
+
+It can be used for reading files into the VFS cache.
+`,
+	})
+}
+
+// Cat a file with --discard
+func rcDiscard(ctx context.Context, in rc.Params) (out rc.Params, err error) {
+	f, remote, err := rc.GetFsAndRemote(ctx, in)
+	if err != nil {
+		return nil, err
+	}
+	o, err := f.NewObject(ctx, remote)
+	if err != nil {
+		return nil, err
+	}
+	offset, err := in.GetInt64("offset")
+	if rc.IsErrParamNotFound(err) {
+		offset = 0
+	} else if err != nil {
+		return nil, err
+	}
+	count, err := in.GetInt64("count")
+	if rc.IsErrParamNotFound(err) {
+		count = -1
+	} else if err != nil {
+		return nil, err
+	}
+	err = CatFile(ctx, o, offset, count, io.Discard)
+	if err != nil {
+		return nil, err
+	}
+	return nil, nil
+}

fs/operations/rc_test.go

@@ -14,12 +14,14 @@ import (
 	"time"
 
 	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/accounting"
 	"github.com/rclone/rclone/fs/cache"
 	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/fs/operations"
 	"github.com/rclone/rclone/fs/rc"
 	"github.com/rclone/rclone/fstest"
 	"github.com/rclone/rclone/lib/diskusage"
+	"github.com/rclone/rclone/lib/random"
 	"github.com/rclone/rclone/lib/rest"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -866,3 +868,66 @@ func TestRcHashsumFile(t *testing.T) {
 	assert.Equal(t, "md5", out["hashType"])
 	assert.Equal(t, []string{"0ef726ce9b1a7692357ff70dd321d595  hashsum-file1"}, out["hashsum"])
 }
+
+// operations/discard: read and discard the contents of a file
+func TestRcDiscard(t *testing.T) {
+	ctx := context.Background()
+	r, call := rcNewRun(t, "operations/discard")
+	r.Mkdir(ctx, r.Fremote)
+
+	fileContents := "file contents to be discarded"
+	file := r.WriteBoth(ctx, "discard-file", fileContents, t1)
+	r.CheckLocalItems(t, file)
+	r.CheckRemoteItems(t, file)
+
+	for _, tt := range []struct {
+		name string
+		in   rc.Params
+		want int64
+	}{{
+		name: "full read",
+		in: rc.Params{
+			"fs":     r.FremoteName,
+			"remote": file.Path,
+		},
+		want: int64(len(fileContents)),
+	}, {
+		name: "start",
+		in: rc.Params{
+			"fs":     r.FremoteName,
+			"remote": file.Path,
+			"count":  2,
+		},
+		want: 2,
+	}, {
+		name: "offset",
+		in: rc.Params{
+			"fs":     r.FremoteName,
+			"remote": file.Path,
+			"offset": 1,
+			"count":  3,
+		},
+		want: 3,
+	}, {
+		name: "end",
+		in: rc.Params{
+			"fs":     r.FremoteName,
+			"remote": file.Path,
+			"offset": -1,
+			"count":  4,
+		},
+		want: 1,
+	}} {
+		t.Run(tt.name, func(t *testing.T) {
+			group := random.String(8)
+			ctx := accounting.WithStatsGroup(ctx, group)
+
+			out, err := call.Fn(ctx, tt.in)
+			require.NoError(t, err)
+			assert.Equal(t, rc.Params(nil), out)
+
+			stats := accounting.Stats(ctx)
+			assert.Equal(t, tt.want, stats.GetBytes())
+		})
+	}
+}