vfs: factor the vfs cache into its own package

cmd/cmount/fs.go

@@ -371,12 +371,7 @@ func (fsys *FS) Write(path string, buff []byte, ofst int64, fh uint64) (n int) {
 	if errc != 0 {
 		return errc
 	}
-	var err error
+	n, err := handle.WriteAt(buff, ofst)
-	if fsys.VFS.Opt.CacheMode < vfs.CacheModeWrites || handle.Node().Mode()&os.ModeAppend == 0 {
-		n, err = handle.WriteAt(buff, ofst)
-	} else {
-		n, err = handle.Write(buff)
-	}
 	if err != nil {
 		return translateError(err)
 	}

cmd/mount/handle.go

@@ -5,7 +5,6 @@ package mount
 import (
 	"context"
 	"io"
-	"os"
 
 	"bazil.org/fuse"
 	fusefs "bazil.org/fuse/fs"
@@ -42,12 +41,7 @@ var _ fusefs.HandleWriter = (*FileHandle)(nil)
 // Write data to the file handle
 func (fh *FileHandle) Write(ctx context.Context, req *fuse.WriteRequest, resp *fuse.WriteResponse) (err error) {
 	defer log.Trace(fh, "len=%d, offset=%d", len(req.Data), req.Offset)("written=%d, err=%v", &resp.Size, &err)
-	var n int
+	n, err := fh.Handle.WriteAt(req.Data, req.Offset)
-	if fh.Handle.Node().VFS().Opt.CacheMode < vfs.CacheModeWrites || fh.Handle.Node().Mode()&os.ModeAppend == 0 {
-		n, err = fh.Handle.WriteAt(req.Data, req.Offset)
-	} else {
-		n, err = fh.Handle.Write(req.Data)
-	}
 	if err != nil {
 		return translateError(err)
 	}

cmd/mount2/file.go

@@ -6,7 +6,6 @@ import (
 	"context"
 	"fmt"
 	"io"
-	"os"
 	"syscall"
 
 	fusefs "github.com/hanwen/go-fuse/v2/fs"
@@ -74,11 +73,7 @@ func (f *FileHandle) Write(ctx context.Context, data []byte, off int64) (written
 	var n int
 	var err error
 	defer log.Trace(f, "off=%d", off)("n=%d, off=%d, errno=%v", &n, &off, &errno)
-	if f.h.Node().VFS().Opt.CacheMode < vfs.CacheModeWrites || f.h.Node().Mode()&os.ModeAppend == 0 {
+	n, err = f.h.WriteAt(data, off)
-		n, err = f.h.WriteAt(data, off)
-	} else {
-		n, err = f.h.Write(data)
-	}
 	return uint32(n), translateError(err)
 }
 

cmd/mountlib/mounttest/fs.go

@@ -25,6 +25,7 @@ import (
 	"github.com/rclone/rclone/fstest"
 	"github.com/rclone/rclone/lib/file"
 	"github.com/rclone/rclone/vfs"
+	"github.com/rclone/rclone/vfs/vfscommon"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -44,11 +45,11 @@ var (
 func RunTests(t *testing.T, fn MountFn) {
 	mountFn = fn
 	flag.Parse()
-	cacheModes := []vfs.CacheMode{
+	cacheModes := []vfscommon.CacheMode{
-		vfs.CacheModeOff,
+		vfscommon.CacheModeOff,
-		vfs.CacheModeMinimal,
+		vfscommon.CacheModeMinimal,
-		vfs.CacheModeWrites,
+		vfscommon.CacheModeWrites,
-		vfs.CacheModeFull,
+		vfscommon.CacheModeFull,
 	}
 	run = newRun()
 	for _, cacheMode := range cacheModes {
@@ -207,7 +208,7 @@ func (r *Run) umount() {
 }
 
 // cacheMode flushes the VFS and changes the CacheMode
-func (r *Run) cacheMode(cacheMode vfs.CacheMode) {
+func (r *Run) cacheMode(cacheMode vfscommon.CacheMode) {
 	if r.skip {
 		log.Printf("FUSE not found so skipping cacheMode")
 		return

cmd/mountlib/mounttest/write.go

@@ -5,10 +5,9 @@ import (
 	"runtime"
 	"testing"
 
+	"github.com/rclone/rclone/vfs/vfscommon"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
-
-	"github.com/rclone/rclone/vfs"
 )
 
 // TestWriteFileNoWrite tests writing a file with no write()'s to it
@@ -91,7 +90,7 @@ func TestWriteFileFsync(t *testing.T) {
 func TestWriteFileDup(t *testing.T) {
 	run.skipIfNoFUSE(t)
 
-	if run.vfs.Opt.CacheMode < vfs.CacheModeWrites {
+	if run.vfs.Opt.CacheMode < vfscommon.CacheModeWrites {
 		t.Skip("not supported on vfs-cache-mode < writes")
 		return
 	}
@@ -136,7 +135,7 @@ func TestWriteFileDup(t *testing.T) {
 func TestWriteFileAppend(t *testing.T) {
 	run.skipIfNoFUSE(t)
 
-	if run.vfs.Opt.CacheMode < vfs.CacheModeWrites {
+	if run.vfs.Opt.CacheMode < vfscommon.CacheModeWrites {
 		t.Skip("not supported on vfs-cache-mode < writes")
 		return
 	}

cmd/mountlib/mounttest/write_unix.go

@@ -6,10 +6,9 @@ import (
 	"runtime"
 	"testing"
 
+	"github.com/rclone/rclone/vfs/vfscommon"
 	"github.com/stretchr/testify/assert"
 	"golang.org/x/sys/unix"
-
-	"github.com/rclone/rclone/vfs"
 )
 
 // TestWriteFileDoubleClose tests double close on write
@@ -45,7 +44,7 @@ func TestWriteFileDoubleClose(t *testing.T) {
 
 	// write to the other dup
 	_, err = unix.Write(fd2, buf)
-	if run.vfs.Opt.CacheMode < vfs.CacheModeWrites {
+	if run.vfs.Opt.CacheMode < vfscommon.CacheModeWrites {
 		// produces an error if cache mode < writes
 		assert.Error(t, err, "input/output error")
 	} else {

vfs/dir.go

@@ -16,6 +16,7 @@ import (
 	"github.com/rclone/rclone/fs/log"
 	"github.com/rclone/rclone/fs/operations"
 	"github.com/rclone/rclone/fs/walk"
+	"github.com/rclone/rclone/vfs/vfscommon"
 )
 
 // Dir represents a directory entry
@@ -150,7 +151,7 @@ func (d *Dir) changeNotify(relativePath string, entryType fs.EntryType) {
 	d.mu.RLock()
 	absPath := path.Join(d.path, relativePath)
 	d.mu.RUnlock()
-	d.invalidateDir(findParent(absPath))
+	d.invalidateDir(vfscommon.FindParent(absPath))
 	if entryType == fs.EntryDirectory {
 		d.invalidateDir(absPath)
 	}
@@ -168,7 +169,7 @@ func (d *Dir) ForgetPath(relativePath string, entryType fs.EntryType) {
 	absPath := path.Join(d.path, relativePath)
 	d.mu.RUnlock()
 	if absPath != "" {
-		d.invalidateDir(findParent(absPath))
+		d.invalidateDir(vfscommon.FindParent(absPath))
 	}
 	if entryType == fs.EntryDirectory {
 		d.forgetDirPath(relativePath)

vfs/file.go

@@ -12,6 +12,7 @@ import (
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/log"
 	"github.com/rclone/rclone/fs/operations"
+	"github.com/rclone/rclone/vfs/vfscommon"
 )
 
 // File represents a file
@@ -103,7 +104,7 @@ func (f *File) Path() string {
 
 // osPath returns the full path of the file in the cache in OS format
 func (f *File) osPath() string {
-	return f.d.vfs.cache.toOSPath(f.Path())
+	return f.d.vfs.cache.ToOSPath(f.Path())
 }
 
 // Sys returns underlying data source (can be nil) - satisfies Node interface
@@ -199,8 +200,8 @@ func (f *File) rename(ctx context.Context, destDir *Dir, newName string) error {
 	}
 
 	// Rename in the cache if it exists
-	if f.d.vfs.Opt.CacheMode != CacheModeOff && f.d.vfs.cache.exists(f.Path()) {
+	if f.d.vfs.Opt.CacheMode != vfscommon.CacheModeOff && f.d.vfs.cache.Exists(f.Path()) {
-		if err := f.d.vfs.cache.rename(f.Path(), newPath); err != nil {
+		if err := f.d.vfs.cache.Rename(f.Path(), newPath); err != nil {
 			fs.Infof(f.Path(), "File.Rename failed in Cache: %v", err)
 		}
 	}
@@ -379,8 +380,8 @@ func (f *File) _applyPendingModTime() error {
 	}
 
 	// set the time of the file in the cache
-	if f.d.vfs.Opt.CacheMode != CacheModeOff {
+	if f.d.vfs.Opt.CacheMode != vfscommon.CacheModeOff {
-		f.d.vfs.cache.setModTime(f._path(), f.pendingModTime)
+		f.d.vfs.cache.SetModTime(f._path(), f.pendingModTime)
 	}
 
 	// set the time of the object
@@ -554,8 +555,8 @@ func (f *File) Remove() error {
 	// Remove the item from the directory listing
 	d.delObject(f.Name())
 	// Remove the object from the cache
-	if d.vfs.Opt.CacheMode >= CacheModeMinimal {
+	if d.vfs.Opt.CacheMode >= vfscommon.CacheModeMinimal {
-		d.vfs.cache.remove(f.Path())
+		d.vfs.cache.Remove(f.Path())
 	}
 	return nil
 }
@@ -646,10 +647,10 @@ func (f *File) Open(flags int) (fd Handle, err error) {
 	d := f.d
 	f.mu.RUnlock()
 	CacheMode := d.vfs.Opt.CacheMode
-	if CacheMode >= CacheModeMinimal && (d.vfs.cache.opens(f.Path()) > 0 || d.vfs.cache.exists(f.Path())) {
+	if CacheMode >= vfscommon.CacheModeMinimal && (d.vfs.cache.Opens(f.Path()) > 0 || d.vfs.cache.Exists(f.Path())) {
 		fd, err = f.openRW(flags)
 	} else if read && write {
-		if CacheMode >= CacheModeMinimal {
+		if CacheMode >= vfscommon.CacheModeMinimal {
 			fd, err = f.openRW(flags)
 		} else {
 			// Open write only and hope the user doesn't
@@ -658,13 +659,13 @@ func (f *File) Open(flags int) (fd Handle, err error) {
 			fd, err = f.openWrite(flags)
 		}
 	} else if write {
-		if CacheMode >= CacheModeWrites {
+		if CacheMode >= vfscommon.CacheModeWrites {
 			fd, err = f.openRW(flags)
 		} else {
 			fd, err = f.openWrite(flags)
 		}
 	} else if read {
-		if CacheMode >= CacheModeFull {
+		if CacheMode >= vfscommon.CacheModeFull {
 			fd, err = f.openRW(flags)
 		} else {
 			fd, err = f.openRead()

vfs/file_test.go

@@ -11,12 +11,13 @@ import (
 	"github.com/rclone/rclone/fstest"
 	"github.com/rclone/rclone/fstest/mockfs"
 	"github.com/rclone/rclone/fstest/mockobject"
+	"github.com/rclone/rclone/vfs/vfscommon"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
-func fileCreate(t *testing.T, r *fstest.Run, mode CacheMode) (*VFS, *File, fstest.Item) {
+func fileCreate(t *testing.T, r *fstest.Run, mode vfscommon.CacheMode) (*VFS, *File, fstest.Item) {
-	opt := DefaultOpt
+	opt := vfscommon.DefaultOpt
 	opt.CacheMode = mode
 	vfs := New(r.Fremote, &opt)
 
@@ -33,7 +34,7 @@ func fileCreate(t *testing.T, r *fstest.Run, mode CacheMode) (*VFS, *File, fstes
 func TestFileMethods(t *testing.T) {
 	r := fstest.NewRun(t)
 	defer r.Finalise()
-	vfs, file, _ := fileCreate(t, r, CacheModeOff)
+	vfs, file, _ := fileCreate(t, r, vfscommon.CacheModeOff)
 
 	// String
 	assert.Equal(t, "dir/file1", file.String())
@@ -88,7 +89,7 @@ func TestFileSetModTime(t *testing.T) {
 		return
 	}
 	defer r.Finalise()
-	vfs, file, file1 := fileCreate(t, r, CacheModeOff)
+	vfs, file, file1 := fileCreate(t, r, vfscommon.CacheModeOff)
 
 	err := file.SetModTime(t2)
 	require.NoError(t, err)
@@ -115,7 +116,7 @@ func fileCheckContents(t *testing.T, file *File) {
 func TestFileOpenRead(t *testing.T) {
 	r := fstest.NewRun(t)
 	defer r.Finalise()
-	_, file, _ := fileCreate(t, r, CacheModeOff)
+	_, file, _ := fileCreate(t, r, vfscommon.CacheModeOff)
 
 	fileCheckContents(t, file)
 }
@@ -168,7 +169,7 @@ func TestFileOpenReadUnknownSize(t *testing.T) {
 func TestFileOpenWrite(t *testing.T) {
 	r := fstest.NewRun(t)
 	defer r.Finalise()
-	vfs, file, _ := fileCreate(t, r, CacheModeOff)
+	vfs, file, _ := fileCreate(t, r, vfscommon.CacheModeOff)
 
 	fd, err := file.openWrite(os.O_WRONLY | os.O_TRUNC)
 	require.NoError(t, err)
@@ -189,7 +190,7 @@ func TestFileOpenWrite(t *testing.T) {
 func TestFileRemove(t *testing.T) {
 	r := fstest.NewRun(t)
 	defer r.Finalise()
-	vfs, file, _ := fileCreate(t, r, CacheModeOff)
+	vfs, file, _ := fileCreate(t, r, vfscommon.CacheModeOff)
 
 	err := file.Remove()
 	require.NoError(t, err)
@@ -204,7 +205,7 @@ func TestFileRemove(t *testing.T) {
 func TestFileRemoveAll(t *testing.T) {
 	r := fstest.NewRun(t)
 	defer r.Finalise()
-	vfs, file, _ := fileCreate(t, r, CacheModeOff)
+	vfs, file, _ := fileCreate(t, r, vfscommon.CacheModeOff)
 
 	err := file.RemoveAll()
 	require.NoError(t, err)
@@ -219,7 +220,7 @@ func TestFileRemoveAll(t *testing.T) {
 func TestFileOpen(t *testing.T) {
 	r := fstest.NewRun(t)
 	defer r.Finalise()
-	_, file, _ := fileCreate(t, r, CacheModeOff)
+	_, file, _ := fileCreate(t, r, vfscommon.CacheModeOff)
 
 	fd, err := file.Open(os.O_RDONLY)
 	require.NoError(t, err)
@@ -242,7 +243,7 @@ func TestFileOpen(t *testing.T) {
 	assert.Equal(t, EPERM, err)
 }
 
-func testFileRename(t *testing.T, mode CacheMode) {
+func testFileRename(t *testing.T, mode vfscommon.CacheMode) {
 	r := fstest.NewRun(t)
 	defer r.Finalise()
 	vfs, file, item := fileCreate(t, r, mode)
@@ -255,10 +256,10 @@ func testFileRename(t *testing.T, mode CacheMode) {
 	require.NoError(t, err)
 
 	// check file in cache
-	if mode != CacheModeOff {
+	if mode != vfscommon.CacheModeOff {
 		// read contents to get file in cache
 		fileCheckContents(t, file)
-		assert.True(t, vfs.cache.exists(item.Path))
+		assert.True(t, vfs.cache.Exists(item.Path))
 	}
 
 	dir := file.Dir()
@@ -274,8 +275,8 @@ func testFileRename(t *testing.T, mode CacheMode) {
 	fstest.CheckItems(t, r.Fremote, item)
 
 	// check file in cache
-	if mode != CacheModeOff {
+	if mode != vfscommon.CacheModeOff {
-		assert.True(t, vfs.cache.exists(item.Path))
+		assert.True(t, vfs.cache.Exists(item.Path))
 	}
 
 	// check file exists in the vfs layer at its new name
@@ -290,8 +291,8 @@ func testFileRename(t *testing.T, mode CacheMode) {
 	fstest.CheckItems(t, r.Fremote, item)
 
 	// check file in cache
-	if mode != CacheModeOff {
+	if mode != vfscommon.CacheModeOff {
-		assert.True(t, vfs.cache.exists(item.Path))
+		assert.True(t, vfs.cache.Exists(item.Path))
 	}
 
 	// now try renaming it with the file open
@@ -308,8 +309,8 @@ func testFileRename(t *testing.T, mode CacheMode) {
 	newItem := fstest.NewItem("newLeaf", string(newContents), item.ModTime)
 
 	// check file has been renamed immediately in the cache
-	if mode != CacheModeOff {
+	if mode != vfscommon.CacheModeOff {
-		assert.True(t, vfs.cache.exists("newLeaf"))
+		assert.True(t, vfs.cache.Exists("newLeaf"))
 	}
 
 	// check file exists in the vfs layer at its new name
@@ -326,9 +327,9 @@ func testFileRename(t *testing.T, mode CacheMode) {
 
 func TestFileRename(t *testing.T) {
 	t.Run("CacheModeOff", func(t *testing.T) {
-		testFileRename(t, CacheModeOff)
+		testFileRename(t, vfscommon.CacheModeOff)
 	})
 	t.Run("CacheModeFull", func(t *testing.T) {
-		testFileRename(t, CacheModeFull)
+		testFileRename(t, vfscommon.CacheModeFull)
 	})
 }

vfs/read_write.go

@@ -12,7 +12,6 @@ import (
 	"github.com/pkg/errors"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/log"
-	"github.com/rclone/rclone/fs/operations"
 	"github.com/rclone/rclone/lib/file"
 )
 
@@ -55,12 +54,12 @@ func newRWFileHandle(d *Dir, f *File, flags int) (fh *RWFileHandle, err error) {
 	}
 
 	// mark the file as open in the cache - must be done before the mkdir
-	fh.d.vfs.cache.open(fh.file.Path())
+	fh.d.vfs.cache.Open(fh.file.Path())
 
 	// Make a place for the file
-	_, err = d.vfs.cache.mkdir(fh.file.Path())
+	_, err = d.vfs.cache.Mkdir(fh.file.Path())
 	if err != nil {
-		fh.d.vfs.cache.close(fh.file.Path())
+		fh.d.vfs.cache.Close(fh.file.Path())
 		return nil, errors.Wrap(err, "open RW handle failed to make cache directory")
 	}
 
@@ -80,16 +79,6 @@ func newRWFileHandle(d *Dir, f *File, flags int) (fh *RWFileHandle, err error) {
 	return fh, nil
 }
 
-// copy an object to or from the remote while accounting for it
-func copyObj(f fs.Fs, dst fs.Object, remote string, src fs.Object) (newDst fs.Object, err error) {
-	if operations.NeedTransfer(context.TODO(), dst, src) {
-		newDst, err = operations.Copy(context.TODO(), f, dst, remote, src)
-	} else {
-		newDst = dst
-	}
-	return newDst, err
-}
-
 // openPending opens the file if there is a pending open
 //
 // call with the lock held
@@ -110,12 +99,9 @@ func (fh *RWFileHandle) openPending(truncate bool) (err error) {
 		// If the remote object exists AND its cached file exists locally AND there are no
 		// other RW handles with it open, then attempt to update it.
 		if o != nil && fh.file.rwOpens() == 0 {
-			cacheObj, err := fh.d.vfs.cache.f.NewObject(context.TODO(), fh.file.Path())
+			err = fh.d.vfs.cache.Check(context.TODO(), o, fh.file.Path())
-			if err == nil && cacheObj != nil {
+			if err != nil {
-				_, err = copyObj(fh.d.vfs.cache.f, cacheObj, fh.file.Path(), o)
+				return errors.Wrap(err, "open RW handle failed to check cache file")
-				if err != nil {
-					return errors.Wrap(err, "open RW handle failed to update cached file")
-				}
 			}
 		}
 
@@ -125,7 +111,7 @@ func (fh *RWFileHandle) openPending(truncate bool) (err error) {
 			// cache file does not exist, so need to fetch it if we have an object to fetch
 			// it from
 			if o != nil {
-				_, err = copyObj(fh.d.vfs.cache.f, nil, fh.file.Path(), o)
+				err = fh.d.vfs.cache.Fetch(context.TODO(), o, fh.file.Path())
 				if err != nil {
 					cause := errors.Cause(err)
 					if cause != fs.ErrorObjectNotFound && cause != fs.ErrorDirNotFound {
@@ -276,22 +262,8 @@ func (fh *RWFileHandle) flushWrites(closeFile bool) error {
 	}
 
 	if isCopied {
-		// Transfer the temp file to the remote
+		o, err := fh.d.vfs.cache.Store(context.TODO(), fh.d.vfs.f, fh.file.getObject(), fh.file.Path())
-		cacheObj, err := fh.d.vfs.cache.f.NewObject(context.TODO(), fh.file.Path())
 		if err != nil {
-			err = errors.Wrap(err, "failed to find cache file")
-			fs.Errorf(fh.logPrefix(), "%v", err)
-			return err
-		}
-
-		objPath := fh.file.Path()
-		objOld := fh.file.getObject()
-		if objOld != nil {
-			objPath = objOld.Remote() // use the path of the actual object if available
-		}
-		o, err := copyObj(fh.d.vfs.f, objOld, objPath, cacheObj)
-		if err != nil {
-			err = errors.Wrap(err, "failed to transfer file from cache to remote")
 			fs.Errorf(fh.logPrefix(), "%v", err)
 			return err
 		}
@@ -322,7 +294,7 @@ func (fh *RWFileHandle) close() (err error) {
 		if fh.opened {
 			fh.file.delRWOpen()
 		}
-		fh.d.vfs.cache.close(fh.file.Path())
+		fh.d.vfs.cache.Close(fh.file.Path())
 	}()
 
 	return fh.flushWrites(true)
@@ -501,6 +473,10 @@ func (fh *RWFileHandle) Write(b []byte) (n int, err error) {
 
 // WriteAt bytes to the file at off
 func (fh *RWFileHandle) WriteAt(b []byte, off int64) (n int, err error) {
+	if fh.flags&os.O_APPEND != 0 {
+		// if append is set, call Write as WriteAt returns an error if append is set
+		return fh.Write(b)
+	}
 	err = fh.writeFn(func() error {
 		n, err = fh.File.WriteAt(b, off)
 		return err

vfs/read_write_test.go

@@ -13,6 +13,7 @@ import (
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/operations"
 	"github.com/rclone/rclone/fstest"
+	"github.com/rclone/rclone/vfs/vfscommon"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -25,8 +26,8 @@ func cleanup(t *testing.T, r *fstest.Run, vfs *VFS) {
 
 // Create a file and open it with the flags passed in
 func rwHandleCreateFlags(t *testing.T, r *fstest.Run, create bool, filename string, flags int) (*VFS, *RWFileHandle) {
-	opt := DefaultOpt
+	opt := vfscommon.DefaultOpt
-	opt.CacheMode = CacheModeFull
+	opt.CacheMode = vfscommon.CacheModeFull
 	vfs := New(r.Fremote, &opt)
 
 	if create {
@@ -661,8 +662,8 @@ func testRWFileHandleOpenTest(t *testing.T, vfs *VFS, test *openTest) {
 
 func TestRWFileHandleOpenTests(t *testing.T) {
 	r := fstest.NewRun(t)
-	opt := DefaultOpt
+	opt := vfscommon.DefaultOpt
-	opt.CacheMode = CacheModeFull
+	opt.CacheMode = vfscommon.CacheModeFull
 	vfs := New(r.Fremote, &opt)
 	defer cleanup(t, r, vfs)
 
@@ -716,8 +717,8 @@ func TestRWCacheRename(t *testing.T) {
 		t.Skip("skip as can't rename files")
 	}
 
-	opt := DefaultOpt
+	opt := vfscommon.DefaultOpt
-	opt.CacheMode = CacheModeFull
+	opt.CacheMode = vfscommon.CacheModeFull
 	vfs := New(r.Fremote, &opt)
 
 	h, err := vfs.OpenFile("rename_me", os.O_WRONLY|os.O_CREATE, 0777)
@@ -732,11 +733,11 @@ func TestRWCacheRename(t *testing.T) {
 	err = fh.Close()
 	require.NoError(t, err)
 
-	assert.True(t, vfs.cache.exists("rename_me"))
+	assert.True(t, vfs.cache.Exists("rename_me"))
 
 	err = vfs.Rename("rename_me", "i_was_renamed")
 	require.NoError(t, err)
 
-	assert.False(t, vfs.cache.exists("rename_me"))
+	assert.False(t, vfs.cache.Exists("rename_me"))
-	assert.True(t, vfs.cache.exists("i_was_renamed"))
+	assert.True(t, vfs.cache.Exists("i_was_renamed"))
 }

vfs/vfs.go

@@ -23,7 +23,6 @@ import (
 	"fmt"
 	"os"
 	"path"
-	"runtime"
 	"strings"
 	"sync"
 	"sync/atomic"
@@ -31,32 +30,10 @@ import (
 
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/log"
+	"github.com/rclone/rclone/vfs/vfscache"
+	"github.com/rclone/rclone/vfs/vfscommon"
 )
 
-// DefaultOpt is the default values uses for Opt
-var DefaultOpt = Options{
-	NoModTime:         false,
-	NoChecksum:        false,
-	NoSeek:            false,
-	DirCacheTime:      5 * 60 * time.Second,
-	PollInterval:      time.Minute,
-	ReadOnly:          false,
-	Umask:             0,
-	UID:               ^uint32(0), // these values instruct WinFSP-FUSE to use the current user
-	GID:               ^uint32(0), // overriden for non windows in mount_unix.go
-	DirPerms:          os.FileMode(0777),
-	FilePerms:         os.FileMode(0666),
-	CacheMode:         CacheModeOff,
-	CacheMaxAge:       3600 * time.Second,
-	CachePollInterval: 60 * time.Second,
-	ChunkSize:         128 * fs.MebiByte,
-	ChunkSizeLimit:    -1,
-	CacheMaxSize:      -1,
-	CaseInsensitive:   runtime.GOOS == "windows" || runtime.GOOS == "darwin", // default to true on Windows and Mac, false otherwise
-	WriteWait:         1000 * time.Millisecond,
-	ReadWait:          5 * time.Millisecond,
-}
-
 // Node represents either a directory (*Dir) or a file (*File)
 type Node interface {
 	os.FileInfo
@@ -175,8 +152,8 @@ var (
 type VFS struct {
 	f         fs.Fs
 	root      *Dir
-	Opt       Options
+	Opt       vfscommon.Options
-	cache     *cache
+	cache     *vfscache.Cache
 	cancel    context.CancelFunc
 	usageMu   sync.Mutex
 	usageTime time.Time
@@ -184,33 +161,9 @@ type VFS struct {
 	pollChan  chan time.Duration
 }
 
-// Options is options for creating the vfs
-type Options struct {
-	NoSeek            bool          // don't allow seeking if set
-	NoChecksum        bool          // don't check checksums if set
-	ReadOnly          bool          // if set VFS is read only
-	NoModTime         bool          // don't read mod times for files
-	DirCacheTime      time.Duration // how long to consider directory listing cache valid
-	PollInterval      time.Duration
-	Umask             int
-	UID               uint32
-	GID               uint32
-	DirPerms          os.FileMode
-	FilePerms         os.FileMode
-	ChunkSize         fs.SizeSuffix // if > 0 read files in chunks
-	ChunkSizeLimit    fs.SizeSuffix // if > ChunkSize double the chunk size after each chunk until reached
-	CacheMode         CacheMode
-	CacheMaxAge       time.Duration
-	CacheMaxSize      fs.SizeSuffix
-	CachePollInterval time.Duration
-	CaseInsensitive   bool
-	WriteWait         time.Duration // time to wait for in-sequence write
-	ReadWait          time.Duration // time to wait for in-sequence read
-}
-
 // New creates a new VFS and root directory.  If opt is nil, then
 // DefaultOpt will be used
-func New(f fs.Fs, opt *Options) *VFS {
+func New(f fs.Fs, opt *vfscommon.Options) *VFS {
 	fsDir := fs.NewDir("", time.Now())
 	vfs := &VFS{
 		f: f,
@@ -220,7 +173,7 @@ func New(f fs.Fs, opt *Options) *VFS {
 	if opt != nil {
 		vfs.Opt = *opt
 	} else {
-		vfs.Opt = DefaultOpt
+		vfs.Opt = vfscommon.DefaultOpt
 	}
 
 	// Mask the permissions with the umask
@@ -255,15 +208,15 @@ func (vfs *VFS) Fs() fs.Fs {
 }
 
 // SetCacheMode change the cache mode
-func (vfs *VFS) SetCacheMode(cacheMode CacheMode) {
+func (vfs *VFS) SetCacheMode(cacheMode vfscommon.CacheMode) {
 	vfs.Shutdown()
 	vfs.cache = nil
-	if cacheMode > CacheModeOff {
+	if cacheMode > vfscommon.CacheModeOff {
 		ctx, cancel := context.WithCancel(context.Background())
-		cache, err := newCache(ctx, vfs.f, &vfs.Opt) // FIXME pass on context or get from Opt?
+		cache, err := vfscache.New(ctx, vfs.f, &vfs.Opt) // FIXME pass on context or get from Opt?
 		if err != nil {
 			fs.Errorf(nil, "Failed to create vfs cache - disabling: %v", err)
-			vfs.Opt.CacheMode = CacheModeOff
+			vfs.Opt.CacheMode = vfscommon.CacheModeOff
 			cancel()
 			return
 		}
@@ -283,10 +236,10 @@ func (vfs *VFS) Shutdown() {
 
 // CleanUp deletes the contents of the on disk cache
 func (vfs *VFS) CleanUp() error {
-	if vfs.Opt.CacheMode == CacheModeOff {
+	if vfs.Opt.CacheMode == vfscommon.CacheModeOff {
 		return nil
 	}
-	return vfs.cache.cleanUp()
+	return vfs.cache.CleanUp()
 }
 
 // FlushDirCache empties the directory cache

vfs/vfs_case_test.go

@@ -6,6 +6,7 @@ import (
 	"testing"
 
 	"github.com/rclone/rclone/fstest"
+	"github.com/rclone/rclone/vfs/vfscommon"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -32,11 +33,11 @@ func TestCaseSensitivity(t *testing.T) {
 	file3 := r.WriteObject(ctx, "FilEb", "data3", t3)
 
 	// Create a case-Sensitive and case-INsensitive VFS
-	optCS := DefaultOpt
+	optCS := vfscommon.DefaultOpt
 	optCS.CaseInsensitive = false
 	vfsCS := New(r.Fremote, &optCS)
 
-	optCI := DefaultOpt
+	optCI := vfscommon.DefaultOpt
 	optCI.CaseInsensitive = true
 	vfsCI := New(r.Fremote, &optCI)
 

vfs/vfs_test.go

@@ -12,6 +12,7 @@ import (
 	_ "github.com/rclone/rclone/backend/all" // import all the backends
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fstest"
+	"github.com/rclone/rclone/vfs/vfscommon"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -100,13 +101,13 @@ func TestVFSNew(t *testing.T) {
 
 	// Check making a VFS with nil options
 	vfs := New(r.Fremote, nil)
-	var defaultOpt = DefaultOpt
+	var defaultOpt = vfscommon.DefaultOpt
 	defaultOpt.DirPerms |= os.ModeDir
 	assert.Equal(t, vfs.Opt, defaultOpt)
 	assert.Equal(t, vfs.f, r.Fremote)
 
 	// Check the initialisation
-	var opt = DefaultOpt
+	var opt = vfscommon.DefaultOpt
 	opt.DirPerms = 0777
 	opt.FilePerms = 0666
 	opt.Umask = 0002

vfs/vfscache/vfscache.go

@@ -1,10 +1,8 @@
-// This deals with caching of files locally
+// Package vfscache deals with caching of files locally for the VFS layer
-
+package vfscache
-package vfs
 
 import (
 	"context"
-	"fmt"
 	"os"
 	"path/filepath"
 	"runtime"
@@ -18,54 +16,14 @@ import (
 	"github.com/rclone/rclone/fs"
 	fscache "github.com/rclone/rclone/fs/cache"
 	"github.com/rclone/rclone/fs/config"
+	"github.com/rclone/rclone/fs/operations"
+	"github.com/rclone/rclone/vfs/vfscommon"
 )
 
-// CacheMode controls the functionality of the cache
+// Cache opened files
-type CacheMode byte
+type Cache struct {
-
-// CacheMode options
-const (
-	CacheModeOff     CacheMode = iota // cache nothing - return errors for writes which can't be satisfied
-	CacheModeMinimal                  // cache only the minimum, eg read/write opens
-	CacheModeWrites                   // cache all files opened with write intent
-	CacheModeFull                     // cache all files opened in any mode
-)
-
-var cacheModeToString = []string{
-	CacheModeOff:     "off",
-	CacheModeMinimal: "minimal",
-	CacheModeWrites:  "writes",
-	CacheModeFull:    "full",
-}
-
-// String turns a CacheMode into a string
-func (l CacheMode) String() string {
-	if l >= CacheMode(len(cacheModeToString)) {
-		return fmt.Sprintf("CacheMode(%d)", l)
-	}
-	return cacheModeToString[l]
-}
-
-// Set a CacheMode
-func (l *CacheMode) Set(s string) error {
-	for n, name := range cacheModeToString {
-		if s != "" && name == s {
-			*l = CacheMode(n)
-			return nil
-		}
-	}
-	return errors.Errorf("Unknown cache mode level %q", s)
-}
-
-// Type of the value
-func (l *CacheMode) Type() string {
-	return "CacheMode"
-}
-
-// cache opened files
-type cache struct {
 	f      fs.Fs                 // fs for the cache directory
-	opt    *Options              // vfs Options
+	opt    *vfscommon.Options    // vfs Options
 	root   string                // root of the cache directory
 	itemMu sync.Mutex            // protects the following variables
 	item   map[string]*cacheItem // files/directories in the cache
@@ -85,11 +43,11 @@ func newCacheItem(isFile bool) *cacheItem {
 	return &cacheItem{atime: time.Now(), isFile: isFile}
 }
 
-// newCache creates a new cache heirachy for f
+// New creates a new cache heirachy for f
 //
 // This starts background goroutines which can be cancelled with the
 // context passed in.
-func newCache(ctx context.Context, f fs.Fs, opt *Options) (*cache, error) {
+func New(ctx context.Context, f fs.Fs, opt *vfscommon.Options) (*Cache, error) {
 	fRoot := filepath.FromSlash(f.Root())
 	if runtime.GOOS == "windows" {
 		if strings.HasPrefix(fRoot, `\\?`) {
@@ -105,7 +63,7 @@ func newCache(ctx context.Context, f fs.Fs, opt *Options) (*cache, error) {
 		return nil, errors.Wrap(err, "failed to create cache remote")
 	}
 
-	c := &cache{
+	c := &Cache{
 		f:    f,
 		opt:  opt,
 		root: root,
@@ -117,15 +75,6 @@ func newCache(ctx context.Context, f fs.Fs, opt *Options) (*cache, error) {
 	return c, nil
 }
 
-// findParent returns the parent directory of name, or "" for the root
-func findParent(name string) string {
-	parent := filepath.Dir(name)
-	if parent == "." || parent == "/" {
-		parent = ""
-	}
-	return parent
-}
-
 // clean returns the cleaned version of name for use in the index map
 func clean(name string) string {
 	name = strings.Trim(name, "/")
@@ -136,17 +85,17 @@ func clean(name string) string {
 	return name
 }
 
-// toOSPath turns a remote relative name into an OS path in the cache
+// ToOSPath turns a remote relative name into an OS path in the cache
-func (c *cache) toOSPath(name string) string {
+func (c *Cache) ToOSPath(name string) string {
 	return filepath.Join(c.root, filepath.FromSlash(name))
 }
 
-// mkdir makes the directory for name in the cache and returns an os
+// Mkdir makes the directory for name in the cache and returns an os
 // path for the file
-func (c *cache) mkdir(name string) (string, error) {
+func (c *Cache) Mkdir(name string) (string, error) {
-	parent := findParent(name)
+	parent := vfscommon.FindParent(name)
 	leaf := filepath.Base(name)
-	parentPath := c.toOSPath(parent)
+	parentPath := c.ToOSPath(parent)
 	err := os.MkdirAll(parentPath, 0700)
 	if err != nil {
 		return "", errors.Wrap(err, "make cache directory failed")
@@ -163,7 +112,7 @@ func (c *cache) mkdir(name string) (string, error) {
 // name should be a remote path not an osPath
 //
 // must be called with itemMu held
-func (c *cache) _get(isFile bool, name string) (item *cacheItem, found bool) {
+func (c *Cache) _get(isFile bool, name string) (item *cacheItem, found bool) {
 	item = c.item[name]
 	found = item != nil
 	if !found {
@@ -173,10 +122,10 @@ func (c *cache) _get(isFile bool, name string) (item *cacheItem, found bool) {
 	return item, found
 }
 
-// opens returns the number of opens that are on the file
+// Opens returns the number of opens that are on the file
 //
 // name should be a remote path not an osPath
-func (c *cache) opens(name string) int {
+func (c *Cache) Opens(name string) int {
 	name = clean(name)
 	c.itemMu.Lock()
 	defer c.itemMu.Unlock()
@@ -190,7 +139,7 @@ func (c *cache) opens(name string) int {
 // get gets name from the cache or creates a new one
 //
 // name should be a remote path not an osPath
-func (c *cache) get(name string) *cacheItem {
+func (c *Cache) get(name string) *cacheItem {
 	name = clean(name)
 	c.itemMu.Lock()
 	item, _ := c._get(true, name)
@@ -204,7 +153,7 @@ func (c *cache) get(name string) *cacheItem {
 // it also sets the size
 //
 // name should be a remote path not an osPath
-func (c *cache) updateStat(name string, when time.Time, size int64) {
+func (c *Cache) updateStat(name string, when time.Time, size int64) {
 	name = clean(name)
 	c.itemMu.Lock()
 	item, found := c._get(true, name)
@@ -219,7 +168,7 @@ func (c *cache) updateStat(name string, when time.Time, size int64) {
 // _open marks name as open, must be called with the lock held
 //
 // name should be a remote path not an osPath
-func (c *cache) _open(isFile bool, name string) {
+func (c *Cache) _open(isFile bool, name string) {
 	for {
 		item, _ := c._get(isFile, name)
 		item.opens++
@@ -228,14 +177,14 @@ func (c *cache) _open(isFile bool, name string) {
 			break
 		}
 		isFile = false
-		name = findParent(name)
+		name = vfscommon.FindParent(name)
 	}
 }
 
-// open marks name as open
+// Open marks name as open
 //
 // name should be a remote path not an osPath
-func (c *cache) open(name string) {
+func (c *Cache) Open(name string) {
 	name = clean(name)
 	c.itemMu.Lock()
 	c._open(true, name)
@@ -245,7 +194,7 @@ func (c *cache) open(name string) {
 // cacheDir marks a directory and its parents as being in the cache
 //
 // name should be a remote path not an osPath
-func (c *cache) cacheDir(name string) {
+func (c *Cache) cacheDir(name string) {
 	name = clean(name)
 	c.itemMu.Lock()
 	defer c.itemMu.Unlock()
@@ -258,13 +207,13 @@ func (c *cache) cacheDir(name string) {
 		if name == "" {
 			break
 		}
-		name = findParent(name)
+		name = vfscommon.FindParent(name)
 	}
 }
 
-// exists checks to see if the file exists in the cache or not
+// Exists checks to see if the file exists in the cache or not
-func (c *cache) exists(name string) bool {
+func (c *Cache) Exists(name string) bool {
-	osPath := c.toOSPath(name)
+	osPath := c.ToOSPath(name)
 	fi, err := os.Stat(osPath)
 	if err != nil {
 		return false
@@ -276,10 +225,10 @@ func (c *cache) exists(name string) bool {
 	return true
 }
 
-// renames the file in cache
+// Rename the file in cache
-func (c *cache) rename(name string, newName string) (err error) {
+func (c *Cache) Rename(name string, newName string) (err error) {
-	osOldPath := c.toOSPath(name)
+	osOldPath := c.ToOSPath(name)
-	osNewPath := c.toOSPath(newName)
+	osNewPath := c.ToOSPath(newName)
 	sfi, err := os.Stat(osOldPath)
 	if err != nil {
 		return errors.Wrapf(err, "Failed to stat source: %s", osOldPath)
@@ -293,7 +242,7 @@ func (c *cache) rename(name string, newName string) (err error) {
 		if !os.IsNotExist(err) {
 			return errors.Wrapf(err, "Failed to stat destination: %s", osNewPath)
 		}
-		parent := findParent(osNewPath)
+		parent := vfscommon.FindParent(osNewPath)
 		err = os.MkdirAll(parent, 0700)
 		if err != nil {
 			return errors.Wrapf(err, "Failed to create parent dir: %s", parent)
@@ -321,7 +270,7 @@ func (c *cache) rename(name string, newName string) (err error) {
 }
 
 // _close marks name as closed - must be called with the lock held
-func (c *cache) _close(isFile bool, name string) {
+func (c *Cache) _close(isFile bool, name string) {
 	for {
 		item, _ := c._get(isFile, name)
 		item.opens--
@@ -329,7 +278,7 @@ func (c *cache) _close(isFile bool, name string) {
 		if item.opens < 0 {
 			fs.Errorf(name, "cache: double close")
 		}
-		osPath := c.toOSPath(name)
+		osPath := c.ToOSPath(name)
 		fi, err := os.Stat(osPath)
 		// Update the size on close
 		if err == nil && !fi.IsDir() {
@@ -339,23 +288,23 @@ func (c *cache) _close(isFile bool, name string) {
 			break
 		}
 		isFile = false
-		name = findParent(name)
+		name = vfscommon.FindParent(name)
 	}
 }
 
-// close marks name as closed
+// Close marks name as closed
 //
 // name should be a remote path not an osPath
-func (c *cache) close(name string) {
+func (c *Cache) Close(name string) {
 	name = clean(name)
 	c.itemMu.Lock()
 	c._close(true, name)
 	c.itemMu.Unlock()
 }
 
-// remove should be called if name is deleted
+// Remove should be called if name is deleted
-func (c *cache) remove(name string) {
+func (c *Cache) Remove(name string) {
-	osPath := c.toOSPath(name)
+	osPath := c.ToOSPath(name)
 	err := os.Remove(osPath)
 	if err != nil && !os.IsNotExist(err) {
 		fs.Errorf(name, "Failed to remove from cache: %v", err)
@@ -366,8 +315,8 @@ func (c *cache) remove(name string) {
 
 // removeDir should be called if dir is deleted and returns true if
 // the directory is gone.
-func (c *cache) removeDir(dir string) bool {
+func (c *Cache) removeDir(dir string) bool {
-	osPath := c.toOSPath(dir)
+	osPath := c.ToOSPath(dir)
 	err := os.Remove(osPath)
 	if err == nil || os.IsNotExist(err) {
 		if err == nil {
@@ -381,22 +330,22 @@ func (c *cache) removeDir(dir string) bool {
 	return false
 }
 
-// setModTime should be called to set the modification time of the cache file
+// SetModTime should be called to set the modification time of the cache file
-func (c *cache) setModTime(name string, modTime time.Time) {
+func (c *Cache) SetModTime(name string, modTime time.Time) {
-	osPath := c.toOSPath(name)
+	osPath := c.ToOSPath(name)
 	err := os.Chtimes(osPath, modTime, modTime)
 	if err != nil {
 		fs.Errorf(name, "Failed to set modification time of cached file: %v", err)
 	}
 }
 
-// cleanUp empties the cache of everything
+// CleanUp empties the cache of everything
-func (c *cache) cleanUp() error {
+func (c *Cache) CleanUp() error {
 	return os.RemoveAll(c.root)
 }
 
 // walk walks the cache calling the function
-func (c *cache) walk(fn func(osPath string, fi os.FileInfo, name string) error) error {
+func (c *Cache) walk(fn func(osPath string, fi os.FileInfo, name string) error) error {
 	return filepath.Walk(c.root, func(osPath string, fi os.FileInfo, err error) error {
 		if err != nil {
 			return err
@@ -419,7 +368,7 @@ func (c *cache) walk(fn func(osPath string, fi os.FileInfo, name string) error)
 // updateStats walks the cache updating any atimes and sizes it finds
 //
 // it also updates used
-func (c *cache) updateStats() error {
+func (c *Cache) updateStats() error {
 	var newUsed int64
 	err := c.walk(func(osPath string, fi os.FileInfo, name string) error {
 		if !fi.IsDir() {
@@ -439,11 +388,11 @@ func (c *cache) updateStats() error {
 }
 
 // purgeOld gets rid of any files that are over age
-func (c *cache) purgeOld(maxAge time.Duration) {
+func (c *Cache) purgeOld(maxAge time.Duration) {
-	c._purgeOld(maxAge, c.remove)
+	c._purgeOld(maxAge, c.Remove)
 }
 
-func (c *cache) _purgeOld(maxAge time.Duration, remove func(name string)) {
+func (c *Cache) _purgeOld(maxAge time.Duration, remove func(name string)) {
 	c.itemMu.Lock()
 	defer c.itemMu.Unlock()
 	cutoff := time.Now().Add(-maxAge)
@@ -462,11 +411,11 @@ func (c *cache) _purgeOld(maxAge time.Duration, remove func(name string)) {
 }
 
 // Purge any empty directories
-func (c *cache) purgeEmptyDirs() {
+func (c *Cache) purgeEmptyDirs() {
 	c._purgeEmptyDirs(c.removeDir)
 }
 
-func (c *cache) _purgeEmptyDirs(removeDir func(name string) bool) {
+func (c *Cache) _purgeEmptyDirs(removeDir func(name string) bool) {
 	c.itemMu.Lock()
 	defer c.itemMu.Unlock()
 	var dirs []string
@@ -499,11 +448,11 @@ func (v cacheNamedItems) Less(i, j int) bool { return v[i].item.atime.Before(v[j
 
 // Remove any files that are over quota starting from the
 // oldest first
-func (c *cache) purgeOverQuota(quota int64) {
+func (c *Cache) purgeOverQuota(quota int64) {
-	c._purgeOverQuota(quota, c.remove)
+	c._purgeOverQuota(quota, c.Remove)
 }
 
-func (c *cache) _purgeOverQuota(quota int64, remove func(name string)) {
+func (c *Cache) _purgeOverQuota(quota int64, remove func(name string)) {
 	c.itemMu.Lock()
 	defer c.itemMu.Unlock()
 
@@ -537,7 +486,7 @@ func (c *cache) _purgeOverQuota(quota int64, remove func(name string)) {
 }
 
 // clean empties the cache of stuff if it can
-func (c *cache) clean() {
+func (c *Cache) clean() {
 	// Cache may be empty so end
 	_, err := os.Stat(c.root)
 	if os.IsNotExist(err) {
@@ -575,7 +524,7 @@ func (c *cache) clean() {
 // cleaner calls clean at regular intervals
 //
 // doesn't return until context is cancelled
-func (c *cache) cleaner(ctx context.Context) {
+func (c *Cache) cleaner(ctx context.Context) {
 	if c.opt.CachePollInterval <= 0 {
 		fs.Debugf(nil, "Cache cleaning thread disabled because poll interval <= 0")
 		return
@@ -595,3 +544,50 @@ func (c *cache) cleaner(ctx context.Context) {
 		}
 	}
 }
+
+// copy an object to or from the remote while accounting for it
+func copyObj(f fs.Fs, dst fs.Object, remote string, src fs.Object) (newDst fs.Object, err error) {
+	if operations.NeedTransfer(context.TODO(), dst, src) {
+		newDst, err = operations.Copy(context.TODO(), f, dst, remote, src)
+	} else {
+		newDst = dst
+	}
+	return newDst, err
+}
+
+// Check the local file is up to date in the cache
+func (c *Cache) Check(ctx context.Context, o fs.Object, remote string) error {
+	cacheObj, err := c.f.NewObject(ctx, remote)
+	if err == nil && cacheObj != nil {
+		_, err = copyObj(c.f, cacheObj, remote, o)
+		if err != nil {
+			return errors.Wrap(err, "failed to update cached file")
+		}
+	}
+	return nil
+}
+
+// Fetch fetches the object to the cache file
+func (c *Cache) Fetch(ctx context.Context, o fs.Object, remote string) error {
+	_, err := copyObj(c.f, nil, remote, o)
+	return err
+}
+
+// Store stores the local cache file to the remote object, returning
+// the new remote object. objOld is the old object if known.
+func (c *Cache) Store(ctx context.Context, f fs.Fs, objOld fs.Object, remote string) (fs.Object, error) {
+	// Transfer the temp file to the remote
+	cacheObj, err := c.f.NewObject(ctx, remote)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to find cache file")
+	}
+
+	if objOld != nil {
+		remote = objOld.Remote() // use the path of the actual object if available
+	}
+	o, err := copyObj(f, objOld, remote, cacheObj)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to transfer file from cache to remote")
+	}
+	return o, nil
+}

vfs/vfscache/vfscache_test.go

@@ -1,4 +1,4 @@
-package vfs
+package vfscache
 
 import (
 	"context"
@@ -11,42 +11,20 @@ import (
 	"time"
 
 	"github.com/djherbis/times"
+	_ "github.com/rclone/rclone/backend/local" // import the local backend
 	"github.com/rclone/rclone/fstest"
-	"github.com/spf13/pflag"
+	"github.com/rclone/rclone/vfs/vfscommon"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
-// Check CacheMode it satisfies the pflag interface
+// TestMain drives the tests
-var _ pflag.Value = (*CacheMode)(nil)
+func TestMain(m *testing.M) {
-
+	fstest.TestMain(m)
-func TestCacheModeString(t *testing.T) {
-	assert.Equal(t, "off", CacheModeOff.String())
-	assert.Equal(t, "full", CacheModeFull.String())
-	assert.Equal(t, "CacheMode(17)", CacheMode(17).String())
-}
-
-func TestCacheModeSet(t *testing.T) {
-	var m CacheMode
-
-	err := m.Set("full")
-	assert.NoError(t, err)
-	assert.Equal(t, CacheModeFull, m)
-
-	err = m.Set("potato")
-	assert.Error(t, err, "Unknown cache mode level")
-
-	err = m.Set("")
-	assert.Error(t, err, "Unknown cache mode level")
-}
-
-func TestCacheModeType(t *testing.T) {
-	var m CacheMode
-	assert.Equal(t, "CacheMode", m.Type())
 }
 
 // convert c.item to a string
-func itemAsString(c *cache) []string {
+func itemAsString(c *Cache) []string {
 	c.itemMu.Lock()
 	defer c.itemMu.Unlock()
 	var out []string
@@ -65,9 +43,9 @@ func TestCacheNew(t *testing.T) {
 	defer cancel()
 
 	// Disable the cache cleaner as it interferes with these tests
-	opt := DefaultOpt
+	opt := vfscommon.DefaultOpt
 	opt.CachePollInterval = 0
-	c, err := newCache(ctx, r.Fremote, &opt)
+	c, err := New(ctx, r.Fremote, &opt)
 	require.NoError(t, err)
 
 	assert.Contains(t, c.root, "vfs")
@@ -75,7 +53,7 @@ func TestCacheNew(t *testing.T) {
 	assert.Equal(t, []string(nil), itemAsString(c))
 
 	// mkdir
-	p, err := c.mkdir("potato")
+	p, err := c.Mkdir("potato")
 	require.NoError(t, err)
 	assert.Equal(t, "potato", filepath.Base(p))
 	assert.Equal(t, []string{
@@ -111,7 +89,7 @@ func TestCacheNew(t *testing.T) {
 		`name="" isFile=false opens=0 size=0`,
 		`name="potato" isFile=true opens=0 size=0`,
 	}, itemAsString(c))
-	c.open("/potato")
+	c.Open("/potato")
 	assert.Equal(t, []string{
 		`name="" isFile=false opens=1 size=0`,
 		`name="potato" isFile=true opens=1 size=0`,
@@ -173,7 +151,7 @@ func TestCacheNew(t *testing.T) {
 		`name="" isFile=false opens=1 size=0`,
 		`name="potato" isFile=true opens=1 size=6`,
 	}, itemAsString(c))
-	c.close("potato/")
+	c.Close("potato/")
 	assert.Equal(t, []string{
 		`name="" isFile=false opens=0 size=0`,
 		`name="potato" isFile=true opens=0 size=5`,
@@ -197,7 +175,7 @@ func TestCacheNew(t *testing.T) {
 	c.clean()
 
 	// cleanup
-	err = c.cleanUp()
+	err = c.CleanUp()
 	require.NoError(t, err)
 	_, err = os.Stat(c.root)
 	assert.True(t, os.IsNotExist(err))
@@ -210,35 +188,35 @@ func TestCacheOpens(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
-	c, err := newCache(ctx, r.Fremote, &DefaultOpt)
+	c, err := New(ctx, r.Fremote, &vfscommon.DefaultOpt)
 	require.NoError(t, err)
 
 	assert.Equal(t, []string(nil), itemAsString(c))
-	c.open("potato")
+	c.Open("potato")
 	assert.Equal(t, []string{
 		`name="" isFile=false opens=1 size=0`,
 		`name="potato" isFile=true opens=1 size=0`,
 	}, itemAsString(c))
-	c.open("potato")
+	c.Open("potato")
 	assert.Equal(t, []string{
 		`name="" isFile=false opens=2 size=0`,
 		`name="potato" isFile=true opens=2 size=0`,
 	}, itemAsString(c))
-	c.close("potato")
+	c.Close("potato")
 	assert.Equal(t, []string{
 		`name="" isFile=false opens=1 size=0`,
 		`name="potato" isFile=true opens=1 size=0`,
 	}, itemAsString(c))
-	c.close("potato")
+	c.Close("potato")
 	assert.Equal(t, []string{
 		`name="" isFile=false opens=0 size=0`,
 		`name="potato" isFile=true opens=0 size=0`,
 	}, itemAsString(c))
 
-	c.open("potato")
+	c.Open("potato")
-	c.open("a//b/c/d/one")
+	c.Open("a//b/c/d/one")
-	c.open("a/b/c/d/e/two")
+	c.Open("a/b/c/d/e/two")
-	c.open("a/b/c/d/e/f/three")
+	c.Open("a/b/c/d/e/f/three")
 	assert.Equal(t, []string{
 		`name="" isFile=false opens=4 size=0`,
 		`name="a" isFile=false opens=3 size=0`,
@@ -252,10 +230,10 @@ func TestCacheOpens(t *testing.T) {
 		`name="a/b/c/d/one" isFile=true opens=1 size=0`,
 		`name="potato" isFile=true opens=1 size=0`,
 	}, itemAsString(c))
-	c.close("potato")
+	c.Close("potato")
-	c.close("a/b/c/d/one")
+	c.Close("a/b/c/d/one")
-	c.close("a/b/c/d/e/two")
+	c.Close("a/b/c/d/e/two")
-	c.close("a/b/c//d/e/f/three")
+	c.Close("a/b/c//d/e/f/three")
 	assert.Equal(t, []string{
 		`name="" isFile=false opens=0 size=0`,
 		`name="a" isFile=false opens=0 size=0`,
@@ -280,13 +258,13 @@ func TestCacheOpenMkdir(t *testing.T) {
 	defer cancel()
 
 	// Disable the cache cleaner as it interferes with these tests
-	opt := DefaultOpt
+	opt := vfscommon.DefaultOpt
 	opt.CachePollInterval = 0
-	c, err := newCache(ctx, r.Fremote, &opt)
+	c, err := New(ctx, r.Fremote, &opt)
 	require.NoError(t, err)
 
 	// open
-	c.open("sub/potato")
+	c.Open("sub/potato")
 
 	assert.Equal(t, []string{
 		`name="" isFile=false opens=1 size=0`,
@@ -295,7 +273,7 @@ func TestCacheOpenMkdir(t *testing.T) {
 	}, itemAsString(c))
 
 	// mkdir
-	p, err := c.mkdir("sub/potato")
+	p, err := c.Mkdir("sub/potato")
 	require.NoError(t, err)
 	assert.Equal(t, "potato", filepath.Base(p))
 	assert.Equal(t, []string{
@@ -318,7 +296,7 @@ func TestCacheOpenMkdir(t *testing.T) {
 	assert.True(t, fi.IsDir())
 
 	// close
-	c.close("sub/potato")
+	c.Close("sub/potato")
 
 	assert.Equal(t, []string{
 		`name="" isFile=false opens=0 size=0`,
@@ -344,7 +322,7 @@ func TestCacheCacheDir(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
-	c, err := newCache(ctx, r.Fremote, &DefaultOpt)
+	c, err := New(ctx, r.Fremote, &vfscommon.DefaultOpt)
 	require.NoError(t, err)
 
 	assert.Equal(t, []string(nil), itemAsString(c))
@@ -379,7 +357,7 @@ func TestCachePurgeOld(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
-	c, err := newCache(ctx, r.Fremote, &DefaultOpt)
+	c, err := New(ctx, r.Fremote, &vfscommon.DefaultOpt)
 	require.NoError(t, err)
 
 	// Test funcs
@@ -400,10 +378,10 @@ func TestCachePurgeOld(t *testing.T) {
 	c._purgeEmptyDirs(removeDir)
 	assert.Equal(t, []string(nil), removed)
 
-	c.open("sub/dir2/potato2")
+	c.Open("sub/dir2/potato2")
-	c.open("sub/dir/potato")
+	c.Open("sub/dir/potato")
-	c.close("sub/dir2/potato2")
+	c.Close("sub/dir2/potato2")
-	c.open("sub/dir/potato")
+	c.Open("sub/dir/potato")
 
 	assert.Equal(t, []string{
 		`name="" isFile=false opens=2 size=0`,
@@ -423,7 +401,7 @@ func TestCachePurgeOld(t *testing.T) {
 		"sub/dir2/",
 	}, removed)
 
-	c.close("sub/dir/potato")
+	c.Close("sub/dir/potato")
 
 	removed = nil
 	removedDir = true
@@ -431,7 +409,7 @@ func TestCachePurgeOld(t *testing.T) {
 	c._purgeEmptyDirs(removeDir)
 	assert.Equal(t, []string(nil), removed)
 
-	c.close("sub/dir/potato")
+	c.Close("sub/dir/potato")
 
 	assert.Equal(t, []string{
 		`name="" isFile=false opens=0 size=0`,
@@ -475,16 +453,16 @@ func TestCachePurgeOverQuota(t *testing.T) {
 	defer cancel()
 
 	// Disable the cache cleaner as it interferes with these tests
-	opt := DefaultOpt
+	opt := vfscommon.DefaultOpt
 	opt.CachePollInterval = 0
-	c, err := newCache(ctx, r.Fremote, &opt)
+	c, err := New(ctx, r.Fremote, &opt)
 	require.NoError(t, err)
 
 	// Test funcs
 	var removed []string
 	remove := func(name string) {
 		removed = append(removed, filepath.ToSlash(name))
-		c.remove(name)
+		c.Remove(name)
 	}
 
 	removed = nil
@@ -500,14 +478,14 @@ func TestCachePurgeOverQuota(t *testing.T) {
 	assert.Equal(t, []string(nil), removed)
 
 	// Make some test files
-	c.open("sub/dir/potato")
+	c.Open("sub/dir/potato")
-	p, err := c.mkdir("sub/dir/potato")
+	p, err := c.Mkdir("sub/dir/potato")
 	require.NoError(t, err)
 	err = ioutil.WriteFile(p, []byte("hello"), 0600)
 	require.NoError(t, err)
 
-	p, err = c.mkdir("sub/dir2/potato2")
+	p, err = c.Mkdir("sub/dir2/potato2")
-	c.open("sub/dir2/potato2")
+	c.Open("sub/dir2/potato2")
 	require.NoError(t, err)
 	err = ioutil.WriteFile(p, []byte("hello2"), 0600)
 	require.NoError(t, err)
@@ -527,8 +505,8 @@ func TestCachePurgeOverQuota(t *testing.T) {
 	assert.Equal(t, []string(nil), removed)
 
 	// Close the files
-	c.close("sub/dir/potato")
+	c.Close("sub/dir/potato")
-	c.close("sub/dir2/potato2")
+	c.Close("sub/dir2/potato2")
 
 	assert.Equal(t, []string{
 		`name="" isFile=false opens=0 size=0`,
@@ -565,12 +543,12 @@ func TestCachePurgeOverQuota(t *testing.T) {
 	}, itemAsString(c))
 
 	// Put potato back
-	c.open("sub/dir/potato")
+	c.Open("sub/dir/potato")
-	p, err = c.mkdir("sub/dir/potato")
+	p, err = c.Mkdir("sub/dir/potato")
 	require.NoError(t, err)
 	err = ioutil.WriteFile(p, []byte("hello"), 0600)
 	require.NoError(t, err)
-	c.close("sub/dir/potato")
+	c.Close("sub/dir/potato")
 
 	// Update the stats to read the total size
 	err = c.updateStats()

vfs/vfscommon/cachemode.go

@@ -0,0 +1,49 @@
+package vfscommon
+
+import (
+	"fmt"
+
+	"github.com/rclone/rclone/lib/errors"
+)
+
+// CacheMode controls the functionality of the cache
+type CacheMode byte
+
+// CacheMode options
+const (
+	CacheModeOff     CacheMode = iota // cache nothing - return errors for writes which can't be satisfied
+	CacheModeMinimal                  // cache only the minimum, eg read/write opens
+	CacheModeWrites                   // cache all files opened with write intent
+	CacheModeFull                     // cache all files opened in any mode
+)
+
+var cacheModeToString = []string{
+	CacheModeOff:     "off",
+	CacheModeMinimal: "minimal",
+	CacheModeWrites:  "writes",
+	CacheModeFull:    "full",
+}
+
+// String turns a CacheMode into a string
+func (l CacheMode) String() string {
+	if l >= CacheMode(len(cacheModeToString)) {
+		return fmt.Sprintf("CacheMode(%d)", l)
+	}
+	return cacheModeToString[l]
+}
+
+// Set a CacheMode
+func (l *CacheMode) Set(s string) error {
+	for n, name := range cacheModeToString {
+		if s != "" && name == s {
+			*l = CacheMode(n)
+			return nil
+		}
+	}
+	return errors.Errorf("Unknown cache mode level %q", s)
+}
+
+// Type of the value
+func (l *CacheMode) Type() string {
+	return "CacheMode"
+}

vfs/vfscommon/cachemode_test.go

@@ -0,0 +1,36 @@
+package vfscommon
+
+import (
+	"testing"
+
+	"github.com/spf13/pflag"
+	"github.com/stretchr/testify/assert"
+)
+
+// Check CacheMode it satisfies the pflag interface
+var _ pflag.Value = (*CacheMode)(nil)
+
+func TestCacheModeString(t *testing.T) {
+	assert.Equal(t, "off", CacheModeOff.String())
+	assert.Equal(t, "full", CacheModeFull.String())
+	assert.Equal(t, "CacheMode(17)", CacheMode(17).String())
+}
+
+func TestCacheModeSet(t *testing.T) {
+	var m CacheMode
+
+	err := m.Set("full")
+	assert.NoError(t, err)
+	assert.Equal(t, CacheModeFull, m)
+
+	err = m.Set("potato")
+	assert.Error(t, err, "Unknown cache mode level")
+
+	err = m.Set("")
+	assert.Error(t, err, "Unknown cache mode level")
+}
+
+func TestCacheModeType(t *testing.T) {
+	var m CacheMode
+	assert.Equal(t, "CacheMode", m.Type())
+}

vfs/vfscommon/options.go

@@ -0,0 +1,57 @@
+package vfscommon
+
+import (
+	"os"
+	"runtime"
+	"time"
+
+	"github.com/rclone/rclone/fs"
+)
+
+// Options is options for creating the vfs
+type Options struct {
+	NoSeek            bool          // don't allow seeking if set
+	NoChecksum        bool          // don't check checksums if set
+	ReadOnly          bool          // if set VFS is read only
+	NoModTime         bool          // don't read mod times for files
+	DirCacheTime      time.Duration // how long to consider directory listing cache valid
+	PollInterval      time.Duration
+	Umask             int
+	UID               uint32
+	GID               uint32
+	DirPerms          os.FileMode
+	FilePerms         os.FileMode
+	ChunkSize         fs.SizeSuffix // if > 0 read files in chunks
+	ChunkSizeLimit    fs.SizeSuffix // if > ChunkSize double the chunk size after each chunk until reached
+	CacheMode         CacheMode
+	CacheMaxAge       time.Duration
+	CacheMaxSize      fs.SizeSuffix
+	CachePollInterval time.Duration
+	CaseInsensitive   bool
+	WriteWait         time.Duration // time to wait for in-sequence write
+	ReadWait          time.Duration // time to wait for in-sequence read
+}
+
+// DefaultOpt is the default values uses for Opt
+var DefaultOpt = Options{
+	NoModTime:         false,
+	NoChecksum:        false,
+	NoSeek:            false,
+	DirCacheTime:      5 * 60 * time.Second,
+	PollInterval:      time.Minute,
+	ReadOnly:          false,
+	Umask:             0,
+	UID:               ^uint32(0), // these values instruct WinFSP-FUSE to use the current user
+	GID:               ^uint32(0), // overriden for non windows in mount_unix.go
+	DirPerms:          os.FileMode(0777),
+	FilePerms:         os.FileMode(0666),
+	CacheMode:         CacheModeOff,
+	CacheMaxAge:       3600 * time.Second,
+	CachePollInterval: 60 * time.Second,
+	ChunkSize:         128 * fs.MebiByte,
+	ChunkSizeLimit:    -1,
+	CacheMaxSize:      -1,
+	CaseInsensitive:   runtime.GOOS == "windows" || runtime.GOOS == "darwin", // default to true on Windows and Mac, false otherwise
+	WriteWait:         1000 * time.Millisecond,
+	ReadWait:          5 * time.Millisecond,
+}

vfs/vfscommon/path.go

@@ -0,0 +1,12 @@
+package vfscommon
+
+import "path/filepath"
+
+// FindParent returns the parent directory of name, or "" for the root
+func FindParent(name string) string {
+	parent := filepath.Dir(name)
+	if parent == "." || parent == "/" {
+		parent = ""
+	}
+	return parent
+}

vfs/vfsflags/vfsflags.go

@@ -4,13 +4,13 @@ package vfsflags
 import (
 	"github.com/rclone/rclone/fs/config/flags"
 	"github.com/rclone/rclone/fs/rc"
-	"github.com/rclone/rclone/vfs"
+	"github.com/rclone/rclone/vfs/vfscommon"
 	"github.com/spf13/pflag"
 )
 
 // Options set by command line flags
 var (
-	Opt       = vfs.DefaultOpt
+	Opt       = vfscommon.DefaultOpt
 	DirPerms  = &FileMode{Mode: &Opt.DirPerms}
 	FilePerms = &FileMode{Mode: &Opt.FilePerms}
 )