go/rclone
1
0
Fork 0
mirror of https://github.com/rclone/rclone.git synced 2025-02-20 07:48:33 +02:00
Code Issues Releases Activity

Fix deletion of some excluded files without --delete-excluded #205

Browse Source 
This only happened if the destination file was present but the source
file was missing.
This commit is contained in:
Nick Craig-Wood 2015-11-12 11:46:04 +00:00
parent 7beb50caa7
commit 80ffbade22
2 changed files with 33 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
fs/operations.go
View File

@ -381,12 +381,18 @@ func DeleteFiles(toBeDeleted ObjectsChan) {
} }
// Read a map of Object.Remote to Object for the given Fs // Read a map of Object.Remote to Object for the given Fs
func readFilesMap(fs Fs) map[string]Object { func readFilesMap(fs Fs, obeyInclude bool) map[string]Object {
files := make(map[string]Object) files := make(map[string]Object)
for o := range fs.List() { for o := range fs.List() {
remote := o.Remote() remote := o.Remote()
if _, ok := files[remote]; !ok { if _, ok := files[remote]; !ok {
files[remote] = o // Make sure we don't delete excluded files if not required
if !obeyInclude || Config.Filter.DeleteExcluded || Config.Filter.Include(remote, o.Size()) {
files[remote] = o
} else {
Debug(o, "Excluded from sync (and deletion)")
}
} else { } else {
Log(o, "Duplicate file detected") Log(o, "Duplicate file detected")
} }
@ -420,7 +426,7 @@ func syncCopyMove(fdst, fsrc Fs, Delete bool, DoMove bool) error {
// Read the destination files first // Read the destination files first
// FIXME could do this in parallel and make it use less memory // FIXME could do this in parallel and make it use less memory
delFiles := readFilesMap(fdst) delFiles := readFilesMap(fdst, true)
// Read source files checking them off against dest files // Read source files checking them off against dest files
toBeChecked := make(ObjectPairChan, Config.Transfers) toBeChecked := make(ObjectPairChan, Config.Transfers)
@ -445,14 +451,10 @@ func syncCopyMove(fdst, fsrc Fs, Delete bool, DoMove bool) error {
go func() { go func() {
for src := range fsrc.List() { for src := range fsrc.List() {
remote := src.Remote() remote := src.Remote()
dst, dstFound := delFiles[remote]
if !Config.Filter.Include(remote, src.Size()) { if !Config.Filter.Include(remote, src.Size()) {
Debug(src, "Excluding from sync") Debug(src, "Excluding from sync")
if dstFound && !Config.Filter.DeleteExcluded {
delete(delFiles, remote)
}
} else { } else {
if dstFound { if dst, dstFound := delFiles[remote]; dstFound {
delete(delFiles, remote) delete(delFiles, remote)
toBeChecked <- ObjectPair{src, dst} toBeChecked <- ObjectPair{src, dst}
} else { } else {
@ -539,11 +541,11 @@ func Check(fdst, fsrc Fs) error {
// Read the destination files first // Read the destination files first
// FIXME could do this in parallel and make it use less memory // FIXME could do this in parallel and make it use less memory
dstFiles := readFilesMap(fdst) dstFiles := readFilesMap(fdst, false)
// Read the source files checking them against dstFiles // Read the source files checking them against dstFiles
// FIXME could do this in parallel and make it use less memory // FIXME could do this in parallel and make it use less memory
srcFiles := readFilesMap(fsrc) srcFiles := readFilesMap(fsrc, false)
// Move all the common files into commonFiles and delete then // Move all the common files into commonFiles and delete then
// from srcFiles and dstFiles // from srcFiles and dstFiles

26
fs/operations_test.go
View File

@ -411,7 +411,7 @@ func TestSyncAfterRemovingAFileAndAddingAFile(t *testing.T) {
// Test with exclude // Test with exclude
func TestSyncWithExclude(t *testing.T) { func TestSyncWithExclude(t *testing.T) {
WriteFile("enormous", "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", t1) // 100 bytes WriteFile("enormous", "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", t1) // 100 bytes
fs.Config.Filter.MaxSize = 80 fs.Config.Filter.MaxSize = 40
defer func() { defer func() {
fs.Config.Filter.MaxSize = 0 fs.Config.Filter.MaxSize = 0
}() }()
@ -424,6 +424,17 @@ func TestSyncWithExclude(t *testing.T) {
{Path: "potato2", Size: 60, ModTime: t1, Md5sum: "d6548b156ea68a4e003e786df99eee76"}, {Path: "potato2", Size: 60, ModTime: t1, Md5sum: "d6548b156ea68a4e003e786df99eee76"},
} }
fstest.CheckListingWithPrecision(t, fremote, items, fs.Config.ModifyWindow) fstest.CheckListingWithPrecision(t, fremote, items, fs.Config.ModifyWindow)
// Now sync the other way round and check enormous doesn't get
// deleted as it is excluded from the sync
items = append(items, fstest.Item{
Path: "enormous", Size: 100, ModTime: t1, Md5sum: "8adc5937e635f6c9af646f0b23560fae",
})
err = fs.Sync(flocal, fremote)
if err != nil {
t.Fatalf("Sync failed: %v", err)
}
fstest.CheckListingWithPrecision(t, flocal, items, fs.Config.ModifyWindow)
} }
// Test with exclude and delete excluded // Test with exclude and delete excluded
@ -444,12 +455,17 @@ func TestSyncWithExcludeAndDeleleteExcluded(t *testing.T) {
} }
fstest.CheckListingWithPrecision(t, fremote, items, fs.Config.ModifyWindow) fstest.CheckListingWithPrecision(t, fremote, items, fs.Config.ModifyWindow)
// Tidy up // Check sync the other way round to make sure enormous gets
reset() // deleted even though it is excluded
err = os.Remove(localName + "/enormous") err = fs.Sync(flocal, fremote)
if err != nil { if err != nil {
t.Fatalf("Remove failed: %v", err) t.Fatalf("Sync failed: %v", err)
} }
fstest.CheckListingWithPrecision(t, flocal, items, fs.Config.ModifyWindow)
// Tidy up - put potato2 back!
reset()
WriteFile("potato2", "------------------------------------------------------------", t1)
err = fs.Sync(fremote, flocal) err = fs.Sync(fremote, flocal)
if err != nil { if err != nil {
t.Fatalf("Sync failed: %v", err) t.Fatalf("Sync failed: %v", err)