From 8f42532b6d8bb457f7a2e21948f2d6fcbdd52d06 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Tue, 9 Jun 2020 20:40:03 +0100 Subject: [PATCH] sync: add --track-renames-strategy leaf See: https://forum.rclone.org/t/how-to-minimize-bandwith-w-r-t-renames-during-sync/16928/22 --- docs/content/docs.md | 24 +++++++++++++----- fs/config/configflags/configflags.go | 2 +- fs/sync/sync.go | 15 ++++++++++- fs/sync/sync_test.go | 38 ++++++++++++++++++++++++++++ 4 files changed, 71 insertions(+), 8 deletions(-) diff --git a/docs/content/docs.md b/docs/content/docs.md index f2dd60522..ee7ec7c19 100644 --- a/docs/content/docs.md +++ b/docs/content/docs.md @@ -1324,13 +1324,25 @@ Note also that `--track-renames` is incompatible with `--delete-before` and will select `--delete-after` instead of `--delete-during`. -### --track-renames-strategy (hash,modtime) ### +### --track-renames-strategy (hash,modtime,leaf,size) ### -This option changes the matching criteria for `--track-renames` to match -by any combination of modtime, hash, size. Matching by size is always enabled -no matter what option is selected here. This also means -that it enables `--track-renames` support for encrypted destinations. -If nothing is specified, the default option is matching by hashes. +This option changes the matching criteria for `--track-renames`. + +The matching is controlled by a comma separated selection of these tokens: + +- `modtime` - the modification time of the file - not supported on all backends +- `hash` - the hash of the file contents - not supported on all backends +- `leaf` - the name of the file not including its directory name +- `size` - the size of the file (this is always enabled) + +So using `--track-renames-strategy modtime,leaf` would match files +based on modification time, the leaf of the file name and the size +only. + +Using `--track-renames-strategy modtime` or `leaf` can enable +`--track-renames` support for encrypted destinations. + +If nothing is specified, the default option is matching by `hash`es. ### --delete-(before,during,after) ### diff --git a/fs/config/configflags/configflags.go b/fs/config/configflags/configflags.go index 084b98fb7..2d2b9064f 100644 --- a/fs/config/configflags/configflags.go +++ b/fs/config/configflags/configflags.go @@ -65,7 +65,7 @@ func AddFlags(flagSet *pflag.FlagSet) { flags.BoolVarP(flagSet, &deleteAfter, "delete-after", "", false, "When synchronizing, delete files on destination after transferring (default)") flags.Int64VarP(flagSet, &fs.Config.MaxDelete, "max-delete", "", -1, "When synchronizing, limit the number of deletes") flags.BoolVarP(flagSet, &fs.Config.TrackRenames, "track-renames", "", fs.Config.TrackRenames, "When synchronizing, track file renames and do a server side move if possible") - flags.StringVarP(flagSet, &fs.Config.TrackRenamesStrategy, "track-renames-strategy", "", fs.Config.TrackRenamesStrategy, "Strategies to use when synchronizing using track-renames hash|modtime") + flags.StringVarP(flagSet, &fs.Config.TrackRenamesStrategy, "track-renames-strategy", "", fs.Config.TrackRenamesStrategy, "Strategies to use when synchronizing using track-renames hash|modtime|leaf") flags.IntVarP(flagSet, &fs.Config.LowLevelRetries, "low-level-retries", "", fs.Config.LowLevelRetries, "Number of low level retries to do.") flags.BoolVarP(flagSet, &fs.Config.UpdateOlder, "update", "u", fs.Config.UpdateOlder, "Skip files that are newer on the destination.") flags.BoolVarP(flagSet, &fs.Config.UseServerModTime, "use-server-modtime", "", fs.Config.UseServerModTime, "Use server modified time instead of object metadata") diff --git a/fs/sync/sync.go b/fs/sync/sync.go index 4a3c2b01a..fc6101b39 100644 --- a/fs/sync/sync.go +++ b/fs/sync/sync.go @@ -76,6 +76,7 @@ type trackRenamesStrategy byte const ( trackRenamesStrategyHash trackRenamesStrategy = 1 << iota trackRenamesStrategyModtime + trackRenamesStrategyLeaf ) func (strategy trackRenamesStrategy) hash() bool { @@ -86,6 +87,10 @@ func (strategy trackRenamesStrategy) modTime() bool { return (strategy & trackRenamesStrategyModtime) != 0 } +func (strategy trackRenamesStrategy) leaf() bool { + return (strategy & trackRenamesStrategyLeaf) != 0 +} + func newSyncCopyMove(ctx context.Context, fdst, fsrc fs.Fs, deleteMode fs.DeleteMode, DoMove bool, deleteEmptySrcDirs bool, copyEmptySrcDirs bool) (*syncCopyMove, error) { if (deleteMode != fs.DeleteModeOff || DoMove) && operations.Overlapping(fdst, fsrc) { return nil, fserrors.FatalError(fs.ErrorOverlapping) @@ -609,6 +614,8 @@ func parseTrackRenamesStrategy(strategies string) (strategy trackRenamesStrategy strategy |= trackRenamesStrategyHash case "modtime": strategy |= trackRenamesStrategyModtime + case "leaf": + strategy |= trackRenamesStrategyLeaf case "size": // ignore default: @@ -638,12 +645,18 @@ func (s *syncCopyMove) renameID(obj fs.Object, renamesStrategy trackRenamesStrat return "" } - fmt.Fprintf(&builder, ",%s", hash) + builder.WriteRune(',') + builder.WriteString(hash) } // for renamesStrategy.modTime() we don't add to the hash but we check the times in // popRenameMap + if renamesStrategy.leaf() { + builder.WriteRune(',') + builder.WriteString(path.Base(obj.Remote())) + } + return builder.String() } diff --git a/fs/sync/sync_test.go b/fs/sync/sync_test.go index 8a74c15b1..95cdfcef6 100644 --- a/fs/sync/sync_test.go +++ b/fs/sync/sync_test.go @@ -1162,6 +1162,44 @@ func TestSyncWithTrackRenamesStrategyModtime(t *testing.T) { } } +func TestSyncWithTrackRenamesStrategyLeaf(t *testing.T) { + r := fstest.NewRun(t) + defer r.Finalise() + + fs.Config.TrackRenames = true + fs.Config.TrackRenamesStrategy = "leaf" + defer func() { + fs.Config.TrackRenames = false + fs.Config.TrackRenamesStrategy = "hash" + }() + + canTrackRenames := operations.CanServerSideMove(r.Fremote) && r.Fremote.Precision() != fs.ModTimeNotSupported + t.Logf("Can track renames: %v", canTrackRenames) + + f1 := r.WriteFile("potato", "Potato Content", t1) + f2 := r.WriteFile("sub/yam", "Yam Content", t2) + + accounting.GlobalStats().ResetCounters() + require.NoError(t, Sync(context.Background(), r.Fremote, r.Flocal, false)) + + fstest.CheckItems(t, r.Fremote, f1, f2) + fstest.CheckItems(t, r.Flocal, f1, f2) + + // Now rename locally. + f2 = r.RenameFile(f2, "yam") + + accounting.GlobalStats().ResetCounters() + require.NoError(t, Sync(context.Background(), r.Fremote, r.Flocal, false)) + + fstest.CheckItems(t, r.Fremote, f1, f2) + + // Check we renamed something if we should have + if canTrackRenames { + renames := accounting.GlobalStats().Renames(0) + assert.Equal(t, canTrackRenames, renames != 0, fmt.Sprintf("canTrackRenames=%v, renames=%d", canTrackRenames, renames)) + } +} + func toyFileTransfers(r *fstest.Run) int64 { remote := r.Fremote.Name() transfers := 1