From c74c3b37da6f97d5619e82e0eecd9bdfdc178bde Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Sat, 6 Jan 2018 17:53:37 +0000 Subject: [PATCH] lsf: add option to print hashes --- cmd/lsf/lsf.go | 12 ++++++++++++ cmd/lsf/lsf_test.go | 10 ++++++++++ fs/hash.go | 26 ++++++++++++++++++++++++++ fs/operations.go | 41 +++++++++++++++++++++++++++++++---------- fs/operations_test.go | 16 ++++++++++++++++ 5 files changed, 95 insertions(+), 10 deletions(-) diff --git a/cmd/lsf/lsf.go b/cmd/lsf/lsf.go index e9d98f841..0f552e3cc 100644 --- a/cmd/lsf/lsf.go +++ b/cmd/lsf/lsf.go @@ -17,6 +17,7 @@ var ( separator string dirSlash bool recurse bool + hashType = fs.HashMD5 ) func init() { @@ -25,6 +26,7 @@ func init() { flags.StringVarP(&format, "format", "F", "p", "Output format - see help for details") flags.StringVarP(&separator, "separator", "s", ";", "Separator for the items in the format.") flags.BoolVarP(&dirSlash, "dir-slash", "d", true, "Append a slash to directory names.") + flags.VarP(&hashType, "hash", "", "Use this hash when `h` is used in the format MD5|SHA-1|DropboxHash") commandDefintion.Flags().BoolVarP(&recurse, "recursive", "R", false, "Recurse into the listing.") } @@ -44,10 +46,18 @@ output: p - path s - size t - modification time + h - hash So if you wanted the path, size and modification time, you would use --format "pst", or maybe --format "tsp" to put the path last. +If you specify "h" in the format you will get the MD5 hash by default, +use the "--hash" flag to change which hash you want. Note that this +can be returned as an empty string if it isn't available on the object +(and for directories), "ERROR" if there was an error reading it from +the object and "UNSUPPORTED" if that object does not support that hash +type. + By default the separator is ";" this can be changed with the --separator flag. Note that separators aren't escaped in the path so putting it last is a good strategy. @@ -76,6 +86,8 @@ func Lsf(fsrc fs.Fs, out io.Writer) error { list.AddModTime() case 's': list.AddSize() + case 'h': + list.AddHash(hashType) default: return errors.Errorf("Unknown format character %q", char) } diff --git a/cmd/lsf/lsf_test.go b/cmd/lsf/lsf_test.go index ec2268e73..7f3fec1dc 100644 --- a/cmd/lsf/lsf_test.go +++ b/cmd/lsf/lsf_test.go @@ -100,6 +100,16 @@ subdir 321 1234 -1 +`, buf.String()) + + buf = new(bytes.Buffer) + format = "hp" + err = Lsf(f, buf) + require.NoError(t, err) + assert.Equal(t, `d41d8cd98f00b204e9800998ecf8427e;file1 +409d6c19451dd39d4a94e42d2ff2c834;file2 +9b4c8a5e36d3be7e2c4b1d75ded8c8a1;file3 +;subdir `, buf.String()) buf = new(bytes.Buffer) diff --git a/fs/hash.go b/fs/hash.go index cd808fbd0..cdd2b0d36 100644 --- a/fs/hash.go +++ b/fs/hash.go @@ -11,6 +11,7 @@ import ( "github.com/ncw/rclone/dropbox/dbhash" "github.com/pkg/errors" + "github.com/spf13/pflag" ) // HashType indicates a standard hashing algorithm @@ -87,6 +88,31 @@ func (h HashType) String() string { } } +// Set a HashType from a flag +func (h *HashType) Set(s string) error { + switch s { + case "None": + *h = HashNone + case "MD5": + *h = HashMD5 + case "SHA-1": + *h = HashSHA1 + case "DropboxHash": + *h = HashDropbox + default: + return errors.Errorf("Unknown hash type %q", s) + } + return nil +} + +// Type of the value +func (h HashType) Type() string { + return "string" +} + +// Check it satisfies the interface +var _ pflag.Value = (*HashType)(nil) + // hashFromTypes will return hashers for all the requested types. // The types must be a subset of SupportedHashes, // and this function must support all types. diff --git a/fs/operations.go b/fs/operations.go index a3487cbc3..1cb39fad9 100644 --- a/fs/operations.go +++ b/fs/operations.go @@ -1068,17 +1068,24 @@ func DropboxHashSum(f Fs, w io.Writer) error { return hashLister(HashDropbox, f, w) } +// hashSum returns the human readable hash for ht passed in. This may +// be UNSUPPORTED or ERROR. +func hashSum(ht HashType, o Object) string { + Stats.Checking(o.Remote()) + sum, err := o.Hash(ht) + Stats.DoneChecking(o.Remote()) + if err == ErrHashUnsupported { + sum = "UNSUPPORTED" + } else if err != nil { + Debugf(o, "Failed to read %v: %v", ht, err) + sum = "ERROR" + } + return sum +} + func hashLister(ht HashType, f Fs, w io.Writer) error { return ListFn(f, func(o Object) { - Stats.Checking(o.Remote()) - sum, err := o.Hash(ht) - Stats.DoneChecking(o.Remote()) - if err == ErrHashUnsupported { - sum = "UNSUPPORTED" - } else if err != nil { - Debugf(o, "Failed to read %v: %v", ht, err) - sum = "ERROR" - } + sum := hashSum(ht, o) syncFprintf(w, "%*s %s\n", HashWidth[ht], sum, o.Remote()) }) } @@ -1792,6 +1799,7 @@ type ListFormat struct { dirSlash bool output []func() string entry DirEntry + hash bool } // SetSeparator changes separator in struct @@ -1816,7 +1824,9 @@ func (l *ListFormat) AddModTime() { // AddSize adds file's size to output func (l *ListFormat) AddSize() { - l.AppendOutput(func() string { return strconv.FormatInt(l.entry.Size(), 10) }) + l.AppendOutput(func() string { + return strconv.FormatInt(l.entry.Size(), 10) + }) } // AddPath adds path to file to output @@ -1831,6 +1841,17 @@ func (l *ListFormat) AddPath() { }) } +// AddHash adds the hash of the type given to the output +func (l *ListFormat) AddHash(ht HashType) { + l.AppendOutput(func() string { + o, ok := l.entry.(Object) + if !ok { + return "" + } + return hashSum(ht, o) + }) +} + // AppendOutput adds string generated by specific function to printed output func (l *ListFormat) AppendOutput(functionToAppend func() string) { if len(l.output) > 0 { diff --git a/fs/operations_test.go b/fs/operations_test.go index 5bfe81443..40bb1e0d2 100644 --- a/fs/operations_test.go +++ b/fs/operations_test.go @@ -999,4 +999,20 @@ func TestListFormat(t *testing.T) { list.SetSeparator("__SEP__") assert.Equal(t, "1__SEP__a__SEP__"+items[0].ModTime().Format("2006-01-02 15:04:05"), fs.ListFormatted(&items[0], &list)) assert.Equal(t, fmt.Sprintf("%d", items[1].Size())+"__SEP__subdir/__SEP__"+items[1].ModTime().Format("2006-01-02 15:04:05"), fs.ListFormatted(&items[1], &list)) + + for _, test := range []struct { + ht fs.HashType + want string + }{ + {fs.HashMD5, "0cc175b9c0f1b6a831c399e269772661"}, + {fs.HashSHA1, "86f7e437faa5a7fce15d1ddcb9eaeaea377667b8"}, + {fs.HashDropbox, "bf5d3affb73efd2ec6c36ad3112dd933efed63c4e1cbffcfa88e2759c144f2d8"}, + } { + list.SetOutput(nil) + list.AddHash(test.ht) + got := fs.ListFormatted(&items[0], &list) + if got != "UNSUPPORTED" { + assert.Equal(t, test.want, got) + } + } }