fs: Add --dump flag, introduce --dump requests, responses and remove --dump-auth, --dump-filters

Now --dump-flag is written as --dump flag. This is a comma separated list which can contain

  * headers - HTTP headers as before
  * bodies  - HTTP bodies as before
  * requests - HTTP request bodies
  * responses - HTTP response bodies
  * auth - HTTP auth
  * filters - Filter rexeps

Leave --dump-headers and --dump-bodies for the time being but remove
the other --dump-* flags as they aren't used very often.

docs/content/docs.md

@@ -888,14 +888,20 @@ here which are used for testing.  These start with remote name eg
 
 Write CPU profile to file.  This can be analysed with `go tool pprof`.
 
-### --dump-auth ###
+#### --dump flag,flag,flag ####
 
-Dump HTTP headers - will contain sensitive info such as
-`Authorization:` headers - use `--dump-headers` to dump without
-`Authorization:` headers.  Can be very verbose.  Useful for debugging
+The `--dump` flag takes a comma separated list of flags to dump info
+about.  These are:
+
+#### --dump headers ####
+
+Dump HTTP headers with `Authorization:` lines removed. May still
+contain sensitive info.  Can be very verbose.  Useful for debugging
 only.
 
-### --dump-bodies ###
+Use `--dump auth` if you do want the `Authorization:` headers.
+
+#### --dump bodies ####
 
 Dump HTTP headers and bodies - may contain sensitive info.  Can be
 very verbose.  Useful for debugging only.
@@ -903,19 +909,28 @@ very verbose.  Useful for debugging only.
 Note that the bodies are buffered in memory so don't use this for
 enormous files.
 
-### --dump-filters ###
+#### --dump requests ####
+
+Like `--dump bodies` but dumps the request bodies and the response
+headers.  Useful for debugging download problems.
+
+#### --dump responses ####
+
+Like `--dump bodies` but dumps the response bodies and the request
+headers. Useful for debugging upload problems.
+
+#### --dump auth ####
+
+Dump HTTP headers - will contain sensitive info such as
+`Authorization:` headers - use `--dump headers` to dump without
+`Authorization:` headers.  Can be very verbose.  Useful for debugging
+only.
+
+#### --dump filters ####
 
 Dump the filters to the output.  Useful to see exactly what include
 and exclude options are filtering on.
 
-### --dump-headers ###
-
-Dump HTTP headers with `Authorization:` lines removed. May still
-contain sensitive info.  Can be very verbose.  Useful for debugging
-only.
-
-Use `--dump-auth` if you do want the `Authorization:` headers.
-
 ### --memprofile=FILE ###
 
 Write memory profile to file. This can be analysed with `go tool pprof`.
@@ -969,7 +984,7 @@ For the filtering options
   * `--max-size`
   * `--min-age`
   * `--max-age`
-  * `--dump-filters`
+  * `--dump filters`
 
 See the [filtering section](/filtering/).
 

docs/content/filtering.md

@@ -400,7 +400,7 @@ these are now excluded from the sync.
 
 Always test first with `--dry-run` and `-v` before using this flag.
 
-### `--dump-filters` - dump the filters to the output ###
+### `--dump filters` - dump the filters to the output ###
 
 This dumps the defined filters to the output as regular expressions.
 

fs/config.go

@@ -90,7 +90,6 @@ var (
 	timeout               = DurationP("timeout", "", 5*60*time.Second, "IO idle timeout")
 	dumpHeaders           = BoolP("dump-headers", "", false, "Dump HTTP headers - may contain sensitive info")
 	dumpBodies            = BoolP("dump-bodies", "", false, "Dump HTTP headers and bodies - may contain sensitive info")
-	dumpAuth              = BoolP("dump-auth", "", false, "Dump HTTP headers with auth info")
 	skipVerify            = BoolP("no-check-certificate", "", false, "Do not verify the server SSL certificate. Insecure.")
 	AskPassword           = BoolP("ask-password", "", true, "Allow prompt for password for encrypted configuration.")
 	deleteBefore          = BoolP("delete-before", "", false, "When synchronizing, delete files on destination before transfering")
@@ -116,6 +115,7 @@ var (
 	immutable             = BoolP("immutable", "", false, "Do not modify files. Fail if existing files have been modified.")
 	autoConfirm           = BoolP("auto-confirm", "", false, "If enabled, do not request console confirmation.")
 	streamingUploadCutoff = SizeSuffix(100 * 1024)
+	dump                  DumpFlags
 	logLevel              = LogLevelNotice
 	statsLogLevel         = LogLevelInfo
 	bwLimit               BwTimetable
@@ -132,6 +132,7 @@ func init() {
 	VarP(&bwLimit, "bwlimit", "", "Bandwidth limit in kBytes/s, or use suffix b|k|M|G or a full timetable.")
 	VarP(&bufferSize, "buffer-size", "", "Buffer size when copying files.")
 	VarP(&streamingUploadCutoff, "streaming-upload-cutoff", "", "Cutoff for switching to chunked upload if file size is unknown. Upload starts after reaching cutoff or when file ends.")
+	VarP(&dump, "dump", "", "List of items to dump from: "+dumpFlagsList)
 }
 
 // crypt internals
@@ -229,9 +230,7 @@ type ConfigInfo struct {
 	Transfers             int
 	ConnectTimeout        time.Duration // Connect timeout
 	Timeout               time.Duration // Data channel timeout
-	DumpHeaders           bool
-	DumpBodies            bool
-	DumpAuth              bool
+	Dump                  DumpFlags
 	Filter                *Filter
 	InsecureSkipVerify    bool // Skip server certificate verification
 	DeleteMode            DeleteMode
@@ -377,9 +376,6 @@ func LoadConfig() {
 	Config.SizeOnly = *sizeOnly
 	Config.IgnoreTimes = *ignoreTimes
 	Config.IgnoreExisting = *ignoreExisting
-	Config.DumpHeaders = *dumpHeaders
-	Config.DumpBodies = *dumpBodies
-	Config.DumpAuth = *dumpAuth
 	Config.InsecureSkipVerify = *skipVerify
 	Config.LowLevelRetries = *lowLevelRetries
 	Config.UpdateOlder = *updateOlder
@@ -398,6 +394,15 @@ func LoadConfig() {
 	Config.AutoConfirm = *autoConfirm
 	Config.BufferSize = bufferSize
 	Config.StreamingUploadCutoff = streamingUploadCutoff
+	Config.Dump = dump
+	if *dumpHeaders {
+		Config.Dump |= DumpHeaders
+		Infof(nil, "--dump-headers is obsolete - please use --dump headers instead")
+	}
+	if *dumpBodies {
+		Config.Dump |= DumpBodies
+		Infof(nil, "--dump-bodies is obsolete - please use --dump bodies instead")
+	}
 
 	Config.TrackRenames = *trackRenames
 
@@ -1460,3 +1465,87 @@ func makeCacheDir() (dir string) {
 	}
 	return filepath.Join(dir, "rclone")
 }
+
+// DumpFlags describes the Dump options in force
+type DumpFlags int
+
+// DumpFlags definitions
+const (
+	DumpHeaders DumpFlags = 1 << iota
+	DumpBodies
+	DumpRequests
+	DumpResponses
+	DumpAuth
+	DumpFilters
+)
+
+var dumpFlags = []struct {
+	flag DumpFlags
+	name string
+}{
+	{DumpHeaders, "headers"},
+	{DumpBodies, "bodies"},
+	{DumpRequests, "requests"},
+	{DumpResponses, "responses"},
+	{DumpAuth, "auth"},
+	{DumpFilters, "filters"},
+}
+
+// list of dump flags used in the help
+var dumpFlagsList string
+
+func init() {
+	// calculate the dump flags list
+	var out []string
+	for _, info := range dumpFlags {
+		out = append(out, info.name)
+	}
+	dumpFlagsList = strings.Join(out, ",")
+}
+
+// String turns a DumpFlags into a string
+func (f DumpFlags) String() string {
+	var out []string
+	for _, info := range dumpFlags {
+		if f&info.flag != 0 {
+			out = append(out, info.name)
+			f &^= info.flag
+		}
+	}
+	if f != 0 {
+		out = append(out, fmt.Sprintf("Unknown-0x%X", int(f)))
+	}
+	return strings.Join(out, ",")
+}
+
+// Set a DumpFlags as a comma separated list of flags
+func (f *DumpFlags) Set(s string) error {
+	var flags DumpFlags
+	parts := strings.Split(s, ",")
+	for _, part := range parts {
+		found := false
+		part = strings.ToLower(strings.TrimSpace(part))
+		if part == "" {
+			continue
+		}
+		for _, info := range dumpFlags {
+			if part == info.name {
+				found = true
+				flags |= info.flag
+			}
+		}
+		if !found {
+			return errors.Errorf("Unknown dump flag %q", part)
+		}
+	}
+	*f = flags
+	return nil
+}
+
+// Type of the value
+func (f *DumpFlags) Type() string {
+	return "string"
+}
+
+// Check it satisfies the interface
+var _ pflag.Value = (*DumpFlags)(nil)

fs/config_test.go

@@ -226,3 +226,50 @@ func hashedKeyCompare(t *testing.T, a, b string, shouldMatch bool) {
 		assert.NotEqual(t, k1, k2)
 	}
 }
+
+func TestDumpFlagsString(t *testing.T) {
+	assert.Equal(t, "", DumpFlags(0).String())
+	assert.Equal(t, "headers", (DumpHeaders).String())
+	assert.Equal(t, "headers,bodies", (DumpHeaders | DumpBodies).String())
+	assert.Equal(t, "headers,bodies,requests,responses,auth,filters", (DumpHeaders | DumpBodies | DumpRequests | DumpResponses | DumpAuth | DumpFilters).String())
+	assert.Equal(t, "headers,Unknown-0x8000", (DumpHeaders | DumpFlags(0x8000)).String())
+}
+
+func TestDumpFlagsSet(t *testing.T) {
+	for _, test := range []struct {
+		in      string
+		want    DumpFlags
+		wantErr string
+	}{
+		{"", DumpFlags(0), ""},
+		{"bodies", DumpBodies, ""},
+		{"bodies,headers,auth", DumpBodies | DumpHeaders | DumpAuth, ""},
+		{"bodies,headers,auth", DumpBodies | DumpHeaders | DumpAuth, ""},
+		{"headers,bodies,requests,responses,auth,filters", DumpHeaders | DumpBodies | DumpRequests | DumpResponses | DumpAuth | DumpFilters, ""},
+		{"headers,bodies,unknown,auth", 0, "Unknown dump flag \"unknown\""},
+	} {
+		f := DumpFlags(-1)
+		initial := f
+		err := f.Set(test.in)
+		if err != nil {
+			if test.wantErr == "" {
+				t.Errorf("Got an error when not expecting one on %q: %v", test.in, err)
+			} else {
+				assert.Contains(t, err.Error(), test.wantErr)
+			}
+			assert.Equal(t, initial, f, test.want)
+		} else {
+			if test.wantErr != "" {
+				t.Errorf("Got no error when expecting one on %q", test.in)
+			} else {
+				assert.Equal(t, test.want, f)
+			}
+		}
+
+	}
+}
+
+func TestDumpFlagsType(t *testing.T) {
+	f := DumpFlags(0)
+	assert.Equal(t, "string", f.Type())
+}

fs/filter.go

@@ -31,7 +31,6 @@ var (
 	maxAge         = StringP("max-age", "", "", "Don't transfer any file older than this in s or suffix ms|s|m|h|d|w|M|y")
 	minSize        = SizeSuffix(-1)
 	maxSize        = SizeSuffix(-1)
-	dumpFilters    = BoolP("dump-filters", "", false, "Dump the filters to the output")
 	//cvsExclude     = BoolP("cvs-exclude", "C", false, "Exclude files in the same way CVS does")
 )
 
@@ -249,7 +248,7 @@ func NewFilter() (f *Filter, err error) {
 		}
 		Debugf(nil, "--max-age %v to %v", duration, f.ModTimeFrom)
 	}
-	if *dumpFilters {
+	if Config.Dump&DumpFilters != 0 {
 		fmt.Println("--- start filters ---")
 		fmt.Println(f.DumpFilters())
 		fmt.Println("--- end filters ---")

fs/http.go

@@ -129,7 +129,7 @@ func (ci *ConfigInfo) Transport() http.RoundTripper {
 		//   t.ExpectContinueTimeout
 		ci.initTransport(t)
 		// Wrap that http.Transport in our own transport
-		transport = NewTransport(t, ci.DumpHeaders, ci.DumpBodies, ci.DumpAuth)
+		transport = NewTransport(t, ci.Dump)
 	})
 	return transport
 }
@@ -146,19 +146,15 @@ func (ci *ConfigInfo) Client() *http.Client {
 // * Does logging
 type Transport struct {
 	*http.Transport
-	logHeader bool
-	logBody   bool
-	logAuth   bool
+	dump DumpFlags
 }
 
 // NewTransport wraps the http.Transport passed in and logs all
 // roundtrips including the body if logBody is set.
-func NewTransport(transport *http.Transport, logHeader, logBody, logAuth bool) *Transport {
+func NewTransport(transport *http.Transport, dump DumpFlags) *Transport {
 	return &Transport{
 		Transport: transport,
-		logHeader: logHeader,
-		logBody:   logBody,
-		logAuth:   logAuth,
+		dump:      dump,
 	}
 }
 
@@ -243,9 +239,9 @@ func (t *Transport) RoundTrip(req *http.Request) (resp *http.Response, err error
 	// Force user agent
 	req.Header.Set("User-Agent", *userAgent)
 	// Logf request
-	if t.logHeader || t.logBody || t.logAuth {
-		buf, _ := httputil.DumpRequestOut(req, t.logBody)
-		if !t.logAuth {
+	if t.dump&(DumpHeaders|DumpBodies|DumpAuth|DumpRequests|DumpResponses) != 0 {
+		buf, _ := httputil.DumpRequestOut(req, t.dump&(DumpBodies|DumpRequests) != 0)
+		if t.dump&DumpAuth == 0 {
 			buf = cleanAuth(buf)
 		}
 		Debugf(nil, "%s", separatorReq)
@@ -256,13 +252,13 @@ func (t *Transport) RoundTrip(req *http.Request) (resp *http.Response, err error
 	// Do round trip
 	resp, err = t.Transport.RoundTrip(req)
 	// Logf response
-	if t.logHeader || t.logBody || t.logAuth {
+	if t.dump&(DumpHeaders|DumpBodies|DumpAuth|DumpRequests|DumpResponses) != 0 {
 		Debugf(nil, "%s", separatorResp)
 		Debugf(nil, "%s (req %p)", "HTTP RESPONSE", req)
 		if err != nil {
 			Debugf(nil, "Error: %v", err)
 		} else {
-			buf, _ := httputil.DumpResponse(resp, t.logBody)
+			buf, _ := httputil.DumpResponse(resp, t.dump&(DumpBodies|DumpResponses) != 0)
 			Debugf(nil, "%s", string(buf))
 		}
 		Debugf(nil, "%s", separatorResp)

fstest/fstest.go

@@ -59,8 +59,12 @@ func Initialise() {
 	if *Verbose {
 		fs.Config.LogLevel = fs.LogLevelDebug
 	}
-	fs.Config.DumpHeaders = *DumpHeaders
-	fs.Config.DumpBodies = *DumpBodies
+	if *DumpHeaders {
+		fs.Config.Dump |= fs.DumpHeaders
+	}
+	if *DumpBodies {
+		fs.Config.Dump |= fs.DumpBodies
+	}
 	fs.Config.LowLevelRetries = *LowLevelRetries
 	fs.Config.UseListR = *UseListR
 }