diff --git a/cmd/cmd.go b/cmd/cmd.go index b540bc215..c8695b292 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -14,7 +14,6 @@ import ( "os" "os/exec" "path" - "regexp" "runtime" "runtime/pprof" "strconv" @@ -49,7 +48,6 @@ var ( cpuProfile = flags.StringP("cpuprofile", "", "", "Write cpu profile to file", "Debugging") memProfile = flags.StringP("memprofile", "", "", "Write memory profile to file", "Debugging") statsInterval = flags.DurationP("stats", "", time.Minute*1, "Interval between printing stats, e.g. 500ms, 60s, 5m (0 to disable)", "Logging") - dataRateUnit = flags.StringP("stats-unit", "", "bytes", "Show data rate in stats as either 'bits' or 'bytes' per second", "Logging") version bool // Errors errorCommandNotFound = errors.New("command not found") @@ -472,13 +470,6 @@ func initConfig() { } }) } - - if m, _ := regexp.MatchString("^(bits|bytes)$", *dataRateUnit); !m { - fs.Errorf(nil, "Invalid unit passed to --stats-unit. Defaulting to bytes.") - ci.DataRateUnit = "bytes" - } else { - ci.DataRateUnit = *dataRateUnit - } } func resolveExitCode(err error) { diff --git a/cmdtest/environment_test.go b/cmdtest/environment_test.go index f0688200d..5a56a692a 100644 --- a/cmdtest/environment_test.go +++ b/cmdtest/environment_test.go @@ -66,7 +66,7 @@ func TestEnvironmentVariables(t *testing.T) { assert.NotContains(t, out, "fileAA1.txt") // depth 4 } - // Test of debug logging while initialising flags from environment (tests #5241 Enhance1) + // Test of debug logging while initialising flags from environment (tests #5341 Enhance1) env = "RCLONE_STATS=173ms" out, err = rcloneEnv(env, "version", "-vv") if assert.NoError(t, err) { diff --git a/fs/config.go b/fs/config.go index 435fbf8e7..bd3d7aa89 100644 --- a/fs/config.go +++ b/fs/config.go @@ -3,6 +3,7 @@ package fs import ( "context" "errors" + "fmt" "net" "os" "strconv" @@ -13,7 +14,7 @@ import ( // Global var ( // globalConfig for rclone - globalConfig = NewConfig() + globalConfig = new(ConfigInfo) // Read a value from the config file // @@ -49,184 +50,693 @@ var ( ConfigEdit = "config_fs_edit" ) +// ConfigOptionsInfo describes the Options in use +var ConfigOptionsInfo = Options{{ + Name: "modify_window", + Default: time.Nanosecond, + Help: "Max time diff to be considered the same", + Groups: "Copy", +}, { + Name: "checkers", + Default: 8, + Help: "Number of checkers to run in parallel", + Groups: "Performance", +}, { + Name: "transfers", + Default: 4, + Help: "Number of file transfers to run in parallel", + Groups: "Performance", +}, { + Name: "checksum", + ShortOpt: "c", + Default: false, + Help: "Check for changes with size & checksum (if available, or fallback to size only).", + Groups: "Copy", +}, { + Name: "size_only", + Default: false, + Help: "Skip based on size only, not modtime or checksum", + Groups: "Copy", +}, { + Name: "ignore_times", + ShortOpt: "I", + Default: false, + Help: "Don't skip items that match size and time - transfer all unconditionally", + Groups: "Copy", +}, { + Name: "ignore_existing", + Default: false, + Help: "Skip all files that exist on destination", + Groups: "Copy", +}, { + Name: "ignore_errors", + Default: false, + Help: "Delete even if there are I/O errors", + Groups: "Sync", +}, { + Name: "dry_run", + ShortOpt: "n", + Default: false, + Help: "Do a trial run with no permanent changes", + Groups: "Config,Important", +}, { + Name: "interactive", + ShortOpt: "i", + Default: false, + Help: "Enable interactive mode", + Groups: "Config,Important", +}, { + Name: "contimeout", + Default: 60 * time.Second, + Help: "Connect timeout", + Groups: "Networking", +}, { + Name: "timeout", + Default: 5 * 60 * time.Second, + Help: "IO idle timeout", + Groups: "Networking", +}, { + Name: "expect_continue_timeout", + Default: 1 * time.Second, + Help: "Timeout when using expect / 100-continue in HTTP", + Groups: "Networking", +}, { + Name: "no_check_certificate", + Default: false, + Help: "Do not verify the server SSL certificate (insecure)", + Groups: "Networking", +}, { + Name: "ask_password", + Default: true, + Help: "Allow prompt for password for encrypted configuration", + Groups: "Config", +}, { + Name: "password_command", + Default: SpaceSepList{}, + Help: "Command for supplying password for encrypted configuration", + Groups: "Config", +}, { + Name: "max_delete", + Default: int64(-1), + Help: "When synchronizing, limit the number of deletes", + Groups: "Sync", +}, { + Name: "max_delete_size", + Default: SizeSuffix(-1), + Help: "When synchronizing, limit the total size of deletes", + Groups: "Sync", +}, { + Name: "track_renames", + Default: false, + Help: "When synchronizing, track file renames and do a server-side move if possible", + Groups: "Sync", +}, { + Name: "track_renames_strategy", + Default: "hash", + Help: "Strategies to use when synchronizing using track-renames hash|modtime|leaf", + Groups: "Sync", +}, { + Name: "retries", + Default: 3, + Help: "Retry operations this many times if they fail", + Groups: "Config", +}, { + Name: "retries_sleep", + Default: time.Duration(0), + Help: "Interval between retrying operations if they fail, e.g. 500ms, 60s, 5m (0 to disable)", + Groups: "Config", +}, { + Name: "low_level_retries", + Default: 10, + Help: "Number of low level retries to do", + Groups: "Config", +}, { + Name: "update", + ShortOpt: "u", + Default: false, + Help: "Skip files that are newer on the destination", + Groups: "Copy", +}, { + Name: "use_server_modtime", + Default: false, + Help: "Use server modified time instead of object metadata", + Groups: "Config", +}, { + Name: "no_gzip_encoding", + Default: false, + Help: "Don't set Accept-Encoding: gzip", + Groups: "Networking", +}, { + Name: "max_depth", + Default: -1, + Help: "If set limits the recursion depth to this", + Groups: "Filter", +}, { + Name: "ignore_size", + Default: false, + Help: "Ignore size when skipping use modtime or checksum", + Groups: "Copy", +}, { + Name: "ignore_checksum", + Default: false, + Help: "Skip post copy check of checksums", + Groups: "Copy", +}, { + Name: "ignore_case_sync", + Default: false, + Help: "Ignore case when synchronizing", + Groups: "Copy", +}, { + Name: "fix_case", + Default: false, + Help: "Force rename of case insensitive dest to match source", + Groups: "Sync", +}, { + Name: "no_traverse", + Default: false, + Help: "Don't traverse destination file system on copy", + Groups: "Copy", +}, { + Name: "check_first", + Default: false, + Help: "Do all the checks before starting transfers", + Groups: "Copy", +}, { + Name: "no_check_dest", + Default: false, + Help: "Don't check the destination, copy regardless", + Groups: "Copy", +}, { + Name: "no_unicode_normalization", + Default: false, + Help: "Don't normalize unicode characters in filenames", + Groups: "Config", +}, { + Name: "no_update_modtime", + Default: false, + Help: "Don't update destination modtime if files identical", + Groups: "Copy", +}, { + Name: "no_update_dir_modtime", + Default: false, + Help: "Don't update directory modification times", + Groups: "Copy", +}, { + Name: "compare_dest", + Default: []string{}, + Help: "Include additional server-side paths during comparison", + Groups: "Copy", +}, { + Name: "copy_dest", + Default: []string{}, + Help: "Implies --compare-dest but also copies files from paths into destination", + Groups: "Copy", +}, { + Name: "backup_dir", + Default: "", + Help: "Make backups into hierarchy based in DIR", + Groups: "Sync", +}, { + Name: "suffix", + Default: "", + Help: "Suffix to add to changed files", + Groups: "Sync", +}, { + Name: "suffix_keep_extension", + Default: false, + Help: "Preserve the extension when using --suffix", + Groups: "Sync", +}, { + Name: "fast_list", + Default: false, + Help: "Use recursive list if available; uses more memory but fewer transactions", + Groups: "Listing", +}, { + Name: "tpslimit", + Default: 0.0, + Help: "Limit HTTP transactions per second to this", + Groups: "Networking", +}, { + Name: "tpslimit_burst", + Default: 1, + Help: "Max burst of transactions for --tpslimit", + Groups: "Networking", +}, { + Name: "user_agent", + Default: "rclone/" + Version, + Help: "Set the user-agent to a specified string", + Groups: "Networking", +}, { + Name: "immutable", + Default: false, + Help: "Do not modify files, fail if existing files have been modified", + Groups: "Copy", +}, { + Name: "auto_confirm", + Default: false, + Help: "If enabled, do not request console confirmation", + Groups: "Config", +}, { + Name: "stats_unit", + Default: "bytes", + Help: "Show data rate in stats as either 'bits' or 'bytes' per second", + Groups: "Logging", +}, { + Name: "stats_file_name_length", + Default: 45, + Help: "Max file name length in stats (0 for no limit)", + Groups: "Logging", +}, { + Name: "log_level", + Default: LogLevelNotice, + Help: "Log level DEBUG|INFO|NOTICE|ERROR", + Groups: "Logging", +}, { + Name: "stats_log_level", + Default: LogLevelInfo, + Help: "Log level to show --stats output DEBUG|INFO|NOTICE|ERROR", + Groups: "Logging", +}, { + Name: "bwlimit", + Default: BwTimetable{}, + Help: "Bandwidth limit in KiB/s, or use suffix B|K|M|G|T|P or a full timetable", + Groups: "Networking", +}, { + Name: "bwlimit_file", + Default: BwTimetable{}, + Help: "Bandwidth limit per file in KiB/s, or use suffix B|K|M|G|T|P or a full timetable", + Groups: "Networking", +}, { + Name: "buffer_size", + Default: SizeSuffix(16 << 20), + Help: "In memory buffer size when reading files for each --transfer", + Groups: "Performance", +}, { + Name: "streaming_upload_cutoff", + Default: SizeSuffix(100 * 1024), + Help: "Cutoff for switching to chunked upload if file size is unknown, upload starts after reaching cutoff or when file ends", + Groups: "Copy", +}, { + Name: "dump", + Default: DumpFlags(0), + Help: "List of items to dump from: " + DumpFlagsList, + Groups: "Debugging", +}, { + Name: "max_transfer", + Default: SizeSuffix(-1), + Help: "Maximum size of data to transfer", + Groups: "Copy", +}, { + Name: "max_duration", + Default: time.Duration(0), + Help: "Maximum duration rclone will transfer data for", + Groups: "Copy", +}, { + Name: "cutoff_mode", + Default: CutoffMode(0), + Help: "Mode to stop transfers when reaching the max transfer limit HARD|SOFT|CAUTIOUS", + Groups: "Copy", +}, { + Name: "max_backlog", + Default: 10000, + Help: "Maximum number of objects in sync or check backlog", + Groups: "Copy,Check", +}, { + Name: "max_stats_groups", + Default: 1000, + Help: "Maximum number of stats groups to keep in memory, on max oldest is discarded", + Groups: "Logging", +}, { + Name: "stats_one_line", + Default: false, + Help: "Make the stats fit on one line", + Groups: "Logging", +}, { + Name: "stats_one_line_date", + Default: false, + Help: "Enable --stats-one-line and add current date/time prefix", + Groups: "Logging", +}, { + Name: "stats_one_line_date_format", + Default: "", + Help: "Enable --stats-one-line-date and use custom formatted date: Enclose date string in double quotes (\"), see https://golang.org/pkg/time/#Time.Format", + Groups: "Logging", +}, { + Name: "error_on_no_transfer", + Default: false, + Help: "Sets exit code 9 if no files are transferred, useful in scripts", + Groups: "Config", +}, { + Name: "progress", + ShortOpt: "P", + Default: false, + Help: "Show progress during transfer", + Groups: "Logging", +}, { + Name: "progress_terminal_title", + Default: false, + Help: "Show progress on the terminal title (requires -P/--progress)", + Groups: "Logging", +}, { + Name: "use_cookies", + Default: false, + Help: "Enable session cookiejar", + Groups: "Networking", +}, { + Name: "use_mmap", + Default: false, + Help: "Use mmap allocator (see docs)", + Groups: "Config", +}, { + Name: "ca_cert", + Default: []string{}, + Help: "CA certificate used to verify servers", + Groups: "Networking", +}, { + Name: "client_cert", + Default: "", + Help: "Client SSL certificate (PEM) for mutual TLS auth", + Groups: "Networking", +}, { + Name: "client_key", + Default: "", + Help: "Client SSL private key (PEM) for mutual TLS auth", + Groups: "Networking", +}, { + Name: "multi_thread_cutoff", + Default: SizeSuffix(256 * 1024 * 1024), + Help: "Use multi-thread downloads for files above this size", + Groups: "Copy", +}, { + Name: "multi_thread_streams", + Default: 4, + Help: "Number of streams to use for multi-thread downloads", + Groups: "Copy", +}, { + Name: "multi_thread_write_buffer_size", + Default: SizeSuffix(128 * 1024), + Help: "In memory buffer size for writing when in multi-thread mode", + Groups: "Copy", +}, { + Name: "multi_thread_chunk_size", + Default: SizeSuffix(64 * 1024 * 1024), + Help: "Chunk size for multi-thread downloads / uploads, if not set by filesystem", + Groups: "Copy", +}, { + Name: "use_json_log", + Default: false, + Help: "Use json log format", + Groups: "Logging", +}, { + Name: "order_by", + Default: "", + Help: "Instructions on how to order the transfers, e.g. 'size,descending'", + Groups: "Copy", +}, { + Name: "refresh_times", + Default: false, + Help: "Refresh the modtime of remote files", + Groups: "Copy", +}, { + Name: "no_console", + Default: false, + Help: "Hide console window (supported on Windows only)", + Groups: "Config", +}, { + Name: "fs_cache_expire_duration", + Default: 300 * time.Second, + Help: "Cache remotes for this long (0 to disable caching)", + Groups: "Config", +}, { + Name: "fs_cache_expire_interval", + Default: 60 * time.Second, + Help: "Interval to check for expired remotes", + Groups: "Config", +}, { + Name: "disable_http2", + Default: false, + Help: "Disable HTTP/2 in the global transport", + Groups: "Networking", +}, { + Name: "human_readable", + Default: false, + Help: "Print numbers in a human-readable format, sizes with suffix Ki|Mi|Gi|Ti|Pi", + Groups: "Config", +}, { + Name: "kv_lock_time", + Default: 1 * time.Second, + Help: "Maximum time to keep key-value database locked by process", + Groups: "Config", +}, { + Name: "disable_http_keep_alives", + Default: false, + Help: "Disable HTTP keep-alives and use each connection once.", + Groups: "Networking", +}, { + Name: "metadata", + ShortOpt: "M", + Default: false, + Help: "If set, preserve metadata when copying objects", + Groups: "Metadata,Copy", +}, { + Name: "server_side_across_configs", + Default: false, + Help: "Allow server-side operations (e.g. copy) to work across different configs", + Groups: "Copy", +}, { + Name: "color", + Default: TerminalColorMode(0), + Help: "When to show colors (and other ANSI codes) AUTO|NEVER|ALWAYS", + Groups: "Config", +}, { + Name: "default_time", + Default: Time(time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC)), + Help: "Time to show if modtime is unknown for files and directories", + Groups: "Config,Listing", +}, { + Name: "inplace", + Default: false, + Help: "Download directly to destination file instead of atomic download to temp/rename", + Groups: "Copy", +}, { + Name: "metadata_mapper", + Default: SpaceSepList{}, + Help: "Program to run to transforming metadata before upload", + Groups: "Metadata", +}, { + Name: "partial_suffix", + Default: ".partial", + Help: "Add partial-suffix to temporary file name when --inplace is not used", + Groups: "Copy", +}} + // ConfigInfo is filesystem config options type ConfigInfo struct { - LogLevel LogLevel - StatsLogLevel LogLevel - UseJSONLog bool - DryRun bool - Interactive bool - CheckSum bool - SizeOnly bool - IgnoreTimes bool - IgnoreExisting bool - IgnoreErrors bool - ModifyWindow time.Duration - Checkers int - Transfers int - ConnectTimeout time.Duration // Connect timeout - Timeout time.Duration // Data channel timeout - ExpectContinueTimeout time.Duration - Dump DumpFlags - InsecureSkipVerify bool // Skip server certificate verification - DeleteMode DeleteMode - MaxDelete int64 - MaxDeleteSize SizeSuffix - TrackRenames bool // Track file renames. - TrackRenamesStrategy string // Comma separated list of strategies used to track renames - Retries int // High-level retries - RetriesInterval time.Duration // --retries-sleep - LowLevelRetries int - UpdateOlder bool // Skip files that are newer on the destination - NoGzip bool // Disable compression - MaxDepth int - IgnoreSize bool - IgnoreChecksum bool - IgnoreCaseSync bool - FixCase bool - NoTraverse bool - CheckFirst bool - NoCheckDest bool - NoUnicodeNormalization bool - NoUpdateModTime bool - NoUpdateDirModTime bool - DataRateUnit string - CompareDest []string - CopyDest []string - BackupDir string - Suffix string - SuffixKeepExtension bool - UseListR bool - BufferSize SizeSuffix - BwLimit BwTimetable - BwLimitFile BwTimetable - TPSLimit float64 - TPSLimitBurst int - BindAddr net.IP - DisableFeatures []string - UserAgent string - Immutable bool - AutoConfirm bool - StreamingUploadCutoff SizeSuffix - StatsFileNameLength int - AskPassword bool - PasswordCommand SpaceSepList - UseServerModTime bool - MaxTransfer SizeSuffix - MaxDuration time.Duration - CutoffMode CutoffMode - MaxBacklog int - MaxStatsGroups int - StatsOneLine bool - StatsOneLineDate bool // If we want a date prefix at all - StatsOneLineDateFormat string // If we want to customize the prefix - ErrorOnNoTransfer bool // Set appropriate exit code if no files transferred - Progress bool - ProgressTerminalTitle bool - Cookie bool - UseMmap bool - CaCert []string // Client Side CA - ClientCert string // Client Side Cert - ClientKey string // Client Side Key - MultiThreadCutoff SizeSuffix - MultiThreadStreams int - MultiThreadSet bool // whether MultiThreadStreams was set (set in fs/config/configflags) - MultiThreadChunkSize SizeSuffix // Chunk size for multi-thread downloads / uploads, if not set by filesystem - MultiThreadWriteBufferSize SizeSuffix - OrderBy string // instructions on how to order the transfer - UploadHeaders []*HTTPOption - DownloadHeaders []*HTTPOption - Headers []*HTTPOption - MetadataSet Metadata // extra metadata to write when uploading - RefreshTimes bool - NoConsole bool - TrafficClass uint8 - FsCacheExpireDuration time.Duration - FsCacheExpireInterval time.Duration - DisableHTTP2 bool - HumanReadable bool - KvLockTime time.Duration // maximum time to keep key-value database locked by process - DisableHTTPKeepAlives bool - Metadata bool - ServerSideAcrossConfigs bool - TerminalColorMode TerminalColorMode - DefaultTime Time // time that directories with no time should display - Inplace bool // Download directly to destination file instead of atomic download to temp/rename - PartialSuffix string - MetadataMapper SpaceSepList + LogLevel LogLevel `config:"log_level"` + StatsLogLevel LogLevel `config:"stats_log_level"` + UseJSONLog bool `config:"use_json_log"` + DryRun bool `config:"dry_run"` + Interactive bool `config:"interactive"` + CheckSum bool `config:"checksum"` + SizeOnly bool `config:"size_only"` + IgnoreTimes bool `config:"ignore_times"` + IgnoreExisting bool `config:"ignore_existing"` + IgnoreErrors bool `config:"ignore_errors"` + ModifyWindow time.Duration `config:"modify_window"` + Checkers int `config:"checkers"` + Transfers int `config:"transfers"` + ConnectTimeout time.Duration `config:"contimeout"` // Connect timeout + Timeout time.Duration `config:"timeout"` // Data channel timeout + ExpectContinueTimeout time.Duration `config:"expect_continue_timeout"` + Dump DumpFlags `config:"dump"` + InsecureSkipVerify bool `config:"no_check_certificate"` // Skip server certificate verification + DeleteMode DeleteMode `config:"delete_mode"` + MaxDelete int64 `config:"max_delete"` + MaxDeleteSize SizeSuffix `config:"max_delete_size"` + TrackRenames bool `config:"track_renames"` // Track file renames. + TrackRenamesStrategy string `config:"track_renames_strategy"` // Comma separated list of strategies used to track renames + Retries int `config:"retries"` // High-level retries + RetriesInterval time.Duration `config:"retries_sleep"` + LowLevelRetries int `config:"low_level_retries"` + UpdateOlder bool `config:"update"` // Skip files that are newer on the destination + NoGzip bool `config:"no_gzip_encoding"` // Disable compression + MaxDepth int `config:"max_depth"` + IgnoreSize bool `config:"ignore_size"` + IgnoreChecksum bool `config:"ignore_checksum"` + IgnoreCaseSync bool `config:"ignore_case_sync"` + FixCase bool `config:"fix_case"` + NoTraverse bool `config:"no_traverse"` + CheckFirst bool `config:"check_first"` + NoCheckDest bool `config:"no_check_dest"` + NoUnicodeNormalization bool `config:"no_unicode_normalization"` + NoUpdateModTime bool `config:"no_update_modtime"` + NoUpdateDirModTime bool `config:"no_update_dir_modtime"` + DataRateUnit string `config:"stats_unit"` + CompareDest []string `config:"compare_dest"` + CopyDest []string `config:"copy_dest"` + BackupDir string `config:"backup_dir"` + Suffix string `config:"suffix"` + SuffixKeepExtension bool `config:"suffix_keep_extension"` + UseListR bool `config:"fast_list"` + BufferSize SizeSuffix `config:"buffer_size"` + BwLimit BwTimetable `config:"bwlimit"` + BwLimitFile BwTimetable `config:"bwlimit_file"` + TPSLimit float64 `config:"tpslimit"` + TPSLimitBurst int `config:"tpslimit_burst"` + BindAddr net.IP `config:"bind_addr"` + DisableFeatures []string `config:"disable"` + UserAgent string `config:"user_agent"` + Immutable bool `config:"immutable"` + AutoConfirm bool `config:"auto_confirm"` + StreamingUploadCutoff SizeSuffix `config:"streaming_upload_cutoff"` + StatsFileNameLength int `config:"stats_file_name_length"` + AskPassword bool `config:"ask_password"` + PasswordCommand SpaceSepList `config:"password_command"` + UseServerModTime bool `config:"use_server_modtime"` + MaxTransfer SizeSuffix `config:"max_transfer"` + MaxDuration time.Duration `config:"max_duration"` + CutoffMode CutoffMode `config:"cutoff_mode"` + MaxBacklog int `config:"max_backlog"` + MaxStatsGroups int `config:"max_stats_groups"` + StatsOneLine bool `config:"stats_one_line"` + StatsOneLineDate bool `config:"stats_one_line_date"` // If we want a date prefix at all + StatsOneLineDateFormat string `config:"stats_one_line_date_format"` // If we want to customize the prefix + ErrorOnNoTransfer bool `config:"error_on_no_transfer"` // Set appropriate exit code if no files transferred + Progress bool `config:"progress"` + ProgressTerminalTitle bool `config:"progress_terminal_title"` + Cookie bool `config:"use_cookies"` + UseMmap bool `config:"use_mmap"` + CaCert []string `config:"ca_cert"` // Client Side CA + ClientCert string `config:"client_cert"` // Client Side Cert + ClientKey string `config:"client_key"` // Client Side Key + MultiThreadCutoff SizeSuffix `config:"multi_thread_cutoff"` + MultiThreadStreams int `config:"multi_thread_streams"` + MultiThreadSet bool `config:"multi_thread_set"` // whether MultiThreadStreams was set (set in fs/config/configflags) + MultiThreadChunkSize SizeSuffix `config:"multi_thread_chunk_size"` // Chunk size for multi-thread downloads / uploads, if not set by filesystem + MultiThreadWriteBufferSize SizeSuffix `config:"multi_thread_write_buffer_size"` + OrderBy string `config:"order_by"` // instructions on how to order the transfer + UploadHeaders []*HTTPOption `config:"upload_headers"` + DownloadHeaders []*HTTPOption `config:"download_headers"` + Headers []*HTTPOption `config:"headers"` + MetadataSet Metadata `config:"metadata_set"` // extra metadata to write when uploading + RefreshTimes bool `config:"refresh_times"` + NoConsole bool `config:"no_console"` + TrafficClass uint8 `config:"traffic_class"` + FsCacheExpireDuration time.Duration `config:"fs_cache_expire_duration"` + FsCacheExpireInterval time.Duration `config:"fs_cache_expire_interval"` + DisableHTTP2 bool `config:"disable_http2"` + HumanReadable bool `config:"human_readable"` + KvLockTime time.Duration `config:"kv_lock_time"` // maximum time to keep key-value database locked by process + DisableHTTPKeepAlives bool `config:"disable_http_keep_alives"` + Metadata bool `config:"metadata"` + ServerSideAcrossConfigs bool `config:"server_side_across_configs"` + TerminalColorMode TerminalColorMode `config:"color"` + DefaultTime Time `config:"default_time"` // time that directories with no time should display + Inplace bool `config:"inplace"` // Download directly to destination file instead of atomic download to temp/rename + PartialSuffix string `config:"partial_suffix"` + MetadataMapper SpaceSepList `config:"metadata_mapper"` } -// NewConfig creates a new config with everything set to the default -// value. These are the ultimate defaults and are overridden by the -// config module. -func NewConfig() *ConfigInfo { - c := new(ConfigInfo) - +func init() { // Set any values which aren't the zero for the type - c.LogLevel = LogLevelNotice - c.StatsLogLevel = LogLevelInfo - c.ModifyWindow = time.Nanosecond - c.Checkers = 8 - c.Transfers = 4 - c.ConnectTimeout = 60 * time.Second - c.Timeout = 5 * 60 * time.Second - c.ExpectContinueTimeout = 1 * time.Second - c.DeleteMode = DeleteModeDefault - c.MaxDelete = -1 - c.MaxDeleteSize = SizeSuffix(-1) - c.Retries = 3 - c.LowLevelRetries = 10 - c.MaxDepth = -1 - c.DataRateUnit = "bytes" - c.BufferSize = SizeSuffix(16 << 20) - c.UserAgent = "rclone/" + Version - c.StreamingUploadCutoff = SizeSuffix(100 * 1024) - c.MaxStatsGroups = 1000 - c.StatsFileNameLength = 45 - c.AskPassword = true - c.TPSLimitBurst = 1 - c.MaxTransfer = -1 - c.MaxBacklog = 10000 - // We do not want to set the default here. We use this variable being empty as part of the fall-through of options. - // c.StatsOneLineDateFormat = "2006/01/02 15:04:05 - " - c.MultiThreadCutoff = SizeSuffix(256 * 1024 * 1024) - c.MultiThreadStreams = 4 - c.MultiThreadChunkSize = SizeSuffix(64 * 1024 * 1024) - c.MultiThreadWriteBufferSize = SizeSuffix(128 * 1024) + globalConfig.DeleteMode = DeleteModeDefault - c.TrackRenamesStrategy = "hash" - c.FsCacheExpireDuration = 300 * time.Second - c.FsCacheExpireInterval = 60 * time.Second - c.KvLockTime = 1 * time.Second - c.DefaultTime = Time(time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC)) - c.PartialSuffix = ".partial" + // Register the config and fill globalConfig with the defaults + RegisterGlobalOptions(OptionsInfo{Name: "main", Opt: globalConfig, Options: ConfigOptionsInfo, Reload: globalConfig.Reload}) - // Perform a simple check for debug flags to enable debug logging during the flag initialization + // initial guess at log level from the flags + globalConfig.LogLevel = initialLogLevel() +} + +// Reload assumes the config has been edited and does what is necessary to make it live +func (ci *ConfigInfo) Reload(ctx context.Context) error { + // Set -vv if --dump is in use + if ci.Dump != 0 && ci.LogLevel != LogLevelDebug { + Logf(nil, "Automatically setting -vv as --dump is enabled") + ci.LogLevel = LogLevelDebug + } + + // If --dry-run or -i then use NOTICE as minimum log level + if (ci.DryRun || ci.Interactive) && ci.StatsLogLevel > LogLevelNotice { + ci.StatsLogLevel = LogLevelNotice + } + + // If --use-json-log then start the JSON logger + if ci.UseJSONLog { + InstallJSONLogger(ci.LogLevel) + } + + // Check --compare-dest and --copy-dest + if len(ci.CompareDest) > 0 && len(ci.CopyDest) > 0 { + return fmt.Errorf("can't use --compare-dest with --copy-dest") + } + + // Check --stats-one-line and dependent flags + switch { + case len(ci.StatsOneLineDateFormat) > 0: + ci.StatsOneLineDate = true + ci.StatsOneLine = true + case ci.StatsOneLineDate: + ci.StatsOneLineDateFormat = "2006/01/02 15:04:05 - " + ci.StatsOneLine = true + } + + // Check --partial-suffix + if len(ci.PartialSuffix) > 16 { + return fmt.Errorf("--partial-suffix: Expecting suffix length not greater than %d but got %d", 16, len(ci.PartialSuffix)) + } + + // Make sure some values are > 0 + nonZero := func(pi *int) { + if *pi <= 0 { + *pi = 1 + } + } + + // Check --stats-unit + if ci.DataRateUnit != "bits" && ci.DataRateUnit != "bytes" { + Errorf(nil, "Unknown unit %q passed to --stats-unit. Defaulting to bytes.", ci.DataRateUnit) + ci.DataRateUnit = "bytes" + } + + // Check these are all > 0 + nonZero(&ci.Retries) + nonZero(&ci.LowLevelRetries) + nonZero(&ci.Transfers) + nonZero(&ci.Checkers) + + return nil +} + +// Initial logging level +// +// Perform a simple check for debug flags to enable debug logging during the flag initialization +func initialLogLevel() LogLevel { + logLevel := LogLevelNotice for argIndex, arg := range os.Args { if strings.HasPrefix(arg, "-vv") && strings.TrimRight(arg, "v") == "-" { - c.LogLevel = LogLevelDebug + logLevel = LogLevelDebug } if arg == "--log-level=DEBUG" || (arg == "--log-level" && len(os.Args) > argIndex+1 && os.Args[argIndex+1] == "DEBUG") { - c.LogLevel = LogLevelDebug + logLevel = LogLevelDebug } if strings.HasPrefix(arg, "--verbose=") { if level, err := strconv.Atoi(arg[10:]); err == nil && level >= 2 { - c.LogLevel = LogLevelDebug + logLevel = LogLevelDebug } } } envValue, found := os.LookupEnv("RCLONE_LOG_LEVEL") if found && envValue == "DEBUG" { - c.LogLevel = LogLevelDebug + logLevel = LogLevelDebug } - - return c + return logLevel } // TimeoutOrInfinite returns ci.Timeout if > 0 or infinite otherwise -func (c *ConfigInfo) TimeoutOrInfinite() time.Duration { - if c.Timeout > 0 { - return c.Timeout +func (ci *ConfigInfo) TimeoutOrInfinite() time.Duration { + if ci.Timeout > 0 { + return ci.Timeout } return ModTimeNotSupported } diff --git a/fs/config/configflags/configflags.go b/fs/config/configflags/configflags.go index ce7425343..056c0d9d2 100644 --- a/fs/config/configflags/configflags.go +++ b/fs/config/configflags/configflags.go @@ -13,9 +13,6 @@ import ( "github.com/rclone/rclone/fs" "github.com/rclone/rclone/fs/config" "github.com/rclone/rclone/fs/config/flags" - fsLog "github.com/rclone/rclone/fs/log" - "github.com/rclone/rclone/fs/rc" - "github.com/sirupsen/logrus" "github.com/spf13/pflag" ) @@ -38,123 +35,30 @@ var ( downloadHeaders []string headers []string metadataSet []string - partialSuffix string ) // AddFlags adds the non filing system specific flags to the command func AddFlags(ci *fs.ConfigInfo, flagSet *pflag.FlagSet) { - rc.AddOption("main", ci) - // NB defaults which aren't the zero for the type should be set in fs/config.go NewConfig + flags.AddFlagsFromOptions(flagSet, "", fs.ConfigOptionsInfo) + + // Add flags we haven't converted into options yet flags.CountVarP(flagSet, &verbose, "verbose", "v", "Print lots more stuff (repeat for more)", "Logging,Important") flags.BoolVarP(flagSet, &quiet, "quiet", "q", false, "Print as little stuff as possible", "Logging") - flags.DurationVarP(flagSet, &ci.ModifyWindow, "modify-window", "", ci.ModifyWindow, "Max time diff to be considered the same", "Copy") - flags.IntVarP(flagSet, &ci.Checkers, "checkers", "", ci.Checkers, "Number of checkers to run in parallel", "Performance") - flags.IntVarP(flagSet, &ci.Transfers, "transfers", "", ci.Transfers, "Number of file transfers to run in parallel", "Performance") flags.StringVarP(flagSet, &configPath, "config", "", config.GetConfigPath(), "Config file", "Config") flags.StringVarP(flagSet, &cacheDir, "cache-dir", "", config.GetCacheDir(), "Directory rclone will use for caching", "Config") flags.StringVarP(flagSet, &tempDir, "temp-dir", "", os.TempDir(), "Directory rclone will use for temporary files", "Config") - flags.BoolVarP(flagSet, &ci.CheckSum, "checksum", "c", ci.CheckSum, "Check for changes with size & checksum (if available, or fallback to size only).", "Copy") - flags.BoolVarP(flagSet, &ci.SizeOnly, "size-only", "", ci.SizeOnly, "Skip based on size only, not modtime or checksum", "Copy") - flags.BoolVarP(flagSet, &ci.IgnoreTimes, "ignore-times", "I", ci.IgnoreTimes, "Don't skip items that match size and time - transfer all unconditionally", "Copy") - flags.BoolVarP(flagSet, &ci.IgnoreExisting, "ignore-existing", "", ci.IgnoreExisting, "Skip all files that exist on destination", "Copy") - flags.BoolVarP(flagSet, &ci.IgnoreErrors, "ignore-errors", "", ci.IgnoreErrors, "Delete even if there are I/O errors", "Sync") - flags.BoolVarP(flagSet, &ci.DryRun, "dry-run", "n", ci.DryRun, "Do a trial run with no permanent changes", "Config,Important") - flags.BoolVarP(flagSet, &ci.Interactive, "interactive", "i", ci.Interactive, "Enable interactive mode", "Config,Important") - flags.DurationVarP(flagSet, &ci.ConnectTimeout, "contimeout", "", ci.ConnectTimeout, "Connect timeout", "Networking") - flags.DurationVarP(flagSet, &ci.Timeout, "timeout", "", ci.Timeout, "IO idle timeout", "Networking") - flags.DurationVarP(flagSet, &ci.ExpectContinueTimeout, "expect-continue-timeout", "", ci.ExpectContinueTimeout, "Timeout when using expect / 100-continue in HTTP", "Networking") flags.BoolVarP(flagSet, &dumpHeaders, "dump-headers", "", false, "Dump HTTP headers - may contain sensitive info", "Debugging") flags.BoolVarP(flagSet, &dumpBodies, "dump-bodies", "", false, "Dump HTTP headers and bodies - may contain sensitive info", "Debugging") - flags.BoolVarP(flagSet, &ci.InsecureSkipVerify, "no-check-certificate", "", ci.InsecureSkipVerify, "Do not verify the server SSL certificate (insecure)", "Networking") - flags.BoolVarP(flagSet, &ci.AskPassword, "ask-password", "", ci.AskPassword, "Allow prompt for password for encrypted configuration", "Config") - flags.FVarP(flagSet, &ci.PasswordCommand, "password-command", "", "Command for supplying password for encrypted configuration", "Config") flags.BoolVarP(flagSet, &deleteBefore, "delete-before", "", false, "When synchronizing, delete files on destination before transferring", "Sync") flags.BoolVarP(flagSet, &deleteDuring, "delete-during", "", false, "When synchronizing, delete files during transfer", "Sync") flags.BoolVarP(flagSet, &deleteAfter, "delete-after", "", false, "When synchronizing, delete files on destination after transferring (default)", "Sync") - flags.Int64VarP(flagSet, &ci.MaxDelete, "max-delete", "", -1, "When synchronizing, limit the number of deletes", "Sync") - flags.FVarP(flagSet, &ci.MaxDeleteSize, "max-delete-size", "", "When synchronizing, limit the total size of deletes", "Sync") - flags.BoolVarP(flagSet, &ci.TrackRenames, "track-renames", "", ci.TrackRenames, "When synchronizing, track file renames and do a server-side move if possible", "Sync") - flags.StringVarP(flagSet, &ci.TrackRenamesStrategy, "track-renames-strategy", "", ci.TrackRenamesStrategy, "Strategies to use when synchronizing using track-renames hash|modtime|leaf", "Sync") - flags.IntVarP(flagSet, &ci.Retries, "retries", "", 3, "Retry operations this many times if they fail", "Config") - flags.DurationVarP(flagSet, &ci.RetriesInterval, "retries-sleep", "", 0, "Interval between retrying operations if they fail, e.g. 500ms, 60s, 5m (0 to disable)", "Config") - flags.IntVarP(flagSet, &ci.LowLevelRetries, "low-level-retries", "", ci.LowLevelRetries, "Number of low level retries to do", "Config") - flags.BoolVarP(flagSet, &ci.UpdateOlder, "update", "u", ci.UpdateOlder, "Skip files that are newer on the destination", "Copy") - flags.BoolVarP(flagSet, &ci.UseServerModTime, "use-server-modtime", "", ci.UseServerModTime, "Use server modified time instead of object metadata", "Config") - flags.BoolVarP(flagSet, &ci.NoGzip, "no-gzip-encoding", "", ci.NoGzip, "Don't set Accept-Encoding: gzip", "Networking") - flags.IntVarP(flagSet, &ci.MaxDepth, "max-depth", "", ci.MaxDepth, "If set limits the recursion depth to this", "Filter") - flags.BoolVarP(flagSet, &ci.IgnoreSize, "ignore-size", "", false, "Ignore size when skipping use modtime or checksum", "Copy") - flags.BoolVarP(flagSet, &ci.IgnoreChecksum, "ignore-checksum", "", ci.IgnoreChecksum, "Skip post copy check of checksums", "Copy") - flags.BoolVarP(flagSet, &ci.IgnoreCaseSync, "ignore-case-sync", "", ci.IgnoreCaseSync, "Ignore case when synchronizing", "Copy") - flags.BoolVarP(flagSet, &ci.FixCase, "fix-case", "", ci.FixCase, "Force rename of case insensitive dest to match source", "Sync") - flags.BoolVarP(flagSet, &ci.NoTraverse, "no-traverse", "", ci.NoTraverse, "Don't traverse destination file system on copy", "Copy") - flags.BoolVarP(flagSet, &ci.CheckFirst, "check-first", "", ci.CheckFirst, "Do all the checks before starting transfers", "Copy") - flags.BoolVarP(flagSet, &ci.NoCheckDest, "no-check-dest", "", ci.NoCheckDest, "Don't check the destination, copy regardless", "Copy") - flags.BoolVarP(flagSet, &ci.NoUnicodeNormalization, "no-unicode-normalization", "", ci.NoUnicodeNormalization, "Don't normalize unicode characters in filenames", "Config") - flags.BoolVarP(flagSet, &ci.NoUpdateModTime, "no-update-modtime", "", ci.NoUpdateModTime, "Don't update destination modtime if files identical", "Copy") - flags.BoolVarP(flagSet, &ci.NoUpdateDirModTime, "no-update-dir-modtime", "", ci.NoUpdateModTime, "Don't update directory modification times", "Copy") - flags.StringArrayVarP(flagSet, &ci.CompareDest, "compare-dest", "", nil, "Include additional comma separated server-side paths during comparison", "Copy") - flags.StringArrayVarP(flagSet, &ci.CopyDest, "copy-dest", "", nil, "Implies --compare-dest but also copies files from paths into destination", "Copy") - flags.StringVarP(flagSet, &ci.BackupDir, "backup-dir", "", ci.BackupDir, "Make backups into hierarchy based in DIR", "Sync") - flags.StringVarP(flagSet, &ci.Suffix, "suffix", "", ci.Suffix, "Suffix to add to changed files", "Sync") - flags.BoolVarP(flagSet, &ci.SuffixKeepExtension, "suffix-keep-extension", "", ci.SuffixKeepExtension, "Preserve the extension when using --suffix", "Sync") - flags.BoolVarP(flagSet, &ci.UseListR, "fast-list", "", ci.UseListR, "Use recursive list if available; uses more memory but fewer transactions", "Listing") - flags.Float64VarP(flagSet, &ci.TPSLimit, "tpslimit", "", ci.TPSLimit, "Limit HTTP transactions per second to this", "Networking") - flags.IntVarP(flagSet, &ci.TPSLimitBurst, "tpslimit-burst", "", ci.TPSLimitBurst, "Max burst of transactions for --tpslimit", "Networking") flags.StringVarP(flagSet, &bindAddr, "bind", "", "", "Local address to bind to for outgoing connections, IPv4, IPv6 or name", "Networking") flags.StringVarP(flagSet, &disableFeatures, "disable", "", "", "Disable a comma separated list of features (use --disable help to see a list)", "Config") - flags.StringVarP(flagSet, &ci.UserAgent, "user-agent", "", ci.UserAgent, "Set the user-agent to a specified string", "Networking") - flags.BoolVarP(flagSet, &ci.Immutable, "immutable", "", ci.Immutable, "Do not modify files, fail if existing files have been modified", "Copy") - flags.BoolVarP(flagSet, &ci.AutoConfirm, "auto-confirm", "", ci.AutoConfirm, "If enabled, do not request console confirmation", "Config") - flags.IntVarP(flagSet, &ci.StatsFileNameLength, "stats-file-name-length", "", ci.StatsFileNameLength, "Max file name length in stats (0 for no limit)", "Logging") - flags.FVarP(flagSet, &ci.LogLevel, "log-level", "", "Log level DEBUG|INFO|NOTICE|ERROR", "Logging") - flags.FVarP(flagSet, &ci.StatsLogLevel, "stats-log-level", "", "Log level to show --stats output DEBUG|INFO|NOTICE|ERROR", "Logging") - flags.FVarP(flagSet, &ci.BwLimit, "bwlimit", "", "Bandwidth limit in KiB/s, or use suffix B|K|M|G|T|P or a full timetable", "Networking") - flags.FVarP(flagSet, &ci.BwLimitFile, "bwlimit-file", "", "Bandwidth limit per file in KiB/s, or use suffix B|K|M|G|T|P or a full timetable", "Networking") - flags.FVarP(flagSet, &ci.BufferSize, "buffer-size", "", "In memory buffer size when reading files for each --transfer", "Performance") - flags.FVarP(flagSet, &ci.StreamingUploadCutoff, "streaming-upload-cutoff", "", "Cutoff for switching to chunked upload if file size is unknown, upload starts after reaching cutoff or when file ends", "Copy") - flags.FVarP(flagSet, &ci.Dump, "dump", "", "List of items to dump from: "+fs.DumpFlagsList, "Debugging") - flags.FVarP(flagSet, &ci.MaxTransfer, "max-transfer", "", "Maximum size of data to transfer", "Copy") - flags.DurationVarP(flagSet, &ci.MaxDuration, "max-duration", "", 0, "Maximum duration rclone will transfer data for", "Copy") - flags.FVarP(flagSet, &ci.CutoffMode, "cutoff-mode", "", "Mode to stop transfers when reaching the max transfer limit HARD|SOFT|CAUTIOUS", "Copy") - flags.IntVarP(flagSet, &ci.MaxBacklog, "max-backlog", "", ci.MaxBacklog, "Maximum number of objects in sync or check backlog", "Copy,Check") - flags.IntVarP(flagSet, &ci.MaxStatsGroups, "max-stats-groups", "", ci.MaxStatsGroups, "Maximum number of stats groups to keep in memory, on max oldest is discarded", "Logging") - flags.BoolVarP(flagSet, &ci.StatsOneLine, "stats-one-line", "", ci.StatsOneLine, "Make the stats fit on one line", "Logging") - flags.BoolVarP(flagSet, &ci.StatsOneLineDate, "stats-one-line-date", "", ci.StatsOneLineDate, "Enable --stats-one-line and add current date/time prefix", "Logging") - flags.StringVarP(flagSet, &ci.StatsOneLineDateFormat, "stats-one-line-date-format", "", ci.StatsOneLineDateFormat, "Enable --stats-one-line-date and use custom formatted date: Enclose date string in double quotes (\"), see https://golang.org/pkg/time/#Time.Format", "Logging") - flags.BoolVarP(flagSet, &ci.ErrorOnNoTransfer, "error-on-no-transfer", "", ci.ErrorOnNoTransfer, "Sets exit code 9 if no files are transferred, useful in scripts", "Config") - flags.BoolVarP(flagSet, &ci.Progress, "progress", "P", ci.Progress, "Show progress during transfer", "Logging") - flags.BoolVarP(flagSet, &ci.ProgressTerminalTitle, "progress-terminal-title", "", ci.ProgressTerminalTitle, "Show progress on the terminal title (requires -P/--progress)", "Logging") - flags.BoolVarP(flagSet, &ci.Cookie, "use-cookies", "", ci.Cookie, "Enable session cookiejar", "Networking") - flags.BoolVarP(flagSet, &ci.UseMmap, "use-mmap", "", ci.UseMmap, "Use mmap allocator (see docs)", "Config") - flags.StringArrayVarP(flagSet, &ci.CaCert, "ca-cert", "", ci.CaCert, "CA certificate used to verify servers", "Networking") - flags.StringVarP(flagSet, &ci.ClientCert, "client-cert", "", ci.ClientCert, "Client SSL certificate (PEM) for mutual TLS auth", "Networking") - flags.StringVarP(flagSet, &ci.ClientKey, "client-key", "", ci.ClientKey, "Client SSL private key (PEM) for mutual TLS auth", "Networking") - flags.FVarP(flagSet, &ci.MultiThreadCutoff, "multi-thread-cutoff", "", "Use multi-thread downloads for files above this size", "Copy") - flags.IntVarP(flagSet, &ci.MultiThreadStreams, "multi-thread-streams", "", ci.MultiThreadStreams, "Number of streams to use for multi-thread downloads", "Copy") - flags.FVarP(flagSet, &ci.MultiThreadWriteBufferSize, "multi-thread-write-buffer-size", "", "In memory buffer size for writing when in multi-thread mode", "Copy") - flags.FVarP(flagSet, &ci.MultiThreadChunkSize, "multi-thread-chunk-size", "", "Chunk size for multi-thread downloads / uploads, if not set by filesystem", "Copy") - flags.BoolVarP(flagSet, &ci.UseJSONLog, "use-json-log", "", ci.UseJSONLog, "Use json log format", "Logging") - flags.StringVarP(flagSet, &ci.OrderBy, "order-by", "", ci.OrderBy, "Instructions on how to order the transfers, e.g. 'size,descending'", "Copy") flags.StringArrayVarP(flagSet, &uploadHeaders, "header-upload", "", nil, "Set HTTP header for upload transactions", "Networking") flags.StringArrayVarP(flagSet, &downloadHeaders, "header-download", "", nil, "Set HTTP header for download transactions", "Networking") flags.StringArrayVarP(flagSet, &headers, "header", "", nil, "Set HTTP header for all transactions", "Networking") flags.StringArrayVarP(flagSet, &metadataSet, "metadata-set", "", nil, "Add metadata key=value when uploading", "Metadata") - flags.BoolVarP(flagSet, &ci.RefreshTimes, "refresh-times", "", ci.RefreshTimes, "Refresh the modtime of remote files", "Copy") - flags.BoolVarP(flagSet, &ci.NoConsole, "no-console", "", ci.NoConsole, "Hide console window (supported on Windows only)", "Config") flags.StringVarP(flagSet, &dscp, "dscp", "", "", "Set DSCP value to connections, value or name, e.g. CS1, LE, DF, AF21", "Networking") - flags.DurationVarP(flagSet, &ci.FsCacheExpireDuration, "fs-cache-expire-duration", "", ci.FsCacheExpireDuration, "Cache remotes for this long (0 to disable caching)", "Config") - flags.DurationVarP(flagSet, &ci.FsCacheExpireInterval, "fs-cache-expire-interval", "", ci.FsCacheExpireInterval, "Interval to check for expired remotes", "Config") - flags.BoolVarP(flagSet, &ci.DisableHTTP2, "disable-http2", "", ci.DisableHTTP2, "Disable HTTP/2 in the global transport", "Networking") - flags.BoolVarP(flagSet, &ci.HumanReadable, "human-readable", "", ci.HumanReadable, "Print numbers in a human-readable format, sizes with suffix Ki|Mi|Gi|Ti|Pi", "Config") - flags.DurationVarP(flagSet, &ci.KvLockTime, "kv-lock-time", "", ci.KvLockTime, "Maximum time to keep key-value database locked by process", "Config") - flags.BoolVarP(flagSet, &ci.DisableHTTPKeepAlives, "disable-http-keep-alives", "", ci.DisableHTTPKeepAlives, "Disable HTTP keep-alives and use each connection once.", "Networking") - flags.BoolVarP(flagSet, &ci.Metadata, "metadata", "M", ci.Metadata, "If set, preserve metadata when copying objects", "Metadata,Copy") - flags.BoolVarP(flagSet, &ci.ServerSideAcrossConfigs, "server-side-across-configs", "", ci.ServerSideAcrossConfigs, "Allow server-side operations (e.g. copy) to work across different configs", "Copy") - flags.FVarP(flagSet, &ci.TerminalColorMode, "color", "", "When to show colors (and other ANSI codes) AUTO|NEVER|ALWAYS", "Config") - flags.FVarP(flagSet, &ci.DefaultTime, "default-time", "", "Time to show if modtime is unknown for files and directories", "Config,Listing") - flags.BoolVarP(flagSet, &ci.Inplace, "inplace", "", ci.Inplace, "Download directly to destination file instead of atomic download to temp/rename", "Copy") - flags.StringVarP(flagSet, &partialSuffix, "partial-suffix", "", ci.PartialSuffix, "Add partial-suffix to temporary file name when --inplace is not used", "Copy") - flags.FVarP(flagSet, &ci.MetadataMapper, "metadata-mapper", "", "Program to run to transforming metadata before upload", "Metadata") } // ParseHeaders converts the strings passed in via the header flags into HTTPOptions @@ -174,8 +78,9 @@ func ParseHeaders(headers []string) []*fs.HTTPOption { return opts } -// SetFlags converts any flags into config which weren't straight forward +// SetFlags sets flags which aren't part of the config system func SetFlags(ci *fs.ConfigInfo) { + // Process obsolete --dump-headers and --dump-bodies flags if dumpHeaders { ci.Dump |= fs.DumpHeaders fs.Logf(nil, "--dump-headers is obsolete - please use --dump headers instead") @@ -184,25 +89,23 @@ func SetFlags(ci *fs.ConfigInfo) { ci.Dump |= fs.DumpBodies fs.Logf(nil, "--dump-bodies is obsolete - please use --dump bodies instead") } - if ci.Dump != 0 && verbose < 2 && ci.LogLevel != fs.LogLevelDebug { - fs.Logf(nil, "Automatically setting -vv as --dump is enabled") - verbose = 2 - } + // Process -v flag if verbose >= 2 { ci.LogLevel = fs.LogLevelDebug } else if verbose >= 1 { ci.LogLevel = fs.LogLevelInfo } - if (ci.DryRun || ci.Interactive) && ci.StatsLogLevel > fs.LogLevelNotice { - ci.StatsLogLevel = fs.LogLevelNotice - } + + // Process -q flag if quiet { if verbose > 0 { log.Fatalf("Can't set -v and -q") } ci.LogLevel = fs.LogLevelError } + + // Can't set log level, -v, -q logLevelFlag := pflag.Lookup("log-level") if logLevelFlag != nil && logLevelFlag.Changed { if verbose > 0 { @@ -212,28 +115,8 @@ func SetFlags(ci *fs.ConfigInfo) { log.Fatalf("Can't set -q and --log-level") } } - if ci.UseJSONLog { - logrus.AddHook(fsLog.NewCallerHook()) - logrus.SetFormatter(&logrus.JSONFormatter{ - TimestampFormat: "2006-01-02T15:04:05.999999-07:00", - }) - logrus.SetLevel(logrus.DebugLevel) - switch ci.LogLevel { - case fs.LogLevelEmergency, fs.LogLevelAlert: - logrus.SetLevel(logrus.PanicLevel) - case fs.LogLevelCritical: - logrus.SetLevel(logrus.FatalLevel) - case fs.LogLevelError: - logrus.SetLevel(logrus.ErrorLevel) - case fs.LogLevelWarning, fs.LogLevelNotice: - logrus.SetLevel(logrus.WarnLevel) - case fs.LogLevelInfo: - logrus.SetLevel(logrus.InfoLevel) - case fs.LogLevelDebug: - logrus.SetLevel(logrus.DebugLevel) - } - } + // Process --delete-before, --delete-during and --delete-after switch { case deleteBefore && (deleteDuring || deleteAfter), deleteDuring && deleteAfter: @@ -248,19 +131,7 @@ func SetFlags(ci *fs.ConfigInfo) { ci.DeleteMode = fs.DeleteModeDefault } - if len(ci.CompareDest) > 0 && len(ci.CopyDest) > 0 { - log.Fatalf(`Can't use --compare-dest with --copy-dest.`) - } - - switch { - case len(ci.StatsOneLineDateFormat) > 0: - ci.StatsOneLineDate = true - ci.StatsOneLine = true - case ci.StatsOneLineDate: - ci.StatsOneLineDateFormat = "2006/01/02 15:04:05 - " - ci.StatsOneLine = true - } - + // Process --bind into IP address if bindAddr != "" { addrs, err := net.LookupIP(bindAddr) if err != nil { @@ -272,6 +143,7 @@ func SetFlags(ci *fs.ConfigInfo) { ci.BindAddr = addrs[0] } + // Process --disable if disableFeatures != "" { if disableFeatures == "help" { log.Fatalf("Possible backend features are: %s\n", strings.Join(new(fs.Features).List(), ", ")) @@ -279,6 +151,7 @@ func SetFlags(ci *fs.ConfigInfo) { ci.DisableFeatures = strings.Split(disableFeatures, ",") } + // Process --headers-upload, --headers-download, --headers if len(uploadHeaders) != 0 { ci.UploadHeaders = ParseHeaders(uploadHeaders) } @@ -288,9 +161,8 @@ func SetFlags(ci *fs.ConfigInfo) { if len(headers) != 0 { ci.Headers = ParseHeaders(headers) } - if len(headers) != 0 { - ci.Headers = ParseHeaders(headers) - } + + // Process --metadata-set if len(metadataSet) != 0 { ci.MetadataSet = make(fs.Metadata, len(metadataSet)) for _, kv := range metadataSet { @@ -302,6 +174,8 @@ func SetFlags(ci *fs.ConfigInfo) { } fs.Debugf(nil, "MetadataUpload %v", ci.MetadataSet) } + + // Process --dscp if len(dscp) != 0 { if value, ok := parseDSCP(dscp); ok { ci.TrafficClass = value << 2 @@ -310,39 +184,24 @@ func SetFlags(ci *fs.ConfigInfo) { } } - // Set path to configuration file + // Process --config path if err := config.SetConfigPath(configPath); err != nil { log.Fatalf("--config: Failed to set %q as config path: %v", configPath, err) } - // Set path to cache dir + // Process --cache-dir path if err := config.SetCacheDir(cacheDir); err != nil { log.Fatalf("--cache-dir: Failed to set %q as cache dir: %v", cacheDir, err) } - // Set path to temp dir + // Process --temp-dir path if err := config.SetTempDir(tempDir); err != nil { log.Fatalf("--temp-dir: Failed to set %q as temp dir: %v", tempDir, err) } - // Set whether multi-thread-streams was set + // Process --multi-thread-streams - set whether multi-thread-streams was set multiThreadStreamsFlag := pflag.Lookup("multi-thread-streams") ci.MultiThreadSet = multiThreadStreamsFlag != nil && multiThreadStreamsFlag.Changed - - if len(partialSuffix) > 16 { - log.Fatalf("--partial-suffix: Expecting suffix length not greater than %d but got %d", 16, len(partialSuffix)) - } - ci.PartialSuffix = partialSuffix - - // Make sure some values are > 0 - nonZero := func(pi *int) { - if *pi <= 0 { - *pi = 1 - } - } - nonZero(&ci.LowLevelRetries) - nonZero(&ci.Transfers) - nonZero(&ci.Checkers) } // parseHeaders converts DSCP names to value diff --git a/fs/log.go b/fs/log.go index d84458a37..40ccdbb27 100644 --- a/fs/log.go +++ b/fs/log.go @@ -57,6 +57,9 @@ func (logLevelChoices) Type() string { // LogPrintPid enables process pid in log var LogPrintPid = false +// InstallJSONLogger is a hook that --use-json-log calls +var InstallJSONLogger = func(logLevel LogLevel) {} + // LogPrint sends the text to the logger of level var LogPrint = func(level LogLevel, text string) { text = fmt.Sprintf("%-6s: %s", level, text) diff --git a/fs/log/caller_hook.go b/fs/log/caller_hook.go index 13941bc02..4d760ae30 100644 --- a/fs/log/caller_hook.go +++ b/fs/log/caller_hook.go @@ -5,9 +5,43 @@ import ( "runtime" "strings" + "github.com/rclone/rclone/fs" "github.com/sirupsen/logrus" ) +var loggerInstalled = false + +// InstallJSONLogger installs the JSON logger at the specified log level +func InstallJSONLogger(logLevel fs.LogLevel) { + if !loggerInstalled { + logrus.AddHook(NewCallerHook()) + loggerInstalled = true + } + logrus.SetFormatter(&logrus.JSONFormatter{ + TimestampFormat: "2006-01-02T15:04:05.999999-07:00", + }) + logrus.SetLevel(logrus.DebugLevel) + switch logLevel { + case fs.LogLevelEmergency, fs.LogLevelAlert: + logrus.SetLevel(logrus.PanicLevel) + case fs.LogLevelCritical: + logrus.SetLevel(logrus.FatalLevel) + case fs.LogLevelError: + logrus.SetLevel(logrus.ErrorLevel) + case fs.LogLevelWarning, fs.LogLevelNotice: + logrus.SetLevel(logrus.WarnLevel) + case fs.LogLevelInfo: + logrus.SetLevel(logrus.InfoLevel) + case fs.LogLevelDebug: + logrus.SetLevel(logrus.DebugLevel) + } +} + +// install hook in fs to call to avoid circular dependency +func init() { + fs.InstallJSONLogger = InstallJSONLogger +} + // CallerHook for log the calling file and line of the fine type CallerHook struct { Field string