From 7a64530e83dc7bbcc3930b58e3bb40452b043b99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sindre=20R=C3=B8kenes=20Myren?= Date: Sun, 3 Sep 2017 12:48:06 +0200 Subject: [PATCH] Added suport for multiline variables from sh Instead of giving an error on multiline results from sh, the results are now stored as is, except that the last newline is stripped away to make the output of most commands easy to use in shell commands. Two helper functions have been added to help deal with multi-line results. In addition, previous PascalCase template function names have been renamed to camelCase for consistency with the sprig lib. --- README.md | 35 ++++-- task.go | 4 +- task_test.go | 20 ++++ testdata/vars/multiline/Taskfile.yml | 43 ++++++++ variables.go | 152 ++++++++++++++------------- 5 files changed, 172 insertions(+), 82 deletions(-) create mode 100644 testdata/vars/multiline/Taskfile.yml diff --git a/README.md b/README.md index b30d7b91..eab8d39c 100644 --- a/README.md +++ b/README.md @@ -198,7 +198,7 @@ write-file: The above syntax is also supported in `deps`. > NOTE: It's also possible to call a task without any param prefixing it -with `^`, but this syntax is deprecaded: +with `^`, but this syntax is deprecated: ```yml a-task: @@ -314,8 +314,9 @@ set-message: #### Dynamic variables -The below syntax (`sh:` prop in a variable) is considered a dynamic -variable. The value will be treated as a command and the output assigned. +The below syntax (`sh:` prop in a variable) is considered a dynamic variable. +The value will be treated as a command and the output assigned. If there is one +or more trailing newlines, the last newline will be trimmed. ```yml build: @@ -345,7 +346,7 @@ GIT_COMMIT: $git log -n 1 --format=%h ### Go's template engine Task parse commands as [Go's template engine][gotemplate] before executing -them. Variables are acessible through dot syntax (`.VARNAME`). +them. Variables are accessible through dot syntax (`.VARNAME`). All functions by the Go's [sprig lib](http://masterminds.github.io/sprig/) are available. The following example gets the current date in a given format: @@ -362,11 +363,13 @@ Task also adds the following functions: "darwin" (macOS) and "freebsd". - `ARCH`: return the architecture Task was compiled to: "386", "amd64", "arm" or "s390x". -- `ToSlash`: Does nothing on Unix, but on Windows converts a string from `\` +- `splitLines`: Splits Unix (\n) and Windows (\r\n) styled newlines. +- `catLines`: Replaces Unix (\n) and Windows (\r\n) styled newlines with a space. +- `toSlash`: Does nothing on Unix, but on Windows converts a string from `\` path format to `/`. -- `FromSlash`: Oposite of `ToSlash`. Does nothing on Unix, but on Windows +- `fromSlash`: Oposite of `toSlash`. Does nothing on Unix, but on Windows converts a string from `\` path format to `/`. -- `ExeExt`: Returns the right executable extension for the current OS +- `exeExt`: Returns the right executable extension for the current OS (`".exe"` for Windows, `""` for others). Example: @@ -377,9 +380,23 @@ print-os: - echo '{{OS}} {{ARCH}}' - echo '{{if eq OS "windows"}}windows-command{{else}}unix-command{{end}}' # This will be path/to/file on Unix but path\to\file on Windows - - echo '{{FromSlash "path/to/file"}}' + - echo '{{fromSlash "path/to/file"}}' +enumerated-file: + vars: + CONTENT: | + foo + bar + cmds: + - | + cat << EOF > output.txt + {{range $i, $line := .CONTENT | splitLines -}} + {{printf "%3d" $i}}: {{$line}} + {{end}}EOF ``` +> NOTE: There are some deprecated function names still available: `ToSlash`, +`FromSlash` and `ExeExt`. These where changed for consistency with sprig lib. + ### Help Running `task --list` (or `task -l`) lists all tasks with a description. @@ -458,7 +475,7 @@ echo: * Or globally with `--silent` or `-s` flag -If you want to supress stdout instead, just redirect a command to `/dev/null`: +If you want to suppress stdout instead, just redirect a command to `/dev/null`: ```yml echo: diff --git a/task.go b/task.go index c633d3ac..c0adc319 100644 --- a/task.go +++ b/task.go @@ -121,8 +121,8 @@ func (e *Executor) RunTask(ctx context.Context, call Call) error { return err } - // FIXME: doing again, since a var may have been overriden using the - // `set:` attribute of a dependecy. Remove this when `set` (that is + // FIXME: doing again, since a var may have been overridden using the + // `set:` attribute of a dependency. Remove this when `set` (that is // deprecated) be removed. t, err = e.CompiledTask(call) if err != nil { diff --git a/task_test.go b/task_test.go index ab67a03b..c8db2f06 100644 --- a/task_test.go +++ b/task_test.go @@ -103,6 +103,26 @@ func TestVars(t *testing.T) { tt.Target = "hello" tt.Run(t) } +func TestMultilineVars(t *testing.T) { + tt := fileContentTest{ + Dir: "testdata/vars/multiline", + Target: "default", + TrimSpace: false, + Files: map[string]string{ + // Note: + // - task does not strip a trailing newline from var entries + // - task strips one trailing newline from shell output + // - the cat command adds a trailing newline + "echo_foobar.txt": "foo\nbar\n", + "echo_n_foobar.txt": "foo\nbar\n", + "echo_n_multiline.txt": "\n\nfoo\n bar\nfoobar\n\nbaz\n\n", + "var_multiline.txt": "\n\nfoo\n bar\nfoobar\n\nbaz\n\n\n", + "var_catlines.txt": " foo bar foobar baz \n", + "var_enumfile.txt": "0:\n1:\n2:foo\n3: bar\n4:foobar\n5:\n6:baz\n7:\n8:\n", + }, + } + tt.Run(t) +} func TestVarsInvalidTmpl(t *testing.T) { const ( diff --git a/testdata/vars/multiline/Taskfile.yml b/testdata/vars/multiline/Taskfile.yml new file mode 100644 index 00000000..79ba86d7 --- /dev/null +++ b/testdata/vars/multiline/Taskfile.yml @@ -0,0 +1,43 @@ +default: + vars: + MULTILINE: "\n\nfoo\n bar\nfoobar\n\nbaz\n\n" + cmds: + - task: file + vars: + CONTENT: + sh: "echo 'foo\nbar'" + FILE: "echo_foobar.txt" + - task: file + vars: + CONTENT: + sh: "echo -n 'foo\nbar'" + FILE: "echo_n_foobar.txt" + - task: file + vars: + CONTENT: + sh: echo -n "{{.MULTILINE}}" + FILE: "echo_n_multiline.txt" + - task: file + vars: + CONTENT: "{{.MULTILINE}}" + FILE: "var_multiline.txt" + - task: file + vars: + CONTENT: "{{.MULTILINE | catLines}}" + FILE: "var_catlines.txt" + - task: enumfile + vars: + LINES: "{{.MULTILINE}}" + FILE: "var_enumfile.txt" +file: + cmds: + - | + cat << EOF > '{{.FILE}}' + {{.CONTENT}} + EOF +enumfile: + cmds: + - | + cat << EOF > '{{.FILE}}' + {{range $i, $line := .LINES| splitLines}}{{$i}}:{{$line}} + {{end}}EOF diff --git a/variables.go b/variables.go index 1d3d1415..14b5a8bf 100644 --- a/variables.go +++ b/variables.go @@ -15,19 +15,30 @@ import ( ) var ( - // TaskvarsFilePath file containing additional variables + // TaskvarsFilePath file containing additional variables. TaskvarsFilePath = "Taskvars" - // ErrMultilineResultCmd is returned when a command returns multiline result - ErrMultilineResultCmd = errors.New("Got multiline result from command") ) -// Vars is a string[string] variables map +var ( + // ErrCantUnmarshalVar is returned for invalid var YAML. + ErrCantUnmarshalVar = errors.New("task: can't unmarshal var value") +) + +// Vars is a string[string] variables map. type Vars map[string]Var -// Var represents either a static or dynamic variable -type Var struct { - Static string - Sh string +func getEnvironmentVariables() Vars { + var ( + env = os.Environ() + m = make(Vars, len(env)) + ) + + for _, e := range env { + keyVal := strings.SplitN(e, "=", 2) + key, val := keyVal[0], keyVal[1] + m[key] = Var{Static: val} + } + return m } func (vs Vars) toStringMap() (m map[string]string) { @@ -43,12 +54,13 @@ func (vs Vars) toStringMap() (m map[string]string) { return } -var ( - // ErrCantUnmarshalVar is returned for invalid var YAML - ErrCantUnmarshalVar = errors.New("task: can't unmarshal var value") -) +// Var represents either a static or dynamic variable. +type Var struct { + Static string + Sh string +} -// UnmarshalYAML implements yaml.Unmarshaler interface +// UnmarshalYAML implements yaml.Unmarshaler interface. func (v *Var) UnmarshalYAML(unmarshal func(interface{}) error) error { var str string if err := unmarshal(&str); err == nil { @@ -67,44 +79,15 @@ func (v *Var) UnmarshalYAML(unmarshal func(interface{}) error) error { v.Sh = sh.Sh return nil } + return ErrCantUnmarshalVar } -var ( - templateFuncs template.FuncMap -) - -func init() { - taskFuncs := template.FuncMap{ - "OS": func() string { return runtime.GOOS }, - "ARCH": func() string { return runtime.GOARCH }, - // historical reasons - "IsSH": func() bool { return true }, - "FromSlash": func(path string) string { - return filepath.FromSlash(path) - }, - "ToSlash": func(path string) string { - return filepath.ToSlash(path) - }, - "ExeExt": func() string { - if runtime.GOOS == "windows" { - return ".exe" - } - return "" - }, - } - - templateFuncs = sprig.TxtFuncMap() - for k, v := range taskFuncs { - templateFuncs[k] = v - } -} - -// getVariables returns fully resolved variables following the priorty order: +// getVariables returns fully resolved variables following the priority order: // 1. Call variables (should already have been resolved) // 2. Environment (should not need to be resolved) // 3. Task variables, resolved with access to: -// - call, taskvars and environement variables +// - call, taskvars and environment variables // 4. Taskvars variables, resolved with access to: // - environment variables func (e *Executor) getVariables(call Call) (Vars, error) { @@ -179,27 +162,13 @@ func (e *Executor) getVariables(call Call) (Vars, error) { return result, nil } -func getEnvironmentVariables() Vars { - var ( - env = os.Environ() - m = make(Vars, len(env)) - ) - - for _, e := range env { - keyVal := strings.SplitN(e, "=", 2) - key, val := keyVal[0], keyVal[1] - m[key] = Var{Static: val} - } - return m -} - func (e *Executor) handleShVar(v Var) (string, error) { - if v.Static != "" { + if v.Static != "" || v.Sh == "" { return v.Static, nil } - e.muDynamicCache.Lock() defer e.muDynamicCache.Unlock() + if result, ok := e.dynamicCache[v.Sh]; ok { return result, nil } @@ -215,19 +184,18 @@ func (e *Executor) handleShVar(v Var) (string, error) { return "", &dynamicVarError{cause: err, cmd: opts.Command} } + // Trim a single trailing newline from the result to make most command + // output easier to use in shell commands. result := strings.TrimSuffix(stdout.String(), "\n") - if strings.ContainsRune(result, '\n') { - return "", ErrMultilineResultCmd - } - result = strings.TrimSpace(result) - e.verbosePrintfln(`task: dynamic variable: "%s", result: "%s"`, v.Sh, result) e.dynamicCache[v.Sh] = result + e.verbosePrintfln(`task: dynamic variable: '%s' result: '%s'`, v.Sh, result) + return result, nil } -// CompiledTask returns a copy of a task, but replacing -// variables in almost all properties using the Go template package +// CompiledTask returns a copy of a task, but replacing variables in almost all +// properties using the Go template package. func (e *Executor) CompiledTask(call Call) (*Task, error) { origTask, ok := e.Tasks[call.Task] if !ok { @@ -289,9 +257,9 @@ func (e *Executor) CompiledTask(call Call) (*Task, error) { } // varReplacer is a help struct that allow us to call "replaceX" funcs multiple -// times, without having to check for error each time. -// The first error that happen will be assigned to r.err, and consecutive -// calls to funcs will just return the zero value. +// times, without having to check for error each time. The first error that +// happen will be assigned to r.err, and consecutive calls to funcs will just +// return the zero value. type varReplacer struct { vars Vars strMap map[string]string @@ -347,3 +315,45 @@ func (r *varReplacer) replaceVars(vars Vars) Vars { } return new } + +var ( + templateFuncs template.FuncMap +) + +func init() { + taskFuncs := template.FuncMap{ + "OS": func() string { return runtime.GOOS }, + "ARCH": func() string { return runtime.GOARCH }, + "catLines": func(s string) string { + s = strings.Replace(s, "\r\n", " ", -1) + return strings.Replace(s, "\n", " ", -1) + }, + "splitLines": func(s string) []string { + s = strings.Replace(s, "\r\n", "\n", -1) + return strings.Split(s, "\n") + }, + "fromSlash": func(path string) string { + return filepath.FromSlash(path) + }, + "toSlash": func(path string) string { + return filepath.ToSlash(path) + }, + "exeExt": func() string { + if runtime.GOOS == "windows" { + return ".exe" + } + return "" + }, + // IsSH is deprecated. + "IsSH": func() bool { return true }, + } + // Deprecated aliases for renamed functions. + taskFuncs["FromSlash"] = taskFuncs["fromSlash"] + taskFuncs["ToSlash"] = taskFuncs["toSlash"] + taskFuncs["ExeExt"] = taskFuncs["exeExt"] + + templateFuncs = sprig.TxtFuncMap() + for k, v := range taskFuncs { + templateFuncs[k] = v + } +}