Improve nested variables support

Closes #76 #89 #77 #83

.gitignore

@@ -15,3 +15,5 @@
 
 ./task
 dist/
+
+.DS_Store

Taskvars.yml

@@ -7,6 +7,7 @@ GO_PACKAGES:
   ./internal/args
   ./internal/compiler
   ./internal/compiler/v1
+  ./internal/compiler/v2
   ./internal/execext
   ./internal/logger
   ./internal/status

internal/compiler/v1/compiler_v1.go

@@ -122,7 +122,7 @@ func (c *CompilerV1) HandleDynamicVar(v taskfile.Var) (string, error) {
 		Stderr:  c.Logger.Stderr,
 	}
 	if err := execext.RunCommand(opts); err != nil {
-		return "", &dynamicVarError{cause: err, cmd: opts.Command}
+		return "", fmt.Errorf(`task: Command "%s" in taskvars file failed: %s`, opts.Command, err)
 	}
 
 	// Trim a single trailing newline from the result to make most command
@@ -134,12 +134,3 @@ func (c *CompilerV1) HandleDynamicVar(v taskfile.Var) (string, error) {
 
 	return result, nil
 }
-
-type dynamicVarError struct {
-	cause error
-	cmd   string
-}
-
-func (err *dynamicVarError) Error() string {
-	return fmt.Sprintf(`task: Command "%s" in taskvars file failed: %s`, err.cmd, err.cause)
-}

internal/compiler/v2/compiler_v2.go

@@ -0,0 +1,104 @@
+package v2
+
+import (
+	"bytes"
+	"fmt"
+	"strings"
+	"sync"
+
+	"github.com/go-task/task/internal/compiler"
+	"github.com/go-task/task/internal/execext"
+	"github.com/go-task/task/internal/logger"
+	"github.com/go-task/task/internal/taskfile"
+	"github.com/go-task/task/internal/templater"
+)
+
+var _ compiler.Compiler = &CompilerV2{}
+
+type CompilerV2 struct {
+	Dir  string
+	Vars taskfile.Vars
+
+	Logger *logger.Logger
+
+	dynamicCache   map[string]string
+	muDynamicCache sync.Mutex
+}
+
+// GetVariables returns fully resolved variables following the priority order:
+// 1. Task variables
+// 2. Call variables
+// 3. Taskvars file variables
+// 4. Environment variables
+func (c *CompilerV2) GetVariables(t *taskfile.Task, call taskfile.Call) (taskfile.Vars, error) {
+	vr := varResolver{c: c, vars: compiler.GetEnviron()}
+	vr.merge(c.Vars)
+	vr.merge(c.Vars)
+	vr.merge(call.Vars)
+	vr.merge(call.Vars)
+	vr.merge(t.Vars)
+	vr.merge(t.Vars)
+	return vr.vars, vr.err
+}
+
+type varResolver struct {
+	c    *CompilerV2
+	vars taskfile.Vars
+	err  error
+}
+
+func (vr *varResolver) merge(vars taskfile.Vars) {
+	if vr.err != nil {
+		return
+	}
+	tr := templater.Templater{Vars: vr.vars}
+	for k, v := range vars {
+		v = taskfile.Var{
+			Static: tr.Replace(v.Static),
+			Sh:     tr.Replace(v.Sh),
+		}
+		static, err := vr.c.HandleDynamicVar(v)
+		if err != nil {
+			vr.err = err
+			return
+		}
+		vr.vars[k] = taskfile.Var{Static: static}
+	}
+	vr.err = tr.Err()
+}
+
+func (c *CompilerV2) HandleDynamicVar(v taskfile.Var) (string, error) {
+	if v.Static != "" || v.Sh == "" {
+		return v.Static, nil
+	}
+
+	c.muDynamicCache.Lock()
+	defer c.muDynamicCache.Unlock()
+
+	if c.dynamicCache == nil {
+		c.dynamicCache = make(map[string]string, 30)
+	}
+	if result, ok := c.dynamicCache[v.Sh]; ok {
+		return result, nil
+	}
+
+	var stdout bytes.Buffer
+	opts := &execext.RunCommandOptions{
+		Command: v.Sh,
+		Dir:     c.Dir,
+		Stdout:  &stdout,
+		Stderr:  c.Logger.Stderr,
+	}
+	if err := execext.RunCommand(opts); err != nil {
+		return "", fmt.Errorf(`task: Command "%s" in taskvars file failed: %s`, opts.Command, err)
+	}
+
+	// Trim a single trailing newline from the result to make most command
+	// output easier to use in shell commands.
+	result := strings.TrimSuffix(stdout.String(), "\n")
+
+	c.dynamicCache[v.Sh] = result
+	c.Logger.VerboseErrf(`task: dynamic variable: '%s' result: '%s'`, v.Sh, result)
+
+	return result, nil
+}

task.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/go-task/task/internal/compiler"
 	compilerv1 "github.com/go-task/task/internal/compiler/v1"
+	compilerv2 "github.com/go-task/task/internal/compiler/v2"
 	"github.com/go-task/task/internal/execext"
 	"github.com/go-task/task/internal/logger"
 	"github.com/go-task/task/internal/taskfile"
@@ -49,37 +50,8 @@ type Executor struct {
 
 // Run runs Task
 func (e *Executor) Run(calls ...taskfile.Call) error {
-	if e.Context == nil {
+	if err := e.setup(); err != nil {
-		e.Context = context.Background()
+		return err
-	}
-	if e.Stdin == nil {
-		e.Stdin = os.Stdin
-	}
-	if e.Stdout == nil {
-		e.Stdout = os.Stdout
-	}
-	if e.Stderr == nil {
-		e.Stderr = os.Stderr
-	}
-	if e.Logger == nil {
-		e.Logger = &logger.Logger{
-			Stdout:  e.Stdout,
-			Stderr:  e.Stderr,
-			Verbose: e.Verbose,
-		}
-	}
-	// TODO: Add version 2
-	if e.Compiler == nil {
-		e.Compiler = &compilerv1.CompilerV1{
-			Dir:    e.Dir,
-			Vars:   e.taskvars,
-			Logger: e.Logger,
-		}
-	}
-
-	e.taskCallCount = make(map[string]*int32, len(e.Taskfile.Tasks))
-	for k := range e.Taskfile.Tasks {
-		e.taskCallCount[k] = new(int32)
 	}
 
 	// check if given tasks exist
@@ -103,6 +75,51 @@ func (e *Executor) Run(calls ...taskfile.Call) error {
 	return nil
 }
 
+func (e *Executor) setup() error {
+	if e.Taskfile.Version == 0 {
+		e.Taskfile.Version = 1
+	}
+	if e.Context == nil {
+		e.Context = context.Background()
+	}
+	if e.Stdin == nil {
+		e.Stdin = os.Stdin
+	}
+	if e.Stdout == nil {
+		e.Stdout = os.Stdout
+	}
+	if e.Stderr == nil {
+		e.Stderr = os.Stderr
+	}
+	e.Logger = &logger.Logger{
+		Stdout:  e.Stdout,
+		Stderr:  e.Stderr,
+		Verbose: e.Verbose,
+	}
+	switch e.Taskfile.Version {
+	case 1:
+		e.Compiler = &compilerv1.CompilerV1{
+			Dir:    e.Dir,
+			Vars:   e.taskvars,
+			Logger: e.Logger,
+		}
+	case 2:
+		e.Compiler = &compilerv2.CompilerV2{
+			Dir:    e.Dir,
+			Vars:   e.taskvars,
+			Logger: e.Logger,
+		}
+	default:
+		return fmt.Errorf(`task: Unrecognized Taskfile version "%d"`, e.Taskfile.Version)
+	}
+
+	e.taskCallCount = make(map[string]*int32, len(e.Taskfile.Tasks))
+	for k := range e.Taskfile.Tasks {
+		e.taskCallCount[k] = new(int32)
+	}
+	return nil
+}
+
 // RunTask runs a task by its name
 func (e *Executor) RunTask(ctx context.Context, call taskfile.Call) error {
 	t, err := e.CompiledTask(call)

task_test.go

@@ -67,9 +67,9 @@ func TestEnv(t *testing.T) {
 	tt.Run(t)
 }
 
-func TestVars(t *testing.T) {
+func TestVarsV1(t *testing.T) {
 	tt := fileContentTest{
-		Dir:       "testdata/vars",
+		Dir:       "testdata/vars/v1",
 		Target:    "default",
 		TrimSpace: true,
 		Files: map[string]string{
@@ -103,30 +103,69 @@ func TestVars(t *testing.T) {
 	tt.Target = "hello"
 	tt.Run(t)
 }
-func TestMultilineVars(t *testing.T) {
+
+func TestVarsV2(t *testing.T) {
 	tt := fileContentTest{
-		Dir:       "testdata/vars/multiline",
+		Dir:       "testdata/vars/v2",
 		Target:    "default",
-		TrimSpace: false,
+		TrimSpace: true,
 		Files: map[string]string{
-			// Note:
+			"foo.txt":              "foo",
-			// - task does not strip a trailing newline from var entries
+			"bar.txt":              "bar",
-			// - task strips one trailing newline from shell output
+			"baz.txt":              "baz",
-			// - the cat command adds a trailing newline
+			"tmpl_foo.txt":         "foo",
-			"echo_foobar.txt":      "foo\nbar\n",
+			"tmpl_bar.txt":         "bar",
-			"echo_n_foobar.txt":    "foo\nbar\n",
+			"tmpl_foo2.txt":        "foo2",
-			"echo_n_multiline.txt": "\n\nfoo\n  bar\nfoobar\n\nbaz\n\n",
+			"tmpl_bar2.txt":        "bar2",
-			"var_multiline.txt":    "\n\nfoo\n  bar\nfoobar\n\nbaz\n\n\n",
+			"shtmpl_foo.txt":       "foo",
-			"var_catlines.txt":     "  foo   bar foobar  baz  \n",
+			"shtmpl_foo2.txt":      "foo2",
-			"var_enumfile.txt":     "0:\n1:\n2:foo\n3:  bar\n4:foobar\n5:\n6:baz\n7:\n8:\n",
+			"nestedtmpl_foo.txt":   "<no value>",
+			"nestedtmpl_foo2.txt":  "foo2",
+			"foo2.txt":             "foo2",
+			"bar2.txt":             "bar2",
+			"baz2.txt":             "baz2",
+			"tmpl2_foo.txt":        "<no value>",
+			"tmpl2_foo2.txt":       "foo2",
+			"tmpl2_bar.txt":        "<no value>",
+			"tmpl2_bar2.txt":       "bar2",
+			"shtmpl2_foo.txt":      "<no value>",
+			"shtmpl2_foo2.txt":     "foo2",
+			"nestedtmpl2_foo2.txt": "<no value>",
+			"override.txt":         "bar",
 		},
 	}
 	tt.Run(t)
+	// Ensure identical results when running hello task directly.
+	tt.Target = "hello"
+	tt.Run(t)
+}
+
+func TestMultilineVars(t *testing.T) {
+	for _, dir := range []string{"testdata/vars/v1/multiline", "testdata/vars/v2/multiline"} {
+		tt := fileContentTest{
+			Dir:       dir,
+			Target:    "default",
+			TrimSpace: false,
+			Files: map[string]string{
+				// Note:
+				// - task does not strip a trailing newline from var entries
+				// - task strips one trailing newline from shell output
+				// - the cat command adds a trailing newline
+				"echo_foobar.txt":      "foo\nbar\n",
+				"echo_n_foobar.txt":    "foo\nbar\n",
+				"echo_n_multiline.txt": "\n\nfoo\n  bar\nfoobar\n\nbaz\n\n",
+				"var_multiline.txt":    "\n\nfoo\n  bar\nfoobar\n\nbaz\n\n\n",
+				"var_catlines.txt":     "  foo   bar foobar  baz  \n",
+				"var_enumfile.txt":     "0:\n1:\n2:foo\n3:  bar\n4:foobar\n5:\n6:baz\n7:\n8:\n",
+			},
+		}
+		tt.Run(t)
+	}
 }
 
 func TestVarsInvalidTmpl(t *testing.T) {
 	const (
-		dir         = "testdata/vars"
+		dir         = "testdata/vars/v1"
 		target      = "invalid-var-tmpl"
 		expectError = "template: :1: unexpected EOF"
 	)

testdata/vars/v2/.gitignore

@@ -0,0 +1 @@
+*.txt

testdata/vars/v2/Taskfile.yml

@@ -0,0 +1,50 @@
+version: 2
+tasks:
+  default:
+    deps: [hello]
+
+  hello:
+    cmds:
+      - echo {{.FOO}} > foo.txt
+      - echo {{.BAR}} > bar.txt
+      - echo {{.BAZ}} > baz.txt
+      - echo '{{.TMPL_FOO}}' > tmpl_foo.txt
+      - echo '{{.TMPL_BAR}}' > tmpl_bar.txt
+      - echo '{{.TMPL_FOO2}}' > tmpl_foo2.txt
+      - echo '{{.TMPL_BAR2}}' > tmpl_bar2.txt
+      - echo '{{.SHTMPL_FOO}}' > shtmpl_foo.txt
+      - echo '{{.SHTMPL_FOO2}}' > shtmpl_foo2.txt
+      - echo '{{.NESTEDTMPL_FOO}}' > nestedtmpl_foo.txt
+      - echo '{{.NESTEDTMPL_FOO2}}' > nestedtmpl_foo2.txt
+      - echo {{.FOO2}} > foo2.txt
+      - echo {{.BAR2}} > bar2.txt
+      - echo {{.BAZ2}} > baz2.txt
+      - echo '{{.TMPL2_FOO}}' > tmpl2_foo.txt
+      - echo '{{.TMPL2_BAR}}' > tmpl2_bar.txt
+      - echo '{{.TMPL2_FOO2}}' > tmpl2_foo2.txt
+      - echo '{{.TMPL2_BAR2}}' > tmpl2_bar2.txt
+      - echo '{{.SHTMPL2_FOO}}' > shtmpl2_foo.txt
+      - echo '{{.SHTMPL2_FOO2}}' > shtmpl2_foo2.txt
+      - echo '{{.NESTEDTMPL2_FOO2}}' > nestedtmpl2_foo2.txt
+      - echo {{.OVERRIDE}} > override.txt
+    vars:
+      FOO: foo
+      BAR: $echo bar
+      BAZ:
+        sh: echo baz
+      TMPL_FOO: "{{.FOO}}"
+      TMPL_BAR: "{{.BAR}}"
+      TMPL_FOO2: "{{.FOO2}}"
+      TMPL_BAR2: "{{.BAR2}}"
+      SHTMPL_FOO:
+        sh: "echo '{{.FOO}}'"
+      SHTMPL_FOO2:
+        sh: "echo '{{.FOO2}}'"
+      NESTEDTMPL_FOO: "{{.TMPL_FOO}}"
+      NESTEDTMPL_FOO2: "{{.TMPL2_FOO2}}"
+      OVERRIDE: "bar"
+
+  invalid-var-tmpl:
+    vars:
+      CHARS: "abcd"
+      INVALID: "{{range .CHARS}}no end"

testdata/vars/v2/Taskvars.yml

@@ -0,0 +1,12 @@
+FOO2: foo2
+BAR2: $echo bar2
+BAZ2:
+  sh: echo baz2
+TMPL2_FOO: "{{.FOO}}"
+TMPL2_BAR: "{{.BAR}}"
+TMPL2_FOO2: "{{.FOO2}}"
+TMPL2_BAR2: "{{.BAR2}}"
+SHTMPL2_FOO2:
+  sh: "echo '{{.FOO2}}'"
+NESTEDTMPL2_FOO2: "{{.TMPL2_FOO2}}"
+OVERRIDE: "foo"

testdata/vars/v2/multiline/Taskfile.yml

@@ -0,0 +1,45 @@
+version: 2
+tasks:
+  default:
+    vars:
+      MULTILINE: "\n\nfoo\n  bar\nfoobar\n\nbaz\n\n"
+    cmds:
+      - task: file
+        vars:
+          CONTENT:
+            sh: "echo 'foo\nbar'"
+          FILE: "echo_foobar.txt"
+      - task: file
+        vars:
+          CONTENT:
+            sh: "echo -n 'foo\nbar'"
+          FILE: "echo_n_foobar.txt"
+      - task: file
+        vars:
+          CONTENT:
+            sh: echo -n "{{.MULTILINE}}"
+          FILE: "echo_n_multiline.txt"
+      - task: file
+        vars:
+          CONTENT: "{{.MULTILINE}}"
+          FILE: "var_multiline.txt"
+      - task: file
+        vars:
+          CONTENT: "{{.MULTILINE | catLines}}"
+          FILE: "var_catlines.txt"
+      - task: enumfile
+        vars:
+          LINES: "{{.MULTILINE}}"
+          FILE: "var_enumfile.txt"
+  file:
+    cmds:
+      - |
+        cat << EOF > '{{.FILE}}'
+        {{.CONTENT}}
+        EOF
+  enumfile:
+    cmds:
+      - |
+        cat << EOF > '{{.FILE}}'
+        {{range $i, $line := .LINES| splitLines}}{{$i}}:{{$line}}
+        {{end}}EOF