feat: looping over dependencies (#1541)

* feat: support for loops in deps

* chore: tests

* docs: looping over deps

docs/docs/usage.md

@@ -1197,6 +1197,43 @@ tasks:
       - echo 'bar'
 ```
 
+### Looping over dependencies
+
+All of the above looping techniques can also be applied to the `deps` property.
+This allows you to combine loops with concurrency:
+
+```yaml
+version: '3'
+
+tasks:
+  default:
+    deps:
+      - for: [foo, bar]
+        task: my-task
+        vars:
+          FILE: '{{.ITEM}}'
+
+  my-task:
+    cmds:
+      - echo '{{.FILE}}'
+```
+
+It is important to note that as `deps` are run in parallel, the order in which
+the iterations are run is not guaranteed and the output may vary. For example,
+the output of the above example may be either:
+
+```shell
+foo
+bar
+```
+
+or
+
+```shell
+bar
+foo
+```
+
 ## Forwarding CLI arguments to commands
 
 If `--` is given in the CLI, all following parameters are added to a special

docs/static/schema.json

@@ -48,17 +48,7 @@
         },
         "deps": {
           "description": "A list of dependencies of this task. Tasks defined here will run in parallel before this task.",
-          "type": "array",
-          "items": {
-            "oneOf": [
-              {
-                "type": "string"
-              },
-              {
-                "$ref": "#/definitions/task_call"
-              }
-            ]
-          }
+          "$ref": "#/definitions/deps"
         },
         "label": {
           "description": "Overrides the name of the task in the output when a task is run. Supports variables.",
@@ -216,10 +206,26 @@
           "$ref": "#/definitions/defer_call"
         },
         {
-          "$ref": "#/definitions/for_call"
+          "$ref": "#/definitions/for_cmds_call"
         }
       ]
     },
+    "deps": {
+      "type": "array",
+      "items": {
+        "oneOf": [
+          {
+            "type": "string"
+          },
+          {
+            "$ref": "#/definitions/task_call"
+          },
+          {
+            "$ref": "#/definitions/for_deps_call"
+          }
+        ]
+      }
+    },
     "set": {
       "type": "string",
       "enum": [
@@ -367,21 +373,11 @@
       "additionalProperties": false,
       "required": ["defer"]
     },
-    "for_call": {
+    "for_cmds_call": {
       "type": "object",
       "properties": {
         "for": {
-          "anyOf": [
-            {
-              "$ref": "#/definitions/for_list"
-            },
-            {
-              "$ref": "#/definitions/for_attribute"
-            },
-            {
-              "$ref": "#/definitions/for_var"
-            }
-          ]
+          "$ref": "#/definitions/for"
         },
         "cmd": {
           "description": "Command to run",
@@ -407,6 +403,45 @@
       "additionalProperties": false,
       "required": ["for"]
     },
+    "for_deps_call": {
+      "type": "object",
+      "properties": {
+        "for": {
+          "$ref": "#/definitions/for"
+        },
+        "silent": {
+          "description": "Silent mode disables echoing of command before Task runs it",
+          "type": "boolean"
+        },
+        "task": {
+          "description": "Task to run",
+          "type": "string"
+        },
+        "vars": {
+          "description": "Values passed to the task called",
+          "$ref": "#/definitions/vars"
+        }
+      },
+      "oneOf": [
+        {"required": ["cmd"]},
+        {"required": ["task"]}
+      ],
+      "additionalProperties": false,
+      "required": ["for"]
+    },
+    "for": {
+      "anyOf": [
+        {
+          "$ref": "#/definitions/for_list"
+        },
+        {
+          "$ref": "#/definitions/for_attribute"
+        },
+        {
+          "$ref": "#/definitions/for_var"
+        }
+      ]
+    },
     "for_list": {
       "description": "A list of values to iterate over",
       "type": "array",

task_test.go

@@ -10,6 +10,7 @@ import (
 	"regexp"
 	"runtime"
 	"strings"
+	"sync"
 	"testing"
 
 	"github.com/Masterminds/semver/v3"
@@ -26,6 +27,21 @@ func init() {
 	_ = os.Setenv("NO_COLOR", "1")
 }
 
+// SyncBuffer is a threadsafe buffer for testing.
+// Some times replace stdout/stderr with a buffer to capture output.
+// stdout and stderr are threadsafe, but a regular bytes.Buffer is not.
+// Using this instead helps prevents race conditions with output.
+type SyncBuffer struct {
+	buf bytes.Buffer
+	mu  sync.Mutex
+}
+
+func (sb *SyncBuffer) Write(p []byte) (n int, err error) {
+	sb.mu.Lock()
+	defer sb.mu.Unlock()
+	return sb.buf.Write(p)
+}
+
 // fileContentTest provides a basic reusable test-case for running a Taskfile
 // and inspect generated files.
 type fileContentTest struct {
@@ -2199,7 +2215,7 @@ func TestForce(t *testing.T) {
 	}
 }
 
-func TestFor(t *testing.T) {
+func TestForCmds(t *testing.T) {
 	tests := []struct {
 		name           string
 		expectedOutput string
@@ -2240,9 +2256,67 @@ func TestFor(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			var buff bytes.Buffer
+			var stdOut bytes.Buffer
+			var stdErr bytes.Buffer
 			e := task.Executor{
-				Dir:    "testdata/for",
+				Dir:    "testdata/for/cmds",
+				Stdout: &stdOut,
+				Stderr: &stdErr,
+				Silent: true,
+				Force:  true,
+			}
+			require.NoError(t, e.Setup())
+			require.NoError(t, e.Run(context.Background(), &ast.Call{Task: test.name}))
+			assert.Equal(t, test.expectedOutput, stdOut.String())
+		})
+	}
+}
+
+func TestForDeps(t *testing.T) {
+	tests := []struct {
+		name                   string
+		expectedOutputContains []string
+	}{
+		{
+			name:                   "loop-explicit",
+			expectedOutputContains: []string{"a\n", "b\n", "c\n"},
+		},
+		{
+			name:                   "loop-sources",
+			expectedOutputContains: []string{"bar\n", "foo\n"},
+		},
+		{
+			name:                   "loop-sources-glob",
+			expectedOutputContains: []string{"bar\n", "foo\n"},
+		},
+		{
+			name:                   "loop-vars",
+			expectedOutputContains: []string{"foo\n", "bar\n"},
+		},
+		{
+			name:                   "loop-vars-sh",
+			expectedOutputContains: []string{"bar\n", "foo\n"},
+		},
+		{
+			name:                   "loop-task",
+			expectedOutputContains: []string{"foo\n", "bar\n"},
+		},
+		{
+			name:                   "loop-task-as",
+			expectedOutputContains: []string{"foo\n", "bar\n"},
+		},
+		{
+			name:                   "loop-different-tasks",
+			expectedOutputContains: []string{"1\n", "2\n", "3\n"},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			// We need to use a sync buffer here as deps are run concurrently
+			var buff SyncBuffer
+			e := task.Executor{
+				Dir:    "testdata/for/deps",
 				Stdout: &buff,
 				Stderr: &buff,
 				Silent: true,
@@ -2250,7 +2324,9 @@ func TestFor(t *testing.T) {
 			}
 			require.NoError(t, e.Setup())
 			require.NoError(t, e.Run(context.Background(), &ast.Call{Task: test.name}))
-			assert.Equal(t, test.expectedOutput, buff.String())
+			for _, expectedOutputContains := range test.expectedOutputContains {
+				assert.Contains(t, buff.buf.String(), expectedOutputContains)
+			}
 		})
 	}
 }

taskfile/ast/dep.go

@@ -9,6 +9,7 @@ import (
 // Dep is a task dependency
 type Dep struct {
 	Task   string
+	For    *For
 	Vars   *Vars
 	Silent bool
 }
@@ -19,6 +20,7 @@ func (d *Dep) DeepCopy() *Dep {
 	}
 	return &Dep{
 		Task:   d.Task,
+		For:    d.For.DeepCopy(),
 		Vars:   d.Vars.DeepCopy(),
 		Silent: d.Silent,
 	}
@@ -38,6 +40,7 @@ func (d *Dep) UnmarshalYAML(node *yaml.Node) error {
 	case yaml.MappingNode:
 		var taskCall struct {
 			Task   string
+			For    *For
 			Vars   *Vars
 			Silent bool
 		}
@@ -45,6 +48,7 @@ func (d *Dep) UnmarshalYAML(node *yaml.Node) error {
 			return err
 		}
 		d.Task = taskCall.Task
+		d.For = taskCall.For
 		d.Vars = taskCall.Vars
 		d.Silent = taskCall.Silent
 		return nil

testdata/for/deps/Taskfile.yml

@@ -0,0 +1,111 @@
+version: "3"
+
+tasks:
+  # Loop over a list of values
+  loop-explicit:
+    deps:
+      - for: ["a", "b", "c"]
+        task: echo
+        vars:
+          TEXT: "{{.ITEM}}"
+
+  # Loop over the task's sources
+  loop-sources:
+    sources:
+      - foo.txt
+      - bar.txt
+    deps:
+      - for: sources
+        task: cat
+        vars:
+          FILE: "{{.ITEM}}"
+
+  # Loop over the task's sources when globbed
+  loop-sources-glob:
+    sources:
+      - "*.txt"
+    deps:
+      - for: sources
+        task: cat
+        vars:
+          FILE: "{{.ITEM}}"
+
+  # Loop over the contents of a variable
+  loop-vars:
+    vars:
+      FOO: foo.txt,bar.txt
+    deps:
+      - for:
+          var: FOO
+          split: ","
+        task: cat
+        vars:
+          FILE: "{{.ITEM}}"
+
+  # Loop over the output of a command (auto splits on " ")
+  loop-vars-sh:
+    vars:
+      FOO:
+        sh: ls *.txt
+    deps:
+      - for:
+          var: FOO
+        task: cat
+        vars:
+          FILE: "{{.ITEM}}"
+
+  # Loop over another task
+  loop-task:
+    vars:
+      FOO: foo.txt bar.txt
+    deps:
+      - for:
+          var: FOO
+        task: looped-task
+        vars:
+          FILE: "{{.ITEM}}"
+
+  # Loop over another task with the variable named differently
+  loop-task-as:
+    vars:
+      FOO: foo.txt bar.txt
+    deps:
+      - for:
+          var: FOO
+          as: FILE
+        task: looped-task
+        vars:
+          FILE: "{{.FILE}}"
+
+  # Loop over different tasks using the variable
+  loop-different-tasks:
+    vars:
+      FOO: "1 2 3"
+    deps:
+      - for:
+          var: FOO
+        task: task-{{.ITEM}}
+
+  looped-task:
+    internal: true
+    cmd: cat "{{.FILE}}"
+
+  task-1:
+    internal: true
+    cmd: echo "1"
+
+  task-2:
+    internal: true
+    cmd: echo "2"
+
+  task-3:
+    internal: true
+    cmd: echo "3"
+
+  echo:
+    cmds:
+      - echo "{{.TEXT}}"
+
+  cat:
+    cmds:
+      - cat "{{.FILE}}"

testdata/for/deps/bar.txt

@@ -0,0 +1 @@
+bar

testdata/for/deps/foo.txt

@@ -0,0 +1 @@
+foo

variables.go

@@ -133,57 +133,10 @@ func (e *Executor) compiledTask(call *ast.Call, evaluateShVars bool) (*ast.Task,
 				continue
 			}
 			if cmd.For != nil {
-				var keys []string
-				var list []any
-				// Get the list from the explicit for list
-				if cmd.For.List != nil && len(cmd.For.List) > 0 {
-					list = cmd.For.List
-				}
-				// Get the list from the task sources
-				if cmd.For.From == "sources" {
-					glist, err := fingerprint.Globs(new.Dir, new.Sources)
+				list, keys, err := itemsFromFor(cmd.For, new.Dir, new.Sources, vars, origTask.Location)
 				if err != nil {
 					return nil, err
 				}
-					// Make the paths relative to the task dir
-					for i, v := range glist {
-						if glist[i], err = filepath.Rel(new.Dir, v); err != nil {
-							return nil, err
-						}
-					}
-					list = asAnySlice(glist)
-				}
-				// Get the list from a variable and split it up
-				if cmd.For.Var != "" {
-					if vars != nil {
-						v := vars.Get(cmd.For.Var)
-						// If the variable is dynamic, then it hasn't been resolved yet
-						// and we can't use it as a list. This happens when fast compiling a task
-						// for use in --list or --list-all etc.
-						if v.Value != nil && v.Sh == "" {
-							switch value := v.Value.(type) {
-							case string:
-								if cmd.For.Split != "" {
-									list = asAnySlice(strings.Split(value, cmd.For.Split))
-								} else {
-									list = asAnySlice(strings.Fields(value))
-								}
-							case []any:
-								list = value
-							case map[string]any:
-								for k, v := range value {
-									keys = append(keys, k)
-									list = append(list, v)
-								}
-							default:
-								return nil, errors.TaskfileInvalidError{
-									URI: origTask.Location.Taskfile,
-									Err: errors.New("var must be a delimiter-separated string or a list"),
-								}
-							}
-						}
-					}
-				}
 				// Name the iterator variable
 				var as string
 				if cmd.For.As != "" {
@@ -231,6 +184,33 @@ func (e *Executor) compiledTask(call *ast.Call, evaluateShVars bool) (*ast.Task,
 			if dep == nil {
 				continue
 			}
+			if dep.For != nil {
+				list, keys, err := itemsFromFor(dep.For, new.Dir, new.Sources, vars, origTask.Location)
+				if err != nil {
+					return nil, err
+				}
+				// Name the iterator variable
+				var as string
+				if dep.For.As != "" {
+					as = dep.For.As
+				} else {
+					as = "ITEM"
+				}
+				// Create a new command for each item in the list
+				for i, loopValue := range list {
+					extra := map[string]any{
+						as: loopValue,
+					}
+					if len(keys) > 0 {
+						extra["KEY"] = keys[i]
+					}
+					newDep := dep.DeepCopy()
+					newDep.Task = r.ReplaceWithExtra(dep.Task, extra)
+					newDep.Vars = r.ReplaceVarsWithExtra(dep.Vars, extra)
+					new.Deps = append(new.Deps, newDep)
+				}
+				continue
+			}
 			newDep := dep.DeepCopy()
 			newDep.Task = templater.Replace(dep.Task, cache)
 			newDep.Vars = templater.ReplaceVars(dep.Vars, cache)
@@ -296,3 +276,64 @@ func asAnySlice[T any](slice []T) []any {
 	}
 	return ret
 }
+
+func itemsFromFor(
+	f *ast.For,
+	dir string,
+	sources []*ast.Glob,
+	vars *ast.Vars,
+	location *ast.Location,
+) ([]any, []string, error) {
+	var keys []string // The list of keys to loop over (only if looping over a map)
+	var values []any  // The list of values to loop over
+	// Get the list from the explicit for list
+	if f.List != nil && len(f.List) > 0 {
+		values = f.List
+	}
+	// Get the list from the task sources
+	if f.From == "sources" {
+		glist, err := fingerprint.Globs(dir, sources)
+		if err != nil {
+			return nil, nil, err
+		}
+		// Make the paths relative to the task dir
+		for i, v := range glist {
+			if glist[i], err = filepath.Rel(dir, v); err != nil {
+				return nil, nil, err
+			}
+		}
+		values = asAnySlice(glist)
+	}
+	// Get the list from a variable and split it up
+	if f.Var != "" {
+		if vars != nil {
+			v := vars.Get(f.Var)
+			// If the variable is dynamic, then it hasn't been resolved yet
+			// and we can't use it as a list. This happens when fast compiling a task
+			// for use in --list or --list-all etc.
+			if v.Value != nil && v.Sh == "" {
+				switch value := v.Value.(type) {
+				case string:
+					if f.Split != "" {
+						values = asAnySlice(strings.Split(value, f.Split))
+					} else {
+						values = asAnySlice(strings.Fields(value))
+					}
+				case []any:
+					values = value
+				case map[string]any:
+					for k, v := range value {
+						keys = append(keys, k)
+						values = append(values, v)
+					}
+				default:
+					return nil, nil, errors.TaskfileInvalidError{
+						URI: location.Taskfile,
+						Err: errors.New("loop var must be a delimiter-separated string, list or a map"),
+					}
+				}
+			}
+		}
+	}
+	return values, keys, nil
+}