diff --git a/docs/docs/usage.md b/docs/docs/usage.md index 7d1f1d73..c04ba8d6 100644 --- a/docs/docs/usage.md +++ b/docs/docs/usage.md @@ -1197,6 +1197,43 @@ tasks: - echo 'bar' ``` +### Looping over dependencies + +All of the above looping techniques can also be applied to the `deps` property. +This allows you to combine loops with concurrency: + +```yaml +version: '3' + +tasks: + default: + deps: + - for: [foo, bar] + task: my-task + vars: + FILE: '{{.ITEM}}' + + my-task: + cmds: + - echo '{{.FILE}}' +``` + +It is important to note that as `deps` are run in parallel, the order in which +the iterations are run is not guaranteed and the output may vary. For example, +the output of the above example may be either: + +```shell +foo +bar +``` + +or + +```shell +bar +foo +``` + ## Forwarding CLI arguments to commands If `--` is given in the CLI, all following parameters are added to a special diff --git a/docs/static/schema.json b/docs/static/schema.json index cb73c478..fb5c3364 100644 --- a/docs/static/schema.json +++ b/docs/static/schema.json @@ -48,17 +48,7 @@ }, "deps": { "description": "A list of dependencies of this task. Tasks defined here will run in parallel before this task.", - "type": "array", - "items": { - "oneOf": [ - { - "type": "string" - }, - { - "$ref": "#/definitions/task_call" - } - ] - } + "$ref": "#/definitions/deps" }, "label": { "description": "Overrides the name of the task in the output when a task is run. Supports variables.", @@ -216,10 +206,26 @@ "$ref": "#/definitions/defer_call" }, { - "$ref": "#/definitions/for_call" + "$ref": "#/definitions/for_cmds_call" } ] }, + "deps": { + "type": "array", + "items": { + "oneOf": [ + { + "type": "string" + }, + { + "$ref": "#/definitions/task_call" + }, + { + "$ref": "#/definitions/for_deps_call" + } + ] + } + }, "set": { "type": "string", "enum": [ @@ -367,21 +373,11 @@ "additionalProperties": false, "required": ["defer"] }, - "for_call": { + "for_cmds_call": { "type": "object", "properties": { "for": { - "anyOf": [ - { - "$ref": "#/definitions/for_list" - }, - { - "$ref": "#/definitions/for_attribute" - }, - { - "$ref": "#/definitions/for_var" - } - ] + "$ref": "#/definitions/for" }, "cmd": { "description": "Command to run", @@ -407,6 +403,45 @@ "additionalProperties": false, "required": ["for"] }, + "for_deps_call": { + "type": "object", + "properties": { + "for": { + "$ref": "#/definitions/for" + }, + "silent": { + "description": "Silent mode disables echoing of command before Task runs it", + "type": "boolean" + }, + "task": { + "description": "Task to run", + "type": "string" + }, + "vars": { + "description": "Values passed to the task called", + "$ref": "#/definitions/vars" + } + }, + "oneOf": [ + {"required": ["cmd"]}, + {"required": ["task"]} + ], + "additionalProperties": false, + "required": ["for"] + }, + "for": { + "anyOf": [ + { + "$ref": "#/definitions/for_list" + }, + { + "$ref": "#/definitions/for_attribute" + }, + { + "$ref": "#/definitions/for_var" + } + ] + }, "for_list": { "description": "A list of values to iterate over", "type": "array", diff --git a/task_test.go b/task_test.go index 4c02565c..b3035cb1 100644 --- a/task_test.go +++ b/task_test.go @@ -10,6 +10,7 @@ import ( "regexp" "runtime" "strings" + "sync" "testing" "github.com/Masterminds/semver/v3" @@ -26,6 +27,21 @@ func init() { _ = os.Setenv("NO_COLOR", "1") } +// SyncBuffer is a threadsafe buffer for testing. +// Some times replace stdout/stderr with a buffer to capture output. +// stdout and stderr are threadsafe, but a regular bytes.Buffer is not. +// Using this instead helps prevents race conditions with output. +type SyncBuffer struct { + buf bytes.Buffer + mu sync.Mutex +} + +func (sb *SyncBuffer) Write(p []byte) (n int, err error) { + sb.mu.Lock() + defer sb.mu.Unlock() + return sb.buf.Write(p) +} + // fileContentTest provides a basic reusable test-case for running a Taskfile // and inspect generated files. type fileContentTest struct { @@ -2199,7 +2215,7 @@ func TestForce(t *testing.T) { } } -func TestFor(t *testing.T) { +func TestForCmds(t *testing.T) { tests := []struct { name string expectedOutput string @@ -2240,9 +2256,67 @@ func TestFor(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - var buff bytes.Buffer + var stdOut bytes.Buffer + var stdErr bytes.Buffer e := task.Executor{ - Dir: "testdata/for", + Dir: "testdata/for/cmds", + Stdout: &stdOut, + Stderr: &stdErr, + Silent: true, + Force: true, + } + require.NoError(t, e.Setup()) + require.NoError(t, e.Run(context.Background(), &ast.Call{Task: test.name})) + assert.Equal(t, test.expectedOutput, stdOut.String()) + }) + } +} + +func TestForDeps(t *testing.T) { + tests := []struct { + name string + expectedOutputContains []string + }{ + { + name: "loop-explicit", + expectedOutputContains: []string{"a\n", "b\n", "c\n"}, + }, + { + name: "loop-sources", + expectedOutputContains: []string{"bar\n", "foo\n"}, + }, + { + name: "loop-sources-glob", + expectedOutputContains: []string{"bar\n", "foo\n"}, + }, + { + name: "loop-vars", + expectedOutputContains: []string{"foo\n", "bar\n"}, + }, + { + name: "loop-vars-sh", + expectedOutputContains: []string{"bar\n", "foo\n"}, + }, + { + name: "loop-task", + expectedOutputContains: []string{"foo\n", "bar\n"}, + }, + { + name: "loop-task-as", + expectedOutputContains: []string{"foo\n", "bar\n"}, + }, + { + name: "loop-different-tasks", + expectedOutputContains: []string{"1\n", "2\n", "3\n"}, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + // We need to use a sync buffer here as deps are run concurrently + var buff SyncBuffer + e := task.Executor{ + Dir: "testdata/for/deps", Stdout: &buff, Stderr: &buff, Silent: true, @@ -2250,7 +2324,9 @@ func TestFor(t *testing.T) { } require.NoError(t, e.Setup()) require.NoError(t, e.Run(context.Background(), &ast.Call{Task: test.name})) - assert.Equal(t, test.expectedOutput, buff.String()) + for _, expectedOutputContains := range test.expectedOutputContains { + assert.Contains(t, buff.buf.String(), expectedOutputContains) + } }) } } diff --git a/taskfile/ast/dep.go b/taskfile/ast/dep.go index 45f83c78..5119d731 100644 --- a/taskfile/ast/dep.go +++ b/taskfile/ast/dep.go @@ -9,6 +9,7 @@ import ( // Dep is a task dependency type Dep struct { Task string + For *For Vars *Vars Silent bool } @@ -19,6 +20,7 @@ func (d *Dep) DeepCopy() *Dep { } return &Dep{ Task: d.Task, + For: d.For.DeepCopy(), Vars: d.Vars.DeepCopy(), Silent: d.Silent, } @@ -38,6 +40,7 @@ func (d *Dep) UnmarshalYAML(node *yaml.Node) error { case yaml.MappingNode: var taskCall struct { Task string + For *For Vars *Vars Silent bool } @@ -45,6 +48,7 @@ func (d *Dep) UnmarshalYAML(node *yaml.Node) error { return err } d.Task = taskCall.Task + d.For = taskCall.For d.Vars = taskCall.Vars d.Silent = taskCall.Silent return nil diff --git a/testdata/for/Taskfile.yml b/testdata/for/cmds/Taskfile.yml similarity index 100% rename from testdata/for/Taskfile.yml rename to testdata/for/cmds/Taskfile.yml diff --git a/testdata/for/bar.txt b/testdata/for/cmds/bar.txt similarity index 100% rename from testdata/for/bar.txt rename to testdata/for/cmds/bar.txt diff --git a/testdata/for/foo.txt b/testdata/for/cmds/foo.txt similarity index 100% rename from testdata/for/foo.txt rename to testdata/for/cmds/foo.txt diff --git a/testdata/for/deps/Taskfile.yml b/testdata/for/deps/Taskfile.yml new file mode 100644 index 00000000..e3c67598 --- /dev/null +++ b/testdata/for/deps/Taskfile.yml @@ -0,0 +1,111 @@ +version: "3" + +tasks: + # Loop over a list of values + loop-explicit: + deps: + - for: ["a", "b", "c"] + task: echo + vars: + TEXT: "{{.ITEM}}" + + # Loop over the task's sources + loop-sources: + sources: + - foo.txt + - bar.txt + deps: + - for: sources + task: cat + vars: + FILE: "{{.ITEM}}" + + # Loop over the task's sources when globbed + loop-sources-glob: + sources: + - "*.txt" + deps: + - for: sources + task: cat + vars: + FILE: "{{.ITEM}}" + + # Loop over the contents of a variable + loop-vars: + vars: + FOO: foo.txt,bar.txt + deps: + - for: + var: FOO + split: "," + task: cat + vars: + FILE: "{{.ITEM}}" + + # Loop over the output of a command (auto splits on " ") + loop-vars-sh: + vars: + FOO: + sh: ls *.txt + deps: + - for: + var: FOO + task: cat + vars: + FILE: "{{.ITEM}}" + + # Loop over another task + loop-task: + vars: + FOO: foo.txt bar.txt + deps: + - for: + var: FOO + task: looped-task + vars: + FILE: "{{.ITEM}}" + + # Loop over another task with the variable named differently + loop-task-as: + vars: + FOO: foo.txt bar.txt + deps: + - for: + var: FOO + as: FILE + task: looped-task + vars: + FILE: "{{.FILE}}" + + # Loop over different tasks using the variable + loop-different-tasks: + vars: + FOO: "1 2 3" + deps: + - for: + var: FOO + task: task-{{.ITEM}} + + looped-task: + internal: true + cmd: cat "{{.FILE}}" + + task-1: + internal: true + cmd: echo "1" + + task-2: + internal: true + cmd: echo "2" + + task-3: + internal: true + cmd: echo "3" + + echo: + cmds: + - echo "{{.TEXT}}" + + cat: + cmds: + - cat "{{.FILE}}" diff --git a/testdata/for/deps/bar.txt b/testdata/for/deps/bar.txt new file mode 100644 index 00000000..5716ca59 --- /dev/null +++ b/testdata/for/deps/bar.txt @@ -0,0 +1 @@ +bar diff --git a/testdata/for/deps/foo.txt b/testdata/for/deps/foo.txt new file mode 100644 index 00000000..257cc564 --- /dev/null +++ b/testdata/for/deps/foo.txt @@ -0,0 +1 @@ +foo diff --git a/variables.go b/variables.go index 39ec98dd..cf218870 100644 --- a/variables.go +++ b/variables.go @@ -133,56 +133,9 @@ func (e *Executor) compiledTask(call *ast.Call, evaluateShVars bool) (*ast.Task, continue } if cmd.For != nil { - var keys []string - var list []any - // Get the list from the explicit for list - if cmd.For.List != nil && len(cmd.For.List) > 0 { - list = cmd.For.List - } - // Get the list from the task sources - if cmd.For.From == "sources" { - glist, err := fingerprint.Globs(new.Dir, new.Sources) - if err != nil { - return nil, err - } - // Make the paths relative to the task dir - for i, v := range glist { - if glist[i], err = filepath.Rel(new.Dir, v); err != nil { - return nil, err - } - } - list = asAnySlice(glist) - } - // Get the list from a variable and split it up - if cmd.For.Var != "" { - if vars != nil { - v := vars.Get(cmd.For.Var) - // If the variable is dynamic, then it hasn't been resolved yet - // and we can't use it as a list. This happens when fast compiling a task - // for use in --list or --list-all etc. - if v.Value != nil && v.Sh == "" { - switch value := v.Value.(type) { - case string: - if cmd.For.Split != "" { - list = asAnySlice(strings.Split(value, cmd.For.Split)) - } else { - list = asAnySlice(strings.Fields(value)) - } - case []any: - list = value - case map[string]any: - for k, v := range value { - keys = append(keys, k) - list = append(list, v) - } - default: - return nil, errors.TaskfileInvalidError{ - URI: origTask.Location.Taskfile, - Err: errors.New("var must be a delimiter-separated string or a list"), - } - } - } - } + list, keys, err := itemsFromFor(cmd.For, new.Dir, new.Sources, vars, origTask.Location) + if err != nil { + return nil, err } // Name the iterator variable var as string @@ -231,6 +184,33 @@ func (e *Executor) compiledTask(call *ast.Call, evaluateShVars bool) (*ast.Task, if dep == nil { continue } + if dep.For != nil { + list, keys, err := itemsFromFor(dep.For, new.Dir, new.Sources, vars, origTask.Location) + if err != nil { + return nil, err + } + // Name the iterator variable + var as string + if dep.For.As != "" { + as = dep.For.As + } else { + as = "ITEM" + } + // Create a new command for each item in the list + for i, loopValue := range list { + extra := map[string]any{ + as: loopValue, + } + if len(keys) > 0 { + extra["KEY"] = keys[i] + } + newDep := dep.DeepCopy() + newDep.Task = r.ReplaceWithExtra(dep.Task, extra) + newDep.Vars = r.ReplaceVarsWithExtra(dep.Vars, extra) + new.Deps = append(new.Deps, newDep) + } + continue + } newDep := dep.DeepCopy() newDep.Task = templater.Replace(dep.Task, cache) newDep.Vars = templater.ReplaceVars(dep.Vars, cache) @@ -296,3 +276,64 @@ func asAnySlice[T any](slice []T) []any { } return ret } + +func itemsFromFor( + f *ast.For, + dir string, + sources []*ast.Glob, + vars *ast.Vars, + location *ast.Location, +) ([]any, []string, error) { + var keys []string // The list of keys to loop over (only if looping over a map) + var values []any // The list of values to loop over + // Get the list from the explicit for list + if f.List != nil && len(f.List) > 0 { + values = f.List + } + // Get the list from the task sources + if f.From == "sources" { + glist, err := fingerprint.Globs(dir, sources) + if err != nil { + return nil, nil, err + } + // Make the paths relative to the task dir + for i, v := range glist { + if glist[i], err = filepath.Rel(dir, v); err != nil { + return nil, nil, err + } + } + values = asAnySlice(glist) + } + // Get the list from a variable and split it up + if f.Var != "" { + if vars != nil { + v := vars.Get(f.Var) + // If the variable is dynamic, then it hasn't been resolved yet + // and we can't use it as a list. This happens when fast compiling a task + // for use in --list or --list-all etc. + if v.Value != nil && v.Sh == "" { + switch value := v.Value.(type) { + case string: + if f.Split != "" { + values = asAnySlice(strings.Split(value, f.Split)) + } else { + values = asAnySlice(strings.Fields(value)) + } + case []any: + values = value + case map[string]any: + for k, v := range value { + keys = append(keys, k) + values = append(values, v) + } + default: + return nil, nil, errors.TaskfileInvalidError{ + URI: location.Taskfile, + Err: errors.New("loop var must be a delimiter-separated string, list or a map"), + } + } + } + } + } + return values, keys, nil +}