From 7ff1b1795e8d8a1f459460280fc9bef868bfed5b Mon Sep 17 00:00:00 2001 From: Pete Davison Date: Thu, 15 Jun 2023 15:04:03 +0000 Subject: [PATCH] feat: for --- docs/static/schema.json | 76 +++++++++++++++++- internal/fingerprint/glob.go | 2 +- internal/fingerprint/sources_checksum.go | 2 +- internal/fingerprint/sources_timestamp.go | 6 +- internal/templater/templater.go | 32 +++++++- task_test.go | 56 ++++++++++++++ taskfile/cmd.go | 10 ++- taskfile/for.go | 68 +++++++++++++++++ testdata/for/Taskfile.yml | 93 +++++++++++++++++++++++ testdata/for/bar.txt | 1 + testdata/for/foo.txt | 1 + variables.go | 52 ++++++++++++- 12 files changed, 387 insertions(+), 12 deletions(-) create mode 100644 taskfile/for.go create mode 100644 testdata/for/Taskfile.yml create mode 100644 testdata/for/bar.txt create mode 100644 testdata/for/foo.txt diff --git a/docs/static/schema.json b/docs/static/schema.json index f4e7ee07..5641c035 100644 --- a/docs/static/schema.json +++ b/docs/static/schema.json @@ -207,6 +207,9 @@ }, { "$ref": "#/definitions/3/task_call" + }, + { + "$ref": "#/definitions/3/for_call" } ] }, @@ -272,7 +275,9 @@ "description": "Hides task name and command from output. The command's output will still be redirected to `STDOUT` and `STDERR`.", "type": "boolean" } - } + }, + "additionalProperties": false, + "required": ["task"] }, "cmd_call": { "type": "object", @@ -318,6 +323,75 @@ "additionalProperties": false, "required": ["cmd"] }, + "for_call": { + "type": "object", + "properties": { + "for": { + "anyOf": [ + { + "$ref": "#/definitions/3/for_list" + }, + { + "$ref": "#/definitions/3/for_source" + }, + { + "$ref": "#/definitions/3/for_var" + } + ] + }, + "cmd": { + "description": "Command to run", + "type": "string" + }, + "task": { + "description": "Task to run", + "type": "string" + }, + "vars": { + "description": "Values passed to the task called", + "$ref": "#/definitions/3/vars" + } + }, + "oneOf": [ + {"required": ["cmd"]}, + {"required": ["task"]} + ], + "additionalProperties": false, + "required": ["for"] + }, + "for_list": { + "description": "List of values to iterate over", + "type": "array", + "items": { + "type": "string" + } + }, + "for_source": { + "description": "List of values to iterate over", + "type": "string", + "enum": ["source"] + }, + "for_var": { + "description": "List of values to iterate over", + "type": "object", + "properties": { + "var": { + "description": "Name of the variable to iterate over", + "type": "string" + }, + "split": { + "description": "String to split the variable on", + "type": "string" + }, + "as": { + "description": "What the loop variable should be named", + "default": "ITEM", + "type": "string" + } + }, + "additionalProperties": false, + "required": ["var"] + }, "precondition": { "anyOf": [ { diff --git a/internal/fingerprint/glob.go b/internal/fingerprint/glob.go index 0b7fc837..ecf0e54c 100644 --- a/internal/fingerprint/glob.go +++ b/internal/fingerprint/glob.go @@ -10,7 +10,7 @@ import ( "github.com/go-task/task/v3/internal/filepathext" ) -func globs(dir string, globs []string) ([]string, error) { +func Globs(dir string, globs []string) ([]string, error) { files := make([]string, 0) for _, g := range globs { f, err := Glob(dir, g) diff --git a/internal/fingerprint/sources_checksum.go b/internal/fingerprint/sources_checksum.go index f38903ca..e9fb2980 100644 --- a/internal/fingerprint/sources_checksum.go +++ b/internal/fingerprint/sources_checksum.go @@ -84,7 +84,7 @@ func (*ChecksumChecker) Kind() string { } func (c *ChecksumChecker) checksum(t *taskfile.Task) (string, error) { - sources, err := globs(t.Dir, t.Sources) + sources, err := Globs(t.Dir, t.Sources) if err != nil { return "", err } diff --git a/internal/fingerprint/sources_timestamp.go b/internal/fingerprint/sources_timestamp.go index 2d53d78f..aafb9bca 100644 --- a/internal/fingerprint/sources_timestamp.go +++ b/internal/fingerprint/sources_timestamp.go @@ -28,11 +28,11 @@ func (checker *TimestampChecker) IsUpToDate(t *taskfile.Task) (bool, error) { return false, nil } - sources, err := globs(t.Dir, t.Sources) + sources, err := Globs(t.Dir, t.Sources) if err != nil { return false, nil } - generates, err := globs(t.Dir, t.Generates) + generates, err := Globs(t.Dir, t.Generates) if err != nil { return false, nil } @@ -90,7 +90,7 @@ func (checker *TimestampChecker) Kind() string { // Value implements the Checker Interface func (checker *TimestampChecker) Value(t *taskfile.Task) (any, error) { - sources, err := globs(t.Dir, t.Sources) + sources, err := Globs(t.Dir, t.Sources) if err != nil { return time.Now(), err } diff --git a/internal/templater/templater.go b/internal/templater/templater.go index 46ff5eb0..c5486281 100644 --- a/internal/templater/templater.go +++ b/internal/templater/templater.go @@ -5,6 +5,8 @@ import ( "strings" "text/template" + "golang.org/x/exp/maps" + "github.com/go-task/task/v3/taskfile" ) @@ -25,6 +27,14 @@ func (r *Templater) ResetCache() { } func (r *Templater) Replace(str string) string { + return r.replace(str, nil) +} + +func (r *Templater) ReplaceWithExtra(str string, extra map[string]any) string { + return r.replace(str, extra) +} + +func (r *Templater) replace(str string, extra map[string]any) string { if r.err != nil || str == "" { return "" } @@ -40,7 +50,15 @@ func (r *Templater) Replace(str string) string { } var b bytes.Buffer - if err = templ.Execute(&b, r.cacheMap); err != nil { + if extra == nil { + err = templ.Execute(&b, r.cacheMap) + } else { + // Copy the map to avoid modifying the cached map + m := maps.Clone(r.cacheMap) + maps.Copy(m, extra) + err = templ.Execute(&b, m) + } + if err != nil { r.err = err return "" } @@ -63,6 +81,14 @@ func (r *Templater) ReplaceSlice(strs []string) []string { } func (r *Templater) ReplaceVars(vars *taskfile.Vars) *taskfile.Vars { + return r.replaceVars(vars, nil) +} + +func (r *Templater) ReplaceVarsWithExtra(vars *taskfile.Vars, extra map[string]any) *taskfile.Vars { + return r.replaceVars(vars, extra) +} + +func (r *Templater) replaceVars(vars *taskfile.Vars, extra map[string]any) *taskfile.Vars { if r.err != nil || vars.Len() == 0 { return nil } @@ -70,9 +96,9 @@ func (r *Templater) ReplaceVars(vars *taskfile.Vars) *taskfile.Vars { var new taskfile.Vars _ = vars.Range(func(k string, v taskfile.Var) error { new.Set(k, taskfile.Var{ - Static: r.Replace(v.Static), + Static: r.ReplaceWithExtra(v.Static, extra), Live: v.Live, - Sh: r.Replace(v.Sh), + Sh: r.ReplaceWithExtra(v.Sh, extra), }) return nil }) diff --git a/task_test.go b/task_test.go index 950a1b36..5c5a153a 100644 --- a/task_test.go +++ b/task_test.go @@ -2207,3 +2207,59 @@ func TestForce(t *testing.T) { }) } } + +func TestFor(t *testing.T) { + tests := []struct { + name string + expectedOutput string + }{ + { + name: "loop-explicit", + expectedOutput: "a\nb\nc\n", + }, + { + name: "loop-sources", + expectedOutput: "bar\nfoo\n", + }, + { + name: "loop-sources-glob", + expectedOutput: "bar\nfoo\n", + }, + { + name: "loop-vars", + expectedOutput: "foo\nbar\n", + }, + { + name: "loop-vars-sh", + expectedOutput: "bar\nfoo\n", + }, + { + name: "loop-task", + expectedOutput: "foo\nbar\n", + }, + { + name: "loop-task-as", + expectedOutput: "foo\nbar\n", + }, + { + name: "loop-different-tasks", + expectedOutput: "1\n2\n3\n", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + var buff bytes.Buffer + e := task.Executor{ + Dir: "testdata/for", + Stdout: &buff, + Stderr: &buff, + Silent: true, + Force: true, + } + require.NoError(t, e.Setup()) + require.NoError(t, e.Run(context.Background(), taskfile.Call{Task: test.name, Direct: true})) + assert.Equal(t, test.expectedOutput, buff.String()) + }) + } +} diff --git a/taskfile/cmd.go b/taskfile/cmd.go index 5bb54421..4157a3a6 100644 --- a/taskfile/cmd.go +++ b/taskfile/cmd.go @@ -11,8 +11,9 @@ import ( // Cmd is a task command type Cmd struct { Cmd string - Silent bool Task string + For *For + Silent bool Set []string Shopt []string Vars *Vars @@ -27,8 +28,9 @@ func (c *Cmd) DeepCopy() *Cmd { } return &Cmd{ Cmd: c.Cmd, - Silent: c.Silent, Task: c.Task, + For: c.For.DeepCopy(), + Silent: c.Silent, Set: deepcopy.Slice(c.Set), Shopt: deepcopy.Slice(c.Shopt), Vars: c.Vars.DeepCopy(), @@ -54,6 +56,7 @@ func (c *Cmd) UnmarshalYAML(node *yaml.Node) error { // A command with additional options var cmdStruct struct { Cmd string + For *For Silent bool Set []string Shopt []string @@ -62,6 +65,7 @@ func (c *Cmd) UnmarshalYAML(node *yaml.Node) error { } if err := node.Decode(&cmdStruct); err == nil && cmdStruct.Cmd != "" { c.Cmd = cmdStruct.Cmd + c.For = cmdStruct.For c.Silent = cmdStruct.Silent c.Set = cmdStruct.Set c.Shopt = cmdStruct.Shopt @@ -95,10 +99,12 @@ func (c *Cmd) UnmarshalYAML(node *yaml.Node) error { var taskCall struct { Task string Vars *Vars + For *For } if err := node.Decode(&taskCall); err == nil && taskCall.Task != "" { c.Task = taskCall.Task c.Vars = taskCall.Vars + c.For = taskCall.For c.Silent = cmdStruct.Silent return nil } diff --git a/taskfile/for.go b/taskfile/for.go new file mode 100644 index 00000000..6f609dea --- /dev/null +++ b/taskfile/for.go @@ -0,0 +1,68 @@ +package taskfile + +import ( + "fmt" + + "gopkg.in/yaml.v3" + + "github.com/go-task/task/v3/internal/deepcopy" +) + +type For struct { + From string + List []string + Var string + Split string + As string +} + +func (f *For) UnmarshalYAML(node *yaml.Node) error { + switch node.Kind { + + case yaml.ScalarNode: + var from string + if err := node.Decode(&from); err != nil { + return err + } + f.From = from + return nil + + case yaml.SequenceNode: + var list []string + if err := node.Decode(&list); err != nil { + return err + } + f.List = list + return nil + + case yaml.MappingNode: + var forStruct struct { + Var string + Split string + As string + } + if err := node.Decode(&forStruct); err == nil && forStruct.Var != "" { + f.Var = forStruct.Var + f.Split = forStruct.Split + f.As = forStruct.As + return nil + } + + return fmt.Errorf("yaml: line %d: invalid keys in for", node.Line) + } + + return fmt.Errorf("yaml: line %d: cannot unmarshal %s into for", node.Line, node.ShortTag()) +} + +func (f *For) DeepCopy() *For { + if f == nil { + return nil + } + return &For{ + From: f.From, + List: deepcopy.Slice(f.List), + Var: f.Var, + Split: f.Split, + As: f.As, + } +} diff --git a/testdata/for/Taskfile.yml b/testdata/for/Taskfile.yml new file mode 100644 index 00000000..0693ed71 --- /dev/null +++ b/testdata/for/Taskfile.yml @@ -0,0 +1,93 @@ +version: "3" + +tasks: + # Loop over a list of values + loop-explicit: + cmds: + - for: ["a", "b", "c"] + cmd: echo "{{.ITEM}}" + + # Loop over the task's sources + loop-sources: + sources: + - foo.txt + - bar.txt + cmds: + - for: source + cmd: cat "{{.ITEM}}" + + # Loop over the task's sources when globbed + loop-sources-glob: + sources: + - "*.txt" + cmds: + - for: source + cmd: cat "{{.ITEM}}" + + # Loop over the contents of a variable + loop-vars: + vars: + FOO: foo.txt,bar.txt + cmds: + - for: + var: FOO + split: "," + cmd: cat "{{.ITEM}}" + + # Loop over the output of a command (auto splits on " ") + loop-vars-sh: + vars: + FOO: + sh: ls *.txt + cmds: + - for: + var: FOO + cmd: cat "{{.ITEM}}" + + # Loop over another task + loop-task: + vars: + FOO: foo.txt bar.txt + cmds: + - for: + var: FOO + task: looped-task + vars: + FILE: "{{.ITEM}}" + + # Loop over another task with the variable named differently + loop-task-as: + vars: + FOO: foo.txt bar.txt + cmds: + - for: + var: FOO + as: FILE + task: looped-task + vars: + FILE: "{{.FILE}}" + + # Loop over different tasks using the variable + loop-different-tasks: + vars: + FOO: "1 2 3" + cmds: + - for: + var: FOO + task: task-{{.ITEM}} + + looped-task: + internal: true + cmd: cat "{{.FILE}}" + + task-1: + internal: true + cmd: echo "1" + + task-2: + internal: true + cmd: echo "2" + + task-3: + internal: true + cmd: echo "3" diff --git a/testdata/for/bar.txt b/testdata/for/bar.txt new file mode 100644 index 00000000..5716ca59 --- /dev/null +++ b/testdata/for/bar.txt @@ -0,0 +1 @@ +bar diff --git a/testdata/for/foo.txt b/testdata/for/foo.txt new file mode 100644 index 00000000..257cc564 --- /dev/null +++ b/testdata/for/foo.txt @@ -0,0 +1 @@ +foo diff --git a/variables.go b/variables.go index 5962d6e4..5d80ebd4 100644 --- a/variables.go +++ b/variables.go @@ -124,10 +124,60 @@ func (e *Executor) compiledTask(call taskfile.Call, evaluateShVars bool) (*taskf if cmd == nil { continue } + if cmd.For != nil { + var list []string + // Get the list from the explicit forh list + if cmd.For.List != nil && len(cmd.For.List) > 0 { + list = cmd.For.List + } + // Get the list from the task sources + if cmd.For.From == "source" { + list, err = fingerprint.Globs(new.Dir, new.Sources) + if err != nil { + return nil, err + } + } + // Get the list from a variable and split it up + if cmd.For.Var != "" { + if vars != nil { + v := vars.Get(cmd.For.Var) + if cmd.For.Split != "" { + list = strings.Split(v.Static, cmd.For.Split) + } else { + list = strings.Fields(v.Static) + } + } + } + // Name the iterator variable + var as string + if cmd.For.As != "" { + as = cmd.For.As + } else { + as = "ITEM" + } + // Create a new command for each item in the list + for _, loopValue := range list { + extra := map[string]any{ + as: loopValue, + } + new.Cmds = append(new.Cmds, &taskfile.Cmd{ + Cmd: r.ReplaceWithExtra(cmd.Cmd, extra), + Task: r.ReplaceWithExtra(cmd.Task, extra), + Silent: cmd.Silent, + Set: cmd.Set, + Shopt: cmd.Shopt, + Vars: r.ReplaceVarsWithExtra(cmd.Vars, extra), + IgnoreError: cmd.IgnoreError, + Defer: cmd.Defer, + Platforms: cmd.Platforms, + }) + } + continue + } new.Cmds = append(new.Cmds, &taskfile.Cmd{ Cmd: r.Replace(cmd.Cmd), - Silent: cmd.Silent, Task: r.Replace(cmd.Task), + Silent: cmd.Silent, Set: cmd.Set, Shopt: cmd.Shopt, Vars: r.ReplaceVars(cmd.Vars),