feat: variable references (#1654)

* feat: add references to the base code instead of the maps experiment

* feat: add template functions to ref resolver

* feat: tests

* docs: variable references

* feat: remove json and yaml keys from map variable experiment

* chore: typo

internal/compiler/compiler.go

@@ -3,15 +3,12 @@ package compiler
 import (
 	"bytes"
 	"context"
-	"encoding/json"
 	"fmt"
 	"os"
 	"path/filepath"
 	"strings"
 	"sync"
 
-	"gopkg.in/yaml.v3"
-
 	"github.com/go-task/task/v3/internal/execext"
 	"github.com/go-task/task/v3/internal/filepathext"
 	"github.com/go-task/task/v3/internal/logger"
@@ -78,18 +75,6 @@ func (c *Compiler) getVariables(t *ast.Task, call *ast.Call, evaluateShVars bool
 			if err := cache.Err(); err != nil {
 				return err
 			}
-			// Evaluate JSON
-			if newVar.Json != "" {
-				if err := json.Unmarshal([]byte(newVar.Json), &newVar.Value); err != nil {
-					return err
-				}
-			}
-			// Evaluate YAML
-			if newVar.Yaml != "" {
-				if err := yaml.Unmarshal([]byte(newVar.Yaml), &newVar.Value); err != nil {
-					return err
-				}
-			}
 			// If the variable is not dynamic, we can set it and return
 			if newVar.Value != nil || newVar.Sh == "" {
 				result.Set(k, ast.Var{Value: newVar.Value})

internal/templater/templater.go

@@ -2,6 +2,7 @@ package templater
 
 import (
 	"bytes"
+	"fmt"
 	"maps"
 	"strings"
 
@@ -40,7 +41,15 @@ func ResolveRef(ref string, cache *Cache) any {
 		cache.cacheMap = cache.Vars.ToCacheMap()
 	}
 
-	val, err := template.ResolveRef(ref, cache.cacheMap)
+	if ref == "." {
+		return cache.cacheMap
+	}
+	t, err := template.New("resolver").Funcs(templateFuncs).Parse(fmt.Sprintf("{{%s}}", ref))
+	if err != nil {
+		cache.err = err
+		return nil
+	}
+	val, err := t.ResolveRef(cache.cacheMap)
 	if err != nil {
 		cache.err = err
 		return nil
@@ -119,8 +128,6 @@ func ReplaceVarWithExtra(v ast.Var, cache *Cache, extra map[string]any) ast.Var
 		Live:  v.Live,
 		Ref:   v.Ref,
 		Dir:   v.Dir,
-		Json:  ReplaceWithExtra(v.Json, cache, extra),
-		Yaml:  ReplaceWithExtra(v.Yaml, cache, extra),
 	}
 }
 

task_test.go

@@ -2426,3 +2426,48 @@ func TestWildcard(t *testing.T) {
 		})
 	}
 }
+
+func TestReference(t *testing.T) {
+	tests := []struct {
+		name           string
+		call           string
+		expectedOutput string
+	}{
+		{
+			name:           "reference in command",
+			call:           "ref-cmd",
+			expectedOutput: "1\n",
+		},
+		{
+			name:           "reference in dependency",
+			call:           "ref-dep",
+			expectedOutput: "1\n",
+		},
+		{
+			name:           "reference using templating resolver",
+			call:           "ref-resolver",
+			expectedOutput: "1\n",
+		},
+		{
+			name:           "reference using templating resolver and dynamic var",
+			call:           "ref-resolver-sh",
+			expectedOutput: "Alice has 3 children called Bob, Charlie, and Diane\n",
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.call, func(t *testing.T) {
+			var buff bytes.Buffer
+			e := task.Executor{
+				Dir:    "testdata/var_references",
+				Stdout: &buff,
+				Stderr: &buff,
+				Silent: true,
+				Force:  true,
+			}
+			require.NoError(t, e.Setup())
+			require.NoError(t, e.Run(context.Background(), &ast.Call{Task: test.call}))
+			assert.Equal(t, test.expectedOutput, buff.String())
+		})
+	}
+}

taskfile/ast/var.go

@@ -83,8 +83,6 @@ type Var struct {
 	Live  any
 	Sh    string
 	Ref   string
-	Json  string
-	Yaml  string
 	Dir   string
 }
 
@@ -103,6 +101,10 @@ func (v *Var) UnmarshalYAML(node *yaml.Node) error {
 					v.Sh = str
 					return nil
 				}
+				if str, ok = strings.CutPrefix(str, "#"); ok {
+					v.Ref = str
+					return nil
+				}
 			}
 			v.Value = value
 			return nil
@@ -114,13 +116,11 @@ func (v *Var) UnmarshalYAML(node *yaml.Node) error {
 			case yaml.MappingNode:
 				key := node.Content[0].Value
 				switch key {
-				case "sh", "ref", "map", "json", "yaml":
+				case "sh", "ref", "map":
 					var m struct {
 						Sh  string
 						Ref string
 						Map any
-						Json string
-						Yaml string
 					}
 					if err := node.Decode(&m); err != nil {
 						return errors.NewTaskfileDecodeError(err, node)
@@ -128,11 +128,9 @@ func (v *Var) UnmarshalYAML(node *yaml.Node) error {
 					v.Sh = m.Sh
 					v.Ref = m.Ref
 					v.Value = m.Map
-					v.Json = m.Json
-					v.Yaml = m.Yaml
 					return nil
 				default:
-					return errors.NewTaskfileDecodeError(nil, node).WithMessage(`%q is not a valid variable type. Try "sh", "ref", "map", "json", "yaml" or using a scalar value`, key)
+					return errors.NewTaskfileDecodeError(nil, node).WithMessage(`%q is not a valid variable type. Try "sh", "ref", "map" or using a scalar value`, key)
 				}
 			default:
 				var value any
@@ -148,17 +146,22 @@ func (v *Var) UnmarshalYAML(node *yaml.Node) error {
 	switch node.Kind {
 
 	case yaml.MappingNode:
-		if len(node.Content) > 2 || node.Content[0].Value != "sh" {
+		key := node.Content[0].Value
-			return errors.NewTaskfileDecodeError(nil, node).WithMessage("maps cannot be assigned to variables")
+		switch key {
-		}
+		case "sh", "ref":
-		var sh struct {
+			var m struct {
 				Sh  string
+				Ref string
 			}
-		if err := node.Decode(&sh); err != nil {
+			if err := node.Decode(&m); err != nil {
 				return errors.NewTaskfileDecodeError(err, node)
 			}
-		v.Sh = sh.Sh
+			v.Sh = m.Sh
+			v.Ref = m.Ref
 			return nil
+		default:
+			return errors.NewTaskfileDecodeError(nil, node).WithMessage("maps cannot be assigned to variables")
+		}
 
 	default:
 		var value any

testdata/var_references/Taskfile.yml

@@ -0,0 +1,74 @@
+version: '3'
+
+vars:
+  GLOBAL_VAR: [1, 2, 2, 2, 3, 3, 4, 5]
+
+tasks:
+  default:
+    - task: ref-cmd
+    - task: ref-dep
+    - task: ref-resolver
+    - task: ref-resolver-sh
+
+  ref-cmd:
+    vars:
+      VAR_REF:
+        ref: .GLOBAL_VAR
+    cmds:
+      - task: print-first
+        vars:
+          VAR:
+            ref: .VAR_REF
+
+  ref-dep:
+    vars:
+      VAR_REF:
+        ref: .GLOBAL_VAR
+    deps:
+      - task: print-first
+        vars:
+          VAR:
+            ref: .VAR_REF
+
+  ref-resolver:
+    vars:
+      VAR_REF:
+        ref: .GLOBAL_VAR
+    cmds:
+      - task: print-var
+        vars:
+          VAR:
+            ref: (index .VAR_REF 0)
+
+  ref-resolver-sh:
+    vars:
+      JSON_STRING:
+        sh: echo '{"name":"Alice","age":30,"children":[{"name":"Bob","age":5},{"name":"Charlie","age":3},{"name":"Diane","age":1}]}'
+      JSON:
+        ref: "fromJson .JSON_STRING"
+      VAR_REF:
+        ref: .JSON
+    cmds:
+      - task: print-story
+        vars:
+          VAR:
+            ref: .VAR_REF
+
+  print-var:
+    cmds:
+      - echo "{{.VAR}}"
+
+  print-first:
+    cmds:
+      - echo "{{index .VAR 0}}"
+
+  print-story:
+    cmds:
+      - >-
+        echo "{{.VAR.name}} has {{len .VAR.children}} children called
+        {{- $children := .VAR.children -}}
+        {{- range $i, $child := $children -}}
+          {{- if lt $i (sub (len $children) 1)}} {{$child.name -}},
+          {{- else}} and {{$child.name -}}
+          {{- end -}}
+        {{- end -}}"

testdata/vars/any2/Taskfile.yml

@@ -10,7 +10,6 @@ tasks:
     - task: ref-dep
     - task: ref-resolver
     - task: json
-    - task: yaml
 
   map:
     vars:
@@ -93,25 +92,13 @@ tasks:
       JSON_STRING:
         sh: cat example.json
       JSON:
-        json: "{{.JSON_STRING}}"
+        ref: "fromJson .JSON_STRING"
     cmds:
       - task: print-story
         vars:
           VAR:
             ref: .JSON
 
-  yaml:
-    vars:
-      YAML_STRING:
-        sh: cat example.yaml
-      YAML:
-        yaml: "{{.YAML_STRING}}"
-    cmds:
-      - task: print-story
-        vars:
-          VAR:
-            ref: .YAML
-
   print-var:
     cmds:
       - echo "{{.VAR}}"

website/docs/experiments/map_variables.mdx

@@ -43,9 +43,9 @@ To enable this experiment, set the environment variable:
 
 :::
 
-This proposal removes support for the `sh` keyword in favour of a new syntax for
+This proposal removes support for the `sh` and `ref` keywords in favour of a new
-dynamically defined variables, This allows you to define a map directly as you
+syntax for dynamically defined variables and references. This allows you to
-would for any other type:
+define a map directly as you would for any other type:
 
 ```yaml
 version: 3
@@ -60,11 +60,26 @@ tasks:
 
 ## Migration
 
-Taskfiles with dynamically defined variables via the `sh` subkey will no longer
+Taskfiles with dynamically defined variables via the `sh` subkey or references
-work with this experiment enabled. In order to keep using dynamically defined
+defined with `ref` will no longer work with this experiment enabled. In order to
-variables, you will need to migrate your Taskfile to use the new syntax.
+keep using these features, you will need to migrate your Taskfile to use the new
+syntax.
 
-Previously, you might have defined a dynamic variable like this:
+### Dynamic Variables
+
+Previously, you had to define dynamic variables using the `sh` subkey. With this
+experiment enabled, you will need to remove the `sh` subkey and define your
+command as a string that begins with a `$`. This will instruct Task to interpret
+the string as a command instead of a literal value and the variable will be
+populated with the output of the command. For example:
+
+<Tabs defaultValue="2"
+  values={[
+    {label: 'Before', value: '1'},
+    {label: 'After', value: '2'}
+  ]}>
+
+<TabItem value="1">
 
 ```yaml
 version: 3
@@ -78,10 +93,8 @@ tasks:
       - 'echo {{.CALCULATED_VAR}}'
 ```
 
-With this experiment enabled, you will need to remove the `sh` subkey and define
+</TabItem>
-your command as a string that begins with a `$`. This will instruct Task to
+<TabItem value="2">
-interpret the string as a command instead of a literal value and the variable
-will be populated with the output of the command. For example:
 
 ```yaml
 version: 3
@@ -89,14 +102,56 @@ version: 3
 tasks:
   foo:
     vars:
-      CALCULATED_VAR: '$echo hello'
+      CALCULATED_VAR: '$echo hello' # <-- Prefix dynamic variable with a `$`
     cmds:
       - 'echo {{.CALCULATED_VAR}}'
 ```
 
-If your current Taskfile contains a string variable that begins with a `$`, you
+</TabItem></Tabs>
-will now need to escape the `$` with a backslash (`\`) to stop Task from
+
-executing it as a command.
+### References
+
+<Tabs defaultValue="2"
+  values={[
+    {label: 'Before', value: '1'},
+    {label: 'After', value: '2'}
+  ]}>
+
+<TabItem value="1">
+
+```yaml
+version: 3
+
+tasks:
+  foo:
+    vars:
+      VAR: 42
+      VAR_REF:
+        ref: '.FOO'
+    cmds:
+      - 'echo {{.VAR_REF}}'
+```
+
+</TabItem>
+<TabItem value="2">
+
+```yaml
+version: 3
+
+tasks:
+  foo:
+    vars:
+      VAR: 42
+      VAR_REF: '#.FOO' # <-- Prefix reference with a `#`
+    cmds:
+      - 'echo {{.VAR_REF}}'
+```
+
+</TabItem></Tabs>
+
+If your current Taskfile contains a string variable that begins with a `$` or a
+`#`, you will now need to escape it with a backslash (`\`) to stop Task from
+interpreting it as a command or reference.
 
 </TabItem>
 <TabItem value="2">
@@ -123,157 +178,12 @@ tasks:
       BAR: true # <-- Other types of variables are still defined directly on the key
       BAZ:
         sh: 'echo Hello Task' # <-- The `sh` subkey is still supported
+      QUX:
+        ref: '.BAZ' # <-- The `ref` subkey is still supported
     cmds:
       - 'echo {{.FOO.a}}'
 ```
 
-## Parsing JSON and YAML
-
-In addition to the new `map` keyword, this proposal also adds support for the
-`json` and `yaml` keywords for parsing JSON and YAML strings into real
-objects/arrays. This is similar to the `fromJSON` template function, but means
-that you only have to parse the JSON/YAML once when you declare the variable,
-instead of every time you want to access a value.
-
-<Tabs defaultValue="2"
-  values={[
-    {label: 'Before', value: '1'},
-    {label: 'After', value: '2'}
-  ]}>
-
-<TabItem value="1">
-
-```yaml
-version: 3
-
-tasks:
-  foo:
-    vars:
-      FOO: '{"a": 1, "b": 2, "c": 3}' # <-- JSON string
-    cmds:
-      - 'echo {{(fromJSON .FOO).a}}' # <-- Parse JSON string every time you want to access a value
-      - 'echo {{(fromJSON .FOO).b}}'
-```
-
-</TabItem>
-<TabItem value="2">
-
-```yaml
-version: 3
-
-tasks:
-  foo:
-    vars:
-      FOO:
-        json: '{"a": 1, "b": 2, "c": 3}' # <-- JSON string parsed once
-    cmds:
-      - 'echo {{.FOO.a}}' # <-- Access values directly
-      - 'echo {{.FOO.b}}'
-```
-
-</TabItem></Tabs>
-
-## Variables by reference
-
-Lastly, this proposal adds support for defining and passing variables by
-reference. This is really important now that variables can be types other than a
-string.
-
-Previously, to send a variable from one task to another, you would have to use
-the templating system. Unfortunately, the templater _always_ outputs a string
-and operations on the passed variable may not have behaved as expected. With
-this proposal, you can now pass variables by reference using the `ref` subkey:
-
-<Tabs defaultValue="2"
-  values={[
-    {label: 'Before', value: '1'},
-    {label: 'After', value: '2'}
-  ]}>
-
-<TabItem value="1">
-
-```yaml
-version: 3
-
-tasks:
-  foo:
-    vars:
-      FOO: [A, B, C] # <-- FOO is defined as an array
-    cmds:
-      - task: bar
-        vars:
-          FOO: '{{.FOO}}' # <-- FOO gets converted to a string when passed to bar
-  bar:
-    cmds:
-      - 'echo {{index .FOO 0}}' # <-- FOO is a string so the task outputs '91' which is the ASCII code for '[' instead of the expected 'A'
-```
-
-</TabItem>
-<TabItem value="2">
-
-```yaml
-version: 3
-
-tasks:
-  foo:
-    vars:
-      FOO: [A, B, C] # <-- FOO is defined as an array
-    cmds:
-      - task: bar
-        vars:
-          FOO:
-            ref: .FOO # <-- FOO gets passed by reference to bar and maintains its type
-  bar:
-    cmds:
-      - 'echo {{index .FOO 0}}' # <-- FOO is still a map so the task outputs 'A' as expected
-```
-
-</TabItem></Tabs>
-
-This means that the type of the variable is maintained when it is passed to
-another Task. This also works the same way when calling `deps` and when defining
-a variable and can be used in any combination:
-
-```yaml
-version: 3
-
-tasks:
-  foo:
-    vars:
-      FOO: [A, B, C] # <-- FOO is defined as an array
-      BAR:
-        ref: .FOO # <-- BAR is defined as a reference to FOO
-    deps:
-      - task: bar
-        vars:
-          BAR:
-            ref: .BAR # <-- BAR gets passed by reference to bar and maintains its type
-  bar:
-    cmds:
-      - 'echo {{index .BAR 0}}' # <-- BAR still refers to FOO so the task outputs 'A'
-```
-
-All references use the same templating syntax as regular templates, so in
-addition to simply calling `.FOO`, you can also pass subkeys (`.FOO.BAR`) or
-indexes (`index .FOO 0`) and use functions (`len .FOO`):
-
-```yaml
-version: 3
-
-tasks:
-  foo:
-    vars:
-      FOO: [A, B, C] # <-- FOO is defined as an array
-    cmds:
-      - task: bar
-        vars:
-          FOO:
-            ref: index .FOO 0 # <-- The element at index 0 is passed by reference to bar
-  bar:
-    cmds:
-      - 'echo {{.MYVAR}}' # <-- FOO is just the letter 'A'
-```
-
 </TabItem></Tabs>
 
 ## Looping over maps

website/docs/reference/templating.mdx

@@ -153,7 +153,7 @@ itself][go-template-functions]:
 
 In addition to the built-in functions, Task also provides a set of functions
 imported via the [slim-sprig][slim-sprig] package. We only provide a very basic
-descroption here for completeness. For detailed usage, please refer to the
+description here for completeness. For detailed usage, please refer to the
 [slim-sprig documentation][slim-sprig]:
 
 #### [String Functions][string-functions]

website/docs/usage.mdx

@@ -3,6 +3,9 @@ slug: /usage/
 sidebar_position: 3
 ---
 
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
 # Usage
 
 ## Getting started
@@ -964,6 +967,43 @@ Maps are not supported by default, but there is an
 you're interested in this functionality, we would appreciate your feedback.
 :pray:
 
+In the meantime, it is technically possible to define a map using a `ref` resolver and a templating function. For example:
+
+```yaml
+version: '3'
+
+tasks:
+  task-with-map:
+    vars:
+      FOO:
+        ref: dict "a" "1" "b" "2" "c" "3"
+    cmds:
+      - echo {{.FOO}}
+```
+
+```txt
+map[a:1 b:2 c:3]
+```
+
+OR by using the same technique with JSON:
+
+```yaml
+version: '3'
+
+tasks:
+  task-with-map:
+    vars:
+      JSON: '{"a": 1, "b": 2, "c": 3}'
+      FOO:
+        ref: "fromJson .JSON"
+    cmds:
+      - echo {{.FOO}}
+```
+
+```txt
+map[a:1 b:2 c:3]
+```
+
 :::
 
 Variables can be set in many places in a Taskfile. When executing
@@ -1047,6 +1087,103 @@ tasks:
 
 This works for all types of variables.
 
+### Referencing other variables
+
+Templating is great for referencing string values if you want to pass
+a value from one task to another. However, the templating engine is only able to
+output strings. If you want to pass something other than a string to another
+task then you will need to use a reference (`ref`) instead.
+
+<Tabs defaultValue="2"
+      values={[
+        { label: 'Templating Engine', value: '1' },
+        { label: 'Reference', value: '2' }
+      ]}>
+
+<TabItem value="1">
+
+```yaml
+version: 3
+
+tasks:
+  foo:
+    vars:
+      FOO: [A, B, C] # <-- FOO is defined as an array
+    cmds:
+      - task: bar
+        vars:
+          FOO: '{{.FOO}}' # <-- FOO gets converted to a string when passed to bar
+  bar:
+    cmds:
+      - 'echo {{index .FOO 0}}' # <-- FOO is a string so the task outputs '91' which is the ASCII code for '[' instead of the expected 'A'
+```
+
+</TabItem>
+<TabItem value="2">
+
+```yaml
+version: 3
+
+tasks:
+  foo:
+    vars:
+      FOO: [A, B, C] # <-- FOO is defined as an array
+    cmds:
+      - task: bar
+        vars:
+          FOO:
+            ref: .FOO # <-- FOO gets passed by reference to bar and maintains its type
+  bar:
+    cmds:
+      - 'echo {{index .FOO 0}}' # <-- FOO is still a map so the task outputs 'A' as expected
+```
+
+</TabItem></Tabs>
+
+This also works the same way when calling `deps` and when defining
+a variable and can be used in any combination:
+
+```yaml
+version: 3
+
+tasks:
+  foo:
+    vars:
+      FOO: [A, B, C] # <-- FOO is defined as an array
+      BAR:
+        ref: .FOO # <-- BAR is defined as a reference to FOO
+    deps:
+      - task: bar
+        vars:
+          BAR:
+            ref: .BAR # <-- BAR gets passed by reference to bar and maintains its type
+  bar:
+    cmds:
+      - 'echo {{index .BAR 0}}' # <-- BAR still refers to FOO so the task outputs 'A'
+```
+
+All references use the same templating syntax as regular templates, so in
+addition to simply calling `.FOO`, you can also pass subkeys (`.FOO.BAR`) or
+indexes (`index .FOO 0`) and use functions (`len .FOO`) as described in the
+[templating-reference][templating-reference]:
+
+```yaml
+version: 3
+
+tasks:
+  foo:
+    vars:
+      FOO: [A, B, C] # <-- FOO is defined as an array
+    cmds:
+      - task: bar
+        vars:
+          FOO:
+            ref: index .FOO 0 # <-- The element at index 0 is passed by reference to bar
+  bar:
+    cmds:
+      - 'echo {{.MYVAR}}' # <-- FOO is just the letter 'A'
+```
+
 ## Looping over values
 
 As of v3.28.0, Task allows you to loop over certain values and execute a command
@@ -1143,8 +1280,7 @@ tasks:
         cmd: cat {{.ITEM}}
 ```
 
-You can also loop over arrays directly (and maps if you have the
+You can also loop over arrays directly and maps:
-[maps experiment](/experiments/map-variables) enabled):
 
 ```yaml
 version: 3

website/static/schema.json

@@ -277,14 +277,6 @@
         "map": {
           "type": "object",
           "description": "The value will be treated as a literal map type and stored in the variable"
-        },
-        "json": {
-          "type": "string",
-          "description": "The value will parsed as a JSON string and stored in the variable"
-        },
-        "yaml": {
-          "type": "string",
-          "description": "The value will parsed as a YAML string and stored in the variable"
         }
       },
       "additionalProperties": false