feat: remote taskfiles (HTTP) (#1152)

* feat: remote taskfiles over http

* feat: allow insecure connections when --insecure flag is provided

* feat: better error handling for fetch errors

* fix: ensure cache directory always exists

* fix: setup logger before everything else

* feat: put remote taskfiles behind an experiment

* feat: --download and --offline flags for remote taskfiles

* feat: node.Read accepts a context

* feat: experiment docs

* chore: changelog

* chore: remove unused optional param from Node interface

* chore: tidy up and generalise NewNode function

* fix: use sha256 in remote checksum

* feat: --download by itself will not run a task

* feat: custom error if remote taskfiles experiment is not enabled

* refactor: BaseNode functional options and simplified FileNode

* fix: use hex encoding for checksum instead of b64

taskfile/included_taskfile.go

@@ -3,6 +3,7 @@ package taskfile
 import (
 	"fmt"
 	"path/filepath"
+	"strings"
 
 	"github.com/go-task/task/v3/internal/execext"
 	"github.com/go-task/task/v3/internal/filepathext"
@@ -148,6 +149,11 @@ func (it *IncludedTaskfile) FullDirPath() (string, error) {
 }
 
 func (it *IncludedTaskfile) resolvePath(path string) (string, error) {
+	// If the file is remote, we don't need to resolve the path
+	if strings.Contains(it.Taskfile, "://") {
+		return path, nil
+	}
+
 	path, err := execext.Expand(path)
 	if err != nil {
 		return "", err

taskfile/read/cache.go

@@ -0,0 +1,58 @@
+package read
+
+import (
+	"crypto/sha256"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+type Cache struct {
+	dir string
+}
+
+func NewCache(dir string) (*Cache, error) {
+	dir = filepath.Join(dir, "remote")
+	if err := os.MkdirAll(dir, 0o755); err != nil {
+		return nil, err
+	}
+	return &Cache{
+		dir: dir,
+	}, nil
+}
+
+func checksum(b []byte) string {
+	h := sha256.New()
+	h.Write(b)
+	return fmt.Sprintf("%x", h.Sum(nil))
+}
+
+func (c *Cache) write(node Node, b []byte) error {
+	return os.WriteFile(c.cacheFilePath(node), b, 0o644)
+}
+
+func (c *Cache) read(node Node) ([]byte, error) {
+	return os.ReadFile(c.cacheFilePath(node))
+}
+
+func (c *Cache) writeChecksum(node Node, checksum string) error {
+	return os.WriteFile(c.checksumFilePath(node), []byte(checksum), 0o644)
+}
+
+func (c *Cache) readChecksum(node Node) string {
+	b, _ := os.ReadFile(c.checksumFilePath(node))
+	return string(b)
+}
+
+func (c *Cache) key(node Node) string {
+	return strings.TrimRight(checksum([]byte(node.Location())), "=")
+}
+
+func (c *Cache) cacheFilePath(node Node) string {
+	return filepath.Join(c.dir, fmt.Sprintf("%s.yaml", c.key(node)))
+}
+
+func (c *Cache) checksumFilePath(node Node) string {
+	return filepath.Join(c.dir, fmt.Sprintf("%s.checksum", c.key(node)))
+}

taskfile/read/node.go

@@ -1,30 +1,39 @@
 package read
 
 import (
+	"context"
 	"strings"
 
-	"github.com/go-task/task/v3/taskfile"
+	"github.com/go-task/task/v3/errors"
+	"github.com/go-task/task/v3/internal/experiments"
 )
 
 type Node interface {
-	Read() (*taskfile.Taskfile, error)
+	Read(ctx context.Context) ([]byte, error)
 	Parent() Node
-	Optional() bool
 	Location() string
+	Optional() bool
+	Remote() bool
 }
 
-func NewNodeFromIncludedTaskfile(parent Node, includedTaskfile taskfile.IncludedTaskfile) (Node, error) {
-	switch getScheme(includedTaskfile.Taskfile) {
-	// TODO: Add support for other schemes.
-	// If no other scheme matches, we assume it's a file.
-	// This also allows users to explicitly set a file:// scheme.
+func NewNode(
+	uri string,
+	insecure bool,
+	opts ...NodeOption,
+) (Node, error) {
+	var node Node
+	var err error
+	switch getScheme(uri) {
+	case "http", "https":
+		node, err = NewHTTPNode(uri, insecure, opts...)
 	default:
-		path, err := includedTaskfile.FullTaskfilePath()
-		if err != nil {
-			return nil, err
-		}
-		return NewFileNode(parent, path, includedTaskfile.Optional)
+		// If no other scheme matches, we assume it's a file
+		node, err = NewFileNode(uri, opts...)
 	}
+	if node.Remote() && !experiments.RemoteTaskfiles {
+		return nil, errors.New("task: Remote taskfiles are not enabled. You can read more about this experiment and how to enable it at https://taskfile.dev/experiments/remote-taskfiles")
+	}
+	return node, err
 }
 
 func getScheme(uri string) string {

taskfile/read/node_base.go

@@ -1,18 +1,47 @@
 package read
 
-// BaseNode is a generic node that implements the Parent() and Optional()
-// methods of the NodeReader interface. It does not implement the Read() method
-// and it designed to be embedded in other node types so that this boilerplate
-// code does not need to be repeated.
-type BaseNode struct {
-	parent   Node
-	optional bool
+type (
+	NodeOption func(*BaseNode)
+	// BaseNode is a generic node that implements the Parent() and Optional()
+	// methods of the NodeReader interface. It does not implement the Read() method
+	// and it designed to be embedded in other node types so that this boilerplate
+	// code does not need to be repeated.
+	BaseNode struct {
+		parent   Node
+		optional bool
+	}
+)
+
+func NewBaseNode(opts ...NodeOption) *BaseNode {
+	node := &BaseNode{
+		parent:   nil,
+		optional: false,
+	}
+
+	// Apply options
+	for _, opt := range opts {
+		opt(node)
+	}
+
+	return node
+}
+
+func WithParent(parent Node) NodeOption {
+	return func(node *BaseNode) {
+		node.parent = parent
+	}
 }
 
 func (node *BaseNode) Parent() Node {
 	return node.parent
 }
 
+func WithOptional(optional bool) NodeOption {
+	return func(node *BaseNode) {
+		node.optional = optional
+	}
+}
+
 func (node *BaseNode) Optional() bool {
 	return node.optional
 }

taskfile/read/node_file.go

@@ -1,34 +1,36 @@
 package read
 
 import (
+	"context"
+	"io"
 	"os"
 	"path/filepath"
 
-	"gopkg.in/yaml.v3"
-
-	"github.com/go-task/task/v3/errors"
 	"github.com/go-task/task/v3/internal/filepathext"
-	"github.com/go-task/task/v3/taskfile"
 )
 
 // A FileNode is a node that reads a taskfile from the local filesystem.
 type FileNode struct {
-	BaseNode
+	*BaseNode
 	Dir        string
 	Entrypoint string
 }
 
-func NewFileNode(parent Node, path string, optional bool) (*FileNode, error) {
-	path, err := exists(path)
+func NewFileNode(uri string, opts ...NodeOption) (*FileNode, error) {
+	base := NewBaseNode(opts...)
+	if uri == "" {
+		d, err := os.Getwd()
+		if err != nil {
+			return nil, err
+		}
+		uri = d
+	}
+	path, err := existsWalk(uri)
 	if err != nil {
 		return nil, err
 	}
-
 	return &FileNode{
-		BaseNode: BaseNode{
-			parent:   parent,
-			optional: optional,
-		},
+		BaseNode:   base,
 		Dir:        filepath.Dir(path),
 		Entrypoint: filepath.Base(path),
 	}, nil
@@ -38,33 +40,15 @@ func (node *FileNode) Location() string {
 	return filepathext.SmartJoin(node.Dir, node.Entrypoint)
 }
 
-func (node *FileNode) Read() (*taskfile.Taskfile, error) {
-	if node.Dir == "" {
-		d, err := os.Getwd()
-		if err != nil {
-			return nil, err
-		}
-		node.Dir = d
-	}
+func (node *FileNode) Remote() bool {
+	return false
+}
 
-	path, err := existsWalk(node.Location())
-	if err != nil {
-		return nil, err
-	}
-	node.Dir = filepath.Dir(path)
-	node.Entrypoint = filepath.Base(path)
-
-	f, err := os.Open(path)
+func (node *FileNode) Read(ctx context.Context) ([]byte, error) {
+	f, err := os.Open(node.Location())
 	if err != nil {
 		return nil, err
 	}
 	defer f.Close()
-
-	var t taskfile.Taskfile
-	if err := yaml.NewDecoder(f).Decode(&t); err != nil {
-		return nil, &errors.TaskfileInvalidError{URI: filepathext.TryAbsToRel(path), Err: err}
-	}
-
-	t.Location = path
-	return &t, nil
+	return io.ReadAll(f)
 }

taskfile/read/node_http.go

@@ -0,0 +1,67 @@
+package read
+
+import (
+	"context"
+	"io"
+	"net/http"
+	"net/url"
+
+	"github.com/go-task/task/v3/errors"
+)
+
+// An HTTPNode is a node that reads a Taskfile from a remote location via HTTP.
+type HTTPNode struct {
+	*BaseNode
+	URL *url.URL
+}
+
+func NewHTTPNode(uri string, insecure bool, opts ...NodeOption) (*HTTPNode, error) {
+	base := NewBaseNode(opts...)
+	url, err := url.Parse(uri)
+	if err != nil {
+		return nil, err
+	}
+	if url.Scheme == "http" && !insecure {
+		return nil, &errors.TaskfileNotSecureError{URI: uri}
+	}
+	return &HTTPNode{
+		BaseNode: base,
+		URL:      url,
+	}, nil
+}
+
+func (node *HTTPNode) Location() string {
+	return node.URL.String()
+}
+
+func (node *HTTPNode) Remote() bool {
+	return true
+}
+
+func (node *HTTPNode) Read(ctx context.Context) ([]byte, error) {
+	req, err := http.NewRequest("GET", node.URL.String(), nil)
+	if err != nil {
+		return nil, errors.TaskfileFetchFailedError{URI: node.URL.String()}
+	}
+
+	resp, err := http.DefaultClient.Do(req.WithContext(ctx))
+	if err != nil {
+		return nil, errors.TaskfileFetchFailedError{URI: node.URL.String()}
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, errors.TaskfileFetchFailedError{
+			URI:            node.URL.String(),
+			HTTPStatusCode: resp.StatusCode,
+		}
+	}
+
+	// Read the entire response body
+	b, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	return b, nil
+}

taskfile/read/taskfile.go

@@ -1,13 +1,17 @@
 package read
 
 import (
+	"context"
 	"fmt"
 	"os"
 	"path/filepath"
 	"runtime"
 
+	"gopkg.in/yaml.v3"
+
 	"github.com/go-task/task/v3/errors"
 	"github.com/go-task/task/v3/internal/filepathext"
+	"github.com/go-task/task/v3/internal/logger"
 	"github.com/go-task/task/v3/internal/sysinfo"
 	"github.com/go-task/task/v3/internal/templater"
 	"github.com/go-task/task/v3/taskfile"
@@ -29,13 +33,112 @@ var (
 	}
 )
 
+func readTaskfile(
+	node Node,
+	download,
+	offline bool,
+	tempDir string,
+	l *logger.Logger,
+) (*taskfile.Taskfile, error) {
+	var b []byte
+
+	cache, err := NewCache(tempDir)
+	if err != nil {
+		return nil, err
+	}
+
+	// If the file is remote, check if we have a cached copy
+	// If we're told to download, skip the cache
+	if node.Remote() && !download {
+		if b, err = cache.read(node); !errors.Is(err, os.ErrNotExist) && err != nil {
+			return nil, err
+		}
+
+		if b != nil {
+			l.VerboseOutf(logger.Magenta, "task: [%s] Fetched cached copy\n", node.Location())
+		}
+	}
+
+	// If the file is remote, we found nothing in the cache and we're not
+	// allowed to download it then we can't do anything.
+	if node.Remote() && b == nil && offline {
+		if b == nil && offline {
+			return nil, &errors.TaskfileCacheNotFound{URI: node.Location()}
+		}
+	}
+
+	// If we still don't have a copy, get the file in the usual way
+	if b == nil {
+		b, err = node.Read(context.Background())
+		if err != nil {
+			return nil, err
+		}
+
+		// If the node was remote, we need to check the checksum
+		if node.Remote() {
+			l.VerboseOutf(logger.Magenta, "task: [%s] Fetched remote copy\n", node.Location())
+
+			// Get the checksums
+			checksum := checksum(b)
+			cachedChecksum := cache.readChecksum(node)
+
+			// If the checksum doesn't exist, prompt the user to continue
+			if cachedChecksum == "" {
+				if cont, err := l.Prompt(logger.Yellow, fmt.Sprintf("The task you are attempting to run depends on the remote Taskfile at %q.\n--- Make sure you trust the source of this Taskfile before continuing ---\nContinue?", node.Location()), "n", "y", "yes"); err != nil {
+					return nil, err
+				} else if !cont {
+					return nil, &errors.TaskfileNotTrustedError{URI: node.Location()}
+				}
+			} else if checksum != cachedChecksum {
+				// If there is a cached hash, but it doesn't match the expected hash, prompt the user to continue
+				if cont, err := l.Prompt(logger.Yellow, fmt.Sprintf("The Taskfile at %q has changed since you last used it!\n--- Make sure you trust the source of this Taskfile before continuing ---\nContinue?", node.Location()), "n", "y", "yes"); err != nil {
+					return nil, err
+				} else if !cont {
+					return nil, &errors.TaskfileNotTrustedError{URI: node.Location()}
+				}
+			}
+
+			// If the hash has changed (or is new), store it in the cache
+			if checksum != cachedChecksum {
+				if err := cache.writeChecksum(node, checksum); err != nil {
+					return nil, err
+				}
+			}
+		}
+	}
+
+	// If the file is remote and we need to cache it
+	if node.Remote() && download {
+		l.VerboseOutf(logger.Magenta, "task: [%s] Caching downloaded file\n", node.Location())
+		// Cache the file for later
+		if err = cache.write(node, b); err != nil {
+			return nil, err
+		}
+	}
+
+	var t taskfile.Taskfile
+	if err := yaml.Unmarshal(b, &t); err != nil {
+		return nil, &errors.TaskfileInvalidError{URI: filepathext.TryAbsToRel(node.Location()), Err: err}
+	}
+	t.Location = node.Location()
+
+	return &t, nil
+}
+
 // Taskfile reads a Taskfile for a given directory
 // Uses current dir when dir is left empty. Uses Taskfile.yml
 // or Taskfile.yaml when entrypoint is left empty
-func Taskfile(node Node) (*taskfile.Taskfile, error) {
+func Taskfile(
+	node Node,
+	insecure bool,
+	download bool,
+	offline bool,
+	tempDir string,
+	l *logger.Logger,
+) (*taskfile.Taskfile, error) {
 	var _taskfile func(Node) (*taskfile.Taskfile, error)
 	_taskfile = func(node Node) (*taskfile.Taskfile, error) {
-		t, err := node.Read()
+		t, err := readTaskfile(node, download, offline, tempDir, l)
 		if err != nil {
 			return nil, err
 		}
@@ -70,7 +173,15 @@ func Taskfile(node Node) (*taskfile.Taskfile, error) {
 				}
 			}
 
-			includeReaderNode, err := NewNodeFromIncludedTaskfile(node, includedTask)
+			uri, err := includedTask.FullTaskfilePath()
+			if err != nil {
+				return err
+			}
+
+			includeReaderNode, err := NewNode(uri, insecure,
+				WithParent(node),
+				WithOptional(includedTask.Optional),
+			)
 			if err != nil {
 				if includedTask.Optional {
 					return nil
@@ -149,17 +260,19 @@ func Taskfile(node Node) (*taskfile.Taskfile, error) {
 				path := filepathext.SmartJoin(node.Dir, fmt.Sprintf("Taskfile_%s.yml", runtime.GOOS))
 				if _, err = os.Stat(path); err == nil {
 					osNode := &FileNode{
-						BaseNode: BaseNode{
-							parent:   node,
-							optional: false,
-						},
+						BaseNode:   NewBaseNode(WithParent(node)),
 						Entrypoint: path,
 						Dir:        node.Dir,
 					}
-					osTaskfile, err := osNode.Read()
+					b, err := osNode.Read(context.Background())
 					if err != nil {
 						return nil, err
 					}
+					var osTaskfile *taskfile.Taskfile
+					if err := yaml.Unmarshal(b, &osTaskfile); err != nil {
+						return nil, &errors.TaskfileInvalidError{URI: filepathext.TryAbsToRel(node.Location()), Err: err}
+					}
+					t.Location = node.Location()
 					if err = taskfile.Merge(t, osTaskfile, nil); err != nil {
 						return nil, err
 					}
@@ -183,6 +296,11 @@ func Taskfile(node Node) (*taskfile.Taskfile, error) {
 	return _taskfile(node)
 }
 
+// exists will check if a file at the given path exists. If it does, it will
+// return the path to it. If it does not, it will search the search for any
+// files at the given path with any of the default Taskfile files names. If any
+// of these match a file, the first matching path will be returned. If no files
+// are found, an error will be returned.
 func exists(path string) (string, error) {
 	fi, err := os.Stat(path)
 	if err != nil {
@@ -202,6 +320,11 @@ func exists(path string) (string, error) {
 	return "", errors.TaskfileNotFoundError{URI: path, Walk: false}
 }
 
+// existsWalk will check if a file at the given path exists by calling the
+// exists function. If a file is not found, it will walk up the directory tree
+// calling the exists function until it finds a file or reaches the root
+// directory. On supported operating systems, it will also check if the user ID
+// of the directory changes and abort if it does.
 func existsWalk(path string) (string, error) {
 	origPath := path
 	owner, err := sysinfo.Owner(path)