feat: remote taskfiles (HTTP) (#1152)

* feat: remote taskfiles over http

* feat: allow insecure connections when --insecure flag is provided

* feat: better error handling for fetch errors

* fix: ensure cache directory always exists

* fix: setup logger before everything else

* feat: put remote taskfiles behind an experiment

* feat: --download and --offline flags for remote taskfiles

* feat: node.Read accepts a context

* feat: experiment docs

* chore: changelog

* chore: remove unused optional param from Node interface

* chore: tidy up and generalise NewNode function

* fix: use sha256 in remote checksum

* feat: --download by itself will not run a task

* feat: custom error if remote taskfiles experiment is not enabled

* refactor: BaseNode functional options and simplified FileNode

* fix: use hex encoding for checksum instead of b64

taskfile/read/taskfile.go

@@ -1,13 +1,17 @@
 package read
 
 import (
+	"context"
 	"fmt"
 	"os"
 	"path/filepath"
 	"runtime"
 
+	"gopkg.in/yaml.v3"
+
 	"github.com/go-task/task/v3/errors"
 	"github.com/go-task/task/v3/internal/filepathext"
+	"github.com/go-task/task/v3/internal/logger"
 	"github.com/go-task/task/v3/internal/sysinfo"
 	"github.com/go-task/task/v3/internal/templater"
 	"github.com/go-task/task/v3/taskfile"
@@ -29,13 +33,112 @@ var (
 	}
 )
 
+func readTaskfile(
+	node Node,
+	download,
+	offline bool,
+	tempDir string,
+	l *logger.Logger,
+) (*taskfile.Taskfile, error) {
+	var b []byte
+
+	cache, err := NewCache(tempDir)
+	if err != nil {
+		return nil, err
+	}
+
+	// If the file is remote, check if we have a cached copy
+	// If we're told to download, skip the cache
+	if node.Remote() && !download {
+		if b, err = cache.read(node); !errors.Is(err, os.ErrNotExist) && err != nil {
+			return nil, err
+		}
+
+		if b != nil {
+			l.VerboseOutf(logger.Magenta, "task: [%s] Fetched cached copy\n", node.Location())
+		}
+	}
+
+	// If the file is remote, we found nothing in the cache and we're not
+	// allowed to download it then we can't do anything.
+	if node.Remote() && b == nil && offline {
+		if b == nil && offline {
+			return nil, &errors.TaskfileCacheNotFound{URI: node.Location()}
+		}
+	}
+
+	// If we still don't have a copy, get the file in the usual way
+	if b == nil {
+		b, err = node.Read(context.Background())
+		if err != nil {
+			return nil, err
+		}
+
+		// If the node was remote, we need to check the checksum
+		if node.Remote() {
+			l.VerboseOutf(logger.Magenta, "task: [%s] Fetched remote copy\n", node.Location())
+
+			// Get the checksums
+			checksum := checksum(b)
+			cachedChecksum := cache.readChecksum(node)
+
+			// If the checksum doesn't exist, prompt the user to continue
+			if cachedChecksum == "" {
+				if cont, err := l.Prompt(logger.Yellow, fmt.Sprintf("The task you are attempting to run depends on the remote Taskfile at %q.\n--- Make sure you trust the source of this Taskfile before continuing ---\nContinue?", node.Location()), "n", "y", "yes"); err != nil {
+					return nil, err
+				} else if !cont {
+					return nil, &errors.TaskfileNotTrustedError{URI: node.Location()}
+				}
+			} else if checksum != cachedChecksum {
+				// If there is a cached hash, but it doesn't match the expected hash, prompt the user to continue
+				if cont, err := l.Prompt(logger.Yellow, fmt.Sprintf("The Taskfile at %q has changed since you last used it!\n--- Make sure you trust the source of this Taskfile before continuing ---\nContinue?", node.Location()), "n", "y", "yes"); err != nil {
+					return nil, err
+				} else if !cont {
+					return nil, &errors.TaskfileNotTrustedError{URI: node.Location()}
+				}
+			}
+
+			// If the hash has changed (or is new), store it in the cache
+			if checksum != cachedChecksum {
+				if err := cache.writeChecksum(node, checksum); err != nil {
+					return nil, err
+				}
+			}
+		}
+	}
+
+	// If the file is remote and we need to cache it
+	if node.Remote() && download {
+		l.VerboseOutf(logger.Magenta, "task: [%s] Caching downloaded file\n", node.Location())
+		// Cache the file for later
+		if err = cache.write(node, b); err != nil {
+			return nil, err
+		}
+	}
+
+	var t taskfile.Taskfile
+	if err := yaml.Unmarshal(b, &t); err != nil {
+		return nil, &errors.TaskfileInvalidError{URI: filepathext.TryAbsToRel(node.Location()), Err: err}
+	}
+	t.Location = node.Location()
+
+	return &t, nil
+}
+
 // Taskfile reads a Taskfile for a given directory
 // Uses current dir when dir is left empty. Uses Taskfile.yml
 // or Taskfile.yaml when entrypoint is left empty
-func Taskfile(node Node) (*taskfile.Taskfile, error) {
+func Taskfile(
+	node Node,
+	insecure bool,
+	download bool,
+	offline bool,
+	tempDir string,
+	l *logger.Logger,
+) (*taskfile.Taskfile, error) {
 	var _taskfile func(Node) (*taskfile.Taskfile, error)
 	_taskfile = func(node Node) (*taskfile.Taskfile, error) {
-		t, err := node.Read()
+		t, err := readTaskfile(node, download, offline, tempDir, l)
 		if err != nil {
 			return nil, err
 		}
@@ -70,7 +173,15 @@ func Taskfile(node Node) (*taskfile.Taskfile, error) {
 				}
 			}
 
-			includeReaderNode, err := NewNodeFromIncludedTaskfile(node, includedTask)
+			uri, err := includedTask.FullTaskfilePath()
+			if err != nil {
+				return err
+			}
+
+			includeReaderNode, err := NewNode(uri, insecure,
+				WithParent(node),
+				WithOptional(includedTask.Optional),
+			)
 			if err != nil {
 				if includedTask.Optional {
 					return nil
@@ -149,17 +260,19 @@ func Taskfile(node Node) (*taskfile.Taskfile, error) {
 				path := filepathext.SmartJoin(node.Dir, fmt.Sprintf("Taskfile_%s.yml", runtime.GOOS))
 				if _, err = os.Stat(path); err == nil {
 					osNode := &FileNode{
-						BaseNode: BaseNode{
-							parent:   node,
-							optional: false,
-						},
+						BaseNode:   NewBaseNode(WithParent(node)),
 						Entrypoint: path,
 						Dir:        node.Dir,
 					}
-					osTaskfile, err := osNode.Read()
+					b, err := osNode.Read(context.Background())
 					if err != nil {
 						return nil, err
 					}
+					var osTaskfile *taskfile.Taskfile
+					if err := yaml.Unmarshal(b, &osTaskfile); err != nil {
+						return nil, &errors.TaskfileInvalidError{URI: filepathext.TryAbsToRel(node.Location()), Err: err}
+					}
+					t.Location = node.Location()
 					if err = taskfile.Merge(t, osTaskfile, nil); err != nil {
 						return nil, err
 					}
@@ -183,6 +296,11 @@ func Taskfile(node Node) (*taskfile.Taskfile, error) {
 	return _taskfile(node)
 }
 
+// exists will check if a file at the given path exists. If it does, it will
+// return the path to it. If it does not, it will search the search for any
+// files at the given path with any of the default Taskfile files names. If any
+// of these match a file, the first matching path will be returned. If no files
+// are found, an error will be returned.
 func exists(path string) (string, error) {
 	fi, err := os.Stat(path)
 	if err != nil {
@@ -202,6 +320,11 @@ func exists(path string) (string, error) {
 	return "", errors.TaskfileNotFoundError{URI: path, Walk: false}
 }
 
+// existsWalk will check if a file at the given path exists by calling the
+// exists function. If a file is not found, it will walk up the directory tree
+// calling the exists function until it finds a file or reaches the root
+// directory. On supported operating systems, it will also check if the user ID
+// of the directory changes and abort if it does.
 func existsWalk(path string) (string, error) {
 	origPath := path
 	owner, err := sysinfo.Owner(path)