feat: remote taskfile improvements (cache/expiry) (#2176)

* feat: cache as node, RemoteNode and cache-first approach

* feat: cache expiry

* feat: pass ctx into reader methods instead of timeout

* docs: updated remote taskfiles experiment doc

* feat: use cache if download fails

cmd/task/task.go

@@ -18,7 +18,6 @@ import (
 	"github.com/go-task/task/v3/internal/flags"
 	"github.com/go-task/task/v3/internal/logger"
 	"github.com/go-task/task/v3/internal/version"
-	"github.com/go-task/task/v3/taskfile"
 	"github.com/go-task/task/v3/taskfile/ast"
 )
 
@@ -121,18 +120,9 @@ func run() error {
 		return err
 	}
 
-	// If the download flag is specified, we should stop execution as soon as
-	// taskfile is downloaded
-	if flags.Download {
-		return nil
-	}
-
 	if flags.ClearCache {
-		cache, err := taskfile.NewCache(e.TempDir.Remote)
+		cachePath := filepath.Join(e.TempDir.Remote, "remote")
-		if err != nil {
+		return os.RemoveAll(cachePath)
-			return err
-		}
-		return cache.Clear()
 	}
 
 	listOptions := task.NewListOptions(

cmd/tmp/main.go

@@ -0,0 +1,38 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"time"
+)
+
+func main() {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond)
+	defer cancel()
+	if err := run(ctx); err != nil {
+		fmt.Println(ctx.Err())
+		fmt.Println(err)
+	}
+}
+
+func run(ctx context.Context) error {
+	req, err := http.NewRequest("GET", "https://taskfile.dev/schema.json", nil)
+	if err != nil {
+		fmt.Println(1)
+		return err
+	}
+
+	resp, err := http.DefaultClient.Do(req.WithContext(ctx))
+	if err != nil {
+		if ctx.Err() != nil {
+			fmt.Println(2)
+			return err
+		}
+		fmt.Println(3)
+		return err
+	}
+	defer resp.Body.Close()
+
+	return nil
+}

errors/errors_taskfile.go

@@ -157,17 +157,12 @@ func (err *TaskfileVersionCheckError) Code() int {
 type TaskfileNetworkTimeoutError struct {
 	URI     string
 	Timeout time.Duration
-	CheckedCache bool
 }
 
 func (err *TaskfileNetworkTimeoutError) Error() string {
-	var cacheText string
-	if err.CheckedCache {
-		cacheText = " and no offline copy was found in the cache"
-	}
 	return fmt.Sprintf(
-		`task: Network connection timed out after %s while attempting to download Taskfile %q%s`,
+		`task: Network connection timed out after %s while attempting to download Taskfile %q`,
-		err.Timeout, err.URI, cacheText,
+		err.Timeout, err.URI,
 	)
 }
 

executor.go

@@ -36,6 +36,7 @@ type (
 		Download            bool
 		Offline             bool
 		Timeout             time.Duration
+		CacheExpiryDuration time.Duration
 		Watch               bool
 		Verbose             bool
 		Silent              bool
@@ -240,6 +241,20 @@ func (o *timeoutOption) ApplyToExecutor(e *Executor) {
 	e.Timeout = o.timeout
 }
 
+// WithCacheExpiryDuration sets the duration after which the cache is considered
+// expired. By default, the cache is considered expired after 24 hours.
+func WithCacheExpiryDuration(duration time.Duration) ExecutorOption {
+	return &cacheExpiryDurationOption{duration: duration}
+}
+
+type cacheExpiryDurationOption struct {
+	duration time.Duration
+}
+
+func (o *cacheExpiryDurationOption) ApplyToExecutor(r *Executor) {
+	r.CacheExpiryDuration = o.duration
+}
+
 // WithWatch tells the [Executor] to keep running in the background and watch
 // for changes to the fingerprint of the tasks that are run. When changes are
 // detected, a new task run is triggered.

internal/flags/flags.go

@@ -73,6 +73,7 @@ var (
 	Offline             bool
 	ClearCache          bool
 	Timeout             time.Duration
+	CacheExpiryDuration time.Duration
 )
 
 func init() {
@@ -131,6 +132,7 @@ func init() {
 		pflag.BoolVar(&Offline, "offline", offline, "Forces Task to only use local or cached Taskfiles.")
 		pflag.DurationVar(&Timeout, "timeout", time.Second*10, "Timeout for downloading remote Taskfiles.")
 		pflag.BoolVar(&ClearCache, "clear-cache", false, "Clear the remote cache.")
+		pflag.DurationVar(&CacheExpiryDuration, "expiry", 0, "Expiry duration for cached remote Taskfiles.")
 	}
 
 	pflag.Parse()
@@ -212,6 +214,7 @@ func (o *flagsOption) ApplyToExecutor(e *task.Executor) {
 		task.WithDownload(Download),
 		task.WithOffline(Offline),
 		task.WithTimeout(Timeout),
+		task.WithCacheExpiryDuration(CacheExpiryDuration),
 		task.WithWatch(Watch),
 		task.WithVerbose(Verbose),
 		task.WithSilent(Silent),

setup.go

@@ -64,6 +64,8 @@ func (e *Executor) getRootNode() (taskfile.Node, error) {
 }
 
 func (e *Executor) readTaskfile(node taskfile.Node) error {
+	ctx, cf := context.WithTimeout(context.Background(), e.Timeout)
+	defer cf()
 	debugFunc := func(s string) {
 		e.Logger.VerboseOutf(logger.Magenta, s)
 	}
@@ -74,13 +76,16 @@ func (e *Executor) readTaskfile(node taskfile.Node) error {
 		taskfile.WithInsecure(e.Insecure),
 		taskfile.WithDownload(e.Download),
 		taskfile.WithOffline(e.Offline),
-		taskfile.WithTimeout(e.Timeout),
 		taskfile.WithTempDir(e.TempDir.Remote),
+		taskfile.WithCacheExpiryDuration(e.CacheExpiryDuration),
 		taskfile.WithDebugFunc(debugFunc),
 		taskfile.WithPromptFunc(promptFunc),
 	)
-	graph, err := reader.Read(node)
+	graph, err := reader.Read(ctx, node)
 	if err != nil {
+		if errors.Is(err, context.DeadlineExceeded) {
+			return &errors.TaskfileNetworkTimeoutError{URI: node.Location(), Timeout: e.Timeout}
+		}
 		return err
 	}
 	if e.Taskfile, err = graph.Merge(); err != nil {

taskfile/cache.go

@@ -1,72 +0,0 @@
-package taskfile
-
-import (
-	"crypto/sha256"
-	"fmt"
-	"os"
-	"path/filepath"
-	"strings"
-)
-
-type Cache struct {
-	dir string
-}
-
-func NewCache(dir string) (*Cache, error) {
-	dir = filepath.Join(dir, "remote")
-	if err := os.MkdirAll(dir, 0o755); err != nil {
-		return nil, err
-	}
-	return &Cache{
-		dir: dir,
-	}, nil
-}
-
-func checksum(b []byte) string {
-	h := sha256.New()
-	h.Write(b)
-	return fmt.Sprintf("%x", h.Sum(nil))
-}
-
-func (c *Cache) write(node Node, b []byte) error {
-	return os.WriteFile(c.cacheFilePath(node), b, 0o644)
-}
-
-func (c *Cache) read(node Node) ([]byte, error) {
-	return os.ReadFile(c.cacheFilePath(node))
-}
-
-func (c *Cache) writeChecksum(node Node, checksum string) error {
-	return os.WriteFile(c.checksumFilePath(node), []byte(checksum), 0o644)
-}
-
-func (c *Cache) readChecksum(node Node) string {
-	b, _ := os.ReadFile(c.checksumFilePath(node))
-	return string(b)
-}
-
-func (c *Cache) key(node Node) string {
-	return strings.TrimRight(checksum([]byte(node.Location())), "=")
-}
-
-func (c *Cache) cacheFilePath(node Node) string {
-	return c.filePath(node, "yaml")
-}
-
-func (c *Cache) checksumFilePath(node Node) string {
-	return c.filePath(node, "checksum")
-}
-
-func (c *Cache) filePath(node Node, suffix string) string {
-	lastDir, filename := node.FilenameAndLastDir()
-	prefix := filename
-	// Means it's not "", nor "." nor "/", so it's a valid directory
-	if len(lastDir) > 1 {
-		prefix = fmt.Sprintf("%s-%s", lastDir, filename)
-	}
-	return filepath.Join(c.dir, fmt.Sprintf("%s.%s.%s", prefix, c.key(node), suffix))
-}
-
-func (c *Cache) Clear() error {
-	return os.RemoveAll(c.dir)
-}

taskfile/node.go

@@ -14,14 +14,18 @@ import (
 )
 
 type Node interface {
-	Read(ctx context.Context) ([]byte, error)
+	Read() ([]byte, error)
 	Parent() Node
 	Location() string
 	Dir() string
-	Remote() bool
 	ResolveEntrypoint(entrypoint string) (string, error)
 	ResolveDir(dir string) (string, error)
-	FilenameAndLastDir() (string, string)
+}
+
+type RemoteNode interface {
+	Node
+	ReadContext(ctx context.Context) ([]byte, error)
+	CacheKey() string
 }
 
 func NewRootNode(
@@ -35,35 +39,35 @@ func NewRootNode(
 	if entrypoint == "-" {
 		return NewStdinNode(dir)
 	}
-	return NewNode(entrypoint, dir, insecure, timeout)
+	return NewNode(entrypoint, dir, insecure)
 }
 
 func NewNode(
 	entrypoint string,
 	dir string,
 	insecure bool,
-	timeout time.Duration,
 	opts ...NodeOption,
 ) (Node, error) {
 	var node Node
 	var err error
+
 	scheme, err := getScheme(entrypoint)
 	if err != nil {
 		return nil, err
 	}
+
 	switch scheme {
 	case "git":
 		node, err = NewGitNode(entrypoint, dir, insecure, opts...)
 	case "http", "https":
-		node, err = NewHTTPNode(entrypoint, dir, insecure, timeout, opts...)
+		node, err = NewHTTPNode(entrypoint, dir, insecure, opts...)
 	default:
 		node, err = NewFileNode(entrypoint, dir, opts...)
-
 	}
-
+	if _, isRemote := node.(RemoteNode); isRemote && !experiments.RemoteTaskfiles.Enabled() {
-	if node.Remote() && !experiments.RemoteTaskfiles.Enabled() {
 		return nil, errors.New("task: Remote taskfiles are not enabled. You can read more about this experiment and how to enable it at https://taskfile.dev/experiments/remote-taskfiles")
 	}
+
 	return node, err
 }
 
@@ -72,6 +76,7 @@ func getScheme(uri string) (string, error) {
 	if u == nil {
 		return "", err
 	}
+
 	if strings.HasSuffix(strings.Split(u.Path, "//")[0], ".git") && (u.Scheme == "git" || u.Scheme == "ssh" || u.Scheme == "https" || u.Scheme == "http") {
 		return "git", nil
 	}
@@ -79,6 +84,7 @@ func getScheme(uri string) (string, error) {
 	if i := strings.Index(uri, "://"); i != -1 {
 		return uri[:i], nil
 	}
+
 	return "", nil
 }
 

taskfile/node_cache.go

@@ -0,0 +1,113 @@
+package taskfile
+
+import (
+	"crypto/sha256"
+	"fmt"
+	"os"
+	"path/filepath"
+	"time"
+)
+
+const remoteCacheDir = "remote"
+
+type CacheNode struct {
+	*BaseNode
+	source RemoteNode
+}
+
+func NewCacheNode(source RemoteNode, dir string) *CacheNode {
+	return &CacheNode{
+		BaseNode: &BaseNode{
+			dir: filepath.Join(dir, remoteCacheDir),
+		},
+		source: source,
+	}
+}
+
+func (node *CacheNode) Read() ([]byte, error) {
+	return os.ReadFile(node.Location())
+}
+
+func (node *CacheNode) Write(data []byte) error {
+	if err := node.CreateCacheDir(); err != nil {
+		return err
+	}
+	return os.WriteFile(node.Location(), data, 0o644)
+}
+
+func (node *CacheNode) ReadTimestamp() time.Time {
+	b, err := os.ReadFile(node.timestampPath())
+	if err != nil {
+		return time.Time{}.UTC()
+	}
+	timestamp, err := time.Parse(time.RFC3339, string(b))
+	if err != nil {
+		return time.Time{}.UTC()
+	}
+	return timestamp.UTC()
+}
+
+func (node *CacheNode) WriteTimestamp(t time.Time) error {
+	if err := node.CreateCacheDir(); err != nil {
+		return err
+	}
+	return os.WriteFile(node.timestampPath(), []byte(t.Format(time.RFC3339)), 0o644)
+}
+
+func (node *CacheNode) ReadChecksum() string {
+	b, _ := os.ReadFile(node.checksumPath())
+	return string(b)
+}
+
+func (node *CacheNode) WriteChecksum(checksum string) error {
+	if err := node.CreateCacheDir(); err != nil {
+		return err
+	}
+	return os.WriteFile(node.checksumPath(), []byte(checksum), 0o644)
+}
+
+func (node *CacheNode) CreateCacheDir() error {
+	if err := os.MkdirAll(node.dir, 0o755); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (node *CacheNode) ChecksumPrompt(checksum string) string {
+	cachedChecksum := node.ReadChecksum()
+	switch {
+
+	// If the checksum doesn't exist, prompt the user to continue
+	case cachedChecksum == "":
+		return taskfileUntrustedPrompt
+
+	// If there is a cached hash, but it doesn't match the expected hash, prompt the user to continue
+	case cachedChecksum != checksum:
+		return taskfileChangedPrompt
+
+	default:
+		return ""
+	}
+}
+
+func (node *CacheNode) Location() string {
+	return node.filePath("yaml")
+}
+
+func (node *CacheNode) checksumPath() string {
+	return node.filePath("checksum")
+}
+
+func (node *CacheNode) timestampPath() string {
+	return node.filePath("timestamp")
+}
+
+func (node *CacheNode) filePath(suffix string) string {
+	return filepath.Join(node.dir, fmt.Sprintf("%s.%s", node.source.CacheKey(), suffix))
+}
+
+func checksum(b []byte) string {
+	h := sha256.New()
+	h.Write(b)
+	return fmt.Sprintf("%x", h.Sum(nil))
+}

taskfile/node_file.go

@@ -1,7 +1,6 @@
 package taskfile
 
 import (
-	"context"
 	"io"
 	"os"
 	"path/filepath"
@@ -34,11 +33,7 @@ func (node *FileNode) Location() string {
 	return node.Entrypoint
 }
 
-func (node *FileNode) Remote() bool {
+func (node *FileNode) Read() ([]byte, error) {
-	return false
-}
-
-func (node *FileNode) Read(ctx context.Context) ([]byte, error) {
 	f, err := os.Open(node.Location())
 	if err != nil {
 		return nil, err
@@ -114,7 +109,3 @@ func (node *FileNode) ResolveDir(dir string) (string, error) {
 	entrypointDir := filepath.Dir(node.Entrypoint)
 	return filepathext.SmartJoin(entrypointDir, path), nil
 }
-
-func (node *FileNode) FilenameAndLastDir() (string, string) {
-	return "", filepath.Base(node.Entrypoint)
-}

taskfile/node_git.go

@@ -71,7 +71,11 @@ func (node *GitNode) Remote() bool {
 	return true
 }
 
-func (node *GitNode) Read(_ context.Context) ([]byte, error) {
+func (node *GitNode) Read() ([]byte, error) {
+	return node.ReadContext(context.Background())
+}
+
+func (node *GitNode) ReadContext(_ context.Context) ([]byte, error) {
 	fs := memfs.New()
 	storer := memory.NewStorage()
 	_, err := git.Clone(storer, fs, &git.CloneOptions{
@@ -121,6 +125,13 @@ func (node *GitNode) ResolveDir(dir string) (string, error) {
 	return filepathext.SmartJoin(entrypointDir, path), nil
 }
 
-func (node *GitNode) FilenameAndLastDir() (string, string) {
+func (node *GitNode) CacheKey() string {
-	return filepath.Base(node.path), filepath.Base(filepath.Dir(node.path))
+	checksum := strings.TrimRight(checksum([]byte(node.Location())), "=")
+	prefix := filepath.Base(filepath.Dir(node.path))
+	lastDir := filepath.Base(node.path)
+	// Means it's not "", nor "." nor "/", so it's a valid directory
+	if len(lastDir) > 1 {
+		prefix = fmt.Sprintf("%s-%s", lastDir, prefix)
+	}
+	return fmt.Sprintf("%s.%s", prefix, checksum)
 }

taskfile/node_git_test.go

@@ -62,24 +62,21 @@ func TestGitNode_httpsWithDir(t *testing.T) {
 	assert.Equal(t, "https://github.com/foo/bar.git//directory/common.yml?ref=main", entrypoint)
 }
 
-func TestGitNode_FilenameAndDir(t *testing.T) {
+func TestGitNode_CacheKey(t *testing.T) {
 	t.Parallel()
 
 	node, err := NewGitNode("https://github.com/foo/bar.git//directory/Taskfile.yml?ref=main", "", false)
 	assert.NoError(t, err)
-	filename, dir := node.FilenameAndLastDir()
+	key := node.CacheKey()
-	assert.Equal(t, "Taskfile.yml", filename)
+	assert.Equal(t, "Taskfile.yml-directory.f1ddddac425a538870230a3e38fc0cded4ec5da250797b6cab62c82477718fbb", key)
-	assert.Equal(t, "directory", dir)
 
 	node, err = NewGitNode("https://github.com/foo/bar.git//Taskfile.yml?ref=main", "", false)
 	assert.NoError(t, err)
-	filename, dir = node.FilenameAndLastDir()
+	key = node.CacheKey()
-	assert.Equal(t, "Taskfile.yml", filename)
+	assert.Equal(t, "Taskfile.yml-..39d28c1ff36f973705ae188b991258bbabaffd6d60bcdde9693d157d00d5e3a4", key)
-	assert.Equal(t, ".", dir)
 
 	node, err = NewGitNode("https://github.com/foo/bar.git//multiple/directory/Taskfile.yml?ref=main", "", false)
 	assert.NoError(t, err)
-	filename, dir = node.FilenameAndLastDir()
+	key = node.CacheKey()
-	assert.Equal(t, "Taskfile.yml", filename)
+	assert.Equal(t, "Taskfile.yml-directory.1b6d145e01406dcc6c0aa572e5a5d1333be1ccf2cae96d18296d725d86197d31", key)
-	assert.Equal(t, "directory", dir)
 }

taskfile/node_http.go

@@ -2,11 +2,12 @@ package taskfile
 
 import (
 	"context"
+	"fmt"
 	"io"
 	"net/http"
 	"net/url"
 	"path/filepath"
-	"time"
+	"strings"
 
 	"github.com/go-task/task/v3/errors"
 	"github.com/go-task/task/v3/internal/execext"
@@ -18,14 +19,12 @@ type HTTPNode struct {
 	*BaseNode
 	URL        *url.URL // stores url pointing actual remote file. (e.g. with Taskfile.yml)
 	entrypoint string   // stores entrypoint url. used for building graph vertices.
-	timeout    time.Duration
 }
 
 func NewHTTPNode(
 	entrypoint string,
 	dir string,
 	insecure bool,
-	timeout time.Duration,
 	opts ...NodeOption,
 ) (*HTTPNode, error) {
 	base := NewBaseNode(dir, opts...)
@@ -41,7 +40,6 @@ func NewHTTPNode(
 		BaseNode:   base,
 		URL:        url,
 		entrypoint: entrypoint,
-		timeout:    timeout,
 	}, nil
 }
 
@@ -49,12 +47,12 @@ func (node *HTTPNode) Location() string {
 	return node.entrypoint
 }
 
-func (node *HTTPNode) Remote() bool {
+func (node *HTTPNode) Read() ([]byte, error) {
-	return true
+	return node.ReadContext(context.Background())
 }
 
-func (node *HTTPNode) Read(ctx context.Context) ([]byte, error) {
+func (node *HTTPNode) ReadContext(ctx context.Context) ([]byte, error) {
-	url, err := RemoteExists(ctx, node.URL, node.timeout)
+	url, err := RemoteExists(ctx, node.URL)
 	if err != nil {
 		return nil, err
 	}
@@ -66,8 +64,8 @@ func (node *HTTPNode) Read(ctx context.Context) ([]byte, error) {
 
 	resp, err := http.DefaultClient.Do(req.WithContext(ctx))
 	if err != nil {
-		if errors.Is(err, context.DeadlineExceeded) {
+		if ctx.Err() != nil {
-			return nil, &errors.TaskfileNetworkTimeoutError{URI: node.URL.String(), Timeout: node.timeout}
+			return nil, err
 		}
 		return nil, errors.TaskfileFetchFailedError{URI: node.URL.String()}
 	}
@@ -116,7 +114,14 @@ func (node *HTTPNode) ResolveDir(dir string) (string, error) {
 	return filepathext.SmartJoin(parent, path), nil
 }
 
-func (node *HTTPNode) FilenameAndLastDir() (string, string) {
+func (node *HTTPNode) CacheKey() string {
+	checksum := strings.TrimRight(checksum([]byte(node.Location())), "=")
 	dir, filename := filepath.Split(node.entrypoint)
-	return filepath.Base(dir), filename
+	lastDir := filepath.Base(dir)
+	prefix := filename
+	// Means it's not "", nor "." nor "/", so it's a valid directory
+	if len(lastDir) > 1 {
+		prefix = fmt.Sprintf("%s-%s", lastDir, filename)
+	}
+	return fmt.Sprintf("%s.%s", prefix, checksum)
 }

taskfile/node_stdin.go

@@ -2,7 +2,6 @@ package taskfile
 
 import (
 	"bufio"
-	"context"
 	"fmt"
 	"os"
 	"strings"
@@ -30,7 +29,7 @@ func (node *StdinNode) Remote() bool {
 	return false
 }
 
-func (node *StdinNode) Read(ctx context.Context) ([]byte, error) {
+func (node *StdinNode) Read() ([]byte, error) {
 	var stdin []byte
 	scanner := bufio.NewScanner(os.Stdin)
 	for scanner.Scan() {
@@ -72,7 +71,3 @@ func (node *StdinNode) ResolveDir(dir string) (string, error) {
 
 	return filepathext.SmartJoin(node.Dir(), path), nil
 }
-
-func (node *StdinNode) FilenameAndLastDir() (string, string) {
-	return "", "__stdin__"
-}

taskfile/reader.go

@@ -43,8 +43,8 @@ type (
 		insecure            bool
 		download            bool
 		offline             bool
-		timeout     time.Duration
 		tempDir             string
+		cacheExpiryDuration time.Duration
 		debugFunc           DebugFunc
 		promptFunc          PromptFunc
 		promptMutex         sync.Mutex
@@ -59,8 +59,8 @@ func NewReader(opts ...ReaderOption) *Reader {
 		insecure:            false,
 		download:            false,
 		offline:             false,
-		timeout:     time.Second * 10,
 		tempDir:             os.TempDir(),
+		cacheExpiryDuration: 0,
 		debugFunc:           nil,
 		promptFunc:          nil,
 		promptMutex:         sync.Mutex{},
@@ -119,20 +119,6 @@ func (o *offlineOption) ApplyToReader(r *Reader) {
 	r.offline = o.offline
 }
 
-// WithTimeout sets the [Reader]'s timeout for fetching remote taskfiles. By
-// default, the timeout is set to 10 seconds.
-func WithTimeout(timeout time.Duration) ReaderOption {
-	return &timeoutOption{timeout: timeout}
-}
-
-type timeoutOption struct {
-	timeout time.Duration
-}
-
-func (o *timeoutOption) ApplyToReader(r *Reader) {
-	r.timeout = o.timeout
-}
-
 // WithTempDir sets the temporary directory that will be used by the [Reader].
 // By default, the reader uses [os.TempDir].
 func WithTempDir(tempDir string) ReaderOption {
@@ -147,6 +133,20 @@ func (o *tempDirOption) ApplyToReader(r *Reader) {
 	r.tempDir = o.tempDir
 }
 
+// WithCacheExpiryDuration sets the duration after which the cache is considered
+// expired. By default, the cache is considered expired after 24 hours.
+func WithCacheExpiryDuration(duration time.Duration) ReaderOption {
+	return &cacheExpiryDurationOption{duration: duration}
+}
+
+type cacheExpiryDurationOption struct {
+	duration time.Duration
+}
+
+func (o *cacheExpiryDurationOption) ApplyToReader(r *Reader) {
+	r.cacheExpiryDuration = o.duration
+}
+
 // WithDebugFunc sets the debug function to be used by the [Reader]. If set,
 // this function will be called with debug messages. This can be useful if the
 // caller wants to log debug messages from the [Reader]. By default, no debug
@@ -186,8 +186,8 @@ func (o *promptFuncOption) ApplyToReader(r *Reader) {
 // through any [ast.Includes] it finds, reading each included Taskfile and
 // building an [ast.TaskfileGraph] as it goes. If any errors occur, they will be
 // returned immediately.
-func (r *Reader) Read(node Node) (*ast.TaskfileGraph, error) {
+func (r *Reader) Read(ctx context.Context, node Node) (*ast.TaskfileGraph, error) {
-	if err := r.include(node); err != nil {
+	if err := r.include(ctx, node); err != nil {
 		return nil, err
 	}
 	return r.graph, nil
@@ -206,7 +206,7 @@ func (r *Reader) promptf(format string, a ...any) error {
 	return nil
 }
 
-func (r *Reader) include(node Node) error {
+func (r *Reader) include(ctx context.Context, node Node) error {
 	// Create a new vertex for the Taskfile
 	vertex := &ast.TaskfileVertex{
 		URI:      node.Location(),
@@ -224,7 +224,7 @@ func (r *Reader) include(node Node) error {
 
 	// Read and parse the Taskfile from the file and add it to the vertex
 	var err error
-	vertex.Taskfile, err = r.readNode(node)
+	vertex.Taskfile, err = r.readNode(ctx, node)
 	if err != nil {
 		return err
 	}
@@ -265,7 +265,7 @@ func (r *Reader) include(node Node) error {
 				return err
 			}
 
-			includeNode, err := NewNode(entrypoint, include.Dir, r.insecure, r.timeout,
+			includeNode, err := NewNode(entrypoint, include.Dir, r.insecure,
 				WithParent(node),
 			)
 			if err != nil {
@@ -276,7 +276,7 @@ func (r *Reader) include(node Node) error {
 			}
 
 			// Recurse into the included Taskfile
-			if err := r.include(includeNode); err != nil {
+			if err := r.include(ctx, includeNode); err != nil {
 				return err
 			}
 
@@ -316,8 +316,8 @@ func (r *Reader) include(node Node) error {
 	return g.Wait()
 }
 
-func (r *Reader) readNode(node Node) (*ast.Taskfile, error) {
+func (r *Reader) readNode(ctx context.Context, node Node) (*ast.Taskfile, error) {
-	b, err := r.loadNodeContent(node)
+	b, err := r.readNodeContent(ctx, node)
 	if err != nil {
 		return nil, err
 	}
@@ -358,72 +358,79 @@ func (r *Reader) readNode(node Node) (*ast.Taskfile, error) {
 	return &tf, nil
 }
 
-func (r *Reader) loadNodeContent(node Node) ([]byte, error) {
+func (r *Reader) readNodeContent(ctx context.Context, node Node) ([]byte, error) {
-	if !node.Remote() {
+	if node, isRemote := node.(RemoteNode); isRemote {
-		ctx, cf := context.WithTimeout(context.Background(), r.timeout)
+		return r.readRemoteNodeContent(ctx, node)
-		defer cf()
+	}
-		return node.Read(ctx)
+	return node.Read()
 }
 
-	cache, err := NewCache(r.tempDir)
+func (r *Reader) readRemoteNodeContent(ctx context.Context, node RemoteNode) ([]byte, error) {
-	if err != nil {
+	cache := NewCacheNode(node, r.tempDir)
-		return nil, err
+	now := time.Now().UTC()
-	}
+	timestamp := cache.ReadTimestamp()
+	expiry := timestamp.Add(r.cacheExpiryDuration)
+	cacheValid := now.Before(expiry)
+	var cacheFound bool
 
+	r.debugf("checking cache for %q in %q\n", node.Location(), cache.Location())
+	cachedBytes, err := cache.Read()
+	switch {
+	// If the cache doesn't exist, we need to download the file
+	case errors.Is(err, os.ErrNotExist):
+		r.debugf("no cache found\n")
+		// If we couldn't find a cached copy, and we are offline, we can't do anything
 		if r.offline {
-		// In offline mode try to use cached copy
+			return nil, &errors.TaskfileCacheNotFoundError{
-		cached, err := cache.read(node)
+				URI: node.Location(),
-		if errors.Is(err, os.ErrNotExist) {
+			}
-			return nil, &errors.TaskfileCacheNotFoundError{URI: node.Location()}
+		}
-		} else if err != nil {
+
+	// If the cache is expired
+	case !cacheValid:
+		r.debugf("cache expired at %s\n", expiry.Format(time.RFC3339))
+		cacheFound = true
+		// If we can't fetch a fresh copy, we should use the cache anyway
+		if r.offline {
+			r.debugf("in offline mode, using expired cache\n")
+			return cachedBytes, nil
+		}
+
+	// Some other error
+	case err != nil:
+		return nil, err
+
+	// Found valid cache
+	default:
+		r.debugf("cache found\n")
+		// Not being forced to redownload, return cache
+		if !r.download {
+			return cachedBytes, nil
+		}
+		cacheFound = true
+	}
+
+	// Try to read the remote file
+	r.debugf("downloading remote file: %s\n", node.Location())
+	downloadedBytes, err := node.ReadContext(ctx)
+	if err != nil {
+		// If the context timed out or was cancelled, but we found a cached version, use that
+		if ctx.Err() != nil && cacheFound {
+			if cacheValid {
+				r.debugf("failed to fetch remote file: %s: using cache\n", ctx.Err().Error())
+			} else {
+				r.debugf("failed to fetch remote file: %s: using expired cache\n", ctx.Err().Error())
+			}
+			return cachedBytes, nil
+		}
 		return nil, err
 	}
-		r.debugf("task: [%s] Fetched cached copy\n", node.Location())
 
-		return cached, nil
+	r.debugf("found remote file at %q\n", node.Location())
-	}
+	checksum := checksum(downloadedBytes)
-
+	prompt := cache.ChecksumPrompt(checksum)
-	ctx, cf := context.WithTimeout(context.Background(), r.timeout)
-	defer cf()
-
-	b, err := node.Read(ctx)
-	if errors.Is(err, &errors.TaskfileNetworkTimeoutError{}) {
-		// If we timed out then we likely have a network issue
-
-		// If a download was requested, then we can't use a cached copy
-		if r.download {
-			return nil, &errors.TaskfileNetworkTimeoutError{URI: node.Location(), Timeout: r.timeout}
-		}
-
-		// Search for any cached copies
-		cached, err := cache.read(node)
-		if errors.Is(err, os.ErrNotExist) {
-			return nil, &errors.TaskfileNetworkTimeoutError{URI: node.Location(), Timeout: r.timeout, CheckedCache: true}
-		} else if err != nil {
-			return nil, err
-		}
-		r.debugf("task: [%s] Network timeout. Fetched cached copy\n", node.Location())
-
-		return cached, nil
-
-	} else if err != nil {
-		return nil, err
-	}
-	r.debugf("task: [%s] Fetched remote copy\n", node.Location())
-
-	// Get the checksums
-	checksum := checksum(b)
-	cachedChecksum := cache.readChecksum(node)
-
-	var prompt string
-	if cachedChecksum == "" {
-		// If the checksum doesn't exist, prompt the user to continue
-		prompt = taskfileUntrustedPrompt
-	} else if checksum != cachedChecksum {
-		// If there is a cached hash, but it doesn't match the expected hash, prompt the user to continue
-		prompt = taskfileChangedPrompt
-	}
 
+	// Prompt the user if required
 	if prompt != "" {
 		if err := func() error {
 			r.promptMutex.Lock()
@@ -432,18 +439,23 @@ func (r *Reader) loadNodeContent(node Node) ([]byte, error) {
 		}(); err != nil {
 			return nil, &errors.TaskfileNotTrustedError{URI: node.Location()}
 		}
+	}
 
 	// Store the checksum
-		if err := cache.writeChecksum(node, checksum); err != nil {
+	if err := cache.WriteChecksum(checksum); err != nil {
+		return nil, err
+	}
+
+	// Store the timestamp
+	if err := cache.WriteTimestamp(now); err != nil {
 		return nil, err
 	}
 
 	// Cache the file
-		r.debugf("task: [%s] Caching downloaded file\n", node.Location())
+	r.debugf("caching %q to %q\n", node.Location(), cache.Location())
-		if err = cache.write(node, b); err != nil {
+	if err = cache.Write(downloadedBytes); err != nil {
 		return nil, err
 	}
-	}
 
-	return b, nil
+	return downloadedBytes, nil
 }

taskfile/taskfile.go

@@ -2,13 +2,13 @@ package taskfile
 
 import (
 	"context"
+	"fmt"
 	"net/http"
 	"net/url"
 	"os"
 	"path/filepath"
 	"slices"
 	"strings"
-	"time"
 
 	"github.com/go-task/task/v3/errors"
 	"github.com/go-task/task/v3/internal/filepathext"
@@ -40,7 +40,7 @@ var (
 // at the given URL with any of the default Taskfile files names. If any of
 // these match a file, the first matching path will be returned. If no files are
 // found, an error will be returned.
-func RemoteExists(ctx context.Context, u *url.URL, timeout time.Duration) (*url.URL, error) {
+func RemoteExists(ctx context.Context, u *url.URL) (*url.URL, error) {
 	// Create a new HEAD request for the given URL to check if the resource exists
 	req, err := http.NewRequestWithContext(ctx, "HEAD", u.String(), nil)
 	if err != nil {
@@ -50,8 +50,8 @@ func RemoteExists(ctx context.Context, u *url.URL, timeout time.Duration) (*url.
 	// Request the given URL
 	resp, err := http.DefaultClient.Do(req)
 	if err != nil {
-		if errors.Is(ctx.Err(), context.DeadlineExceeded) {
+		if ctx.Err() != nil {
-			return nil, &errors.TaskfileNetworkTimeoutError{URI: u.String(), Timeout: timeout}
+			return nil, fmt.Errorf("checking remote file: %w", ctx.Err())
 		}
 		return nil, errors.TaskfileFetchFailedError{URI: u.String()}
 	}

website/docs/experiments/remote_taskfiles.mdx

@@ -2,6 +2,9 @@
 slug: /experiments/remote-taskfiles/
 ---
 
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
 # Remote Taskfiles (#1317)
 
 :::caution
@@ -20,33 +23,151 @@ To enable this experiment, set the environment variable:
 
 :::
 
-This experiment allows you to specify a remote Taskfile URL when including a
+:::danger
-Taskfile. For example:
+Never run remote Taskfiles from sources that you do not trust.
+:::
 
-```yaml
+This experiment allows you to use Taskfiles which are stored in remote
-version: '3'
+locations. This applies to both the root Taskfile (aka. Entrypoint) and also
+when including Taskfiles.
 
-includes:
+Task uses "nodes" to reference remote Taskfiles. There are a few different types
-  my-remote-namespace: https://raw.githubusercontent.com/my-org/my-repo/main/Taskfile.yml
+of node which you can use:
-```
 
-This works exactly the same way that including a local file does. Any tasks in
+<Tabs groupId="method" queryString>
-the remote Taskfile will be available to run from your main Taskfile via the
+<TabItem value="http" label="HTTP/HTTPS">
-namespace `my-remote-namespace`. For example, if the remote file contains the
+
-following:
+`https://raw.githubusercontent.com/go-task/task/main/website/static/Taskfile.yml`
+
+This is the most basic type of remote node and works by downloading the file
+from the specified URL. The file must be a valid Taskfile and can be of any
+name. If a file is not found at the specified URL, Task will append each of the
+[supported file names][supported-file-names] in turn until it finds a valid
+file. If it still does not find a valid Taskfile, an error is returned.
+
+</TabItem>
+<TabItem value="git-http" label="Git over HTTP">
+
+`https://github.com/go-task/task.git//website/static/Taskfile.yml?ref=main`
+
+This type of node works by downloading the file from a Git repository over
+HTTP/HTTPS. The first part of the URL is the base URL of the Git repository.
+This is the same URL that you would use to clone the repo over HTTP.
+
+- You can optionally add the path to the Taskfile in the repository by appending
+`//<path>` to the URL.
+- You can also optionally specify a branch or tag to use by appending
+`?ref=<ref>` to the end of the URL. If you omit a reference, the default branch
+will be used.
+
+</TabItem>
+<TabItem value="git-ssh" label="Git over SSH">
+
+`git@github.com/go-task/task.git//website/static/Taskfile.yml?ref=main`
+
+This type of node works by downloading the file from a Git repository over SSH.
+The first part of the URL is the user and base URL of the Git repository. This
+is the same URL that you would use to clone the repo over SSH.
+
+To use Git over SSH, you need to make sure that your SSH agent has your private
+SSH keys added so that they can be used during authentication.
+
+- You can optionally add the path to the Taskfile in the repository by appending
+`//<path>` to the URL.
+- You can also optionally specify a branch or tag to use by appending
+`?ref=<ref>` to the end of the URL. If you omit a reference, the default branch
+will be used.
+
+</TabItem>
+</Tabs>
+
+Task has an [example remote Taskfile][example-remote-taskfile] in our repository
+that you can use for testing and that we will use throughout this document:
 
 ```yaml
 version: '3'
 
 tasks:
-  hello:
+  default:
-    silent: true
     cmds:
-      - echo "Hello from the remote Taskfile!"
+      - task: hello
+
+  hello:
+    cmds:
+      - echo "Hello Task!"
 ```
 
-and you run `task my-remote-namespace:hello`, it will print the text: "Hello
+## Specifying a remote entrypoint
-from the remote Taskfile!" to your console.
+
+By default, Task will look for one of the [supported file
+names][supported-file-names] on your local filesystem. If you want to use a
+remote file instead, you can pass its URI into the `--taskfile`/`-t` flag just
+like you would to specify a different local file. For example:
+
+<Tabs groupId="method" queryString>
+<TabItem value="http" label="HTTP/HTTPS">
+```shell
+$ task --taskfile https://raw.githubusercontent.com/go-task/task/main/website/static/Taskfile.yml
+task: [hello] echo "Hello Task!"
+Hello Task!
+```
+</TabItem>
+<TabItem value="git-http" label="Git over HTTP">
+```shell
+$ task --taskfile https://github.com/go-task/task.git//website/static/Taskfile.yml?ref=main
+task: [hello] echo "Hello Task!"
+Hello Task!
+```
+</TabItem>
+<TabItem value="git-ssh" label="Git over SSH">
+```shell
+$ task --taskfile git@github.com/go-task/task.git//website/static/Taskfile.yml?ref=main
+task: [hello] echo "Hello Task!"
+Hello Task!
+```
+</TabItem>
+</Tabs>
+
+## Including remote Taskfiles
+
+Including a remote file works exactly the same way that including a local file
+does. You just need to replace the local path with a remote URI. Any tasks in
+the remote Taskfile will be available to run from your main Taskfile.
+
+<Tabs groupId="method" queryString>
+<TabItem value="http" label="HTTP/HTTPS">
+```yaml
+version: '3'
+
+includes:
+  my-remote-namespace: https://raw.githubusercontent.com/go-task/task/main/website/static/Taskfile.yml
+```
+</TabItem>
+<TabItem value="git-http" label="Git over HTTP">
+```yaml
+version: '3'
+
+includes:
+  my-remote-namespace: https://github.com/go-task/task.git//website/static/Taskfile.yml?ref=main
+```
+</TabItem>
+<TabItem value="git-ssh" label="Git over SSH">
+```yaml
+version: '3'
+
+includes:
+  my-remote-namespace: git@github.com/go-task/task.git//website/static/Taskfile.yml?ref=main
+```
+</TabItem>
+</Tabs>
+
+```shell
+$ task my-remote-namespace:hello
+task: [hello] echo "Hello Task!"
+Hello Task!
+```
+
+### Authenticating using environment variables
 
 The Taskfile location is processed by the templating system, so you can
 reference environment variables in your URL if you need to add authentication.
@@ -59,19 +180,6 @@ includes:
   my-remote-namespace: https://{{.TOKEN}}@raw.githubusercontent.com/my-org/my-repo/main/Taskfile.yml
 ```
 
-`TOKEN=my-token task my-remote-namespace:hello` will be resolved by Task to
-`https://my-token@raw.githubusercontent.com/my-org/my-repo/main/Taskfile.yml`
-
-## Git nodes
-
-You can also include a Taskfile from a Git node. We currently support ssh-style and http / https addresses like `git@example.com/foo/bar.git//Taskfiles.yml?ref=v1` and `https://example.com/foo/bar.git//Taskfiles.yml?ref=v1`.
-
-You need to follow this pattern : `<baseUrl>.git//<path>?ref=<ref>`.
-The `ref` parameter, optional, can be a branch name or a tag, if not provided it'll pick up the default branch.
-The `path` is the path to the Taskfile in the repository.
-
-If you want to use the SSH protocol, you need to make sure that your ssh-agent has your private ssh keys added so that they can be used during authentication.
-
 ## Security
 
 Running commands from sources that you do not control is always a potential
@@ -104,20 +212,26 @@ flag. Before enabling this flag, you should:
 Task currently supports both `http` and `https` URLs. However, the `http`
 requests will not execute by default unless you run the task with the
 `--insecure` flag. This is to protect you from accidentally running a remote
-Taskfile that is via an unencrypted connection. Sources that are not protected
+Taskfile that is downloaded via an unencrypted connection. Sources that are not
-by TLS are vulnerable to [man-in-the-middle attacks][man-in-the-middle-attacks]
+protected by TLS are vulnerable to [man-in-the-middle
-and should be avoided unless you know what you are doing.
+attacks][man-in-the-middle-attacks] and should be avoided unless you know what
+you are doing.
 
 ## Caching & Running Offline
 
 Whenever you run a remote Taskfile, the latest copy will be downloaded from the
-internet and cached locally. If for whatever reason, you lose access to the
+internet and cached locally. This cached file will be used for all future
-internet, you will still be able to run your tasks by specifying the `--offline`
+invocations of the Taskfile until the cache expires. Once it expires, Task will
-flag. This will tell Task to use the latest cached version of the file instead
+download the latest copy of the file and update the cache. By default, the cache
-of trying to download it. You are able to use the `--download` flag to update
+is set to expire immediately. This means that Task will always fetch the latest
-the cached version of the remote files without running any tasks. You are able
+version. However, the cache expiry duration can be modified by setting the
-to use the `--clear-cache` flag to clear all cached version of the remote files
+`--expiry` flag.
-without running any tasks.
+
+If for any reason you lose access to the internet or you are running Task in
+offline mode (via the `--offline` flag or `TASK_OFFLINE` environment variable),
+Task will run the any available cached files _even if they are expired_. This
+means that you should never be stuck without the ability to run your tasks as
+long as you have downloaded a remote Taskfile at least once.
 
 By default, Task will timeout requests to download remote files after 10 seconds
 and look for a cached copy instead. This timeout can be configured by setting
@@ -129,7 +243,14 @@ By default, the cache is stored in the Task temp directory, represented by the
 override the location of the cache by setting the `TASK_REMOTE_DIR` environment
 variable. This way, you can share the cache between different projects.
 
+You can force Task to ignore the cache and download the latest version
+by using the `--download` flag.
+
+You can use the `--clear-cache` flag to clear all cached remote files.
+
 {/* prettier-ignore-start */}
 [enabling-experiments]: ./experiments.mdx#enabling-experiments
 [man-in-the-middle-attacks]: https://en.wikipedia.org/wiki/Man-in-the-middle_attack
+[supported-file-names]: https://taskfile.dev/usage/#supported-file-names
+[example-remote-taskfile]: https://raw.githubusercontent.com/go-task/task/main/website/static/Taskfile.yml
 {/* prettier-ignore-end */}

website/docs/reference/package.mdx

@@ -117,7 +117,8 @@ Taskfiles) by calling the `Read` method on the reader and pass the `Node` as an
 argument:
 
 ```go
-tfg, err := reader.Read(node)
+ctx := context.Background()
+tfg, err := reader.Read(ctx, node)
 // handle error
 ```