diff --git a/cmd/task/task.go b/cmd/task/task.go index 62d5be90..88eb8a86 100644 --- a/cmd/task/task.go +++ b/cmd/task/task.go @@ -18,7 +18,6 @@ import ( "github.com/go-task/task/v3/internal/flags" "github.com/go-task/task/v3/internal/logger" "github.com/go-task/task/v3/internal/version" - "github.com/go-task/task/v3/taskfile" "github.com/go-task/task/v3/taskfile/ast" ) @@ -121,18 +120,9 @@ func run() error { return err } - // If the download flag is specified, we should stop execution as soon as - // taskfile is downloaded - if flags.Download { - return nil - } - if flags.ClearCache { - cache, err := taskfile.NewCache(e.TempDir.Remote) - if err != nil { - return err - } - return cache.Clear() + cachePath := filepath.Join(e.TempDir.Remote, "remote") + return os.RemoveAll(cachePath) } listOptions := task.NewListOptions( diff --git a/cmd/tmp/main.go b/cmd/tmp/main.go new file mode 100644 index 00000000..7b26556a --- /dev/null +++ b/cmd/tmp/main.go @@ -0,0 +1,38 @@ +package main + +import ( + "context" + "fmt" + "net/http" + "time" +) + +func main() { + ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond) + defer cancel() + if err := run(ctx); err != nil { + fmt.Println(ctx.Err()) + fmt.Println(err) + } +} + +func run(ctx context.Context) error { + req, err := http.NewRequest("GET", "https://taskfile.dev/schema.json", nil) + if err != nil { + fmt.Println(1) + return err + } + + resp, err := http.DefaultClient.Do(req.WithContext(ctx)) + if err != nil { + if ctx.Err() != nil { + fmt.Println(2) + return err + } + fmt.Println(3) + return err + } + defer resp.Body.Close() + + return nil +} diff --git a/errors/errors_taskfile.go b/errors/errors_taskfile.go index ad6d1898..cbb160ae 100644 --- a/errors/errors_taskfile.go +++ b/errors/errors_taskfile.go @@ -155,19 +155,14 @@ func (err *TaskfileVersionCheckError) Code() int { // TaskfileNetworkTimeoutError is returned when the user attempts to use a remote // Taskfile but a network connection could not be established within the timeout. type TaskfileNetworkTimeoutError struct { - URI string - Timeout time.Duration - CheckedCache bool + URI string + Timeout time.Duration } func (err *TaskfileNetworkTimeoutError) Error() string { - var cacheText string - if err.CheckedCache { - cacheText = " and no offline copy was found in the cache" - } return fmt.Sprintf( - `task: Network connection timed out after %s while attempting to download Taskfile %q%s`, - err.Timeout, err.URI, cacheText, + `task: Network connection timed out after %s while attempting to download Taskfile %q`, + err.Timeout, err.URI, ) } diff --git a/executor.go b/executor.go index 329ddc13..cb846464 100644 --- a/executor.go +++ b/executor.go @@ -27,26 +27,27 @@ type ( // within them. Executor struct { // Flags - Dir string - Entrypoint string - TempDir TempDir - Force bool - ForceAll bool - Insecure bool - Download bool - Offline bool - Timeout time.Duration - Watch bool - Verbose bool - Silent bool - AssumeYes bool - AssumeTerm bool // Used for testing - Dry bool - Summary bool - Parallel bool - Color bool - Concurrency int - Interval time.Duration + Dir string + Entrypoint string + TempDir TempDir + Force bool + ForceAll bool + Insecure bool + Download bool + Offline bool + Timeout time.Duration + CacheExpiryDuration time.Duration + Watch bool + Verbose bool + Silent bool + AssumeYes bool + AssumeTerm bool // Used for testing + Dry bool + Summary bool + Parallel bool + Color bool + Concurrency int + Interval time.Duration // I/O Stdin io.Reader @@ -240,6 +241,20 @@ func (o *timeoutOption) ApplyToExecutor(e *Executor) { e.Timeout = o.timeout } +// WithCacheExpiryDuration sets the duration after which the cache is considered +// expired. By default, the cache is considered expired after 24 hours. +func WithCacheExpiryDuration(duration time.Duration) ExecutorOption { + return &cacheExpiryDurationOption{duration: duration} +} + +type cacheExpiryDurationOption struct { + duration time.Duration +} + +func (o *cacheExpiryDurationOption) ApplyToExecutor(r *Executor) { + r.CacheExpiryDuration = o.duration +} + // WithWatch tells the [Executor] to keep running in the background and watch // for changes to the fingerprint of the tasks that are run. When changes are // detected, a new task run is triggered. diff --git a/internal/flags/flags.go b/internal/flags/flags.go index 3069ec3e..d104b5d2 100644 --- a/internal/flags/flags.go +++ b/internal/flags/flags.go @@ -40,39 +40,40 @@ Options: ` var ( - Version bool - Help bool - Init bool - Completion string - List bool - ListAll bool - ListJson bool - TaskSort string - Status bool - NoStatus bool - Insecure bool - Force bool - ForceAll bool - Watch bool - Verbose bool - Silent bool - AssumeYes bool - Dry bool - Summary bool - ExitCode bool - Parallel bool - Concurrency int - Dir string - Entrypoint string - Output ast.Output - Color bool - Interval time.Duration - Global bool - Experiments bool - Download bool - Offline bool - ClearCache bool - Timeout time.Duration + Version bool + Help bool + Init bool + Completion string + List bool + ListAll bool + ListJson bool + TaskSort string + Status bool + NoStatus bool + Insecure bool + Force bool + ForceAll bool + Watch bool + Verbose bool + Silent bool + AssumeYes bool + Dry bool + Summary bool + ExitCode bool + Parallel bool + Concurrency int + Dir string + Entrypoint string + Output ast.Output + Color bool + Interval time.Duration + Global bool + Experiments bool + Download bool + Offline bool + ClearCache bool + Timeout time.Duration + CacheExpiryDuration time.Duration ) func init() { @@ -131,6 +132,7 @@ func init() { pflag.BoolVar(&Offline, "offline", offline, "Forces Task to only use local or cached Taskfiles.") pflag.DurationVar(&Timeout, "timeout", time.Second*10, "Timeout for downloading remote Taskfiles.") pflag.BoolVar(&ClearCache, "clear-cache", false, "Clear the remote cache.") + pflag.DurationVar(&CacheExpiryDuration, "expiry", 0, "Expiry duration for cached remote Taskfiles.") } pflag.Parse() @@ -212,6 +214,7 @@ func (o *flagsOption) ApplyToExecutor(e *task.Executor) { task.WithDownload(Download), task.WithOffline(Offline), task.WithTimeout(Timeout), + task.WithCacheExpiryDuration(CacheExpiryDuration), task.WithWatch(Watch), task.WithVerbose(Verbose), task.WithSilent(Silent), diff --git a/setup.go b/setup.go index a577414a..13f03dd4 100644 --- a/setup.go +++ b/setup.go @@ -64,6 +64,8 @@ func (e *Executor) getRootNode() (taskfile.Node, error) { } func (e *Executor) readTaskfile(node taskfile.Node) error { + ctx, cf := context.WithTimeout(context.Background(), e.Timeout) + defer cf() debugFunc := func(s string) { e.Logger.VerboseOutf(logger.Magenta, s) } @@ -74,13 +76,16 @@ func (e *Executor) readTaskfile(node taskfile.Node) error { taskfile.WithInsecure(e.Insecure), taskfile.WithDownload(e.Download), taskfile.WithOffline(e.Offline), - taskfile.WithTimeout(e.Timeout), taskfile.WithTempDir(e.TempDir.Remote), + taskfile.WithCacheExpiryDuration(e.CacheExpiryDuration), taskfile.WithDebugFunc(debugFunc), taskfile.WithPromptFunc(promptFunc), ) - graph, err := reader.Read(node) + graph, err := reader.Read(ctx, node) if err != nil { + if errors.Is(err, context.DeadlineExceeded) { + return &errors.TaskfileNetworkTimeoutError{URI: node.Location(), Timeout: e.Timeout} + } return err } if e.Taskfile, err = graph.Merge(); err != nil { diff --git a/taskfile/cache.go b/taskfile/cache.go deleted file mode 100644 index 2b57c17d..00000000 --- a/taskfile/cache.go +++ /dev/null @@ -1,72 +0,0 @@ -package taskfile - -import ( - "crypto/sha256" - "fmt" - "os" - "path/filepath" - "strings" -) - -type Cache struct { - dir string -} - -func NewCache(dir string) (*Cache, error) { - dir = filepath.Join(dir, "remote") - if err := os.MkdirAll(dir, 0o755); err != nil { - return nil, err - } - return &Cache{ - dir: dir, - }, nil -} - -func checksum(b []byte) string { - h := sha256.New() - h.Write(b) - return fmt.Sprintf("%x", h.Sum(nil)) -} - -func (c *Cache) write(node Node, b []byte) error { - return os.WriteFile(c.cacheFilePath(node), b, 0o644) -} - -func (c *Cache) read(node Node) ([]byte, error) { - return os.ReadFile(c.cacheFilePath(node)) -} - -func (c *Cache) writeChecksum(node Node, checksum string) error { - return os.WriteFile(c.checksumFilePath(node), []byte(checksum), 0o644) -} - -func (c *Cache) readChecksum(node Node) string { - b, _ := os.ReadFile(c.checksumFilePath(node)) - return string(b) -} - -func (c *Cache) key(node Node) string { - return strings.TrimRight(checksum([]byte(node.Location())), "=") -} - -func (c *Cache) cacheFilePath(node Node) string { - return c.filePath(node, "yaml") -} - -func (c *Cache) checksumFilePath(node Node) string { - return c.filePath(node, "checksum") -} - -func (c *Cache) filePath(node Node, suffix string) string { - lastDir, filename := node.FilenameAndLastDir() - prefix := filename - // Means it's not "", nor "." nor "/", so it's a valid directory - if len(lastDir) > 1 { - prefix = fmt.Sprintf("%s-%s", lastDir, filename) - } - return filepath.Join(c.dir, fmt.Sprintf("%s.%s.%s", prefix, c.key(node), suffix)) -} - -func (c *Cache) Clear() error { - return os.RemoveAll(c.dir) -} diff --git a/taskfile/node.go b/taskfile/node.go index 486a0a16..fb9abdc9 100644 --- a/taskfile/node.go +++ b/taskfile/node.go @@ -14,14 +14,18 @@ import ( ) type Node interface { - Read(ctx context.Context) ([]byte, error) + Read() ([]byte, error) Parent() Node Location() string Dir() string - Remote() bool ResolveEntrypoint(entrypoint string) (string, error) ResolveDir(dir string) (string, error) - FilenameAndLastDir() (string, string) +} + +type RemoteNode interface { + Node + ReadContext(ctx context.Context) ([]byte, error) + CacheKey() string } func NewRootNode( @@ -35,35 +39,35 @@ func NewRootNode( if entrypoint == "-" { return NewStdinNode(dir) } - return NewNode(entrypoint, dir, insecure, timeout) + return NewNode(entrypoint, dir, insecure) } func NewNode( entrypoint string, dir string, insecure bool, - timeout time.Duration, opts ...NodeOption, ) (Node, error) { var node Node var err error + scheme, err := getScheme(entrypoint) if err != nil { return nil, err } + switch scheme { case "git": node, err = NewGitNode(entrypoint, dir, insecure, opts...) case "http", "https": - node, err = NewHTTPNode(entrypoint, dir, insecure, timeout, opts...) + node, err = NewHTTPNode(entrypoint, dir, insecure, opts...) default: node, err = NewFileNode(entrypoint, dir, opts...) - } - - if node.Remote() && !experiments.RemoteTaskfiles.Enabled() { + if _, isRemote := node.(RemoteNode); isRemote && !experiments.RemoteTaskfiles.Enabled() { return nil, errors.New("task: Remote taskfiles are not enabled. You can read more about this experiment and how to enable it at https://taskfile.dev/experiments/remote-taskfiles") } + return node, err } @@ -72,6 +76,7 @@ func getScheme(uri string) (string, error) { if u == nil { return "", err } + if strings.HasSuffix(strings.Split(u.Path, "//")[0], ".git") && (u.Scheme == "git" || u.Scheme == "ssh" || u.Scheme == "https" || u.Scheme == "http") { return "git", nil } @@ -79,6 +84,7 @@ func getScheme(uri string) (string, error) { if i := strings.Index(uri, "://"); i != -1 { return uri[:i], nil } + return "", nil } diff --git a/taskfile/node_cache.go b/taskfile/node_cache.go new file mode 100644 index 00000000..b489161b --- /dev/null +++ b/taskfile/node_cache.go @@ -0,0 +1,113 @@ +package taskfile + +import ( + "crypto/sha256" + "fmt" + "os" + "path/filepath" + "time" +) + +const remoteCacheDir = "remote" + +type CacheNode struct { + *BaseNode + source RemoteNode +} + +func NewCacheNode(source RemoteNode, dir string) *CacheNode { + return &CacheNode{ + BaseNode: &BaseNode{ + dir: filepath.Join(dir, remoteCacheDir), + }, + source: source, + } +} + +func (node *CacheNode) Read() ([]byte, error) { + return os.ReadFile(node.Location()) +} + +func (node *CacheNode) Write(data []byte) error { + if err := node.CreateCacheDir(); err != nil { + return err + } + return os.WriteFile(node.Location(), data, 0o644) +} + +func (node *CacheNode) ReadTimestamp() time.Time { + b, err := os.ReadFile(node.timestampPath()) + if err != nil { + return time.Time{}.UTC() + } + timestamp, err := time.Parse(time.RFC3339, string(b)) + if err != nil { + return time.Time{}.UTC() + } + return timestamp.UTC() +} + +func (node *CacheNode) WriteTimestamp(t time.Time) error { + if err := node.CreateCacheDir(); err != nil { + return err + } + return os.WriteFile(node.timestampPath(), []byte(t.Format(time.RFC3339)), 0o644) +} + +func (node *CacheNode) ReadChecksum() string { + b, _ := os.ReadFile(node.checksumPath()) + return string(b) +} + +func (node *CacheNode) WriteChecksum(checksum string) error { + if err := node.CreateCacheDir(); err != nil { + return err + } + return os.WriteFile(node.checksumPath(), []byte(checksum), 0o644) +} + +func (node *CacheNode) CreateCacheDir() error { + if err := os.MkdirAll(node.dir, 0o755); err != nil { + return err + } + return nil +} + +func (node *CacheNode) ChecksumPrompt(checksum string) string { + cachedChecksum := node.ReadChecksum() + switch { + + // If the checksum doesn't exist, prompt the user to continue + case cachedChecksum == "": + return taskfileUntrustedPrompt + + // If there is a cached hash, but it doesn't match the expected hash, prompt the user to continue + case cachedChecksum != checksum: + return taskfileChangedPrompt + + default: + return "" + } +} + +func (node *CacheNode) Location() string { + return node.filePath("yaml") +} + +func (node *CacheNode) checksumPath() string { + return node.filePath("checksum") +} + +func (node *CacheNode) timestampPath() string { + return node.filePath("timestamp") +} + +func (node *CacheNode) filePath(suffix string) string { + return filepath.Join(node.dir, fmt.Sprintf("%s.%s", node.source.CacheKey(), suffix)) +} + +func checksum(b []byte) string { + h := sha256.New() + h.Write(b) + return fmt.Sprintf("%x", h.Sum(nil)) +} diff --git a/taskfile/node_file.go b/taskfile/node_file.go index e0dedcb5..a94a7cf4 100644 --- a/taskfile/node_file.go +++ b/taskfile/node_file.go @@ -1,7 +1,6 @@ package taskfile import ( - "context" "io" "os" "path/filepath" @@ -34,11 +33,7 @@ func (node *FileNode) Location() string { return node.Entrypoint } -func (node *FileNode) Remote() bool { - return false -} - -func (node *FileNode) Read(ctx context.Context) ([]byte, error) { +func (node *FileNode) Read() ([]byte, error) { f, err := os.Open(node.Location()) if err != nil { return nil, err @@ -114,7 +109,3 @@ func (node *FileNode) ResolveDir(dir string) (string, error) { entrypointDir := filepath.Dir(node.Entrypoint) return filepathext.SmartJoin(entrypointDir, path), nil } - -func (node *FileNode) FilenameAndLastDir() (string, string) { - return "", filepath.Base(node.Entrypoint) -} diff --git a/taskfile/node_git.go b/taskfile/node_git.go index 72ac7e1a..fd15ce4e 100644 --- a/taskfile/node_git.go +++ b/taskfile/node_git.go @@ -71,7 +71,11 @@ func (node *GitNode) Remote() bool { return true } -func (node *GitNode) Read(_ context.Context) ([]byte, error) { +func (node *GitNode) Read() ([]byte, error) { + return node.ReadContext(context.Background()) +} + +func (node *GitNode) ReadContext(_ context.Context) ([]byte, error) { fs := memfs.New() storer := memory.NewStorage() _, err := git.Clone(storer, fs, &git.CloneOptions{ @@ -121,6 +125,13 @@ func (node *GitNode) ResolveDir(dir string) (string, error) { return filepathext.SmartJoin(entrypointDir, path), nil } -func (node *GitNode) FilenameAndLastDir() (string, string) { - return filepath.Base(node.path), filepath.Base(filepath.Dir(node.path)) +func (node *GitNode) CacheKey() string { + checksum := strings.TrimRight(checksum([]byte(node.Location())), "=") + prefix := filepath.Base(filepath.Dir(node.path)) + lastDir := filepath.Base(node.path) + // Means it's not "", nor "." nor "/", so it's a valid directory + if len(lastDir) > 1 { + prefix = fmt.Sprintf("%s-%s", lastDir, prefix) + } + return fmt.Sprintf("%s.%s", prefix, checksum) } diff --git a/taskfile/node_git_test.go b/taskfile/node_git_test.go index 2df39c54..1b88a083 100644 --- a/taskfile/node_git_test.go +++ b/taskfile/node_git_test.go @@ -62,24 +62,21 @@ func TestGitNode_httpsWithDir(t *testing.T) { assert.Equal(t, "https://github.com/foo/bar.git//directory/common.yml?ref=main", entrypoint) } -func TestGitNode_FilenameAndDir(t *testing.T) { +func TestGitNode_CacheKey(t *testing.T) { t.Parallel() node, err := NewGitNode("https://github.com/foo/bar.git//directory/Taskfile.yml?ref=main", "", false) assert.NoError(t, err) - filename, dir := node.FilenameAndLastDir() - assert.Equal(t, "Taskfile.yml", filename) - assert.Equal(t, "directory", dir) + key := node.CacheKey() + assert.Equal(t, "Taskfile.yml-directory.f1ddddac425a538870230a3e38fc0cded4ec5da250797b6cab62c82477718fbb", key) node, err = NewGitNode("https://github.com/foo/bar.git//Taskfile.yml?ref=main", "", false) assert.NoError(t, err) - filename, dir = node.FilenameAndLastDir() - assert.Equal(t, "Taskfile.yml", filename) - assert.Equal(t, ".", dir) + key = node.CacheKey() + assert.Equal(t, "Taskfile.yml-..39d28c1ff36f973705ae188b991258bbabaffd6d60bcdde9693d157d00d5e3a4", key) node, err = NewGitNode("https://github.com/foo/bar.git//multiple/directory/Taskfile.yml?ref=main", "", false) assert.NoError(t, err) - filename, dir = node.FilenameAndLastDir() - assert.Equal(t, "Taskfile.yml", filename) - assert.Equal(t, "directory", dir) + key = node.CacheKey() + assert.Equal(t, "Taskfile.yml-directory.1b6d145e01406dcc6c0aa572e5a5d1333be1ccf2cae96d18296d725d86197d31", key) } diff --git a/taskfile/node_http.go b/taskfile/node_http.go index 6197f327..16e0ee40 100644 --- a/taskfile/node_http.go +++ b/taskfile/node_http.go @@ -2,11 +2,12 @@ package taskfile import ( "context" + "fmt" "io" "net/http" "net/url" "path/filepath" - "time" + "strings" "github.com/go-task/task/v3/errors" "github.com/go-task/task/v3/internal/execext" @@ -18,14 +19,12 @@ type HTTPNode struct { *BaseNode URL *url.URL // stores url pointing actual remote file. (e.g. with Taskfile.yml) entrypoint string // stores entrypoint url. used for building graph vertices. - timeout time.Duration } func NewHTTPNode( entrypoint string, dir string, insecure bool, - timeout time.Duration, opts ...NodeOption, ) (*HTTPNode, error) { base := NewBaseNode(dir, opts...) @@ -41,7 +40,6 @@ func NewHTTPNode( BaseNode: base, URL: url, entrypoint: entrypoint, - timeout: timeout, }, nil } @@ -49,12 +47,12 @@ func (node *HTTPNode) Location() string { return node.entrypoint } -func (node *HTTPNode) Remote() bool { - return true +func (node *HTTPNode) Read() ([]byte, error) { + return node.ReadContext(context.Background()) } -func (node *HTTPNode) Read(ctx context.Context) ([]byte, error) { - url, err := RemoteExists(ctx, node.URL, node.timeout) +func (node *HTTPNode) ReadContext(ctx context.Context) ([]byte, error) { + url, err := RemoteExists(ctx, node.URL) if err != nil { return nil, err } @@ -66,8 +64,8 @@ func (node *HTTPNode) Read(ctx context.Context) ([]byte, error) { resp, err := http.DefaultClient.Do(req.WithContext(ctx)) if err != nil { - if errors.Is(err, context.DeadlineExceeded) { - return nil, &errors.TaskfileNetworkTimeoutError{URI: node.URL.String(), Timeout: node.timeout} + if ctx.Err() != nil { + return nil, err } return nil, errors.TaskfileFetchFailedError{URI: node.URL.String()} } @@ -116,7 +114,14 @@ func (node *HTTPNode) ResolveDir(dir string) (string, error) { return filepathext.SmartJoin(parent, path), nil } -func (node *HTTPNode) FilenameAndLastDir() (string, string) { +func (node *HTTPNode) CacheKey() string { + checksum := strings.TrimRight(checksum([]byte(node.Location())), "=") dir, filename := filepath.Split(node.entrypoint) - return filepath.Base(dir), filename + lastDir := filepath.Base(dir) + prefix := filename + // Means it's not "", nor "." nor "/", so it's a valid directory + if len(lastDir) > 1 { + prefix = fmt.Sprintf("%s-%s", lastDir, filename) + } + return fmt.Sprintf("%s.%s", prefix, checksum) } diff --git a/taskfile/node_stdin.go b/taskfile/node_stdin.go index 20d33137..387f50fe 100644 --- a/taskfile/node_stdin.go +++ b/taskfile/node_stdin.go @@ -2,7 +2,6 @@ package taskfile import ( "bufio" - "context" "fmt" "os" "strings" @@ -30,7 +29,7 @@ func (node *StdinNode) Remote() bool { return false } -func (node *StdinNode) Read(ctx context.Context) ([]byte, error) { +func (node *StdinNode) Read() ([]byte, error) { var stdin []byte scanner := bufio.NewScanner(os.Stdin) for scanner.Scan() { @@ -72,7 +71,3 @@ func (node *StdinNode) ResolveDir(dir string) (string, error) { return filepathext.SmartJoin(node.Dir(), path), nil } - -func (node *StdinNode) FilenameAndLastDir() (string, string) { - return "", "__stdin__" -} diff --git a/taskfile/reader.go b/taskfile/reader.go index 9de57c99..edcf54e0 100644 --- a/taskfile/reader.go +++ b/taskfile/reader.go @@ -39,15 +39,15 @@ type ( // A Reader will recursively read Taskfiles from a given [Node] and build a // [ast.TaskfileGraph] from them. Reader struct { - graph *ast.TaskfileGraph - insecure bool - download bool - offline bool - timeout time.Duration - tempDir string - debugFunc DebugFunc - promptFunc PromptFunc - promptMutex sync.Mutex + graph *ast.TaskfileGraph + insecure bool + download bool + offline bool + tempDir string + cacheExpiryDuration time.Duration + debugFunc DebugFunc + promptFunc PromptFunc + promptMutex sync.Mutex } ) @@ -55,15 +55,15 @@ type ( // options. func NewReader(opts ...ReaderOption) *Reader { r := &Reader{ - graph: ast.NewTaskfileGraph(), - insecure: false, - download: false, - offline: false, - timeout: time.Second * 10, - tempDir: os.TempDir(), - debugFunc: nil, - promptFunc: nil, - promptMutex: sync.Mutex{}, + graph: ast.NewTaskfileGraph(), + insecure: false, + download: false, + offline: false, + tempDir: os.TempDir(), + cacheExpiryDuration: 0, + debugFunc: nil, + promptFunc: nil, + promptMutex: sync.Mutex{}, } r.Options(opts...) return r @@ -119,20 +119,6 @@ func (o *offlineOption) ApplyToReader(r *Reader) { r.offline = o.offline } -// WithTimeout sets the [Reader]'s timeout for fetching remote taskfiles. By -// default, the timeout is set to 10 seconds. -func WithTimeout(timeout time.Duration) ReaderOption { - return &timeoutOption{timeout: timeout} -} - -type timeoutOption struct { - timeout time.Duration -} - -func (o *timeoutOption) ApplyToReader(r *Reader) { - r.timeout = o.timeout -} - // WithTempDir sets the temporary directory that will be used by the [Reader]. // By default, the reader uses [os.TempDir]. func WithTempDir(tempDir string) ReaderOption { @@ -147,6 +133,20 @@ func (o *tempDirOption) ApplyToReader(r *Reader) { r.tempDir = o.tempDir } +// WithCacheExpiryDuration sets the duration after which the cache is considered +// expired. By default, the cache is considered expired after 24 hours. +func WithCacheExpiryDuration(duration time.Duration) ReaderOption { + return &cacheExpiryDurationOption{duration: duration} +} + +type cacheExpiryDurationOption struct { + duration time.Duration +} + +func (o *cacheExpiryDurationOption) ApplyToReader(r *Reader) { + r.cacheExpiryDuration = o.duration +} + // WithDebugFunc sets the debug function to be used by the [Reader]. If set, // this function will be called with debug messages. This can be useful if the // caller wants to log debug messages from the [Reader]. By default, no debug @@ -186,8 +186,8 @@ func (o *promptFuncOption) ApplyToReader(r *Reader) { // through any [ast.Includes] it finds, reading each included Taskfile and // building an [ast.TaskfileGraph] as it goes. If any errors occur, they will be // returned immediately. -func (r *Reader) Read(node Node) (*ast.TaskfileGraph, error) { - if err := r.include(node); err != nil { +func (r *Reader) Read(ctx context.Context, node Node) (*ast.TaskfileGraph, error) { + if err := r.include(ctx, node); err != nil { return nil, err } return r.graph, nil @@ -206,7 +206,7 @@ func (r *Reader) promptf(format string, a ...any) error { return nil } -func (r *Reader) include(node Node) error { +func (r *Reader) include(ctx context.Context, node Node) error { // Create a new vertex for the Taskfile vertex := &ast.TaskfileVertex{ URI: node.Location(), @@ -224,7 +224,7 @@ func (r *Reader) include(node Node) error { // Read and parse the Taskfile from the file and add it to the vertex var err error - vertex.Taskfile, err = r.readNode(node) + vertex.Taskfile, err = r.readNode(ctx, node) if err != nil { return err } @@ -265,7 +265,7 @@ func (r *Reader) include(node Node) error { return err } - includeNode, err := NewNode(entrypoint, include.Dir, r.insecure, r.timeout, + includeNode, err := NewNode(entrypoint, include.Dir, r.insecure, WithParent(node), ) if err != nil { @@ -276,7 +276,7 @@ func (r *Reader) include(node Node) error { } // Recurse into the included Taskfile - if err := r.include(includeNode); err != nil { + if err := r.include(ctx, includeNode); err != nil { return err } @@ -316,8 +316,8 @@ func (r *Reader) include(node Node) error { return g.Wait() } -func (r *Reader) readNode(node Node) (*ast.Taskfile, error) { - b, err := r.loadNodeContent(node) +func (r *Reader) readNode(ctx context.Context, node Node) (*ast.Taskfile, error) { + b, err := r.readNodeContent(ctx, node) if err != nil { return nil, err } @@ -358,72 +358,79 @@ func (r *Reader) readNode(node Node) (*ast.Taskfile, error) { return &tf, nil } -func (r *Reader) loadNodeContent(node Node) ([]byte, error) { - if !node.Remote() { - ctx, cf := context.WithTimeout(context.Background(), r.timeout) - defer cf() - return node.Read(ctx) +func (r *Reader) readNodeContent(ctx context.Context, node Node) ([]byte, error) { + if node, isRemote := node.(RemoteNode); isRemote { + return r.readRemoteNodeContent(ctx, node) + } + return node.Read() +} + +func (r *Reader) readRemoteNodeContent(ctx context.Context, node RemoteNode) ([]byte, error) { + cache := NewCacheNode(node, r.tempDir) + now := time.Now().UTC() + timestamp := cache.ReadTimestamp() + expiry := timestamp.Add(r.cacheExpiryDuration) + cacheValid := now.Before(expiry) + var cacheFound bool + + r.debugf("checking cache for %q in %q\n", node.Location(), cache.Location()) + cachedBytes, err := cache.Read() + switch { + // If the cache doesn't exist, we need to download the file + case errors.Is(err, os.ErrNotExist): + r.debugf("no cache found\n") + // If we couldn't find a cached copy, and we are offline, we can't do anything + if r.offline { + return nil, &errors.TaskfileCacheNotFoundError{ + URI: node.Location(), + } + } + + // If the cache is expired + case !cacheValid: + r.debugf("cache expired at %s\n", expiry.Format(time.RFC3339)) + cacheFound = true + // If we can't fetch a fresh copy, we should use the cache anyway + if r.offline { + r.debugf("in offline mode, using expired cache\n") + return cachedBytes, nil + } + + // Some other error + case err != nil: + return nil, err + + // Found valid cache + default: + r.debugf("cache found\n") + // Not being forced to redownload, return cache + if !r.download { + return cachedBytes, nil + } + cacheFound = true } - cache, err := NewCache(r.tempDir) + // Try to read the remote file + r.debugf("downloading remote file: %s\n", node.Location()) + downloadedBytes, err := node.ReadContext(ctx) if err != nil { + // If the context timed out or was cancelled, but we found a cached version, use that + if ctx.Err() != nil && cacheFound { + if cacheValid { + r.debugf("failed to fetch remote file: %s: using cache\n", ctx.Err().Error()) + } else { + r.debugf("failed to fetch remote file: %s: using expired cache\n", ctx.Err().Error()) + } + return cachedBytes, nil + } return nil, err } - if r.offline { - // In offline mode try to use cached copy - cached, err := cache.read(node) - if errors.Is(err, os.ErrNotExist) { - return nil, &errors.TaskfileCacheNotFoundError{URI: node.Location()} - } else if err != nil { - return nil, err - } - r.debugf("task: [%s] Fetched cached copy\n", node.Location()) - - return cached, nil - } - - ctx, cf := context.WithTimeout(context.Background(), r.timeout) - defer cf() - - b, err := node.Read(ctx) - if errors.Is(err, &errors.TaskfileNetworkTimeoutError{}) { - // If we timed out then we likely have a network issue - - // If a download was requested, then we can't use a cached copy - if r.download { - return nil, &errors.TaskfileNetworkTimeoutError{URI: node.Location(), Timeout: r.timeout} - } - - // Search for any cached copies - cached, err := cache.read(node) - if errors.Is(err, os.ErrNotExist) { - return nil, &errors.TaskfileNetworkTimeoutError{URI: node.Location(), Timeout: r.timeout, CheckedCache: true} - } else if err != nil { - return nil, err - } - r.debugf("task: [%s] Network timeout. Fetched cached copy\n", node.Location()) - - return cached, nil - - } else if err != nil { - return nil, err - } - r.debugf("task: [%s] Fetched remote copy\n", node.Location()) - - // Get the checksums - checksum := checksum(b) - cachedChecksum := cache.readChecksum(node) - - var prompt string - if cachedChecksum == "" { - // If the checksum doesn't exist, prompt the user to continue - prompt = taskfileUntrustedPrompt - } else if checksum != cachedChecksum { - // If there is a cached hash, but it doesn't match the expected hash, prompt the user to continue - prompt = taskfileChangedPrompt - } + r.debugf("found remote file at %q\n", node.Location()) + checksum := checksum(downloadedBytes) + prompt := cache.ChecksumPrompt(checksum) + // Prompt the user if required if prompt != "" { if err := func() error { r.promptMutex.Lock() @@ -432,18 +439,23 @@ func (r *Reader) loadNodeContent(node Node) ([]byte, error) { }(); err != nil { return nil, &errors.TaskfileNotTrustedError{URI: node.Location()} } - - // Store the checksum - if err := cache.writeChecksum(node, checksum); err != nil { - return nil, err - } - - // Cache the file - r.debugf("task: [%s] Caching downloaded file\n", node.Location()) - if err = cache.write(node, b); err != nil { - return nil, err - } } - return b, nil + // Store the checksum + if err := cache.WriteChecksum(checksum); err != nil { + return nil, err + } + + // Store the timestamp + if err := cache.WriteTimestamp(now); err != nil { + return nil, err + } + + // Cache the file + r.debugf("caching %q to %q\n", node.Location(), cache.Location()) + if err = cache.Write(downloadedBytes); err != nil { + return nil, err + } + + return downloadedBytes, nil } diff --git a/taskfile/taskfile.go b/taskfile/taskfile.go index 3502638a..d902559a 100644 --- a/taskfile/taskfile.go +++ b/taskfile/taskfile.go @@ -2,13 +2,13 @@ package taskfile import ( "context" + "fmt" "net/http" "net/url" "os" "path/filepath" "slices" "strings" - "time" "github.com/go-task/task/v3/errors" "github.com/go-task/task/v3/internal/filepathext" @@ -40,7 +40,7 @@ var ( // at the given URL with any of the default Taskfile files names. If any of // these match a file, the first matching path will be returned. If no files are // found, an error will be returned. -func RemoteExists(ctx context.Context, u *url.URL, timeout time.Duration) (*url.URL, error) { +func RemoteExists(ctx context.Context, u *url.URL) (*url.URL, error) { // Create a new HEAD request for the given URL to check if the resource exists req, err := http.NewRequestWithContext(ctx, "HEAD", u.String(), nil) if err != nil { @@ -50,8 +50,8 @@ func RemoteExists(ctx context.Context, u *url.URL, timeout time.Duration) (*url. // Request the given URL resp, err := http.DefaultClient.Do(req) if err != nil { - if errors.Is(ctx.Err(), context.DeadlineExceeded) { - return nil, &errors.TaskfileNetworkTimeoutError{URI: u.String(), Timeout: timeout} + if ctx.Err() != nil { + return nil, fmt.Errorf("checking remote file: %w", ctx.Err()) } return nil, errors.TaskfileFetchFailedError{URI: u.String()} } diff --git a/website/docs/experiments/remote_taskfiles.mdx b/website/docs/experiments/remote_taskfiles.mdx index 45e6617e..b9c2d1f2 100644 --- a/website/docs/experiments/remote_taskfiles.mdx +++ b/website/docs/experiments/remote_taskfiles.mdx @@ -2,6 +2,9 @@ slug: /experiments/remote-taskfiles/ --- +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + # Remote Taskfiles (#1317) :::caution @@ -20,33 +23,151 @@ To enable this experiment, set the environment variable: ::: -This experiment allows you to specify a remote Taskfile URL when including a -Taskfile. For example: +:::danger +Never run remote Taskfiles from sources that you do not trust. +::: -```yaml -version: '3' +This experiment allows you to use Taskfiles which are stored in remote +locations. This applies to both the root Taskfile (aka. Entrypoint) and also +when including Taskfiles. -includes: - my-remote-namespace: https://raw.githubusercontent.com/my-org/my-repo/main/Taskfile.yml -``` +Task uses "nodes" to reference remote Taskfiles. There are a few different types +of node which you can use: -This works exactly the same way that including a local file does. Any tasks in -the remote Taskfile will be available to run from your main Taskfile via the -namespace `my-remote-namespace`. For example, if the remote file contains the -following: + + + +`https://raw.githubusercontent.com/go-task/task/main/website/static/Taskfile.yml` + +This is the most basic type of remote node and works by downloading the file +from the specified URL. The file must be a valid Taskfile and can be of any +name. If a file is not found at the specified URL, Task will append each of the +[supported file names][supported-file-names] in turn until it finds a valid +file. If it still does not find a valid Taskfile, an error is returned. + + + + +`https://github.com/go-task/task.git//website/static/Taskfile.yml?ref=main` + +This type of node works by downloading the file from a Git repository over +HTTP/HTTPS. The first part of the URL is the base URL of the Git repository. +This is the same URL that you would use to clone the repo over HTTP. + +- You can optionally add the path to the Taskfile in the repository by appending +`//` to the URL. +- You can also optionally specify a branch or tag to use by appending +`?ref=` to the end of the URL. If you omit a reference, the default branch +will be used. + + + + +`git@github.com/go-task/task.git//website/static/Taskfile.yml?ref=main` + +This type of node works by downloading the file from a Git repository over SSH. +The first part of the URL is the user and base URL of the Git repository. This +is the same URL that you would use to clone the repo over SSH. + +To use Git over SSH, you need to make sure that your SSH agent has your private +SSH keys added so that they can be used during authentication. + +- You can optionally add the path to the Taskfile in the repository by appending +`//` to the URL. +- You can also optionally specify a branch or tag to use by appending +`?ref=` to the end of the URL. If you omit a reference, the default branch +will be used. + + + + +Task has an [example remote Taskfile][example-remote-taskfile] in our repository +that you can use for testing and that we will use throughout this document: ```yaml version: '3' tasks: - hello: - silent: true + default: cmds: - - echo "Hello from the remote Taskfile!" + - task: hello + + hello: + cmds: + - echo "Hello Task!" ``` -and you run `task my-remote-namespace:hello`, it will print the text: "Hello -from the remote Taskfile!" to your console. +## Specifying a remote entrypoint + +By default, Task will look for one of the [supported file +names][supported-file-names] on your local filesystem. If you want to use a +remote file instead, you can pass its URI into the `--taskfile`/`-t` flag just +like you would to specify a different local file. For example: + + + +```shell +$ task --taskfile https://raw.githubusercontent.com/go-task/task/main/website/static/Taskfile.yml +task: [hello] echo "Hello Task!" +Hello Task! +``` + + +```shell +$ task --taskfile https://github.com/go-task/task.git//website/static/Taskfile.yml?ref=main +task: [hello] echo "Hello Task!" +Hello Task! +``` + + +```shell +$ task --taskfile git@github.com/go-task/task.git//website/static/Taskfile.yml?ref=main +task: [hello] echo "Hello Task!" +Hello Task! +``` + + + +## Including remote Taskfiles + +Including a remote file works exactly the same way that including a local file +does. You just need to replace the local path with a remote URI. Any tasks in +the remote Taskfile will be available to run from your main Taskfile. + + + +```yaml +version: '3' + +includes: + my-remote-namespace: https://raw.githubusercontent.com/go-task/task/main/website/static/Taskfile.yml +``` + + +```yaml +version: '3' + +includes: + my-remote-namespace: https://github.com/go-task/task.git//website/static/Taskfile.yml?ref=main +``` + + +```yaml +version: '3' + +includes: + my-remote-namespace: git@github.com/go-task/task.git//website/static/Taskfile.yml?ref=main +``` + + + +```shell +$ task my-remote-namespace:hello +task: [hello] echo "Hello Task!" +Hello Task! +``` + +### Authenticating using environment variables The Taskfile location is processed by the templating system, so you can reference environment variables in your URL if you need to add authentication. @@ -59,19 +180,6 @@ includes: my-remote-namespace: https://{{.TOKEN}}@raw.githubusercontent.com/my-org/my-repo/main/Taskfile.yml ``` -`TOKEN=my-token task my-remote-namespace:hello` will be resolved by Task to -`https://my-token@raw.githubusercontent.com/my-org/my-repo/main/Taskfile.yml` - -## Git nodes - -You can also include a Taskfile from a Git node. We currently support ssh-style and http / https addresses like `git@example.com/foo/bar.git//Taskfiles.yml?ref=v1` and `https://example.com/foo/bar.git//Taskfiles.yml?ref=v1`. - -You need to follow this pattern : `.git//?ref=`. -The `ref` parameter, optional, can be a branch name or a tag, if not provided it'll pick up the default branch. -The `path` is the path to the Taskfile in the repository. - -If you want to use the SSH protocol, you need to make sure that your ssh-agent has your private ssh keys added so that they can be used during authentication. - ## Security Running commands from sources that you do not control is always a potential @@ -104,20 +212,26 @@ flag. Before enabling this flag, you should: Task currently supports both `http` and `https` URLs. However, the `http` requests will not execute by default unless you run the task with the `--insecure` flag. This is to protect you from accidentally running a remote -Taskfile that is via an unencrypted connection. Sources that are not protected -by TLS are vulnerable to [man-in-the-middle attacks][man-in-the-middle-attacks] -and should be avoided unless you know what you are doing. +Taskfile that is downloaded via an unencrypted connection. Sources that are not +protected by TLS are vulnerable to [man-in-the-middle +attacks][man-in-the-middle-attacks] and should be avoided unless you know what +you are doing. ## Caching & Running Offline Whenever you run a remote Taskfile, the latest copy will be downloaded from the -internet and cached locally. If for whatever reason, you lose access to the -internet, you will still be able to run your tasks by specifying the `--offline` -flag. This will tell Task to use the latest cached version of the file instead -of trying to download it. You are able to use the `--download` flag to update -the cached version of the remote files without running any tasks. You are able -to use the `--clear-cache` flag to clear all cached version of the remote files -without running any tasks. +internet and cached locally. This cached file will be used for all future +invocations of the Taskfile until the cache expires. Once it expires, Task will +download the latest copy of the file and update the cache. By default, the cache +is set to expire immediately. This means that Task will always fetch the latest +version. However, the cache expiry duration can be modified by setting the +`--expiry` flag. + +If for any reason you lose access to the internet or you are running Task in +offline mode (via the `--offline` flag or `TASK_OFFLINE` environment variable), +Task will run the any available cached files _even if they are expired_. This +means that you should never be stuck without the ability to run your tasks as +long as you have downloaded a remote Taskfile at least once. By default, Task will timeout requests to download remote files after 10 seconds and look for a cached copy instead. This timeout can be configured by setting @@ -129,7 +243,14 @@ By default, the cache is stored in the Task temp directory, represented by the override the location of the cache by setting the `TASK_REMOTE_DIR` environment variable. This way, you can share the cache between different projects. +You can force Task to ignore the cache and download the latest version +by using the `--download` flag. + +You can use the `--clear-cache` flag to clear all cached remote files. + {/* prettier-ignore-start */} [enabling-experiments]: ./experiments.mdx#enabling-experiments [man-in-the-middle-attacks]: https://en.wikipedia.org/wiki/Man-in-the-middle_attack +[supported-file-names]: https://taskfile.dev/usage/#supported-file-names +[example-remote-taskfile]: https://raw.githubusercontent.com/go-task/task/main/website/static/Taskfile.yml {/* prettier-ignore-end */} diff --git a/website/docs/reference/package.mdx b/website/docs/reference/package.mdx index ebffbd7d..4777f5df 100644 --- a/website/docs/reference/package.mdx +++ b/website/docs/reference/package.mdx @@ -117,7 +117,8 @@ Taskfiles) by calling the `Read` method on the reader and pass the `Node` as an argument: ```go -tfg, err := reader.Read(node) +ctx := context.Background() +tfg, err := reader.Read(ctx, node) // handle error ```