go/task
go/task
1
0
Fork 0
mirror of https://github.com/go-task/task.git synced 2025-02-03 13:22:11 +02:00
Code Issues Releases Activity

feat: dag reader

Browse Source
This commit is contained in:
Pete Davison 2024-01-01 23:12:28 +00:00
parent 1890722b75
commit a50580b5a1
6 changed files with 145 additions and 128 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@ -10,6 +10,9 @@
# Output of the go coverage tool, specifically when used with LiteIDE
*.out
# Graphvis files
*.gv
# Project-local glide cache, RE: https://github.com/Masterminds/glide/issues/736
.glide/

1
go.mod
View File

@ -5,6 +5,7 @@ go 1.21
require (
github.com/Masterminds/semver/v3 v3.2.1
github.com/davecgh/go-spew v1.1.1
github.com/dominikbraun/graph v0.23.0
github.com/fatih/color v1.16.0
github.com/go-task/slim-sprig/v3 v3.0.0
github.com/joho/godotenv v1.5.1

2
go.sum
View File

@ -4,6 +4,8 @@ github.com/creack/pty v1.1.21 h1:1/QdRyBaHHJP61QkWMXlOIBfsgdDeeKfK8SYVUWJKf0=
github.com/creack/pty v1.1.21/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dominikbraun/graph v0.23.0 h1:TdZB4pPqCLFxYhdyMFb1TBdFxp8XLcJfTTBQucVPgCo=
github.com/dominikbraun/graph v0.23.0/go.mod h1:yOjYyogZLY1LSG9E33JWZJiq5k83Qy2C6POAuiViluc=
github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=

7
setup.go
View File

@ -63,8 +63,7 @@ func (e *Executor) getRootNode() (taskfile.Node, error) {
}
func (e *Executor) readTaskfile(node taskfile.Node) error {
var err error
e.Taskfile, err = taskfile.Read(
reader := taskfile.NewReader(
node,
e.Insecure,
e.Download,
@ -73,9 +72,13 @@ func (e *Executor) readTaskfile(node taskfile.Node) error {
e.TempDir,
e.Logger,
)
graph, err := reader.Read()
if err != nil {
return err
}
if err := graph.Visualize("./taskfile-dag.gv"); err != nil {
return err
}
return nil
}

41
taskfile/ast/graph.go Normal file
View File

@ -0,0 +1,41 @@
package ast
import (
"os"
"github.com/dominikbraun/graph"
"github.com/dominikbraun/graph/draw"
)
type TaskfileGraph struct {
graph.Graph[string, *TaskfileVertex]
}
// A TaskfileVertex is a vertex on the Taskfile DAG.
type TaskfileVertex struct {
URI string
Taskfile *Taskfile
}
func taskfileHash(vertex *TaskfileVertex) string {
return vertex.URI
}
func NewTaskfileGraph() *TaskfileGraph {
return &TaskfileGraph{
graph.New(taskfileHash,
graph.Directed(),
graph.PreventCycles(),
graph.Rooted(),
),
}
}
func (r *TaskfileGraph) Visualize(filename string) error {
f, err := os.Create(filename)
if err != nil {
return err
}
defer f.Close()
return draw.DOT(r.Graph, f)
}

219
taskfile/reader.go
View File

@ -6,6 +6,8 @@ import (
"os"
"time"
"github.com/dominikbraun/graph"
"golang.org/x/sync/errgroup"
"gopkg.in/yaml.v3"
"github.com/go-task/task/v3/errors"
@ -24,32 +26,83 @@ Continue?`
Continue?`
)
// Read reads a Read for a given directory
// Uses current dir when dir is left empty. Uses Read.yml
// or Read.yaml when entrypoint is left empty
func Read(
// A Reader will recursively read Taskfiles from a given source using a directed
// acyclic graph (DAG).
type Reader struct {
graph *ast.TaskfileGraph
node Node
insecure bool
download bool
offline bool
timeout time.Duration
tempDir string
logger *logger.Logger
}
func NewReader(
node Node,
insecure bool,
download bool,
offline bool,
timeout time.Duration,
tempDir string,
l *logger.Logger,
) (*ast.Taskfile, error) {
var _taskfile func(Node) (*ast.Taskfile, error)
_taskfile = func(node Node) (*ast.Taskfile, error) {
tf, err := readTaskfile(node, download, offline, timeout, tempDir, l)
if err != nil {
return nil, err
}
logger *logger.Logger,
) *Reader {
return &Reader{
graph: ast.NewTaskfileGraph(),
node: node,
insecure: insecure,
download: download,
offline: offline,
timeout: timeout,
tempDir: tempDir,
logger: logger,
}
}
// Check that the Taskfile is set and has a schema version
if tf == nil || tf.Version == nil {
return nil, &errors.TaskfileVersionCheckError{URI: node.Location()}
}
func (r *Reader) Read() (*ast.TaskfileGraph, error) {
// Recursively loop through each Taskfile, adding vertices/edges to the graph
if err := r.include(r.node); err != nil {
return nil, err
}
err = tf.Includes.Range(func(namespace string, include ast.Include) error {
cache := &templater.Cache{Vars: tf.Vars}
return r.graph, nil
}
func (r *Reader) include(node Node) error {
// Create a new vertex for the Taskfile
vertex := &ast.TaskfileVertex{
URI: node.Location(),
Taskfile: nil,
}
// Add the included Taskfile to the DAG
// If the vertex already exists, we return early since its Taskfile has
// already been read and its children explored
if err := r.graph.AddVertex(vertex); err == graph.ErrVertexAlreadyExists {
return nil
} else if err != nil {
return err
}
// Read and parse the Taskfile from the file and add it to the vertex
var err error
vertex.Taskfile, err = r.readNode(node)
if err != nil {
if node.Optional() {
return nil
}
return err
}
// Create an error group to wait for all included Taskfiles to be read
var g errgroup.Group
// Loop over each included taskfile
vertex.Taskfile.Includes.Range(func(namespace string, include ast.Include) error {
// Start a goroutine to process each included Taskfile
g.Go(func() error {
cache := &templater.Cache{Vars: vertex.Taskfile.Vars}
include = ast.Include{
Namespace: include.Namespace,
Taskfile: templater.Replace(include.Taskfile, cache),
@ -74,117 +127,53 @@ func Read(
return err
}
includeReaderNode, err := NewNode(l, entrypoint, dir, insecure, timeout,
includeNode, err := NewNode(r.logger, entrypoint, dir, r.insecure, r.timeout,
WithParent(node),
WithOptional(include.Optional),
)
if err != nil {
if include.Optional {
return nil
}
return err
}
if err := checkCircularIncludes(includeReaderNode); err != nil {
// Recurse into the included Taskfile
if err := r.include(includeNode); err != nil {
return err
}
includedTaskfile, err := _taskfile(includeReaderNode)
if err != nil {
if include.Optional {
return nil
}
return err
}
if len(includedTaskfile.Dotenv) > 0 {
return ErrIncludedTaskfilesCantHaveDotenvs
}
if include.AdvancedImport {
// nolint: errcheck
includedTaskfile.Vars.Range(func(k string, v ast.Var) error {
o := v
o.Dir = dir
includedTaskfile.Vars.Set(k, o)
return nil
})
// nolint: errcheck
includedTaskfile.Env.Range(func(k string, v ast.Var) error {
o := v
o.Dir = dir
includedTaskfile.Env.Set(k, o)
return nil
})
for _, task := range includedTaskfile.Tasks.Values() {
task.Dir = filepathext.SmartJoin(dir, task.Dir)
if task.IncludeVars == nil {
task.IncludeVars = &ast.Vars{}
}
task.IncludeVars.Merge(include.Vars)
task.IncludedTaskfileVars = includedTaskfile.Vars
}
}
if err = tf.Merge(includedTaskfile, &include); err != nil {
return err
}
return nil
// Create an edge between the Taskfiles
return r.graph.AddEdge(node.Location(), includeNode.Location())
})
if err != nil {
return nil, err
}
return nil
})
for _, task := range tf.Tasks.Values() {
// If the task is not defined, create a new one
if task == nil {
task = &ast.Task{}
}
// Set the location of the taskfile for each task
if task.Location.Taskfile == "" {
task.Location.Taskfile = tf.Location
}
}
return tf, nil
}
return _taskfile(node)
// Wait for all the go routines to finish
return g.Wait()
}
func readTaskfile(
node Node,
download,
offline bool,
timeout time.Duration,
tempDir string,
l *logger.Logger,
) (*ast.Taskfile, error) {
func (r *Reader) readNode(node Node) (*ast.Taskfile, error) {
var b []byte
var err error
var cache *Cache
if node.Remote() {
cache, err = NewCache(tempDir)
cache, err = NewCache(r.tempDir)
if err != nil {
return nil, err
}
}
// If the file is remote and we're in offline mode, check if we have a cached copy
if node.Remote() && offline {
if node.Remote() && r.offline {
if b, err = cache.read(node); errors.Is(err, os.ErrNotExist) {
return nil, &errors.TaskfileCacheNotFoundError{URI: node.Location()}
} else if err != nil {
return nil, err
}
l.VerboseOutf(logger.Magenta, "task: [%s] Fetched cached copy\n", node.Location())
r.logger.VerboseOutf(logger.Magenta, "task: [%s] Fetched cached copy\n", node.Location())
} else {
downloaded := false
ctx, cf := context.WithTimeout(context.Background(), timeout)
ctx, cf := context.WithTimeout(context.Background(), r.timeout)
defer cf()
// Read the file
@ -192,16 +181,16 @@ func readTaskfile(
// If we timed out then we likely have a network issue
if node.Remote() && errors.Is(ctx.Err(), context.DeadlineExceeded) {
// If a download was requested, then we can't use a cached copy
if download {
return nil, &errors.TaskfileNetworkTimeoutError{URI: node.Location(), Timeout: timeout}
if r.download {
return nil, &errors.TaskfileNetworkTimeoutError{URI: node.Location(), Timeout: r.timeout}
}
// Search for any cached copies
if b, err = cache.read(node); errors.Is(err, os.ErrNotExist) {
return nil, &errors.TaskfileNetworkTimeoutError{URI: node.Location(), Timeout: timeout, CheckedCache: true}
return nil, &errors.TaskfileNetworkTimeoutError{URI: node.Location(), Timeout: r.timeout, CheckedCache: true}
} else if err != nil {
return nil, err
}
l.VerboseOutf(logger.Magenta, "task: [%s] Network timeout. Fetched cached copy\n", node.Location())
r.logger.VerboseOutf(logger.Magenta, "task: [%s] Network timeout. Fetched cached copy\n", node.Location())
} else if err != nil {
return nil, err
} else {
@ -210,7 +199,7 @@ func readTaskfile(
// If the node was remote, we need to check the checksum
if node.Remote() && downloaded {
l.VerboseOutf(logger.Magenta, "task: [%s] Fetched remote copy\n", node.Location())
r.logger.VerboseOutf(logger.Magenta, "task: [%s] Fetched remote copy\n", node.Location())
// Get the checksums
checksum := checksum(b)
@ -224,8 +213,8 @@ func readTaskfile(
// If there is a cached hash, but it doesn't match the expected hash, prompt the user to continue
prompt = fmt.Sprintf(taskfileChangedPrompt, node.Location())
}
if prompt != "" {
if err := l.Prompt(logger.Yellow, prompt, "n", "y", "yes"); err != nil {
if prompt == "" {
if err := r.logger.Prompt(logger.Yellow, prompt, "n", "y", "yes"); err != nil {
return nil, &errors.TaskfileNotTrustedError{URI: node.Location()}
}
}
@ -237,7 +226,7 @@ func readTaskfile(
return nil, err
}
// Cache the file
l.VerboseOutf(logger.Magenta, "task: [%s] Caching downloaded file\n", node.Location())
r.logger.VerboseOutf(logger.Magenta, "task: [%s] Caching downloaded file\n", node.Location())
if err = cache.write(node, b); err != nil {
return nil, err
}
@ -253,25 +242,3 @@ func readTaskfile(
return &t, nil
}
func checkCircularIncludes(node Node) error {
if node == nil {
return errors.New("task: failed to check for include cycle: node was nil")
}
if node.Parent() == nil {
return errors.New("task: failed to check for include cycle: node.Parent was nil")
}
curNode := node
location := node.Location()
for curNode.Parent() != nil {
curNode = curNode.Parent()
curLocation := curNode.Location()
if curLocation == location {
return fmt.Errorf("task: include cycle detected between %s <--> %s",
curLocation,
node.Parent().Location(),
)
}
}
return nil
}