1
0
mirror of https://github.com/go-task/task.git synced 2025-08-10 22:42:19 +02:00

refactor: decouple fingerprinting from executor (#1039)

This commit is contained in:
Pete Davison
2023-03-10 18:27:30 +00:00
committed by GitHub
parent c64f8818be
commit 0838d48ee3
24 changed files with 734 additions and 241 deletions

View File

@@ -0,0 +1,20 @@
package fingerprint
import (
"context"
"github.com/go-task/task/v3/taskfile"
)
// StatusCheckable defines any type that can check if the status of a task is up-to-date.
type StatusCheckable interface {
IsUpToDate(ctx context.Context, t *taskfile.Task) (bool, error)
}
// SourcesCheckable defines any type that can check if the sources of a task are up-to-date.
type SourcesCheckable interface {
IsUpToDate(t *taskfile.Task) (bool, error)
Value(t *taskfile.Task) (interface{}, error)
OnError(t *taskfile.Task) error
Kind() string
}

View File

@@ -0,0 +1,132 @@
// Code generated by MockGen. DO NOT EDIT.
// Source: checker.go
// Package fingerprint is a generated GoMock package.
package fingerprint
import (
context "context"
reflect "reflect"
taskfile "github.com/go-task/task/v3/taskfile"
gomock "github.com/golang/mock/gomock"
)
// MockStatusCheckable is a mock of StatusCheckable interface.
type MockStatusCheckable struct {
ctrl *gomock.Controller
recorder *MockStatusCheckableMockRecorder
}
// MockStatusCheckableMockRecorder is the mock recorder for MockStatusCheckable.
type MockStatusCheckableMockRecorder struct {
mock *MockStatusCheckable
}
// NewMockStatusCheckable creates a new mock instance.
func NewMockStatusCheckable(ctrl *gomock.Controller) *MockStatusCheckable {
mock := &MockStatusCheckable{ctrl: ctrl}
mock.recorder = &MockStatusCheckableMockRecorder{mock}
return mock
}
// EXPECT returns an object that allows the caller to indicate expected use.
func (m *MockStatusCheckable) EXPECT() *MockStatusCheckableMockRecorder {
return m.recorder
}
// IsUpToDate mocks base method.
func (m *MockStatusCheckable) IsUpToDate(ctx context.Context, t *taskfile.Task) (bool, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "IsUpToDate", ctx, t)
ret0, _ := ret[0].(bool)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// IsUpToDate indicates an expected call of IsUpToDate.
func (mr *MockStatusCheckableMockRecorder) IsUpToDate(ctx, t interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsUpToDate", reflect.TypeOf((*MockStatusCheckable)(nil).IsUpToDate), ctx, t)
}
// MockSourcesCheckable is a mock of SourcesCheckable interface.
type MockSourcesCheckable struct {
ctrl *gomock.Controller
recorder *MockSourcesCheckableMockRecorder
}
// MockSourcesCheckableMockRecorder is the mock recorder for MockSourcesCheckable.
type MockSourcesCheckableMockRecorder struct {
mock *MockSourcesCheckable
}
// NewMockSourcesCheckable creates a new mock instance.
func NewMockSourcesCheckable(ctrl *gomock.Controller) *MockSourcesCheckable {
mock := &MockSourcesCheckable{ctrl: ctrl}
mock.recorder = &MockSourcesCheckableMockRecorder{mock}
return mock
}
// EXPECT returns an object that allows the caller to indicate expected use.
func (m *MockSourcesCheckable) EXPECT() *MockSourcesCheckableMockRecorder {
return m.recorder
}
// IsUpToDate mocks base method.
func (m *MockSourcesCheckable) IsUpToDate(t *taskfile.Task) (bool, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "IsUpToDate", t)
ret0, _ := ret[0].(bool)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// IsUpToDate indicates an expected call of IsUpToDate.
func (mr *MockSourcesCheckableMockRecorder) IsUpToDate(t interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsUpToDate", reflect.TypeOf((*MockSourcesCheckable)(nil).IsUpToDate), t)
}
// Kind mocks base method.
func (m *MockSourcesCheckable) Kind() string {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "Kind")
ret0, _ := ret[0].(string)
return ret0
}
// Kind indicates an expected call of Kind.
func (mr *MockSourcesCheckableMockRecorder) Kind() *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Kind", reflect.TypeOf((*MockSourcesCheckable)(nil).Kind))
}
// OnError mocks base method.
func (m *MockSourcesCheckable) OnError(t *taskfile.Task) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "OnError", t)
ret0, _ := ret[0].(error)
return ret0
}
// OnError indicates an expected call of OnError.
func (mr *MockSourcesCheckableMockRecorder) OnError(t interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "OnError", reflect.TypeOf((*MockSourcesCheckable)(nil).OnError), t)
}
// Value mocks base method.
func (m *MockSourcesCheckable) Value(t *taskfile.Task) (interface{}, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "Value", t)
ret0, _ := ret[0].(interface{})
ret1, _ := ret[1].(error)
return ret0, ret1
}
// Value indicates an expected call of Value.
func (mr *MockSourcesCheckableMockRecorder) Value(t interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Value", reflect.TypeOf((*MockSourcesCheckable)(nil).Value), t)
}

View File

@@ -0,0 +1,51 @@
package fingerprint
import (
"os"
"sort"
"github.com/mattn/go-zglob"
"github.com/go-task/task/v3/internal/execext"
"github.com/go-task/task/v3/internal/filepathext"
)
func globs(dir string, globs []string) ([]string, error) {
files := make([]string, 0)
for _, g := range globs {
f, err := Glob(dir, g)
if err != nil {
continue
}
files = append(files, f...)
}
sort.Strings(files)
return files, nil
}
func Glob(dir string, g string) ([]string, error) {
files := make([]string, 0)
g = filepathext.SmartJoin(dir, g)
g, err := execext.Expand(g)
if err != nil {
return nil, err
}
fs, err := zglob.GlobFollowSymlinks(g)
if err != nil {
return nil, err
}
for _, f := range fs {
info, err := os.Stat(f)
if err != nil {
return nil, err
}
if info.IsDir() {
continue
}
files = append(files, f)
}
return files, nil
}

View File

@@ -0,0 +1,16 @@
package fingerprint
import "fmt"
func NewSourcesChecker(method, tempDir string, dry bool) (SourcesCheckable, error) {
switch method {
case "timestamp":
return NewTimestampChecker(tempDir, dry), nil
case "checksum":
return NewChecksumChecker(tempDir, dry), nil
case "none":
return NoneChecker{}, nil
default:
return nil, fmt.Errorf(`task: invalid method "%s"`, method)
}
}

View File

@@ -0,0 +1,121 @@
package fingerprint
import (
"crypto/md5"
"fmt"
"io"
"os"
"path/filepath"
"regexp"
"strings"
"github.com/go-task/task/v3/internal/filepathext"
"github.com/go-task/task/v3/taskfile"
)
// ChecksumChecker validates if a task is up to date by calculating its source
// files checksum
type ChecksumChecker struct {
tempDir string
dry bool
}
func NewChecksumChecker(tempDir string, dry bool) *ChecksumChecker {
return &ChecksumChecker{
tempDir: tempDir,
dry: dry,
}
}
func (checker *ChecksumChecker) IsUpToDate(t *taskfile.Task) (bool, error) {
if len(t.Sources) == 0 {
return false, nil
}
checksumFile := checker.checksumFilePath(t)
data, _ := os.ReadFile(checksumFile)
oldMd5 := strings.TrimSpace(string(data))
sources, err := globs(t.Dir, t.Sources)
if err != nil {
return false, err
}
newMd5, err := checker.checksum(sources...)
if err != nil {
return false, nil
}
if !checker.dry {
_ = os.MkdirAll(filepathext.SmartJoin(checker.tempDir, "checksum"), 0o755)
if err = os.WriteFile(checksumFile, []byte(newMd5+"\n"), 0o644); err != nil {
return false, err
}
}
if len(t.Generates) > 0 {
// For each specified 'generates' field, check whether the files actually exist
for _, g := range t.Generates {
generates, err := Glob(t.Dir, g)
if os.IsNotExist(err) {
return false, nil
}
if err != nil {
return false, err
}
if len(generates) == 0 {
return false, nil
}
}
}
return oldMd5 == newMd5, nil
}
func (checker *ChecksumChecker) Value(t *taskfile.Task) (interface{}, error) {
return checker.checksum()
}
func (checker *ChecksumChecker) OnError(t *taskfile.Task) error {
if len(t.Sources) == 0 {
return nil
}
return os.Remove(checker.checksumFilePath(t))
}
func (*ChecksumChecker) Kind() string {
return "checksum"
}
func (c *ChecksumChecker) checksum(files ...string) (string, error) {
h := md5.New()
for _, f := range files {
// also sum the filename, so checksum changes for renaming a file
if _, err := io.Copy(h, strings.NewReader(filepath.Base(f))); err != nil {
return "", err
}
f, err := os.Open(f)
if err != nil {
return "", err
}
if _, err = io.Copy(h, f); err != nil {
return "", err
}
f.Close()
}
return fmt.Sprintf("%x", h.Sum(nil)), nil
}
func (checker *ChecksumChecker) checksumFilePath(t *taskfile.Task) string {
return filepath.Join(checker.tempDir, "checksum", normalizeFilename(t.Name()))
}
var checksumFilenameRegexp = regexp.MustCompile("[^A-z0-9]")
// replaces invalid characters on filenames with "-"
func normalizeFilename(f string) string {
return checksumFilenameRegexp.ReplaceAllString(f, "-")
}

View File

@@ -0,0 +1,21 @@
package fingerprint
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestNormalizeFilename(t *testing.T) {
tests := []struct {
In, Out string
}{
{"foobarbaz", "foobarbaz"},
{"foo/bar/baz", "foo-bar-baz"},
{"foo@bar/baz", "foo-bar-baz"},
{"foo1bar2baz3", "foo1bar2baz3"},
}
for _, test := range tests {
assert.Equal(t, test.Out, normalizeFilename(test.In))
}
}

View File

@@ -0,0 +1,23 @@
package fingerprint
import "github.com/go-task/task/v3/taskfile"
// NoneChecker is a no-op Checker.
// It will always report that the task is not up-to-date.
type NoneChecker struct{}
func (NoneChecker) IsUpToDate(t *taskfile.Task) (bool, error) {
return false, nil
}
func (NoneChecker) Value(t *taskfile.Task) (interface{}, error) {
return "", nil
}
func (NoneChecker) OnError(t *taskfile.Task) error {
return nil
}
func (NoneChecker) Kind() string {
return "none"
}

View File

@@ -0,0 +1,151 @@
package fingerprint
import (
"os"
"path/filepath"
"time"
"github.com/go-task/task/v3/taskfile"
)
// TimestampChecker checks if any source change compared with the generated files,
// using file modifications timestamps.
type TimestampChecker struct {
tempDir string
dry bool
}
func NewTimestampChecker(tempDir string, dry bool) *TimestampChecker {
return &TimestampChecker{
tempDir: tempDir,
dry: dry,
}
}
// IsUpToDate implements the Checker interface
func (checker *TimestampChecker) IsUpToDate(t *taskfile.Task) (bool, error) {
if len(t.Sources) == 0 {
return false, nil
}
sources, err := globs(t.Dir, t.Sources)
if err != nil {
return false, nil
}
generates, err := globs(t.Dir, t.Generates)
if err != nil {
return false, nil
}
timestampFile := checker.timestampFilePath(t)
// If the file exists, add the file path to the generates.
// If the generate file is old, the task will be executed.
_, err = os.Stat(timestampFile)
if err == nil {
generates = append(generates, timestampFile)
} else {
// Create the timestamp file for the next execution when the file does not exist.
if !checker.dry {
if err := os.MkdirAll(filepath.Dir(timestampFile), 0o755); err != nil {
return false, err
}
f, err := os.Create(timestampFile)
if err != nil {
return false, err
}
f.Close()
}
}
taskTime := time.Now()
// Compare the time of the generates and sources. If the generates are old, the task will be executed.
// Get the max time of the generates.
generateMaxTime, err := getMaxTime(generates...)
if err != nil || generateMaxTime.IsZero() {
return false, nil
}
// Check if any of the source files is newer than the max time of the generates.
shouldUpdate, err := anyFileNewerThan(sources, generateMaxTime)
if err != nil {
return false, nil
}
// Modify the metadata of the file to the the current time.
if !checker.dry {
if err := os.Chtimes(timestampFile, taskTime, taskTime); err != nil {
return false, err
}
}
return !shouldUpdate, nil
}
func (checker *TimestampChecker) Kind() string {
return "timestamp"
}
// Value implements the Checker Interface
func (checker *TimestampChecker) Value(t *taskfile.Task) (interface{}, error) {
sources, err := globs(t.Dir, t.Sources)
if err != nil {
return time.Now(), err
}
sourcesMaxTime, err := getMaxTime(sources...)
if err != nil {
return time.Now(), err
}
if sourcesMaxTime.IsZero() {
return time.Unix(0, 0), nil
}
return sourcesMaxTime, nil
}
func getMaxTime(files ...string) (time.Time, error) {
var t time.Time
for _, f := range files {
info, err := os.Stat(f)
if err != nil {
return time.Time{}, err
}
t = maxTime(t, info.ModTime())
}
return t, nil
}
func maxTime(a, b time.Time) time.Time {
if a.After(b) {
return a
}
return b
}
// If the modification time of any of the files is newer than the the given time, returns true.
// This function is lazy, as it stops when it finds a file newer than the given time.
func anyFileNewerThan(files []string, givenTime time.Time) (bool, error) {
for _, f := range files {
info, err := os.Stat(f)
if err != nil {
return false, err
}
if info.ModTime().After(givenTime) {
return true, nil
}
}
return false, nil
}
// OnError implements the Checker interface
func (*TimestampChecker) OnError(t *taskfile.Task) error {
return nil
}
func (checker *TimestampChecker) timestampFilePath(t *taskfile.Task) string {
return filepath.Join(checker.tempDir, "timestamp", normalizeFilename(t.Task))
}

View File

@@ -0,0 +1,36 @@
package fingerprint
import (
"context"
"github.com/go-task/task/v3/internal/env"
"github.com/go-task/task/v3/internal/execext"
"github.com/go-task/task/v3/internal/logger"
"github.com/go-task/task/v3/taskfile"
)
type StatusChecker struct {
logger *logger.Logger
}
func NewStatusChecker(logger *logger.Logger) StatusCheckable {
return &StatusChecker{
logger: logger,
}
}
func (checker *StatusChecker) IsUpToDate(ctx context.Context, t *taskfile.Task) (bool, error) {
for _, s := range t.Status {
err := execext.RunCommand(ctx, &execext.RunCommandOptions{
Command: s,
Dir: t.Dir,
Env: env.Get(t),
})
if err != nil {
checker.logger.VerboseOutf(logger.Yellow, "task: status command %s exited non-zero: %s", s, err)
return false, nil
}
checker.logger.VerboseOutf(logger.Yellow, "task: status command %s exited zero", s)
}
return true, nil
}

View File

@@ -0,0 +1,132 @@
package fingerprint
import (
"context"
"github.com/go-task/task/v3/internal/logger"
"github.com/go-task/task/v3/taskfile"
)
type (
CheckerOption func(*CheckerConfig)
CheckerConfig struct {
method string
dry bool
tempDir string
logger *logger.Logger
statusChecker StatusCheckable
sourcesChecker SourcesCheckable
}
)
func WithMethod(method string) CheckerOption {
return func(config *CheckerConfig) {
config.method = method
}
}
func WithDry(dry bool) CheckerOption {
return func(config *CheckerConfig) {
config.dry = dry
}
}
func WithTempDir(tempDir string) CheckerOption {
return func(config *CheckerConfig) {
config.tempDir = tempDir
}
}
func WithLogger(logger *logger.Logger) CheckerOption {
return func(config *CheckerConfig) {
config.logger = logger
}
}
func WithStatusChecker(checker StatusCheckable) CheckerOption {
return func(config *CheckerConfig) {
config.statusChecker = checker
}
}
func WithSourcesChecker(checker SourcesCheckable) CheckerOption {
return func(config *CheckerConfig) {
config.sourcesChecker = checker
}
}
func IsTaskUpToDate(
ctx context.Context,
t *taskfile.Task,
opts ...CheckerOption,
) (bool, error) {
var statusUpToDate bool
var sourcesUpToDate bool
var err error
// Default config
config := &CheckerConfig{
method: "none",
tempDir: "",
dry: false,
logger: nil,
statusChecker: nil,
sourcesChecker: nil,
}
// Apply functional options
for _, opt := range opts {
opt(config)
}
// If no status checker was given, set up the default one
if config.statusChecker == nil {
config.statusChecker = NewStatusChecker(config.logger)
}
// If no sources checker was given, set up the default one
if config.sourcesChecker == nil {
config.sourcesChecker, err = NewSourcesChecker(config.method, config.tempDir, config.dry)
if err != nil {
return false, err
}
}
statusIsSet := len(t.Status) != 0
sourcesIsSet := len(t.Sources) != 0
// If status is set, check if it is up-to-date
if statusIsSet {
statusUpToDate, err = config.statusChecker.IsUpToDate(ctx, t)
if err != nil {
return false, err
}
}
// If sources is set, check if they are up-to-date
if sourcesIsSet {
sourcesUpToDate, err = config.sourcesChecker.IsUpToDate(t)
if err != nil {
return false, err
}
}
// If both status and sources are set, the task is up-to-date if both are up-to-date
if statusIsSet && sourcesIsSet {
return statusUpToDate && sourcesUpToDate, nil
}
// If only status is set, the task is up-to-date if the status is up-to-date
if statusIsSet {
return statusUpToDate, nil
}
// If only sources is set, the task is up-to-date if the sources are up-to-date
if sourcesIsSet {
return sourcesUpToDate, nil
}
// If no status or sources are set, the task should always run
// i.e. it is never considered "up-to-date"
return false, nil
}

View File

@@ -0,0 +1,174 @@
package fingerprint
import (
"context"
"testing"
"github.com/golang/mock/gomock"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/go-task/task/v3/taskfile"
)
// TruthTable
//
// | Status up-to-date | Sources up-to-date | Task is up-to-date |
// | ----------------- | ------------------ | ------------------ |
// | not set | not set | false |
// | not set | true | true |
// | not set | false | false |
// | true | not set | true |
// | true | true | true |
// | true | false | false |
// | false | not set | false |
// | false | true | false |
// | false | false | false |
func TestIsTaskUpToDate(t *testing.T) {
tests := []struct {
name string
task *taskfile.Task
setupMockStatusChecker func(m *MockStatusCheckable)
setupMockSourcesChecker func(m *MockSourcesCheckable)
expected bool
}{
{
name: "expect FALSE when no status or sources are defined",
task: &taskfile.Task{
Status: nil,
Sources: nil,
},
setupMockStatusChecker: nil,
setupMockSourcesChecker: nil,
expected: false,
},
{
name: "expect TRUE when no status is defined and sources are up-to-date",
task: &taskfile.Task{
Status: nil,
Sources: []string{"sources"},
},
setupMockStatusChecker: nil,
setupMockSourcesChecker: func(m *MockSourcesCheckable) {
m.EXPECT().IsUpToDate(gomock.Any()).Return(true, nil)
},
expected: true,
},
{
name: "expect FALSE when no status is defined and sources are NOT up-to-date",
task: &taskfile.Task{
Status: nil,
Sources: []string{"sources"},
},
setupMockStatusChecker: nil,
setupMockSourcesChecker: func(m *MockSourcesCheckable) {
m.EXPECT().IsUpToDate(gomock.Any()).Return(false, nil)
},
expected: false,
},
{
name: "expect TRUE when status is up-to-date and sources are not defined",
task: &taskfile.Task{
Status: []string{"status"},
Sources: nil,
},
setupMockStatusChecker: func(m *MockStatusCheckable) {
m.EXPECT().IsUpToDate(gomock.Any(), gomock.Any()).Return(true, nil)
},
setupMockSourcesChecker: nil,
expected: true,
},
{
name: "expect TRUE when status and sources are up-to-date",
task: &taskfile.Task{
Status: []string{"status"},
Sources: []string{"sources"},
},
setupMockStatusChecker: func(m *MockStatusCheckable) {
m.EXPECT().IsUpToDate(gomock.Any(), gomock.Any()).Return(true, nil)
},
setupMockSourcesChecker: func(m *MockSourcesCheckable) {
m.EXPECT().IsUpToDate(gomock.Any()).Return(true, nil)
},
expected: true,
},
{
name: "expect FALSE when status is up-to-date, but sources are NOT up-to-date",
task: &taskfile.Task{
Status: []string{"status"},
Sources: []string{"sources"},
},
setupMockStatusChecker: func(m *MockStatusCheckable) {
m.EXPECT().IsUpToDate(gomock.Any(), gomock.Any()).Return(true, nil)
},
setupMockSourcesChecker: func(m *MockSourcesCheckable) {
m.EXPECT().IsUpToDate(gomock.Any()).Return(false, nil)
},
expected: false,
},
{
name: "expect FALSE when status is NOT up-to-date and sources are not defined",
task: &taskfile.Task{
Status: []string{"status"},
Sources: nil,
},
setupMockStatusChecker: func(m *MockStatusCheckable) {
m.EXPECT().IsUpToDate(gomock.Any(), gomock.Any()).Return(false, nil)
},
setupMockSourcesChecker: nil,
expected: false,
},
{
name: "expect FALSE when status is NOT up-to-date, but sources are up-to-date",
task: &taskfile.Task{
Status: []string{"status"},
Sources: []string{"sources"},
},
setupMockStatusChecker: func(m *MockStatusCheckable) {
m.EXPECT().IsUpToDate(gomock.Any(), gomock.Any()).Return(false, nil)
},
setupMockSourcesChecker: func(m *MockSourcesCheckable) {
m.EXPECT().IsUpToDate(gomock.Any()).Return(true, nil)
},
expected: false,
},
{
name: "expect FALSE when status and sources are NOT up-to-date",
task: &taskfile.Task{
Status: []string{"status"},
Sources: []string{"sources"},
},
setupMockStatusChecker: func(m *MockStatusCheckable) {
m.EXPECT().IsUpToDate(gomock.Any(), gomock.Any()).Return(false, nil)
},
setupMockSourcesChecker: func(m *MockSourcesCheckable) {
m.EXPECT().IsUpToDate(gomock.Any()).Return(false, nil)
},
expected: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
ctrl := gomock.NewController(t)
mockStatusChecker := NewMockStatusCheckable(ctrl)
if tt.setupMockStatusChecker != nil {
tt.setupMockStatusChecker(mockStatusChecker)
}
mockSourcesChecker := NewMockSourcesCheckable(ctrl)
if tt.setupMockSourcesChecker != nil {
tt.setupMockSourcesChecker(mockSourcesChecker)
}
result, err := IsTaskUpToDate(
context.Background(),
tt.task,
WithStatusChecker(mockStatusChecker),
WithSourcesChecker(mockSourcesChecker),
)
require.NoError(t, err)
assert.Equal(t, tt.expected, result)
})
}
}