Task dependencies to survive restarts

model/queue.go

@@ -23,9 +23,11 @@ import (
 
 // Task defines scheduled pipeline Task.
 type Task struct {
-	ID     string            `meddler:"task_id"`
+	ID           string            `meddler:"task_id"`
-	Data   []byte            `meddler:"task_data"`
+	Data         []byte            `meddler:"task_data"`
-	Labels map[string]string `meddler:"task_labels,json"`
+	Labels       map[string]string `meddler:"task_labels,json"`
+	Dependencies []string          `meddler:"task_dependencies,json"`
+	RunOn        []string          `meddler:"task_run_on,json"`
 }
 
 // TaskStore defines storage for scheduled Tasks.
@@ -39,13 +41,18 @@ type TaskStore interface {
 // ensures the task Queue can be restored when the system starts.
 func WithTaskStore(q queue.Queue, s TaskStore) queue.Queue {
 	tasks, _ := s.TaskList()
+	toEnqueue := []*queue.Task{}
 	for _, task := range tasks {
-		q.Push(context.Background(), &queue.Task{
+		toEnqueue = append(toEnqueue, &queue.Task{
-			ID:     task.ID,
+			ID:           task.ID,
-			Data:   task.Data,
+			Data:         task.Data,
-			Labels: task.Labels,
+			Labels:       task.Labels,
+			Dependencies: task.Dependencies,
+			RunOn:        task.RunOn,
+			DepStatus:    make(map[string]bool),
 		})
 	}
+	q.PushAtOnce(context.Background(), toEnqueue)
 	return &persistentQueue{q, s}
 }
 
@@ -57,9 +64,11 @@ type persistentQueue struct {
 // Push pushes a task to the tail of this queue.
 func (q *persistentQueue) Push(c context.Context, task *queue.Task) error {
 	q.store.TaskInsert(&Task{
-		ID:     task.ID,
+		ID:           task.ID,
-		Data:   task.Data,
+		Data:         task.Data,
-		Labels: task.Labels,
+		Labels:       task.Labels,
+		Dependencies: task.Dependencies,
+		RunOn:        task.RunOn,
 	})
 	err := q.Queue.Push(c, task)
 	if err != nil {
@@ -72,9 +81,11 @@ func (q *persistentQueue) Push(c context.Context, task *queue.Task) error {
 func (q *persistentQueue) PushAtOnce(c context.Context, tasks []*queue.Task) error {
 	for _, task := range tasks {
 		q.store.TaskInsert(&Task{
-			ID:     task.ID,
+			ID:           task.ID,
-			Data:   task.Data,
+			Data:         task.Data,
-			Labels: task.Labels,
+			Labels:       task.Labels,
+			Dependencies: task.Dependencies,
+			RunOn:        task.RunOn,
 		})
 	}
 	err := q.Queue.PushAtOnce(c, tasks)

store/datastore/ddl/mysql/ddl_gen.go

@@ -172,6 +172,14 @@ var migrations = []struct {
 		name: "populate-build-config",
 		stmt: populateBuildConfig,
 	},
+	{
+		name: "alter-table-add-task-dependencies",
+		stmt: alterTableAddTaskDependencies,
+	},
+	{
+		name: "alter-table-add-task-run-on",
+		stmt: alterTableAddTaskRunOn,
+	},
 }
 
 // Migrate performs the database migration. If the migration fails
@@ -673,3 +681,15 @@ var populateBuildConfig = `
 INSERT INTO build_config (config_id, build_id)
 SELECT build_config_id, build_id FROM builds
 `
+
+//
+// 022_add_task_columns.sql
+//
+
+var alterTableAddTaskDependencies = `
+ALTER TABLE tasks ADD COLUMN task_dependencies MEDIUMBLOB
+`
+
+var alterTableAddTaskRunOn = `
+ALTER TABLE tasks ADD COLUMN task_run_on MEDIUMBLOB
+`

store/datastore/ddl/mysql/files/022_add_task_columns.sql

@@ -0,0 +1,6 @@
+-- name: alter-table-add-task-dependencies
+ALTER TABLE tasks ADD COLUMN task_dependencies MEDIUMBLOB
+
+-- name: alter-table-add-task-run-on
+
+ALTER TABLE tasks ADD COLUMN task_run_on MEDIUMBLOB

store/datastore/ddl/postgres/ddl_gen.go

@@ -172,6 +172,14 @@ var migrations = []struct {
 		name: "populate-build-config",
 		stmt: populateBuildConfig,
 	},
+	{
+		name: "alter-table-add-task-dependencies",
+		stmt: alterTableAddTaskDependencies,
+	},
+	{
+		name: "alter-table-add-task-run-on",
+		stmt: alterTableAddTaskRunOn,
+	},
 }
 
 // Migrate performs the database migration. If the migration fails
@@ -675,3 +683,15 @@ var populateBuildConfig = `
 INSERT INTO build_config (config_id, build_id)
 SELECT build_config_id, build_id FROM builds
 `
+
+//
+// 022_add_task_columns.sql
+//
+
+var alterTableAddTaskDependencies = `
+ALTER TABLE tasks ADD COLUMN task_dependencies BYTEA
+`
+
+var alterTableAddTaskRunOn = `
+ALTER TABLE tasks ADD COLUMN task_run_on BYTEA
+`

store/datastore/ddl/postgres/files/022_add_task_columns.sql

@@ -0,0 +1,6 @@
+-- name: alter-table-add-task-dependencies
+ALTER TABLE tasks ADD COLUMN task_dependencies BYTEA
+
+-- name: alter-table-add-task-run-on
+
+ALTER TABLE tasks ADD COLUMN task_run_on BYTEA

store/datastore/ddl/sqlite/ddl_gen.go

@@ -176,6 +176,14 @@ var migrations = []struct {
 		name: "populate-build-config",
 		stmt: populateBuildConfig,
 	},
+	{
+		name: "alter-table-add-task-dependencies",
+		stmt: alterTableAddTaskDependencies,
+	},
+	{
+		name: "alter-table-add-task-run-on",
+		stmt: alterTableAddTaskRunOn,
+	},
 }
 
 // Migrate performs the database migration. If the migration fails
@@ -674,3 +682,15 @@ var populateBuildConfig = `
 INSERT INTO build_config (config_id, build_id)
 SELECT build_config_id, build_id FROM builds
 `
+
+//
+// 022_add_task_columns.sql
+//
+
+var alterTableAddTaskDependencies = `
+ALTER TABLE tasks ADD COLUMN task_dependencies BLOB
+`
+
+var alterTableAddTaskRunOn = `
+ALTER TABLE tasks ADD COLUMN task_run_on BLOB
+`

store/datastore/ddl/sqlite/files/022_add_task_columns.sql

@@ -0,0 +1,6 @@
+-- name: alter-table-add-task-dependencies
+ALTER TABLE tasks ADD COLUMN task_dependencies BLOB
+
+-- name: alter-table-add-task-run-on
+
+ALTER TABLE tasks ADD COLUMN task_run_on BLOB

store/datastore/sql/mysql/files/task.sql

@@ -4,6 +4,8 @@ SELECT
  task_id
 ,task_data
 ,task_labels
+,task_dependencies
+,task_run_on
 FROM tasks
 
 -- name: task-delete

store/datastore/sql/mysql/sql_gen.go

@@ -554,6 +554,8 @@ SELECT
  task_id
 ,task_data
 ,task_labels
+,task_dependencies
+,task_run_on
 FROM tasks
 `
 

store/datastore/sql/postgres/files/tasks.sql

@@ -4,6 +4,8 @@ SELECT
  task_id
 ,task_data
 ,task_labels
+,task_dependencies
+,task_run_on
 FROM tasks
 
 -- name: task-delete

store/datastore/sql/postgres/sql_gen.go

@@ -559,6 +559,8 @@ SELECT
  task_id
 ,task_data
 ,task_labels
+,task_dependencies
+,task_run_on
 FROM tasks
 `
 

store/datastore/sql/sqlite/files/task.sql

@@ -4,6 +4,8 @@ SELECT
  task_id
 ,task_data
 ,task_labels
+,task_dependencies
+,task_run_on
 FROM tasks
 
 -- name: task-delete

store/datastore/sql/sqlite/sql_gen.go

@@ -554,6 +554,8 @@ SELECT
  task_id
 ,task_data
 ,task_labels
+,task_dependencies
+,task_run_on
 FROM tasks
 `