persist and compare yaml for gating

drone/server/server.go

@@ -391,6 +391,7 @@ func setupEvilGlobals(c *cli.Context, v store.Store) {
 
 	// storage
 	droneserver.Config.Storage.Files = v
+	droneserver.Config.Storage.Config = v
 
 	// services
 	droneserver.Config.Services.Queue = setupQueue(c, v)

model/build.go

@@ -4,6 +4,7 @@ package model
 type Build struct {
 	ID        int64  `json:"id"            meddler:"build_id,pk"`
 	RepoID    int64  `json:"-"             meddler:"build_repo_id"`
+	ConfigID  int64  `json:"-"             meddler:"build_config_id"`
 	Number    int    `json:"number"        meddler:"build_number"`
 	Parent    int    `json:"parent"        meddler:"build_parent"`
 	Event     string `json:"event"         meddler:"build_event"`

model/config.go

@@ -1,24 +1,18 @@
 package model
 
-// Config defines system configuration parameters.
+// ConfigStore persists pipeline configuration to storage.
+type ConfigStore interface {
+	ConfigLoad(int64) (*Config, error)
+	ConfigFind(*Repo, string) (*Config, error)
+	ConfigUpdate(*Config) error
+	ConfigInsert(*Config) error
+}
+
+// Config represents a pipeline configuration.
 type Config struct {
-	Open   bool            // Enables open registration
-	Secret string          // Secret token used to authenticate agents
-	Admins map[string]bool // Administrative users
-	Orgs   map[string]bool // Organization whitelist
-}
-
-// IsAdmin returns true if the user is a member of the administrator list.
-func (c *Config) IsAdmin(user *User) bool {
-	return c.Admins[user.Login]
-}
-
-// IsMember returns true if the user is a member of the whitelisted teams.
-func (c *Config) IsMember(teams []*Team) bool {
-	for _, team := range teams {
-		if c.Orgs[team.Login] {
-			return true
-		}
-	}
-	return false
+	ID       int64  `json:"-"        meddler:"config_id,pk"`
+	RepoID   int64  `json:"-"        meddler:"config_repo_id"`
+	Data     string `json:"data"     meddler:"config_data"`
+	Hash     string `json:"hash"     meddler:"config_hash"`
+	Approved bool   `json:"approved" meddler:"config_approved"`
 }

model/repo.go

@@ -26,6 +26,7 @@ type Repo struct {
 	IsTrusted   bool   `json:"trusted"                  meddler:"repo_trusted"`
 	IsStarred   bool   `json:"starred,omitempty"        meddler:"-"`
 	IsGated     bool   `json:"gated"                    meddler:"repo_gated"`
+	IsGatedConf bool   `json:"gated_conf"               meddler:"repo_gated_conf"`
 	AllowPull   bool   `json:"allow_pr"                 meddler:"repo_allow_pr"`
 	AllowPush   bool   `json:"allow_push"               meddler:"repo_allow_push"`
 	AllowDeploy bool   `json:"allow_deploys"            meddler:"repo_allow_deploys"`

model/settings.go

@@ -0,0 +1,24 @@
+package model
+
+// Settings defines system configuration parameters.
+type Settings struct {
+	Open   bool            // Enables open registration
+	Secret string          // Secret token used to authenticate agents
+	Admins map[string]bool // Administrative users
+	Orgs   map[string]bool // Organization whitelist
+}
+
+// IsAdmin returns true if the user is a member of the administrator list.
+func (c *Settings) IsAdmin(user *User) bool {
+	return c.Admins[user.Login]
+}
+
+// IsMember returns true if the user is a member of the whitelisted teams.
+func (c *Settings) IsMember(teams []*Team) bool {
+	for _, team := range teams {
+		if c.Orgs[team.Login] {
+			return true
+		}
+	}
+	return false
+}

router/middleware/config.go

@@ -19,8 +19,8 @@ func Config(cli *cli.Context) gin.HandlerFunc {
 }
 
 // helper function to create the configuration from the CLI context.
-func setupConfig(c *cli.Context) *model.Config {
-	return &model.Config{
+func setupConfig(c *cli.Context) *model.Settings {
+	return &model.Settings{
 		Open:   c.Bool("open"),
 		Secret: c.String("agent-secret"),
 		Admins: sliceToMap2(c.StringSlice("admin")),

router/middleware/session/user.go

@@ -45,7 +45,7 @@ func SetUser() gin.HandlerFunc {
 		})
 		if err == nil {
 			confv := c.MustGet("config")
-			if conf, ok := confv.(*model.Config); ok {
+			if conf, ok := confv.(*model.Settings); ok {
 				user.Admin = conf.IsAdmin(user)
 			}
 			c.Set("user", user)

server/build.go

@@ -174,12 +174,16 @@ func PostApproval(c *gin.Context) {
 	//
 
 	// fetch the build file from the database
-	raw, err := remote_.File(user, repo, build, repo.Config)
+	conf, err := Config.Storage.Config.ConfigLoad(build.ConfigID)
 	if err != nil {
 		logrus.Errorf("failure to get build config for %s. %s", repo.FullName, err)
 		c.AbortWithError(404, err)
 		return
 	}
+	if !conf.Approved {
+		conf.Approved = true
+		Config.Storage.Config.ConfigUpdate(conf)
+	}
 
 	netrc, err := remote_.Netrc(user, repo)
 	if err != nil {
@@ -222,7 +226,7 @@ func PostApproval(c *gin.Context) {
 		Secs:  secs,
 		Regs:  regs,
 		Link:  httputil.GetURL(c.Request),
-		Yaml:  string(raw),
+		Yaml:  conf.Data,
 	}
 	items, err := b.Build()
 	if err != nil {
@@ -394,12 +398,16 @@ func PostBuild(c *gin.Context) {
 	}
 
 	// fetch the .drone.yml file from the database
-	raw, err := remote_.File(user, repo, build, repo.Config)
+	conf, err := Config.Storage.Config.ConfigLoad(build.ConfigID)
 	if err != nil {
 		logrus.Errorf("failure to get build config for %s. %s", repo.FullName, err)
 		c.AbortWithError(404, err)
 		return
 	}
+	if !conf.Approved {
+		conf.Approved = true
+		Config.Storage.Config.ConfigUpdate(conf)
+	}
 
 	netrc, err := remote_.Netrc(user, repo)
 	if err != nil {
@@ -493,7 +501,7 @@ func PostBuild(c *gin.Context) {
 		Secs:  secs,
 		Regs:  regs,
 		Link:  httputil.GetURL(c.Request),
-		Yaml:  string(raw),
+		Yaml:  conf.Data,
 	}
 	// TODO inject environment varibles !!!!!! buildParams
 	items, err := b.Build()

server/hook.go

@@ -2,6 +2,7 @@ package server
 
 import (
 	"context"
+	"crypto/sha256"
 	"encoding/json"
 	"fmt"
 	"regexp"
@@ -131,12 +132,38 @@ func PostHook(c *gin.Context) {
 	}
 
 	// fetch the build file from the database
-	raw, err := remote_.File(user, repo, build, repo.Config)
+	confb, err := remote_.File(user, repo, build, repo.Config)
 	if err != nil {
 		logrus.Errorf("failure to get build config for %s. %s", repo.FullName, err)
 		c.AbortWithError(404, err)
 		return
 	}
+	sha := shasum(confb)
+	conf, err := Config.Storage.Config.ConfigFind(repo, sha)
+	if err != nil {
+		conf = &model.Config{
+			RepoID:   repo.ID,
+			Data:     string(confb),
+			Hash:     sha,
+			Approved: false,
+		}
+		if user.Login == repo.Owner || build.Event != model.EventPull {
+			conf.Approved = true
+		}
+		err = Config.Storage.Config.ConfigInsert(conf)
+		if err != nil {
+			logrus.Errorf("failure to persist config for %s. %s", repo.FullName, err)
+			c.AbortWithError(500, err)
+			return
+		}
+	}
+	if !conf.Approved {
+		if user.Login == repo.Owner || build.Event != model.EventPull || !repo.IsGatedConf {
+			conf.Approved = true
+			Config.Storage.Config.ConfigUpdate(conf)
+		}
+	}
+	build.ConfigID = conf.ID
 
 	netrc, err := remote_.Netrc(user, repo)
 	if err != nil {
@@ -145,7 +172,7 @@ func PostHook(c *gin.Context) {
 	}
 
 	// verify the branches can be built vs skipped
-	branches, err := yaml.ParseBytes(raw)
+	branches, err := yaml.ParseString(conf.Data)
 	if err == nil {
 		if !branches.Branches.Match(build.Branch) && build.Event != model.EventTag && build.Event != model.EventDeploy {
 			c.String(200, "Branch does not match restrictions defined in yaml")
@@ -168,7 +195,7 @@ func PostHook(c *gin.Context) {
 	build.Verified = true
 	build.Status = model.StatusPending
 
-	if repo.IsGated {
+	if repo.IsGated || repo.IsGatedConf {
 		allowed, _ := Config.Services.Senders.SenderAllowed(user, repo, build)
 		if !allowed {
 			build.Status = model.StatusBlocked
@@ -212,7 +239,7 @@ func PostHook(c *gin.Context) {
 		Secs:  secs,
 		Regs:  regs,
 		Link:  httputil.GetURL(c.Request),
-		Yaml:  string(raw),
+		Yaml:  conf.Data,
 	}
 	items, err := b.Build()
 	if err != nil {
@@ -442,7 +469,7 @@ func (b *builder) Build() ([]*buildItem, error) {
 			linter.WithTrusted(b.Repo.IsTrusted),
 		).Lint(parsed)
 		if lerr != nil {
-			return nil, err
+			return nil, lerr
 		}
 
 		var registries []compiler.Registry
@@ -511,3 +538,8 @@ func (b *builder) Build() ([]*buildItem, error) {
 
 	return items, nil
 }
+
+func shasum(raw []byte) string {
+	sum := sha256.Sum256(raw)
+	return fmt.Sprintf("%x", sum)
+}

server/login.go

@@ -161,7 +161,7 @@ type tokenPayload struct {
 }
 
 // ToConfig returns the config from the Context
-func ToConfig(c *gin.Context) *model.Config {
+func ToConfig(c *gin.Context) *model.Settings {
 	v := c.MustGet("config")
-	return v.(*model.Config)
+	return v.(*model.Settings)
 }

server/rpc.go

@@ -41,8 +41,9 @@ var Config = struct {
 		// Repos  model.RepoStore
 		// Builds model.BuildStore
 		// Logs   model.LogStore
-		Files model.FileStore
-		Procs model.ProcStore
+		Config model.ConfigStore
+		Files  model.FileStore
+		Procs  model.ProcStore
 		// Registries model.RegistryStore
 		// Secrets model.SecretStore
 	}

store/datastore/config.go

@@ -0,0 +1,29 @@
+package datastore
+
+import (
+	"github.com/drone/drone/model"
+	"github.com/drone/drone/store/datastore/sql"
+	"github.com/russross/meddler"
+)
+
+func (db *datastore) ConfigLoad(id int64) (*model.Config, error) {
+	stmt := sql.Lookup(db.driver, "config-find-repo-id")
+	conf := new(model.Config)
+	err := meddler.QueryRow(db, conf, stmt, id)
+	return conf, err
+}
+
+func (db *datastore) ConfigFind(repo *model.Repo, hash string) (*model.Config, error) {
+	stmt := sql.Lookup(db.driver, "config-find-repo-hash")
+	conf := new(model.Config)
+	err := meddler.QueryRow(db, conf, stmt, repo.ID, hash)
+	return conf, err
+}
+
+func (db *datastore) ConfigUpdate(config *model.Config) error {
+	return meddler.Update(db, "config", config)
+}
+
+func (db *datastore) ConfigInsert(config *model.Config) error {
+	return meddler.Insert(db, "config", config)
+}

store/datastore/config_test.go

@@ -0,0 +1,106 @@
+package datastore
+
+import (
+	"testing"
+
+	"github.com/drone/drone/model"
+)
+
+func TestConfig(t *testing.T) {
+	s := newTest()
+	defer func() {
+		s.Exec("delete from config")
+		s.Close()
+	}()
+
+	var (
+		data = "pipeline: [ { image: golang, commands: [ go build, go test ] } ]"
+		hash = "8d8647c9aa90d893bfb79dddbe901f03e258588121e5202632f8ae5738590b26"
+	)
+
+	if err := s.ConfigInsert(
+		&model.Config{
+			RepoID:   2,
+			Data:     data,
+			Hash:     hash,
+			Approved: false,
+		},
+	); err != nil {
+		t.Errorf("Unexpected error: insert config: %s", err)
+		return
+	}
+
+	config, err := s.ConfigFind(&model.Repo{ID: 2}, hash)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	if got, want := config.ID, int64(1); got != want {
+		t.Errorf("Want config id %d, got %d", want, got)
+	}
+	if got, want := config.RepoID, int64(2); got != want {
+		t.Errorf("Want config repo id %d, got %d", want, got)
+	}
+	if got, want := config.Data, data; got != want {
+		t.Errorf("Want config data %s, got %s", want, got)
+	}
+	if got, want := config.Hash, hash; got != want {
+		t.Errorf("Want config hash %s, got %s", want, got)
+	}
+	if got, want := config.Approved, false; got != want {
+		t.Errorf("Want config approved %v, got %v", want, got)
+	}
+
+	config.Approved = true
+	err = s.ConfigUpdate(config)
+	if err != nil {
+		t.Errorf("Want config updated, got error %q", err)
+		return
+	}
+
+	updated, err := s.ConfigFind(&model.Repo{ID: 2}, hash)
+	if err != nil {
+		t.Errorf("Want config find, got error %q", err)
+		return
+	}
+	if got, want := updated.Approved, true; got != want {
+		t.Errorf("Want config approved updated %v, got %v", want, got)
+	}
+}
+
+//
+// func TestConfigIndexes(t *testing.T) {
+// 	s := newTest()
+// 	defer func() {
+// 		s.Exec("delete from config")
+// 		s.Close()
+// 	}()
+//
+// 	if err := s.FileCreate(
+// 		&model.File{
+// 			BuildID: 1,
+// 			ProcID:  1,
+// 			Name:    "hello.txt",
+// 			Size:    11,
+// 			Mime:    "text/plain",
+// 		},
+// 		bytes.NewBufferString("hello world"),
+// 	); err != nil {
+// 		t.Errorf("Unexpected error: insert file: %s", err)
+// 		return
+// 	}
+//
+// 	// fail due to duplicate file name
+// 	if err := s.FileCreate(
+// 		&model.File{
+// 			BuildID: 1,
+// 			ProcID:  1,
+// 			Name:    "hello.txt",
+// 			Mime:    "text/plain",
+// 			Size:    11,
+// 		},
+// 		bytes.NewBufferString("hello world"),
+// 	); err == nil {
+// 		t.Errorf("Unexpected error: dupliate pid")
+// 	}
+// }

store/datastore/ddl/mysql/16.sql

@@ -0,0 +1,21 @@
+-- +migrate Up
+
+CREATE TABLE config (
+ config_id       INTEGER PRIMARY KEY AUTO_INCREMENT
+,config_repo_id  INTEGER
+,config_hash     VARCHAR(250)
+,config_data     MEDIUMBLOB
+,config_approved BOOLEAN
+
+,UNIQUE(config_hash, config_repo_id)
+);
+
+ALTER TABLE builds ADD COLUMN build_config_id INTEGER;
+UPDATE builds set build_config_id = 0;
+
+ALTER TABLE repos ADD COLUMN repo_gated_conf BOOLEAN;
+UPDATE repos SET repo_gated_conf = 0;
+
+-- +migrate Down
+
+DROP TABLE config;

store/datastore/ddl/postgres/16.sql

@@ -0,0 +1,21 @@
+-- +migrate Up
+
+CREATE TABLE config (
+ config_id       SERIAL PRIMARY KEY
+,config_repo_id  INTEGER
+,config_hash     VARCHAR(250)
+,config_data     BYTEA
+,config_approved BOOLEAN
+
+,UNIQUE(config_hash, config_repo_id)
+);
+
+ALTER TABLE builds ADD COLUMN build_config_id INTEGER;
+UPDATE builds set build_config_id = 0;
+
+ALTER TABLE repos ADD COLUMN repo_gated_conf BOOLEAN;
+UPDATE repos SET repo_gated_conf = 0;
+
+-- +migrate Down
+
+DROP TABLE config;

store/datastore/ddl/sqlite3/16.sql

@@ -0,0 +1,21 @@
+-- +migrate Up
+
+CREATE TABLE config (
+ config_id       INTEGER PRIMARY KEY AUTOINCREMENT
+,config_repo_id  INTEGER
+,config_hash     TEXT
+,config_data     BLOB
+,config_approved BOOLEAN
+
+,UNIQUE(config_hash, config_repo_id)
+);
+
+ALTER TABLE builds ADD COLUMN build_config_id INTEGER;
+UPDATE builds set build_config_id = 0;
+
+ALTER TABLE repos ADD COLUMN repo_gated_conf BOOLEAN;
+UPDATE repos SET repo_gated_conf = 0;
+
+-- +migrate Down
+
+DROP TABLE config;

store/datastore/sql/postgres/files/config.sql

@@ -0,0 +1,22 @@
+-- name: config-find-id
+
+SELECT
+ config_id
+,config_repo_id
+,config_hash
+,config_data
+,config_approved
+FROM config
+WHERE config_id = $1
+
+-- name: config-find-repo-hash
+
+SELECT
+ config_id
+,config_repo_id
+,config_hash
+,config_data
+,config_approved
+FROM config
+WHERE config_repo_id = $1
+  AND config_hash    = $2

store/datastore/sql/postgres/sql_gen.go

@@ -6,6 +6,8 @@ func Lookup(name string) string {
 }
 
 var index = map[string]string{
+	"config-find-id":            configFindId,
+	"config-find-repo-hash":     configFindRepoHash,
 	"count-users":               countUsers,
 	"count-repos":               countRepos,
 	"count-builds":              countBuilds,
@@ -33,6 +35,29 @@ var index = map[string]string{
 	"task-delete":               taskDelete,
 }
 
+var configFindId = `
+SELECT
+ config_id
+,config_repo_id
+,config_hash
+,config_data
+,config_approved
+FROM config
+WHERE config_id = $1
+`
+
+var configFindRepoHash = `
+SELECT
+ config_id
+,config_repo_id
+,config_hash
+,config_data
+,config_approved
+FROM config
+WHERE config_repo_id = $1
+  AND config_hash    = $2
+`
+
 var countUsers = `
 SELECT reltuples
 FROM pg_class WHERE relname = 'users';

store/datastore/sql/sqlite/files/config.sql

@@ -0,0 +1,22 @@
+-- name: config-find-id
+
+SELECT
+ config_id
+,config_repo_id
+,config_hash
+,config_data
+,config_approved
+FROM config
+WHERE config_id = ?
+
+-- name: config-find-repo-hash
+
+SELECT
+ config_id
+,config_repo_id
+,config_hash
+,config_data
+,config_approved
+FROM config
+WHERE config_repo_id = ?
+  AND config_hash    = ?

store/datastore/sql/sqlite/sql_gen.go

@@ -6,6 +6,8 @@ func Lookup(name string) string {
 }
 
 var index = map[string]string{
+	"config-find-id":            configFindId,
+	"config-find-repo-hash":     configFindRepoHash,
 	"count-users":               countUsers,
 	"count-repos":               countRepos,
 	"count-builds":              countBuilds,
@@ -33,6 +35,29 @@ var index = map[string]string{
 	"task-delete":               taskDelete,
 }
 
+var configFindId = `
+SELECT
+ config_id
+,config_repo_id
+,config_hash
+,config_data
+,config_approved
+FROM config
+WHERE config_id = ?
+`
+
+var configFindRepoHash = `
+SELECT
+ config_id
+,config_repo_id
+,config_hash
+,config_data
+,config_approved
+FROM config
+WHERE config_repo_id = ?
+  AND config_hash    = ?
+`
+
 var countUsers = `
 SELECT count(1)
 FROM users

store/store.go

@@ -92,6 +92,11 @@ type Store interface {
 	// new functions
 	//
 
+	ConfigLoad(int64) (*model.Config, error)
+	ConfigFind(*model.Repo, string) (*model.Config, error)
+	ConfigUpdate(*model.Config) error
+	ConfigInsert(*model.Config) error
+
 	SenderFind(*model.Repo, string) (*model.Sender, error)
 	SenderList(*model.Repo) ([]*model.Sender, error)
 	SenderCreate(*model.Sender) error