improve yaml parsing and interpreter

drone/secert_add.go

@@ -34,6 +34,10 @@ var secretAddCmd = cli.Command{
 			Usage: "inject the secret for these image types",
 			Value: &cli.StringSlice{},
 		},
+		cli.StringFlag{
+			Name:  "input",
+			Usage: "input secret value from a file",
+		},
 	},
 }
 
@@ -60,8 +64,10 @@ func secretAdd(c *cli.Context) error {
 		return fmt.Errorf("Please specify the --image parameter")
 	}
 
-	// allow secret value to come from a file when prefixed with the @ symbol,
-	// similar to curl conventions.
+	// TODO(bradrydzewski) below we use an @ sybmol to denote that the secret
+	// value should be loaded from a file (inspired by curl). I'd prefer to use
+	// a --input flag to explicitly specify a filepath instead.
+
 	if strings.HasPrefix(secret.Value, "@") {
 		path := secret.Value[1:]
 		out, ferr := ioutil.ReadFile(path)

engine/compiler/compile.go

@@ -52,15 +52,6 @@ func (c *Compiler) Compile(in []byte) (*runner.Spec, error) {
 		}
 	}
 
-	// cache section
-	if root.Cache != nil {
-		node, ok := root.Cache.(*yaml.ContainerNode)
-		if ok && !node.Disabled {
-			config.Containers = append(config.Containers, &node.Container)
-			tree.Append(parse.NewRunNode().SetName(node.Container.Name))
-		}
-	}
-
 	// clone section
 	if root.Clone != nil {
 		node, ok := root.Clone.(*yaml.ContainerNode)

engine/compiler/parse/node_root.go

@@ -11,7 +11,6 @@ type RootNode struct {
 
 	Pod      Node
 	Build    Node
-	Cache    Node
 	Clone    Node
 	Script   []Node
 	Volumes  []Node
@@ -110,7 +109,6 @@ func (n *RootNode) Walk(fn WalkFunc) (err error) {
 	var nodes []Node
 	nodes = append(nodes, n)
 	nodes = append(nodes, n.Build)
-	nodes = append(nodes, n.Cache)
 	nodes = append(nodes, n.Clone)
 	nodes = append(nodes, n.Script...)
 	nodes = append(nodes, n.Volumes...)

engine/compiler/parse/parse.go

@@ -45,16 +45,6 @@ func Parse(in []byte) (*RootNode, error) {
 		}
 	}
 
-	// add the cache section
-	{
-		cc := root.NewCacheNode()
-		cc.Container = out.Cache.ToContainer()
-		cc.Conditions = out.Cache.ToConditions()
-		cc.Container.Name = "cache"
-		cc.Vargs = out.Cache.Vargs
-		root.Cache = cc
-	}
-
 	// add the clone section
 	{
 		cc := root.NewCloneNode()

engine/compiler/parse/parse_test.go

@@ -22,7 +22,6 @@ func TestParse(t *testing.T) {
 				g.Assert(out.Path).Equal("src/github.com/octocat/hello-world")
 				g.Assert(out.Build.(*BuildNode).Context).Equal(".")
 				g.Assert(out.Build.(*BuildNode).Dockerfile).Equal("Dockerfile")
-				g.Assert(out.Cache.(*ContainerNode).Vargs["mount"]).Equal("node_modules")
 				g.Assert(out.Clone.(*ContainerNode).Container.Image).Equal("git")
 				g.Assert(out.Clone.(*ContainerNode).Vargs["depth"]).Equal(1)
 				g.Assert(out.Volumes[0].(*VolumeNode).Name).Equal("custom")

yaml/branch.go

@@ -3,12 +3,14 @@ package yaml
 import (
 	"path/filepath"
 
+	"github.com/drone/drone/yaml/types"
+
 	"gopkg.in/yaml.v2"
 )
 
 type Branch struct {
-	Include []string `yaml:"include"`
-	Exclude []string `yaml:"exclude"`
+	Include []string
+	Exclude []string
 }
 
 // ParseBranch parses the branch section of the Yaml document.
@@ -21,16 +23,16 @@ func ParseBranchString(in string) *Branch {
 	return ParseBranch([]byte(in))
 }
 
-// Matches returns true if the branch matches the include patterns and
-// does not match any of the exclude patterns.
+// Matches returns true if the branch matches the include patterns and does not
+// match any of the exclude patterns.
 func (b *Branch) Matches(branch string) bool {
 	// when no includes or excludes automatically match
 	if len(b.Include) == 0 && len(b.Exclude) == 0 {
 		return true
 	}
 
-	// exclusions are processed first. So we can include everything and
-	// then selectively exclude certain sub-patterns.
+	// exclusions are processed first. So we can include everything and then
+	// selectively exclude certain sub-patterns.
 	for _, pattern := range b.Exclude {
 		if pattern == branch {
 			return false
@@ -55,13 +57,13 @@ func (b *Branch) Matches(branch string) bool {
 func parseBranch(in []byte) *Branch {
 	out1 := struct {
 		Branch struct {
-			Include stringOrSlice `yaml:"include"`
-			Exclude stringOrSlice `yaml:"exclude"`
+			Include types.StringOrSlice `yaml:"include"`
+			Exclude types.StringOrSlice `yaml:"exclude"`
 		} `yaml:"branches"`
 	}{}
 
 	out2 := struct {
-		Include stringOrSlice `yaml:"branches"`
+		Include types.StringOrSlice `yaml:"branches"`
 	}{}
 
 	yaml.Unmarshal(in, &out1)

yaml/build.go

@@ -0,0 +1,26 @@
+package yaml
+
+// Build represents Docker image build instructions.
+type Build struct {
+	Context    string
+	Dockerfile string
+	Args       map[string]string
+}
+
+// UnmarshalYAML implements custom Yaml unmarshaling.
+func (b *Build) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	err := unmarshal(&b.Context)
+	if err == nil {
+		return nil
+	}
+	out := struct {
+		Context    string
+		Dockerfile string
+		Args       map[string]string
+	}{}
+	err = unmarshal(&out)
+	b.Context = out.Context
+	b.Args = out.Args
+	b.Dockerfile = out.Dockerfile
+	return err
+}

yaml/build_test.go

@@ -0,0 +1,38 @@
+package yaml
+
+import (
+	"testing"
+
+	"github.com/franela/goblin"
+	"gopkg.in/yaml.v2"
+)
+
+func TestBuild(t *testing.T) {
+	g := goblin.Goblin(t)
+
+	g.Describe("Build", func() {
+		g.Describe("given a yaml file", func() {
+
+			g.It("should unmarshal", func() {
+				in := []byte(".")
+				out := Build{}
+				err := yaml.Unmarshal(in, &out)
+				if err != nil {
+					g.Fail(err)
+				}
+				g.Assert(out.Context).Equal(".")
+			})
+
+			g.It("should unmarshal shorthand", func() {
+				in := []byte("{ context: ., dockerfile: Dockerfile }")
+				out := Build{}
+				err := yaml.Unmarshal(in, &out)
+				if err != nil {
+					g.Fail(err)
+				}
+				g.Assert(out.Context).Equal(".")
+				g.Assert(out.Dockerfile).Equal("Dockerfile")
+			})
+		})
+	})
+}

yaml/config.go

@@ -0,0 +1,67 @@
+package yaml
+
+import "gopkg.in/yaml.v2"
+
+// Workspace represents the build workspace.
+type Workspace struct {
+	Base string
+	Path string
+}
+
+// Config represents the build configuration Yaml document.
+type Config struct {
+	Image     string
+	Build     *Build
+	Workspace *Workspace
+	Pipeline  []*Container
+	Services  []*Container
+	Volumes   []*Volume
+	Networks  []*Network
+}
+
+// ParseString parses the Yaml configuration document.
+func ParseString(data string) (*Config, error) {
+	return Parse([]byte(data))
+}
+
+// Parse parses Yaml configuration document.
+func Parse(data []byte) (*Config, error) {
+	v := struct {
+		Image     string
+		Build     *Build
+		Workspace *Workspace
+		Services  containerList
+		Pipeline  containerList
+		Networks  networkList
+		Volumes   volumeList
+	}{}
+
+	err := yaml.Unmarshal(data, &v)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, c := range v.Services.containers {
+		c.Detached = true
+	}
+
+	return &Config{
+		Image:     v.Image,
+		Build:     v.Build,
+		Workspace: v.Workspace,
+		Services:  v.Services.containers,
+		Pipeline:  v.Pipeline.containers,
+		Networks:  v.Networks.networks,
+		Volumes:   v.Volumes.volumes,
+	}, nil
+}
+
+type config struct {
+	Image     string
+	Build     *Build
+	Workspace *Workspace
+	Services  containerList
+	Pipeline  containerList
+	Networks  networkList
+	Volumes   volumeList
+}

yaml/config_test.go

@@ -0,0 +1,83 @@
+package yaml
+
+import (
+	"testing"
+
+	"github.com/franela/goblin"
+)
+
+func TestParse(t *testing.T) {
+	g := goblin.Goblin(t)
+
+	g.Describe("Parser", func() {
+		g.Describe("Given a yaml file", func() {
+
+			g.It("Should unmarshal a string", func() {
+				out, err := ParseString(sampleYaml)
+				if err != nil {
+					g.Fail(err)
+				}
+				g.Assert(out.Image).Equal("hello-world")
+				g.Assert(out.Workspace.Base).Equal("/go")
+				g.Assert(out.Workspace.Path).Equal("src/github.com/octocat/hello-world")
+				g.Assert(out.Build.Context).Equal(".")
+				g.Assert(out.Build.Dockerfile).Equal("Dockerfile")
+				g.Assert(out.Volumes[0].Name).Equal("custom")
+				g.Assert(out.Volumes[0].Driver).Equal("blockbridge")
+				g.Assert(out.Networks[0].Name).Equal("custom")
+				g.Assert(out.Networks[0].Driver).Equal("overlay")
+				g.Assert(out.Services[0].Name).Equal("database")
+				g.Assert(out.Services[0].Image).Equal("mysql")
+				g.Assert(out.Pipeline[0].Name).Equal("test")
+				g.Assert(out.Pipeline[0].Image).Equal("golang")
+				g.Assert(out.Pipeline[0].Commands).Equal([]string{"go install", "go test"})
+				g.Assert(out.Pipeline[1].Name).Equal("build")
+				g.Assert(out.Pipeline[1].Image).Equal("golang")
+				g.Assert(out.Pipeline[1].Commands).Equal([]string{"go build"})
+				g.Assert(out.Pipeline[2].Name).Equal("notify")
+				g.Assert(out.Pipeline[2].Image).Equal("slack")
+			})
+		})
+	})
+}
+
+var sampleYaml = `
+image: hello-world
+build:
+  context: .
+  dockerfile: Dockerfile
+
+workspace:
+  path: src/github.com/octocat/hello-world
+  base: /go
+
+pipeline:
+  test:
+    image: golang
+    commands:
+      - go install
+      - go test
+  build:
+    image: golang
+    commands:
+      - go build
+    when:
+      event: push
+  notify:
+    image: slack
+    channel: dev
+    when:
+      event: failure
+
+services:
+  database:
+    image: mysql
+
+networks:
+  custom:
+    driver: overlay
+
+volumes:
+  custom:
+    driver: blockbridge
+`

yaml/constraint.go

@@ -0,0 +1,49 @@
+package yaml
+
+// Constraints define constraints for container execution.
+type Constraints struct {
+	Platform    []string
+	Environment []string
+	Event       []string
+	Branch      []string
+	Status      []string
+	Matrix      map[string]string
+}
+
+//
+// // Constraint defines an individual contraint.
+// type Constraint struct {
+// 	Include []string
+// 	Exclude []string
+// }
+//
+// // Match returns true if the branch matches the include patterns and does not
+// // match any of the exclude patterns.
+// func (c *Constraint) Match(v string) bool {
+// 	// when no includes or excludes automatically match
+// 	if len(c.Include) == 0 && len(c.Exclude) == 0 {
+// 		return true
+// 	}
+//
+// 	// exclusions are processed first. So we can include everything and then
+// 	// selectively exclude certain sub-patterns.
+// 	for _, pattern := range c.Exclude {
+// 		if pattern == v {
+// 			return false
+// 		}
+// 		if ok, _ := filepath.Match(pattern, v); ok {
+// 			return false
+// 		}
+// 	}
+//
+// 	for _, pattern := range c.Include {
+// 		if pattern == v {
+// 			return true
+// 		}
+// 		if ok, _ := filepath.Match(pattern, v); ok {
+// 			return true
+// 		}
+// 	}
+//
+// 	return false
+// }

yaml/container.go

@@ -0,0 +1,171 @@
+package yaml
+
+import (
+	"fmt"
+
+	"github.com/drone/drone/yaml/types"
+	"gopkg.in/yaml.v2"
+)
+
+// Auth defines Docker authentication credentials.
+type Auth struct {
+	Username string
+	Password string
+	Email    string
+}
+
+// Container defines a Docker container.
+type Container struct {
+	ID             string
+	Name           string
+	Image          string
+	Build          string
+	Pull           bool
+	AuthConfig     Auth
+	Detached       bool
+	Privileged     bool
+	WorkingDir     string
+	Environment    map[string]string
+	Entrypoint     []string
+	Command        []string
+	Commands       []string
+	ExtraHosts     []string
+	Volumes        []string
+	VolumesFrom    []string
+	Devices        []string
+	Network        string
+	DNS            []string
+	DNSSearch      []string
+	MemSwapLimit   int64
+	MemLimit       int64
+	CPUQuota       int64
+	CPUShares      int64
+	CPUSet         string
+	OomKillDisable bool
+	Constraints    Constraints
+
+	Vargs map[string]interface{}
+}
+
+// container is an intermediate type used for decoding a container in a format
+// compatible with docker-compose.yml.
+
+// this file has a bunch of custom types that are annoying to work with, which
+// is why this is used for intermediate purposes and converted to something
+// easier to work with.
+type container struct {
+	Name           string              `yaml:"name"`
+	Image          string              `yaml:"image"`
+	Build          string              `yaml:"build"`
+	Pull           bool                `yaml:"pull"`
+	Privileged     bool                `yaml:"privileged"`
+	Environment    types.MapEqualSlice `yaml:"environment"`
+	Entrypoint     types.StringOrSlice `yaml:"entrypoint"`
+	Command        types.StringOrSlice `yaml:"command"`
+	Commands       types.StringOrSlice `yaml:"commands"`
+	ExtraHosts     types.StringOrSlice `yaml:"extra_hosts"`
+	Volumes        types.StringOrSlice `yaml:"volumes"`
+	VolumesFrom    types.StringOrSlice `yaml:"volumes_from"`
+	Devices        types.StringOrSlice `yaml:"devices"`
+	Network        string              `yaml:"network_mode"`
+	DNS            types.StringOrSlice `yaml:"dns"`
+	DNSSearch      types.StringOrSlice `yaml:"dns_search"`
+	MemSwapLimit   int64               `yaml:"memswap_limit"`
+	MemLimit       int64               `yaml:"mem_limit"`
+	CPUQuota       int64               `yaml:"cpu_quota"`
+	CPUShares      int64               `yaml:"cpu_shares"`
+	CPUSet         string              `yaml:"cpuset"`
+	OomKillDisable bool                `yaml:"oom_kill_disable"`
+
+	AuthConfig struct {
+		Username string `yaml:"username"`
+		Password string `yaml:"password"`
+		Email    string `yaml:"email"`
+		Token    string `yaml:"registry_token"`
+	} `yaml:"auth_config"`
+
+	Constraints struct {
+		Platform    types.StringOrSlice `yaml:"platform"`
+		Environment types.StringOrSlice `yaml:"environment"`
+		Event       types.StringOrSlice `yaml:"event"`
+		Branch      types.StringOrSlice `yaml:"branch"`
+		Status      types.StringOrSlice `yaml:"status"`
+		Matrix      map[string]string   `yaml:"matrix"`
+	} `yaml:"when"`
+
+	Vargs map[string]interface{} `yaml:",inline"`
+}
+
+// containerList is an intermediate type used for decoding a slice of containers
+// in a format compatible with docker-compose.yml
+type containerList struct {
+	containers []*Container
+}
+
+// UnmarshalYAML implements custom Yaml unmarshaling.
+func (c *containerList) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	slice := yaml.MapSlice{}
+	err := unmarshal(&slice)
+	if err != nil {
+		return err
+	}
+
+	for _, s := range slice {
+		cc := container{}
+
+		out, merr := yaml.Marshal(s.Value)
+		if err != nil {
+			return merr
+		}
+
+		err = yaml.Unmarshal(out, &cc)
+		if err != nil {
+			return err
+		}
+		if cc.Name == "" {
+			cc.Name = fmt.Sprintf("%v", s.Key)
+		}
+		if cc.Image == "" {
+			cc.Image = fmt.Sprintf("%v", s.Key)
+		}
+		c.containers = append(c.containers, &Container{
+			Name:           cc.Name,
+			Image:          cc.Image,
+			Build:          cc.Build,
+			Pull:           cc.Pull,
+			Privileged:     cc.Privileged,
+			Environment:    cc.Environment.Map(),
+			Entrypoint:     cc.Entrypoint.Slice(),
+			Command:        cc.Command.Slice(),
+			Commands:       cc.Commands.Slice(),
+			ExtraHosts:     cc.ExtraHosts.Slice(),
+			Volumes:        cc.Volumes.Slice(),
+			VolumesFrom:    cc.VolumesFrom.Slice(),
+			Devices:        cc.Devices.Slice(),
+			Network:        cc.Network,
+			DNS:            cc.DNS.Slice(),
+			DNSSearch:      cc.DNSSearch.Slice(),
+			MemSwapLimit:   cc.MemSwapLimit,
+			MemLimit:       cc.MemLimit,
+			CPUQuota:       cc.CPUQuota,
+			CPUShares:      cc.CPUShares,
+			CPUSet:         cc.CPUSet,
+			OomKillDisable: cc.OomKillDisable,
+			Vargs:          cc.Vargs,
+			AuthConfig: Auth{
+				Username: cc.AuthConfig.Username,
+				Password: cc.AuthConfig.Password,
+				Email:    cc.AuthConfig.Email,
+			},
+			Constraints: Constraints{
+				Platform:    cc.Constraints.Platform.Slice(),
+				Environment: cc.Constraints.Environment.Slice(),
+				Event:       cc.Constraints.Event.Slice(),
+				Branch:      cc.Constraints.Branch.Slice(),
+				Status:      cc.Constraints.Status.Slice(),
+				Matrix:      cc.Constraints.Matrix,
+			},
+		})
+	}
+	return err
+}

yaml/container_test.go

@@ -0,0 +1,97 @@
+package yaml
+
+import (
+	"testing"
+
+	"github.com/franela/goblin"
+	"gopkg.in/yaml.v2"
+)
+
+func TestContainerNode(t *testing.T) {
+	g := goblin.Goblin(t)
+
+	g.Describe("Containers", func() {
+		g.Describe("given a yaml file", func() {
+
+			g.It("should unmarshal", func() {
+				in := []byte(sampleContainer)
+				out := containerList{}
+				err := yaml.Unmarshal(in, &out)
+				if err != nil {
+					g.Fail(err)
+				}
+				g.Assert(len(out.containers)).Equal(1)
+
+				c := out.containers[0]
+				g.Assert(c.Name).Equal("foo")
+				g.Assert(c.Image).Equal("golang")
+				g.Assert(c.Build).Equal(".")
+				g.Assert(c.Pull).Equal(true)
+				g.Assert(c.Privileged).Equal(true)
+				g.Assert(c.Entrypoint).Equal([]string{"/bin/sh"})
+				g.Assert(c.Command).Equal([]string{"yes"})
+				g.Assert(c.Commands).Equal([]string{"whoami"})
+				g.Assert(c.ExtraHosts).Equal([]string{"foo.com"})
+				g.Assert(c.Volumes).Equal([]string{"/foo:/bar"})
+				g.Assert(c.VolumesFrom).Equal([]string{"foo"})
+				g.Assert(c.Devices).Equal([]string{"/dev/tty0"})
+				g.Assert(c.Network).Equal("bridge")
+				g.Assert(c.DNS).Equal([]string{"8.8.8.8"})
+				g.Assert(c.MemSwapLimit).Equal(int64(1))
+				g.Assert(c.MemLimit).Equal(int64(2))
+				g.Assert(c.CPUQuota).Equal(int64(3))
+				g.Assert(c.CPUSet).Equal("1,2")
+				g.Assert(c.OomKillDisable).Equal(true)
+				g.Assert(c.AuthConfig.Username).Equal("octocat")
+				g.Assert(c.AuthConfig.Password).Equal("password")
+				g.Assert(c.AuthConfig.Email).Equal("octocat@github.com")
+				g.Assert(c.Vargs["access_key"]).Equal("970d28f4dd477bc184fbd10b376de753")
+				g.Assert(c.Vargs["secret_key"]).Equal("9c5785d3ece6a9cdefa42eb99b58986f9095ff1c")
+			})
+
+			g.It("should unmarshal named", func() {
+				in := []byte("foo: { name: bar }")
+				out := containerList{}
+				err := yaml.Unmarshal(in, &out)
+				if err != nil {
+					g.Fail(err)
+				}
+				g.Assert(len(out.containers)).Equal(1)
+				g.Assert(out.containers[0].Name).Equal("bar")
+			})
+
+		})
+	})
+}
+
+var sampleContainer = `
+foo:
+  image: golang
+  build: .
+  pull: true
+  privileged: true
+  environment:
+    FOO: BAR
+  entrypoint: /bin/sh
+  command: "yes"
+  commands: whoami
+  extra_hosts: foo.com
+  volumes: /foo:/bar
+  volumes_from: foo
+  devices: /dev/tty0
+  network_mode: bridge
+  dns: 8.8.8.8
+  memswap_limit: 1
+  mem_limit: 2
+  cpu_quota: 3
+  cpuset: 1,2
+  oom_kill_disable: true
+
+  auth_config:
+    username: octocat
+    password: password
+    email: octocat@github.com
+
+  access_key: 970d28f4dd477bc184fbd10b376de753
+  secret_key: 9c5785d3ece6a9cdefa42eb99b58986f9095ff1c
+`

yaml/interpreter/error.go

@@ -0,0 +1,37 @@
+package interpreter
+
+import (
+	"errors"
+	"fmt"
+)
+
+var (
+	// ErrSkip is used as a return value when container execution should be
+	// skipped at runtime. It is not returned as an error by any function.
+	ErrSkip = errors.New("Skip")
+
+	// ErrTerm is used as a return value when the runner should terminate
+	// execution and exit. It is not returned as an error by any function.
+	ErrTerm = errors.New("Terminate")
+)
+
+// An ExitError reports an unsuccessful exit.
+type ExitError struct {
+	Name string
+	Code int
+}
+
+// Error reteurns the error message in string format.
+func (e *ExitError) Error() string {
+	return fmt.Sprintf("%s : exit code %d", e.Name, e.Code)
+}
+
+// An OomError reports the process received an OOMKill from the kernel.
+type OomError struct {
+	Name string
+}
+
+// Error reteurns the error message in string format.
+func (e *OomError) Error() string {
+	return fmt.Sprintf("%s : received oom kill", e.Name)
+}

yaml/interpreter/error_test.go

@@ -0,0 +1,26 @@
+package interpreter
+
+import (
+	"testing"
+
+	"github.com/franela/goblin"
+)
+
+func TestErrors(t *testing.T) {
+	g := goblin.Goblin(t)
+
+	g.Describe("Error messages", func() {
+
+		g.It("should include OOM details", func() {
+			err := OomError{Name: "golang"}
+			got, want := err.Error(), "golang : received oom kill"
+			g.Assert(got).Equal(want)
+		})
+
+		g.It("should include Exit code", func() {
+			err := ExitError{Name: "golang", Code: 255}
+			got, want := err.Error(), "golang : exit code 255"
+			g.Assert(got).Equal(want)
+		})
+	})
+}

yaml/interpreter/pipe.go

@@ -0,0 +1,49 @@
+package interpreter
+
+import "fmt"
+
+// Pipe returns a buffered pipe that is connected to the console output.
+type Pipe struct {
+	lines chan *Line
+	eof   chan bool
+}
+
+// Next returns the next Line of console output.
+func (p *Pipe) Next() *Line {
+	select {
+	case line := <-p.lines:
+		return line
+	case <-p.eof:
+		return nil
+	}
+}
+
+// Close closes the pipe of console output.
+func (p *Pipe) Close() {
+	go func() {
+		p.eof <- true
+	}()
+}
+
+func newPipe(buffer int) *Pipe {
+	return &Pipe{
+		lines: make(chan *Line, buffer),
+		eof:   make(chan bool),
+	}
+}
+
+// Line is a line of console output.
+type Line struct {
+	Proc string `json:"proc,omitempty"`
+	Time int64  `json:"time,omitempty"`
+	Type int    `json:"type,omitempty"`
+	Pos  int    `json:"pos,omityempty"`
+	Out  string `json:"out,omitempty"`
+}
+
+func (l *Line) String() string {
+	return fmt.Sprintf("[%s:L%v:%vs] %s", l.Proc, l.Pos, l.Time, l.Out)
+}
+
+// TODO(bradrydzewski) consider an alternate buffer impelmentation based on the
+// x.crypto ssh buffer https://github.com/golang/crypto/blob/master/ssh/buffer.go

yaml/interpreter/pipe_test.go

@@ -0,0 +1,54 @@
+package interpreter
+
+import (
+	"sync"
+	"testing"
+
+	"github.com/franela/goblin"
+)
+
+func TestPipe(t *testing.T) {
+	g := goblin.Goblin(t)
+
+	g.Describe("Pipe", func() {
+		g.It("should get next line from buffer", func() {
+			line := &Line{
+				Proc: "redis",
+				Pos:  1,
+				Out:  "starting redis server",
+			}
+			pipe := newPipe(10)
+			pipe.lines <- line
+			next := pipe.Next()
+			g.Assert(next).Equal(line)
+		})
+
+		g.It("should get null line on buffer closed", func() {
+			pipe := newPipe(10)
+
+			var wg sync.WaitGroup
+			wg.Add(1)
+
+			go func() {
+				next := pipe.Next()
+				g.Assert(next == nil).IsTrue("line should be nil")
+				wg.Done()
+			}()
+
+			pipe.Close()
+			wg.Wait()
+		})
+
+		g.Describe("Line output", func() {
+			g.It("should prefix string() with metadata", func() {
+				line := Line{
+					Proc: "redis",
+					Time: 60,
+					Pos:  1,
+					Out:  "starting redis server",
+				}
+				g.Assert(line.String()).Equal("[redis:L1:60s] starting redis server")
+			})
+		})
+	})
+}

yaml/interpreter/pipeline.go

@@ -0,0 +1,348 @@
+package interpreter
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"strings"
+
+	"github.com/drone/drone/yaml"
+
+	"github.com/samalba/dockerclient"
+)
+
+// element represents a link in the linked list.
+type element struct {
+	*yaml.Container
+	next *element
+}
+
+// Pipeline represents a build pipeline.
+type Pipeline struct {
+	conf *yaml.Config
+	head *element
+	tail *element
+	next chan (error)
+	done chan (error)
+	err  error
+
+	containers []string
+	volumes    []string
+	networks   []string
+
+	client dockerclient.Client
+}
+
+// Load loads the pipeline from the Yaml configuration file.
+func Load(conf *yaml.Config) *Pipeline {
+	pipeline := Pipeline{
+		conf: conf,
+		next: make(chan error),
+		done: make(chan error),
+	}
+
+	var containers []*yaml.Container
+	containers = append(containers, conf.Services...)
+	containers = append(containers, conf.Pipeline...)
+
+	for i, c := range containers {
+		next := &element{Container: c}
+		if i == 0 {
+			pipeline.head = next
+			pipeline.tail = next
+		} else {
+			pipeline.tail.next = next
+			pipeline.tail = next
+		}
+	}
+
+	go func() {
+		pipeline.next <- nil
+	}()
+
+	return &pipeline
+}
+
+// Done returns when the process is done executing.
+func (p *Pipeline) Done() <-chan error {
+	return p.done
+}
+
+// Err returns the error for the current process.
+func (p *Pipeline) Err() error {
+	return p.err
+}
+
+// Next returns the next step in the process.
+func (p *Pipeline) Next() <-chan error {
+	return p.next
+}
+
+// Exec executes the current step.
+func (p *Pipeline) Exec() {
+	err := p.exec(p.head.Container)
+	if err != nil {
+		p.err = err
+	}
+	p.step()
+}
+
+// Skip skips the current step.
+func (p *Pipeline) Skip() {
+	p.step()
+}
+
+// Head returns the head item in the list.
+func (p *Pipeline) Head() *yaml.Container {
+	return p.head.Container
+}
+
+// Tail returns the tail item in the list.
+func (p *Pipeline) Tail() *yaml.Container {
+	return p.tail.Container
+}
+
+// Stop stops the pipeline.
+func (p *Pipeline) Stop() {
+	p.close(ErrTerm)
+	return
+}
+
+// Setup prepares the build pipeline environment.
+func (p *Pipeline) Setup() error {
+	return nil
+}
+
+// Teardown removes the pipeline environment.
+func (p *Pipeline) Teardown() {
+	for _, id := range p.containers {
+		p.client.StopContainer(id, 1)
+		p.client.KillContainer(id, "9")
+		p.client.RemoveContainer(id, true, true)
+	}
+	for _, id := range p.networks {
+		p.client.RemoveNetwork(id)
+	}
+	for _, id := range p.volumes {
+		p.client.RemoveVolume(id)
+	}
+}
+
+// step steps through the pipeline to head.next
+func (p *Pipeline) step() {
+	if p.head == p.tail {
+		p.close(nil)
+		return
+	}
+	go func() {
+		p.head = p.head.next
+		p.next <- nil
+	}()
+}
+
+// close closes open channels and signals the pipeline is done.
+func (p *Pipeline) close(err error) {
+	go func() {
+		p.done <- nil
+		close(p.next)
+		close(p.done)
+	}()
+}
+
+func (p *Pipeline) exec(c *yaml.Container) error {
+	conf := toContainerConfig(c)
+	auth := toAuthConfig(c)
+
+	// check for the image and pull if not exists or if configured to always
+	// pull the latest version.
+	_, err := p.client.InspectImage(c.Image)
+	if err == nil || c.Pull {
+		err = p.client.PullImage(c.Image, auth)
+		if err != nil {
+			return err
+		}
+	}
+
+	// creates and starts the container.
+	id, err := p.client.CreateContainer(conf, c.ID, auth)
+	if err != nil {
+		return err
+	}
+	p.containers = append(p.containers, id)
+
+	err = p.client.StartContainer(c.ID, &conf.HostConfig)
+	if err != nil {
+		return err
+	}
+
+	// stream the container logs
+	go func() {
+		rc, rerr := toLogs(p.client, c.ID)
+		if rerr != nil {
+			return
+		}
+		defer rc.Close()
+
+		num := 0
+		// now := time.Now().UTC()
+		scanner := bufio.NewScanner(rc)
+		for scanner.Scan() {
+			// r.pipe.lines <- &Line{
+			// 	Proc: c.Name,
+			// 	Time: int64(time.Since(now).Seconds()),
+			// 	Pos:  num,
+			// 	Out:  scanner.Text(),
+			// }
+			num++
+		}
+	}()
+
+	// if the container is run in detached mode we can exit without waiting
+	// for execution to complete.
+	if c.Detached {
+		return nil
+	}
+
+	<-p.client.Wait(c.ID)
+
+	res, err := p.client.InspectContainer(c.ID)
+	if err != nil {
+		return err
+	}
+
+	if res.State.OOMKilled {
+		return &OomError{c.Name}
+	} else if res.State.ExitCode != 0 {
+		return &ExitError{c.Name, res.State.ExitCode}
+	}
+	return nil
+}
+
+func toLogs(client dockerclient.Client, id string) (io.ReadCloser, error) {
+	opts := &dockerclient.LogOptions{
+		Follow: true,
+		Stdout: true,
+		Stderr: true,
+	}
+
+	piper, pipew := io.Pipe()
+	go func() {
+		defer pipew.Close()
+
+		// sometimes the docker logs fails due to parsing errors. this routine will
+		// check for such a failure and attempt to resume if necessary.
+		for i := 0; i < 5; i++ {
+			if i > 0 {
+				opts.Tail = 1
+			}
+
+			rc, err := client.ContainerLogs(id, opts)
+			if err != nil {
+				return
+			}
+			defer rc.Close()
+
+			// use Docker StdCopy
+			// internal.StdCopy(pipew, pipew, rc)
+
+			// check to see if the container is still running. If not,  we can safely
+			// exit and assume there are no more logs left to stream.
+			v, err := client.InspectContainer(id)
+			if err != nil || !v.State.Running {
+				return
+			}
+		}
+	}()
+	return piper, nil
+}
+
+// helper function that converts the Continer data structure to the exepcted
+// dockerclient.ContainerConfig.
+func toContainerConfig(c *yaml.Container) *dockerclient.ContainerConfig {
+	config := &dockerclient.ContainerConfig{
+		Image:      c.Image,
+		Env:        toEnvironmentSlice(c.Environment),
+		Cmd:        c.Command,
+		Entrypoint: c.Entrypoint,
+		WorkingDir: c.WorkingDir,
+		HostConfig: dockerclient.HostConfig{
+			Privileged:       c.Privileged,
+			NetworkMode:      c.Network,
+			Memory:           c.MemLimit,
+			CpuShares:        c.CPUShares,
+			CpuQuota:         c.CPUQuota,
+			CpusetCpus:       c.CPUSet,
+			MemorySwappiness: -1,
+			OomKillDisable:   c.OomKillDisable,
+		},
+	}
+
+	if len(config.Entrypoint) == 0 {
+		config.Entrypoint = nil
+	}
+	if len(config.Cmd) == 0 {
+		config.Cmd = nil
+	}
+	if len(c.ExtraHosts) > 0 {
+		config.HostConfig.ExtraHosts = c.ExtraHosts
+	}
+	if len(c.DNS) != 0 {
+		config.HostConfig.Dns = c.DNS
+	}
+	if len(c.DNSSearch) != 0 {
+		config.HostConfig.DnsSearch = c.DNSSearch
+	}
+	if len(c.VolumesFrom) != 0 {
+		config.HostConfig.VolumesFrom = c.VolumesFrom
+	}
+
+	config.Volumes = map[string]struct{}{}
+	for _, path := range c.Volumes {
+		if strings.Index(path, ":") == -1 {
+			config.Volumes[path] = struct{}{}
+			continue
+		}
+		parts := strings.Split(path, ":")
+		config.Volumes[parts[1]] = struct{}{}
+		config.HostConfig.Binds = append(config.HostConfig.Binds, path)
+	}
+
+	for _, path := range c.Devices {
+		if strings.Index(path, ":") == -1 {
+			continue
+		}
+		parts := strings.Split(path, ":")
+		device := dockerclient.DeviceMapping{
+			PathOnHost:        parts[0],
+			PathInContainer:   parts[1],
+			CgroupPermissions: "rwm",
+		}
+		config.HostConfig.Devices = append(config.HostConfig.Devices, device)
+	}
+
+	return config
+}
+
+// helper function that converts the AuthConfig data structure to the exepcted
+// dockerclient.AuthConfig.
+func toAuthConfig(c *yaml.Container) *dockerclient.AuthConfig {
+	if c.AuthConfig.Username == "" &&
+		c.AuthConfig.Password == "" {
+		return nil
+	}
+	return &dockerclient.AuthConfig{
+		Email:    c.AuthConfig.Email,
+		Username: c.AuthConfig.Username,
+		Password: c.AuthConfig.Password,
+	}
+}
+
+// helper function that converts a key value map of environment variables to a
+// string slice in key=value format.
+func toEnvironmentSlice(env map[string]string) []string {
+	var envs []string
+	for k, v := range env {
+		envs = append(envs, fmt.Sprintf("%s=%s", k, v))
+	}
+	return envs
+}

yaml/interpreter/pipeline_test.go

@@ -0,0 +1,70 @@
+package interpreter
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/drone/drone/yaml"
+)
+
+func TestInterpreter(t *testing.T) {
+
+	conf, err := yaml.ParseString(sampleYaml)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	pipeline := Load(conf)
+
+	for {
+		select {
+		case <-pipeline.Done():
+			fmt.Println("GOT DONE")
+			return
+
+		case <-pipeline.Next():
+			pipeline.Exec()
+		}
+	}
+}
+
+var sampleYaml = `
+image: hello-world
+build:
+  context: .
+  dockerfile: Dockerfile
+
+workspace:
+  path: src/github.com/octocat/hello-world
+  base: /go
+
+pipeline:
+  test:
+    image: golang
+    commands:
+      - go install
+      - go test
+  build:
+    image: golang
+    commands:
+      - go build
+    when:
+      event: push
+  notify:
+    image: slack
+    channel: dev
+    when:
+      event: failure
+
+services:
+  database:
+    image: mysql
+
+networks:
+  custom:
+    driver: overlay
+
+volumes:
+  custom:
+    driver: blockbridge
+`

yaml/network.go

@@ -0,0 +1,51 @@
+package yaml
+
+import (
+	"fmt"
+
+	"gopkg.in/yaml.v2"
+)
+
+// Network defines a Docker network.
+type Network struct {
+	Name       string
+	Driver     string
+	DriverOpts map[string]string `yaml:"driver_opts"`
+}
+
+// networkList is an intermediate type used for decoding a slice of networks
+// in a format compatible with docker-compose.yml
+type networkList struct {
+	networks []*Network
+}
+
+// UnmarshalYAML implements custom Yaml unmarshaling.
+func (n *networkList) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	slice := yaml.MapSlice{}
+	err := unmarshal(&slice)
+	if err != nil {
+		return err
+	}
+
+	for _, s := range slice {
+		nn := Network{}
+
+		out, merr := yaml.Marshal(s.Value)
+		if merr != nil {
+			return merr
+		}
+
+		err = yaml.Unmarshal(out, &nn)
+		if err != nil {
+			return err
+		}
+		if nn.Name == "" {
+			nn.Name = fmt.Sprintf("%v", s.Key)
+		}
+		if nn.Driver == "" {
+			nn.Driver = "bridge"
+		}
+		n.networks = append(n.networks, &nn)
+	}
+	return err
+}

yaml/network_test.go

@@ -0,0 +1,51 @@
+package yaml
+
+import (
+	"testing"
+
+	"github.com/franela/goblin"
+	"gopkg.in/yaml.v2"
+)
+
+func TestNetworks(t *testing.T) {
+	g := goblin.Goblin(t)
+
+	g.Describe("Networks", func() {
+		g.Describe("given a yaml file", func() {
+
+			g.It("should unmarshal", func() {
+				in := []byte("foo: { driver: overlay }")
+				out := networkList{}
+				err := yaml.Unmarshal(in, &out)
+				if err != nil {
+					g.Fail(err)
+				}
+				g.Assert(len(out.networks)).Equal(1)
+				g.Assert(out.networks[0].Name).Equal("foo")
+				g.Assert(out.networks[0].Driver).Equal("overlay")
+			})
+
+			g.It("should unmarshal named", func() {
+				in := []byte("foo: { name: bar }")
+				out := networkList{}
+				err := yaml.Unmarshal(in, &out)
+				if err != nil {
+					g.Fail(err)
+				}
+				g.Assert(len(out.networks)).Equal(1)
+				g.Assert(out.networks[0].Name).Equal("bar")
+			})
+
+			g.It("should unmarshal and use default driver", func() {
+				in := []byte("foo: { name: bar }")
+				out := networkList{}
+				err := yaml.Unmarshal(in, &out)
+				if err != nil {
+					g.Fail(err)
+				}
+				g.Assert(len(out.networks)).Equal(1)
+				g.Assert(out.networks[0].Driver).Equal("bridge")
+			})
+		})
+	})
+}

yaml/types/map.go

@@ -0,0 +1,38 @@
+package types
+
+import "strings"
+
+// MapEqualSlice is a custom Yaml type that can hold a map or slice of strings
+// in key=value format.
+type MapEqualSlice struct {
+	parts map[string]string
+}
+
+// UnmarshalYAML implements custom Yaml unmarshaling.
+func (s *MapEqualSlice) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	s.parts = map[string]string{}
+	err := unmarshal(&s.parts)
+	if err == nil {
+		return nil
+	}
+
+	var slice []string
+	err = unmarshal(&slice)
+	if err != nil {
+		return err
+	}
+	for _, v := range slice {
+		parts := strings.SplitN(v, "=", 2)
+		if len(parts) == 2 {
+			key := parts[0]
+			val := parts[1]
+			s.parts[key] = val
+		}
+	}
+	return nil
+}
+
+// Map returns the Yaml information as a map.
+func (s *MapEqualSlice) Map() map[string]string {
+	return s.parts
+}

yaml/types/map_test.go

@@ -0,0 +1,44 @@
+package types
+
+import (
+	"testing"
+
+	"github.com/franela/goblin"
+	"gopkg.in/yaml.v2"
+)
+
+func TestMapEqualSlice(t *testing.T) {
+	g := goblin.Goblin(t)
+
+	g.Describe("Yaml map equal slice", func() {
+
+		g.It("should unmarshal a map", func() {
+			in := []byte("foo: bar")
+			out := MapEqualSlice{}
+			err := yaml.Unmarshal(in, &out)
+			if err != nil {
+				g.Fail(err)
+			}
+			g.Assert(len(out.Map())).Equal(1)
+			g.Assert(out.Map()["foo"]).Equal("bar")
+		})
+
+		g.It("should unmarshal a map equal slice", func() {
+			in := []byte("[ foo=bar ]")
+			out := MapEqualSlice{}
+			err := yaml.Unmarshal(in, &out)
+			if err != nil {
+				g.Fail(err)
+			}
+			g.Assert(len(out.parts)).Equal(1)
+			g.Assert(out.parts["foo"]).Equal("bar")
+		})
+
+		g.It("should throw error when invalid map equal slice", func() {
+			in := []byte("foo") // string value should fail parse
+			out := MapEqualSlice{}
+			err := yaml.Unmarshal(in, &out)
+			g.Assert(err != nil).IsTrue("expects error")
+		})
+	})
+}

yaml/types/slice.go

@@ -1,11 +1,12 @@
-package yaml
+package types
 
-// stringOrSlice represents a string or an array of strings.
-type stringOrSlice struct {
+// StringOrSlice is a custom Yaml type that can hold a string or slice of strings.
+type StringOrSlice struct {
 	parts []string
 }
 
-func (s *stringOrSlice) UnmarshalYAML(unmarshal func(interface{}) error) error {
+// UnmarshalYAML implements custom Yaml unmarshaling.
+func (s *StringOrSlice) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	var sliceType []string
 	err := unmarshal(&sliceType)
 	if err == nil {
@@ -23,6 +24,7 @@ func (s *stringOrSlice) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	return err
 }
 
-func (s stringOrSlice) Slice() []string {
+// Slice returns the slice of strings.
+func (s StringOrSlice) Slice() []string {
 	return s.parts
 }

yaml/types/slice_test.go

@@ -1,4 +1,4 @@
-package yaml
+package types
 
 import (
 	"testing"
@@ -7,26 +7,26 @@ import (
 	"gopkg.in/yaml.v2"
 )
 
-func TestTypes(t *testing.T) {
+func TestStringSlice(t *testing.T) {
 	g := goblin.Goblin(t)
 
-	g.Describe("Yaml types", func() {
+	g.Describe("Yaml string slice", func() {
 		g.Describe("given a yaml file", func() {
 
 			g.It("should unmarshal a string", func() {
 				in := []byte("foo")
-				out := stringOrSlice{}
+				out := StringOrSlice{}
 				err := yaml.Unmarshal(in, &out)
 				if err != nil {
 					g.Fail(err)
 				}
-				g.Assert(len(out.parts)).Equal(1)
-				g.Assert(out.parts[0]).Equal("foo")
+				g.Assert(len(out.Slice())).Equal(1)
+				g.Assert(out.Slice()[0]).Equal("foo")
 			})
 
 			g.It("should unmarshal a string slice", func() {
 				in := []byte("[ foo ]")
-				out := stringOrSlice{}
+				out := StringOrSlice{}
 				err := yaml.Unmarshal(in, &out)
 				if err != nil {
 					g.Fail(err)
@@ -37,7 +37,7 @@ func TestTypes(t *testing.T) {
 
 			g.It("should throw error when invalid string slice", func() {
 				in := []byte("{ }") // string value should fail parse
-				out := stringOrSlice{}
+				out := StringOrSlice{}
 				err := yaml.Unmarshal(in, &out)
 				g.Assert(err != nil).IsTrue("expects error")
 			})

yaml/volume.go

@@ -0,0 +1,51 @@
+package yaml
+
+import (
+	"fmt"
+
+	"gopkg.in/yaml.v2"
+)
+
+// Volume defines a Docker volume.
+type Volume struct {
+	Name       string
+	Driver     string
+	DriverOpts map[string]string `yaml:"driver_opts"`
+	External   bool
+}
+
+// volumeList is an intermediate type used for decoding a slice of volumes
+// in a format compatible with docker-compose.yml
+type volumeList struct {
+	volumes []*Volume
+}
+
+// UnmarshalYAML implements custom Yaml unmarshaling.
+func (v *volumeList) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	slice := yaml.MapSlice{}
+	err := unmarshal(&slice)
+	if err != nil {
+		return err
+	}
+
+	for _, s := range slice {
+		vv := Volume{}
+		out, merr := yaml.Marshal(s.Value)
+		if merr != nil {
+			return merr
+		}
+
+		err = yaml.Unmarshal(out, &vv)
+		if err != nil {
+			return err
+		}
+		if vv.Name == "" {
+			vv.Name = fmt.Sprintf("%v", s.Key)
+		}
+		if vv.Driver == "" {
+			vv.Driver = "local"
+		}
+		v.volumes = append(v.volumes, &vv)
+	}
+	return err
+}

yaml/volume_test.go

@@ -0,0 +1,51 @@
+package yaml
+
+import (
+	"testing"
+
+	"github.com/franela/goblin"
+	"gopkg.in/yaml.v2"
+)
+
+func TestVolumes(t *testing.T) {
+	g := goblin.Goblin(t)
+
+	g.Describe("Volumes", func() {
+		g.Describe("given a yaml file", func() {
+
+			g.It("should unmarshal", func() {
+				in := []byte("foo: { driver: blockbridge }")
+				out := volumeList{}
+				err := yaml.Unmarshal(in, &out)
+				if err != nil {
+					g.Fail(err)
+				}
+				g.Assert(len(out.volumes)).Equal(1)
+				g.Assert(out.volumes[0].Name).Equal("foo")
+				g.Assert(out.volumes[0].Driver).Equal("blockbridge")
+			})
+
+			g.It("should unmarshal named", func() {
+				in := []byte("foo: { name: bar }")
+				out := volumeList{}
+				err := yaml.Unmarshal(in, &out)
+				if err != nil {
+					g.Fail(err)
+				}
+				g.Assert(len(out.volumes)).Equal(1)
+				g.Assert(out.volumes[0].Name).Equal("bar")
+			})
+
+			g.It("should unmarshal and use default driver", func() {
+				in := []byte("foo: { name: bar }")
+				out := volumeList{}
+				err := yaml.Unmarshal(in, &out)
+				if err != nil {
+					g.Fail(err)
+				}
+				g.Assert(len(out.volumes)).Equal(1)
+				g.Assert(out.volumes[0].Driver).Equal("local")
+			})
+		})
+	})
+}