Merge branch 'master' into bitbucketserver

Moving git username and password to the url
Removing un-needed setting of Allows
Moving log.Fatal to log.Error
Removing panics
Moving to https for gravatar

# Conflicts:
#	remote/remote.go

Dockerfile

@@ -21,4 +21,9 @@ ENV DATABASE_CONFIG=/var/lib/drone/drone.sqlite
 
 ADD drone_static /drone_static
 
+# Alpine Linux doesn't use pam, which means that there is no /etc/nsswitch.conf,
+# but Go and CGO rely on /etc/nsswitch.conf to check the order of DNS resolving.
+# To fix this we just create /etc/nsswitch.conf and add the following line:
+#RUN echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf
+
 ENTRYPOINT ["/drone_static"]

Dockerfile.env

@@ -1,10 +0,0 @@
-# build environment used in .drone.yml
-#
-#     docker build --rm=true -t drone/golang:1.5 -f Dockerfile.env .
-
-FROM golang:1.5
-ADD contrib/*.sh /usr/local/bin/
-RUN chmod +x /usr/local/bin/setup-sassc.sh && \
-    chmod +x /usr/local/bin/setup-sqlite.sh && \
-    /usr/local/bin/setup-sassc.sh && \
-    /usr/local/bin/setup-sqlite.sh

api/build.go

@@ -18,7 +18,6 @@ import (
 	"github.com/gin-gonic/gin"
 
 	"github.com/drone/drone/model"
-	"github.com/drone/drone/router/middleware/context"
 	"github.com/drone/drone/router/middleware/session"
 )
 
@@ -130,7 +129,7 @@ func GetBuildLogs(c *gin.Context) {
 }
 
 func DeleteBuild(c *gin.Context) {
-	engine_ := context.Engine(c)
+	engine_ := engine.FromContext(c)
 	repo := session.Repo(c)
 
 	// parse the build number and job sequence number from
@@ -281,7 +280,7 @@ func PostBuild(c *gin.Context) {
 	// on status change notifications
 	last, _ := store.GetBuildLastBefore(c, repo, build.Branch, build.ID)
 
-	engine_ := context.Engine(c)
+	engine_ := engine.FromContext(c)
 	go engine_.Schedule(c.Copy(), &engine.Task{
 		User:      user,
 		Repo:      repo,

api/node.go

@@ -6,8 +6,8 @@ import (
 
 	"github.com/gin-gonic/gin"
 
+	"github.com/drone/drone/engine"
 	"github.com/drone/drone/model"
-	"github.com/drone/drone/router/middleware/context"
 	"github.com/drone/drone/store"
 )
 
@@ -25,7 +25,7 @@ func GetNode(c *gin.Context) {
 }
 
 func PostNode(c *gin.Context) {
-	engine := context.Engine(c)
+	engine := engine.FromContext(c)
 
 	in := struct {
 		Addr string `json:"address"`
@@ -63,7 +63,7 @@ func PostNode(c *gin.Context) {
 }
 
 func DeleteNode(c *gin.Context) {
-	engine := context.Engine(c)
+	engine := engine.FromContext(c)
 
 	id, _ := strconv.Atoi(c.Param("node"))
 	node, err := store.GetNode(c, int64(id))

bus/bus.go

@@ -0,0 +1,40 @@
+package bus
+
+//go:generate mockery -name Bus -output mock -case=underscore
+
+import "golang.org/x/net/context"
+
+// Bus represents an event bus implementation that
+// allows a publisher to broadcast Event notifications
+// to a list of subscribers.
+type Bus interface {
+	// Publish broadcasts an event to all subscribers.
+	Publish(*Event)
+
+	// Subscribe adds the channel to the list of
+	// subscribers. Each subscriber in the list will
+	// receive broadcast events.
+	Subscribe(chan *Event)
+
+	// Unsubscribe removes the channel from the list
+	// of subscribers.
+	Unsubscribe(chan *Event)
+}
+
+// Publish broadcasts an event to all subscribers.
+func Publish(c context.Context, event *Event) {
+	FromContext(c).Publish(event)
+}
+
+// Subscribe adds the channel to the list of
+// subscribers. Each subscriber in the list will
+// receive broadcast events.
+func Subscribe(c context.Context, eventc chan *Event) {
+	FromContext(c).Subscribe(eventc)
+}
+
+// Unsubscribe removes the channel from the
+// list of subscribers.
+func Unsubscribe(c context.Context, eventc chan *Event) {
+	FromContext(c).Unsubscribe(eventc)
+}

bus/bus_impl.go

@@ -0,0 +1,46 @@
+package bus
+
+import (
+	"sync"
+)
+
+type eventbus struct {
+	sync.Mutex
+	subs map[chan *Event]bool
+}
+
+// New creates a simple event bus that manages a list of
+// subscribers to which events are published.
+func New() Bus {
+	return newEventbus()
+}
+
+func newEventbus() *eventbus {
+	return &eventbus{
+		subs: make(map[chan *Event]bool),
+	}
+}
+
+func (b *eventbus) Subscribe(c chan *Event) {
+	b.Lock()
+	b.subs[c] = true
+	b.Unlock()
+}
+
+func (b *eventbus) Unsubscribe(c chan *Event) {
+	b.Lock()
+	delete(b.subs, c)
+	b.Unlock()
+}
+
+func (b *eventbus) Publish(event *Event) {
+	b.Lock()
+	defer b.Unlock()
+
+	for s := range b.subs {
+		go func(c chan *Event) {
+			defer recover()
+			c <- event
+		}(s)
+	}
+}

bus/bus_impl_test.go

@@ -0,0 +1,73 @@
+package bus
+
+import (
+	"sync"
+	"testing"
+
+	"github.com/drone/drone/model"
+	. "github.com/franela/goblin"
+	"github.com/gin-gonic/gin"
+)
+
+func TestBus(t *testing.T) {
+	g := Goblin(t)
+	g.Describe("Event bus", func() {
+
+		g.It("Should unsubscribe", func() {
+			c := new(gin.Context)
+			b := newEventbus()
+			ToContext(c, b)
+
+			c1 := make(chan *Event)
+			c2 := make(chan *Event)
+			Subscribe(c, c1)
+			Subscribe(c, c2)
+
+			g.Assert(len(b.subs)).Equal(2)
+		})
+
+		g.It("Should subscribe", func() {
+			c := new(gin.Context)
+			b := newEventbus()
+			ToContext(c, b)
+
+			c1 := make(chan *Event)
+			c2 := make(chan *Event)
+			Subscribe(c, c1)
+			Subscribe(c, c2)
+
+			g.Assert(len(b.subs)).Equal(2)
+
+			Unsubscribe(c, c1)
+			Unsubscribe(c, c2)
+
+			g.Assert(len(b.subs)).Equal(0)
+		})
+
+		g.It("Should publish", func() {
+			c := new(gin.Context)
+			b := New()
+			ToContext(c, b)
+
+			e1 := NewEvent(Started, &model.Repo{}, &model.Build{}, &model.Job{})
+			e2 := NewEvent(Started, &model.Repo{}, &model.Build{}, &model.Job{})
+			c1 := make(chan *Event)
+
+			Subscribe(c, c1)
+
+			var wg sync.WaitGroup
+			wg.Add(1)
+
+			var r1, r2 *Event
+			go func() {
+				r1 = <-c1
+				r2 = <-c1
+				wg.Done()
+			}()
+			Publish(c, e1)
+			Publish(c, e2)
+			wg.Wait()
+		})
+	})
+
+}

bus/context.go

@@ -0,0 +1,21 @@
+package bus
+
+import "golang.org/x/net/context"
+
+const key = "bus"
+
+// Setter defines a context that enables setting values.
+type Setter interface {
+	Set(string, interface{})
+}
+
+// FromContext returns the Bus associated with this context.
+func FromContext(c context.Context) Bus {
+	return c.Value(key).(Bus)
+}
+
+// ToContext adds the Bus to this context if it supports
+// the Setter interface.
+func ToContext(c Setter, b Bus) {
+	c.Set(key, b)
+}

bus/types.go

@@ -0,0 +1,32 @@
+package bus
+
+import "github.com/drone/drone/model"
+
+// EventType defines the possible types of build events.
+type EventType string
+
+const (
+	Enqueued  EventType = "enqueued"
+	Started   EventType = "started"
+	Finished  EventType = "finished"
+	Cancelled EventType = "cancelled"
+)
+
+// Event represents a build event.
+type Event struct {
+	Type  EventType   `json:"type"`
+	Repo  model.Repo  `json:"repo"`
+	Build model.Build `json:"build"`
+	Job   model.Job   `json:"job"`
+}
+
+// NewEvent creates a new Event for the build, using copies of
+// the build data to avoid possible mutation or race conditions.
+func NewEvent(t EventType, r *model.Repo, b *model.Build, j *model.Job) *Event {
+	return &Event{
+		Type:  t,
+		Repo:  *r,
+		Build: *b,
+		Job:   *j,
+	}
+}

drone.go

@@ -1,55 +1,53 @@
 package main
 
 import (
-	"flag"
+	"net/http"
+	"time"
 
-	"github.com/drone/drone/engine"
-	"github.com/drone/drone/remote"
 	"github.com/drone/drone/router"
-	"github.com/drone/drone/router/middleware/cache"
-	"github.com/drone/drone/router/middleware/context"
-	"github.com/drone/drone/router/middleware/header"
-	"github.com/drone/drone/shared/envconfig"
-	"github.com/drone/drone/shared/server"
-	"github.com/drone/drone/store/datastore"
+	"github.com/drone/drone/router/middleware"
 
 	"github.com/Sirupsen/logrus"
+	"github.com/gin-gonic/contrib/ginrus"
+	"github.com/ianschenck/envflag"
+	_ "github.com/joho/godotenv/autoload"
 )
 
 var (
-	dotenv = flag.String("config", ".env", "")
-	debug  = flag.Bool("debug", false, "")
+	addr = envflag.String("SERVER_ADDR", ":8000", "")
+	cert = envflag.String("SERVER_CERT", "", "")
+	key  = envflag.String("SERVER_KEY", "", "")
+
+	debug = envflag.Bool("DEBUG", false, "")
 )
 
 func main() {
-	flag.Parse()
+	envflag.Parse()
 
 	// debug level if requested by user
 	if *debug {
 		logrus.SetLevel(logrus.DebugLevel)
+	} else {
+		logrus.SetLevel(logrus.WarnLevel)
 	}
 
-	// Load the configuration from env file
-	env := envconfig.Load(*dotenv)
-
-	// Setup the database driver
-	store_ := datastore.Load(env)
-
-	// setup the remote driver
-	remote_ := remote.Load(env)
-
-	// setup the runner
-	engine_ := engine.Load(env, store_)
-
 	// setup the server and start the listener
-	server_ := server.Load(env)
-	server_.Run(
-		router.Load(
-			header.Version,
-			cache.Default(),
-			context.SetStore(store_),
-			context.SetRemote(remote_),
-			context.SetEngine(engine_),
-		),
+	handler := router.Load(
+		ginrus.Ginrus(logrus.StandardLogger(), time.RFC3339, true),
+		middleware.Version,
+		middleware.Cache(),
+		middleware.Store(),
+		middleware.Remote(),
+		middleware.Engine(),
 	)
+
+	if *cert != "" {
+		logrus.Fatal(
+			http.ListenAndServeTLS(*addr, *cert, *key, handler),
+		)
+	} else {
+		logrus.Fatal(
+			http.ListenAndServe(*addr, handler),
+		)
+	}
 }

engine/compiler/builtin/alias.go

@@ -0,0 +1,29 @@
+package builtin
+
+import (
+	"fmt"
+
+	"github.com/drone/drone/engine/compiler/parse"
+)
+
+type aliasOp struct {
+	visitor
+	index  map[string]string
+	prefix string
+	suffix int
+}
+
+func NewAliasOp(prefix string) Visitor {
+	return &aliasOp{
+		index: map[string]string{},
+		prefix: prefix,
+	}
+}
+
+func (v *aliasOp) VisitContainer(node *parse.ContainerNode) error {
+	v.suffix++
+
+	node.Container.Alias = node.Container.Name
+	node.Container.Name = fmt.Sprintf("%s_%d", v.prefix, v.suffix)
+	return nil
+}

engine/compiler/builtin/args.go

@@ -0,0 +1,90 @@
+package builtin
+
+import (
+	"fmt"
+	"reflect"
+	"strconv"
+	"strings"
+
+	"github.com/drone/drone/engine/compiler/parse"
+
+	json "github.com/ghodss/yaml"
+	"gopkg.in/yaml.v2"
+)
+
+type argsOps struct {
+	visitor
+}
+
+// NewArgsOp returns a transformer that provides the plugin node
+// with the custom arguments from the Yaml file.
+func NewArgsOp() Visitor {
+	return &argsOps{}
+}
+
+func (v *argsOps) VisitContainer(node *parse.ContainerNode) error {
+	switch node.NodeType {
+	case parse.NodePlugin, parse.NodeCache, parse.NodeClone:
+		break // no-op
+	default:
+		return nil
+	}
+	if node.Container.Environment == nil {
+		node.Container.Environment = map[string]string{}
+	}
+	return argsToEnv(node.Vargs, node.Container.Environment)
+}
+
+// argsToEnv uses reflection to convert a map[string]interface to a list
+// of environment variables.
+func argsToEnv(from map[string]interface{}, to map[string]string) error {
+
+	for k, v := range from {
+		t := reflect.TypeOf(v)
+		vv := reflect.ValueOf(v)
+
+		k = "PLUGIN_" + strings.ToUpper(k)
+
+		switch t.Kind() {
+		case reflect.Bool:
+			to[k] = strconv.FormatBool(vv.Bool())
+
+		case reflect.String:
+			to[k] = vv.String()
+
+		case reflect.Int, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Int8:
+			to[k] = fmt.Sprintf("%v", vv.Int())
+
+		case reflect.Float32, reflect.Float64:
+			to[k] = fmt.Sprintf("%v", vv.Float())
+
+		// case reflect.Int, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Int8:
+		// 	to[k] = strconv.FormatInt(vv.Int(), 16)
+
+		// case reflect.Float32, reflect.Float64:
+		// 	to[k] = strconv.FormatFloat(vv.Float(), 'E', -1, 64)
+
+		case reflect.Map:
+			yml, _ := yaml.Marshal(vv.Interface())
+			out, _ := json.YAMLToJSON(yml)
+			to[k] = string(out)
+
+		case reflect.Slice:
+			out, _ := yaml.Marshal(vv.Interface())
+
+			in := []string{}
+			err := yaml.Unmarshal(out, &in)
+			if err == nil {
+				to[k] = strings.Join(in, ",")
+			} else {
+				out, err = json.YAMLToJSON(out)
+				if err != nil {
+					println(err.Error())
+				}
+				to[k] = string(out)
+			}
+		}
+	}
+
+	return nil
+}

engine/compiler/builtin/args_test.go

@@ -0,0 +1,46 @@
+package builtin
+
+import (
+	"testing"
+
+	"github.com/drone/drone/engine/compiler/parse"
+	"github.com/drone/drone/engine/runner"
+
+	"github.com/franela/goblin"
+)
+
+func Test_args(t *testing.T) {
+
+	g := goblin.Goblin(t)
+	g.Describe("plugins arguments", func() {
+
+		g.It("should ignore non-plugin containers", func() {
+			root := parse.NewRootNode()
+			c := root.NewShellNode()
+			c.Container = runner.Container{}
+			c.Vargs = map[string]interface{}{
+				"depth": 50,
+			}
+
+			ops := NewArgsOp()
+			ops.VisitContainer(c)
+
+			g.Assert(c.Container.Environment["PLUGIN_DEPTH"]).Equal("")
+		})
+
+		g.It("should include args as environment variable", func() {
+			root := parse.NewRootNode()
+			c := root.NewPluginNode()
+			c.Container = runner.Container{}
+			c.Vargs = map[string]interface{}{
+				"depth": 50,
+			}
+
+			ops := NewArgsOp()
+			ops.VisitContainer(c)
+
+			g.Assert(c.Container.Environment["PLUGIN_DEPTH"]).Equal("50")
+		})
+	})
+
+}

engine/compiler/builtin/build.go

@@ -0,0 +1,40 @@
+package builtin
+
+import (
+	"github.com/drone/drone/engine/compiler/parse"
+	"github.com/drone/drone/engine/runner"
+)
+
+// BuildOp is a transform operation that converts the build section of the Yaml
+// to a step in the pipeline responsible for building the Docker image.
+func BuildOp(node parse.Node) error {
+	build, ok := node.(*parse.BuildNode)
+	if !ok {
+		return nil
+	}
+	if build.Context == "" {
+		return nil
+	}
+
+	root := node.Root()
+	builder := root.NewContainerNode()
+
+	command := []string{
+		"build",
+		"--force-rm",
+		"-f", build.Dockerfile,
+		"-t", root.Image,
+		build.Context,
+	}
+
+	builder.Container = runner.Container{
+		Image:      "docker:apline",
+		Volumes:    []string{"/var/run/docker.sock:/var/run/docker.sock"},
+		Entrypoint: []string{"/usr/local/bin/docker"},
+		Command:    command,
+		WorkingDir: root.Path,
+	}
+
+	root.Services = append(root.Services, builder)
+	return nil
+}

engine/compiler/builtin/cache.go

@@ -0,0 +1,61 @@
+package builtin
+
+import (
+	"github.com/drone/drone/engine/runner"
+	"github.com/drone/drone/engine/compiler/parse"
+)
+
+type cacheOp struct {
+	visitor
+	enable bool
+	plugin string
+	mount  string
+}
+
+// NewCacheOp returns a transformer that configures the default cache plugin.
+func NewCacheOp(plugin, mount string, enable bool) Visitor {
+	return &cacheOp{
+		mount: mount,
+		enable: enable,
+		plugin: plugin,
+	}
+}
+
+func (v *cacheOp) VisitContainer(node *parse.ContainerNode) error {
+	if node.Type() != parse.NodeCache {
+		return nil
+	}
+	if len(node.Vargs) == 0 || v.enable == false {
+		node.Disabled = true
+		return nil
+	}
+
+	if node.Container.Name == "" {
+		node.Container.Name = "cache"
+	}
+	if node.Container.Image == "" {
+		node.Container.Image = v.plugin
+	}
+
+	// discard any other cache properties except the image name.
+	// everything else is discard for security reasons.
+	node.Container = runner.Container{
+		Name: node.Container.Name,
+		Alias: node.Container.Alias,
+		Image: node.Container.Image,
+		Volumes: []string{
+			v.mount + ":/cache",
+		},
+	}
+
+	// this is a hack until I can come up with a better solution.
+	// this copies the clone name, and appends at the end of the
+	// build. When it is executed a second time the build should
+	// have a completed status, so it knows to cache instead
+	// of restore.
+	cache := node.Root().NewCacheNode()
+	cache.Vargs = node.Vargs
+	cache.Container = node.Container
+	node.Root().Script = append(node.Root().Script, cache)
+	return nil
+}

engine/compiler/builtin/cache_test.go

@@ -0,0 +1,37 @@
+package builtin
+
+// import (
+// 	"testing"
+
+// 	"github.com/libcd/libcd"
+// 	"github.com/libcd/libyaml/parse"
+
+// 	"github.com/franela/goblin"
+// )
+
+// func Test_cache(t *testing.T) {
+// 	root := parse.NewRootNode()
+
+// 	g := goblin.Goblin(t)
+// 	g.Describe("cache", func() {
+
+// 		g.It("should use default when nil", func() {
+// 			op := NewCacheOp("plugins/cache:latest", "/tmp/cache")
+
+// 			op.VisitRoot(root)
+// 			g.Assert(root.Cache.(*parse.ContainerNode).Container.Image).Equal("plugins/cache:latest")
+// 			g.Assert(root.Cache.(*parse.ContainerNode).Container.Volumes[0]).Equal("/tmp/cache:/cache")
+// 		})
+
+// 		g.It("should use user-defined cache plugin", func() {
+// 			op := NewCacheOp("plugins/cache:latest", "/tmp/cache")
+// 			cache := root.NewCacheNode()
+// 			cache.Container = libcd.Container{}
+// 			cache.Container.Image = "custom/cacher:latest"
+// 			root.Cache = cache
+
+// 			op.VisitRoot(root)
+// 			g.Assert(cache.Container.Image).Equal("custom/cacher:latest")
+// 		})
+// 	})
+// }

engine/compiler/builtin/clone.go

@@ -0,0 +1,45 @@
+package builtin
+
+import (
+	"github.com/drone/drone/engine/compiler/parse"
+	"github.com/drone/drone/engine/runner"
+)
+
+type cloneOp struct {
+	visitor
+	plugin string
+	enable bool
+}
+
+// NewCloneOp returns a transformer that configures the default clone plugin.
+func NewCloneOp(plugin string, enable bool) Visitor {
+	return &cloneOp{
+		enable: enable,
+		plugin: plugin,
+	}
+}
+
+func (v *cloneOp) VisitContainer(node *parse.ContainerNode) error {
+	if node.Type() != parse.NodeClone {
+		return nil
+	}
+	if v.enable == false {
+		node.Disabled = true
+		return nil
+	}
+
+	if node.Container.Name == "" {
+		node.Container.Name = "clone"
+	}
+	if node.Container.Image == "" {
+		node.Container.Image = v.plugin
+	}
+
+	// discard any other cache properties except the image name.
+	// everything else is discard for security reasons.
+	node.Container = runner.Container{
+		Name: node.Container.Name,
+		Image: node.Container.Image,
+	}
+	return nil
+}

engine/compiler/builtin/clone_test.go

@@ -0,0 +1,36 @@
+package builtin
+
+// import (
+// 	"testing"
+
+// 	"github.com/libcd/libcd"
+// 	"github.com/libcd/libyaml/parse"
+
+// 	"github.com/franela/goblin"
+// )
+
+// func Test_clone(t *testing.T) {
+// 	root := parse.NewRootNode()
+
+// 	g := goblin.Goblin(t)
+// 	g.Describe("clone", func() {
+
+// 		g.It("should use default when nil", func() {
+// 			op := NewCloneOp("plugins/git:latest")
+
+// 			op.VisitRoot(root)
+// 			g.Assert(root.Clone.(*parse.ContainerNode).Container.Image).Equal("plugins/git:latest")
+// 		})
+
+// 		g.It("should use user-defined clone plugin", func() {
+// 			op := NewCloneOp("plugins/git:latest")
+// 			clone := root.NewCloneNode()
+// 			clone.Container = libcd.Container{}
+// 			clone.Container.Image = "custom/hg:latest"
+// 			root.Clone = clone
+
+// 			op.VisitRoot(root)
+// 			g.Assert(clone.Container.Image).Equal("custom/hg:latest")
+// 		})
+// 	})
+// }

engine/compiler/builtin/envs.go

@@ -0,0 +1,57 @@
+package builtin
+
+import (
+	"os"
+	"strings"
+
+	"github.com/drone/drone/engine/compiler/parse"
+)
+
+var (
+	httpProxy  = os.Getenv("HTTP_PROXY")
+	httpsProxy = os.Getenv("HTTPS_PROXY")
+	noProxy    = os.Getenv("NO_PROXY")
+)
+
+type envOp struct {
+	visitor
+	envs map[string]string
+}
+
+// NewEnvOp returns a transformer that sets default environment variables
+// for each container, service and plugin.
+func NewEnvOp(envs map[string]string) Visitor {
+	return &envOp{
+		envs: envs,
+	}
+}
+
+func (v *envOp) VisitContainer(node *parse.ContainerNode) error {
+	if node.Container.Environment == nil {
+		node.Container.Environment = map[string]string{}
+	}
+	v.defaultEnv(node)
+	v.defaultEnvProxy(node)
+	return nil
+}
+
+func (v *envOp) defaultEnv(node *parse.ContainerNode) {
+	for k, v := range v.envs {
+		node.Container.Environment[k] = v
+	}
+}
+
+func (v *envOp) defaultEnvProxy(node *parse.ContainerNode) {
+	if httpProxy != "" {
+		node.Container.Environment["HTTP_PROXY"] = httpProxy
+		node.Container.Environment["http_proxy"] = strings.ToUpper(httpProxy)
+	}
+	if httpsProxy != "" {
+		node.Container.Environment["HTTPS_PROXY"] = httpsProxy
+		node.Container.Environment["https_proxy"] = strings.ToUpper(httpsProxy)
+	}
+	if noProxy != "" {
+		node.Container.Environment["NO_PROXY"] = noProxy
+		node.Container.Environment["no_proxy"] = strings.ToUpper(noProxy)
+	}
+}

engine/compiler/builtin/envs_test.go

@@ -0,0 +1,45 @@
+package builtin
+
+import (
+	"testing"
+
+	"github.com/drone/drone/engine/compiler/parse"
+	"github.com/drone/drone/engine/runner"
+
+	"github.com/franela/goblin"
+)
+
+func Test_env(t *testing.T) {
+	root := parse.NewRootNode()
+
+	g := goblin.Goblin(t)
+	g.Describe("environment variables", func() {
+
+		g.It("should be copied", func() {
+			envs := map[string]string{"CI": "drone"}
+
+			c := root.NewContainerNode()
+			c.Container = runner.Container{}
+			op := NewEnvOp(envs)
+
+			op.VisitContainer(c)
+			g.Assert(c.Container.Environment["CI"]).Equal("drone")
+		})
+
+		g.It("should include http proxy variables", func() {
+			httpProxy = "foo"
+			httpsProxy = "bar"
+			noProxy = "baz"
+
+			c := root.NewContainerNode()
+			c.Container = runner.Container{}
+			op := NewEnvOp(map[string]string{})
+
+			op.VisitContainer(c)
+			g.Assert(c.Container.Environment["HTTP_PROXY"]).Equal("foo")
+			g.Assert(c.Container.Environment["HTTPS_PROXY"]).Equal("bar")
+			g.Assert(c.Container.Environment["NO_PROXY"]).Equal("baz")
+		})
+
+	})
+}

engine/compiler/builtin/filter.go

@@ -0,0 +1,128 @@
+package builtin
+
+import (
+	"path/filepath"
+
+	"github.com/drone/drone/engine/compiler/parse"
+)
+
+type filterOp struct {
+	visitor
+	status   string
+	branch   string
+	event    string
+	environ  string
+	platform string
+	matrix   map[string]string
+}
+
+// NewFilterOp returns a transformer that filters (ie removes) steps
+// from the process based on conditional logic in the yaml.
+func NewFilterOp(status, branch, event, env string, matrix map[string]string) Visitor {
+	return &filterOp{
+		status:  status,
+		branch:  branch,
+		event:   event,
+		environ: env,
+		matrix:  matrix,
+	}
+}
+
+func (v *filterOp) VisitContainer(node *parse.ContainerNode) error {
+	v.visitStatus(node)
+	v.visitBranch(node)
+	v.visitEvent(node)
+	v.visitMatrix(node)
+	v.visitPlatform(node)
+	return nil
+}
+
+// visitStatus is a helpfer function that converts an on_change status
+// filter to either success or failure based on the prior build status.
+func (v *filterOp) visitStatus(node *parse.ContainerNode) {
+	if len(node.Conditions.Status) == 0 {
+		node.Conditions.Status = []string{"success"}
+		return
+	}
+	for _, status := range node.Conditions.Status {
+		if status != "change" && status != "changed" && status != "changes" {
+			continue
+		}
+		var want []string
+		switch v.status {
+		case "success":
+			want = append(want, "failure")
+		case "failure", "error", "killed":
+			want = append(want, "success")
+		default:
+			want = []string{"success", "failure"}
+		}
+		node.Conditions.Status = append(node.Conditions.Status, want...)
+		break
+	}
+}
+
+// visitBranch is a helper function that disables container steps when
+// the branch conditions are not satisfied.
+func (v *filterOp) visitBranch(node *parse.ContainerNode) {
+	if len(node.Conditions.Branch) == 0 {
+		return
+	}
+	for _, pattern := range node.Conditions.Branch {
+		if ok, _ := filepath.Match(pattern, v.branch); ok {
+			return
+		}
+	}
+	node.Disabled = true
+}
+
+// visitEnvironment is a helper function that disables container steps
+// when the deployment environment conditions are not satisfied.
+func (v *filterOp) visitEnvironment(node *parse.ContainerNode) {
+	if len(node.Conditions.Environment) == 0 {
+		return
+	}
+	for _, pattern := range node.Conditions.Environment {
+		if ok, _ := filepath.Match(pattern, v.environ); ok {
+			return
+		}
+	}
+	node.Disabled = true
+}
+
+// visitEvent is a helper function that disables container steps
+// when the build event conditions are not satisfied.
+func (v *filterOp) visitEvent(node *parse.ContainerNode) {
+	if len(node.Conditions.Event) == 0 {
+		return
+	}
+	for _, pattern := range node.Conditions.Event {
+		if ok, _ := filepath.Match(pattern, v.event); ok {
+			return
+		}
+	}
+	node.Disabled = true
+}
+
+func (v *filterOp) visitMatrix(node *parse.ContainerNode) {
+	for key, val := range node.Conditions.Matrix {
+		if v.matrix[key] != val {
+			node.Disabled = true
+			break
+		}
+	}
+}
+
+// visitPlatform is a helper function that disables container steps
+// when the build event conditions are not satisfied.
+func (v *filterOp) visitPlatform(node *parse.ContainerNode) {
+	if len(node.Conditions.Platform) == 0 {
+		return
+	}
+	for _, pattern := range node.Conditions.Platform {
+		if ok, _ := filepath.Match(pattern, v.platform); ok {
+			return
+		}
+	}
+	node.Disabled = true
+}

engine/compiler/builtin/filter_test.go

@@ -0,0 +1,130 @@
+package builtin
+
+// import (
+// 	"testing"
+
+// 	"github.com/franela/goblin"
+// )
+
+// func TestFilter(t *testing.T) {
+// 	g := goblin.Goblin(t)
+// 	g.Describe("Filters", func() {
+
+// 		g.It("Should match no branch filter", func() {
+// 			c := &Container{}
+// 			FilterBranch("feature/foo")(nil, c)
+// 			g.Assert(c.Disabled).IsFalse()
+// 		})
+
+// 		g.It("Should match branch", func() {
+// 			c := &Container{}
+// 			c.Conditions.Branch.parts = []string{"feature/*"}
+// 			FilterBranch("feature/foo")(nil, c)
+// 			g.Assert(c.Disabled).IsFalse()
+// 		})
+
+// 		g.It("Should match branch wildcard", func() {
+// 			c := &Container{}
+// 			c.Conditions.Branch.parts = []string{"feature/*"}
+// 			FilterBranch("feature/foo")(nil, c)
+// 			g.Assert(c.Disabled).IsFalse()
+// 		})
+
+// 		g.It("Should disable when branch filter doesn't match", func() {
+// 			c := &Container{}
+// 			c.Conditions.Branch.parts = []string{"feature/*", "develop"}
+// 			FilterBranch("master")(nil, c)
+// 			g.Assert(c.Disabled).IsTrue()
+// 		})
+
+// 		g.It("Should match no platform filter", func() {
+// 			c := &Container{}
+// 			FilterPlatform("linux_amd64")(nil, c)
+// 			g.Assert(c.Disabled).IsFalse()
+// 		})
+
+// 		g.It("Should match platform", func() {
+// 			c := &Container{}
+// 			c.Conditions.Platform.parts = []string{"linux_amd64"}
+// 			FilterPlatform("linux_amd64")(nil, c)
+// 			g.Assert(c.Disabled).IsFalse()
+// 		})
+
+// 		g.It("Should disable when platform filter doesn't match", func() {
+// 			c := &Container{}
+// 			c.Conditions.Platform.parts = []string{"linux_arm", "linux_arm64"}
+// 			FilterPlatform("linux_amd64")(nil, c)
+// 			g.Assert(c.Disabled).IsTrue()
+// 		})
+
+// 		g.It("Should match no environment filter", func() {
+// 			c := &Container{}
+// 			FilterEnvironment("production")(nil, c)
+// 			g.Assert(c.Disabled).IsFalse()
+// 		})
+
+// 		g.It("Should match environment", func() {
+// 			c := &Container{}
+// 			c.Conditions.Environment.parts = []string{"production"}
+// 			FilterEnvironment("production")(nil, c)
+// 			g.Assert(c.Disabled).IsFalse()
+// 		})
+
+// 		g.It("Should disable when environment filter doesn't match", func() {
+// 			c := &Container{}
+// 			c.Conditions.Environment.parts = []string{"develop", "staging"}
+// 			FilterEnvironment("production")(nil, c)
+// 			g.Assert(c.Disabled).IsTrue()
+// 		})
+
+// 		g.It("Should match no event filter", func() {
+// 			c := &Container{}
+// 			FilterEvent("push")(nil, c)
+// 			g.Assert(c.Disabled).IsFalse()
+// 		})
+
+// 		g.It("Should match event", func() {
+// 			c := &Container{}
+// 			c.Conditions.Event.parts = []string{"push"}
+// 			FilterEvent("push")(nil, c)
+// 			g.Assert(c.Disabled).IsFalse()
+// 		})
+
+// 		g.It("Should disable when event filter doesn't match", func() {
+// 			c := &Container{}
+// 			c.Conditions.Event.parts = []string{"push", "tag"}
+// 			FilterEvent("pull_request")(nil, c)
+// 			g.Assert(c.Disabled).IsTrue()
+// 		})
+
+// 		g.It("Should match matrix", func() {
+// 			c := &Container{}
+// 			c.Conditions.Matrix = map[string]string{
+// 				"go":    "1.5",
+// 				"redis": "3.0",
+// 			}
+// 			matrix := map[string]string{
+// 				"go":    "1.5",
+// 				"redis": "3.0",
+// 				"node":  "5.0.0",
+// 			}
+// 			FilterMatrix(matrix)(nil, c)
+// 			g.Assert(c.Disabled).IsFalse()
+// 		})
+
+// 		g.It("Should disable when event filter doesn't match", func() {
+// 			c := &Container{}
+// 			c.Conditions.Matrix = map[string]string{
+// 				"go":    "1.5",
+// 				"redis": "3.0",
+// 			}
+// 			matrix := map[string]string{
+// 				"go":    "1.4.2",
+// 				"redis": "3.0",
+// 				"node":  "5.0.0",
+// 			}
+// 			FilterMatrix(matrix)(nil, c)
+// 			g.Assert(c.Disabled).IsTrue()
+// 		})
+// 	})
+// }

engine/compiler/builtin/normalize.go

@@ -0,0 +1,63 @@
+package builtin
+
+import (
+	"path/filepath"
+	"strings"
+
+	"github.com/drone/drone/engine/compiler/parse"
+)
+
+type normalizeOp struct {
+	visitor
+	namespace string
+}
+
+// NewNormalizeOp returns a transformer that normalizes the container image
+// names and plugin names to their fully qualified values.
+func NewNormalizeOp(namespace string) Visitor {
+	return &normalizeOp{
+		namespace: namespace,
+	}
+}
+
+func (v *normalizeOp) VisitContainer(node *parse.ContainerNode) error {
+	v.normalizeName(node)
+	v.normalizeImage(node)
+	switch node.NodeType {
+	case parse.NodePlugin, parse.NodeCache, parse.NodeClone:
+		v.normalizePlugin(node)
+	}
+	return nil
+}
+
+// normalize the container image to the fully qualified name.
+func (v *normalizeOp) normalizeImage(node *parse.ContainerNode) {
+	if strings.Contains(node.Container.Image, ":") {
+		return
+	}
+	node.Container.Image = node.Container.Image + ":latest"
+}
+
+// normalize the plugin entrypoint and command values.
+func (v *normalizeOp) normalizePlugin(node *parse.ContainerNode) {
+	if strings.Contains(node.Container.Image, "/") {
+		return
+	}
+	node.Container.Image = filepath.Join(v.namespace, node.Container.Image)
+}
+
+// normalize the container name to ensrue a value is set.
+func (v *normalizeOp) normalizeName(node *parse.ContainerNode) {
+	if node.Container.Name != "" {
+		return
+	}
+
+	parts := strings.Split(node.Container.Image, "/")
+	if len(parts) != 0 {
+		node.Container.Name = parts[len(parts)-1]
+	}
+	parts = strings.Split(node.Container.Image, ":")
+	if len(parts) != 0 {
+		node.Container.Name = parts[0]
+	}
+}

engine/compiler/builtin/normalize_test.go

@@ -0,0 +1,69 @@
+package builtin
+
+import (
+	"testing"
+
+	"github.com/drone/drone/engine/compiler/parse"
+	"github.com/drone/drone/engine/runner"
+
+	"github.com/franela/goblin"
+)
+
+func Test_normalize(t *testing.T) {
+	root := parse.NewRootNode()
+
+	g := goblin.Goblin(t)
+	g.Describe("normalizing", func() {
+
+		g.Describe("images", func() {
+
+			g.It("should append tag if empty", func() {
+				c := root.NewContainerNode()
+				c.Container = runner.Container{Image: "golang"}
+				op := NewNormalizeOp("")
+
+				op.VisitContainer(c)
+				g.Assert(c.Container.Image).Equal("golang:latest")
+			})
+
+			g.It("should not override existing tag", func() {
+				c := root.NewContainerNode()
+				c.Container = runner.Container{Image: "golang:1.5"}
+				op := NewNormalizeOp("")
+
+				op.VisitContainer(c)
+				g.Assert(c.Container.Image).Equal("golang:1.5")
+			})
+		})
+
+		g.Describe("plugins", func() {
+
+			g.It("should prepend namespace", func() {
+				c := root.NewPluginNode()
+				c.Container = runner.Container{Image: "git"}
+				op := NewNormalizeOp("plugins")
+
+				op.VisitContainer(c)
+				g.Assert(c.Container.Image).Equal("plugins/git:latest")
+			})
+
+			g.It("should not override existing namespace", func() {
+				c := root.NewPluginNode()
+				c.Container = runner.Container{Image: "index.docker.io/drone/git"}
+				op := NewNormalizeOp("plugins")
+
+				op.VisitContainer(c)
+				g.Assert(c.Container.Image).Equal("index.docker.io/drone/git:latest")
+			})
+
+			g.It("should ignore shell or service types", func() {
+				c := root.NewShellNode()
+				c.Container = runner.Container{Image: "golang"}
+				op := NewNormalizeOp("plugins")
+
+				op.VisitContainer(c)
+				g.Assert(c.Container.Image).Equal("golang:latest")
+			})
+		})
+	})
+}

engine/compiler/builtin/pod.go

@@ -0,0 +1,50 @@
+package builtin
+
+import (
+	"fmt"
+
+	"github.com/drone/drone/engine/compiler/parse"
+	"github.com/drone/drone/engine/runner"
+)
+
+type podOp struct {
+	visitor
+	name string
+}
+
+// NewPodOp returns a transformer that configures an ambassador container
+// providing shared networking and container volumes.
+func NewPodOp(name string) Visitor {
+	return &podOp{
+		name: name,
+	}
+}
+
+func (v *podOp) VisitContainer(node *parse.ContainerNode) error {
+	if node.Container.Network == "" {
+		parent := fmt.Sprintf("container:%s", v.name)
+		node.Container.Network = parent
+	}
+	node.Container.VolumesFrom = append(node.Container.VolumesFrom, v.name)
+	return nil
+}
+
+func (v *podOp) VisitRoot(node *parse.RootNode) error {
+	service := node.NewServiceNode()
+	service.Container = runner.Container{
+		Name:       v.name,
+		Alias:      "ambassador",
+		Image:      "busybox",
+		Entrypoint: []string{"/bin/sleep"},
+		Command:    []string{"86400"},
+		Volumes:    []string{node.Path, node.Base},
+		// Entrypoint: []string{"/bin/sh", "-c"},
+		// Volumes:    []string{node.Base},
+		// Command:    []string{
+		// 	fmt.Sprintf("mkdir -p %s; sleep 86400", node.Path),
+		// },
+	}
+
+	node.Pod = service
+	return nil
+}

engine/compiler/builtin/pull.go

@@ -0,0 +1,26 @@
+package builtin
+
+import (
+	"github.com/drone/drone/engine/compiler/parse"
+)
+
+type pullOp struct {
+	visitor
+	pull bool
+}
+
+// NewPullOp returns a transformer that configures plugins to automatically
+// pull the latest images at runtime.
+func NewPullOp(pull bool) Visitor {
+	return &pullOp{
+		pull: pull,
+	}
+}
+
+func (v *pullOp) VisitContainer(node *parse.ContainerNode) error {
+	switch node.NodeType {
+	case parse.NodePlugin, parse.NodeCache, parse.NodeClone:
+		node.Container.Pull = v.pull
+	}
+	return nil
+}

engine/compiler/builtin/pull_test.go

@@ -0,0 +1,45 @@
+package builtin
+
+import (
+	"testing"
+
+	"github.com/drone/drone/engine/compiler/parse"
+	"github.com/drone/drone/engine/runner"
+
+	"github.com/franela/goblin"
+)
+
+func Test_pull(t *testing.T) {
+	root := parse.NewRootNode()
+
+	g := goblin.Goblin(t)
+	g.Describe("pull image", func() {
+
+		g.It("should be enabled for plugins", func() {
+			c := root.NewPluginNode()
+			c.Container = runner.Container{}
+			op := NewPullOp(true)
+
+			op.VisitContainer(c)
+			g.Assert(c.Container.Pull).IsTrue()
+		})
+
+		g.It("should be disabled for plugins", func() {
+			c := root.NewPluginNode()
+			c.Container = runner.Container{}
+			op := NewPullOp(false)
+
+			op.VisitContainer(c)
+			g.Assert(c.Container.Pull).IsFalse()
+		})
+
+		g.It("should be disabled for non-plugins", func() {
+			c := root.NewShellNode()
+			c.Container = runner.Container{}
+			op := NewPullOp(true)
+
+			op.VisitContainer(c)
+			g.Assert(c.Container.Pull).IsFalse()
+		})
+	})
+}

engine/compiler/builtin/shell.go

@@ -0,0 +1,85 @@
+package builtin
+
+import (
+	"bytes"
+	"encoding/base64"
+	"fmt"
+
+	"github.com/drone/drone/engine/compiler/parse"
+)
+
+const (
+	Freebsd_amd64 = "freebsd_amd64"
+	Linux_adm64   = "linux_amd64"
+	Windows_amd64 = "windows_amd64"
+)
+
+type shellOp struct {
+	visitor
+	platform string
+}
+
+// NewShellOp returns a transformer that converts the shell node to
+// a runnable container.
+func NewShellOp(platform string) Visitor {
+	return &shellOp{
+		platform: platform,
+	}
+}
+
+func (v *shellOp) VisitContainer(node *parse.ContainerNode) error {
+	if node.NodeType != parse.NodeShell {
+		return nil
+	}
+
+	node.Container.Entrypoint = []string{
+		"/bin/sh", "-c",
+	}
+	node.Container.Command = []string{
+		"echo $CI_CMDS | base64 -d | /bin/sh -e",
+	}
+	if node.Container.Environment == nil {
+		node.Container.Environment = map[string]string{}
+	}
+	node.Container.Environment["HOME"] = "/root"
+	node.Container.Environment["SHELL"] = "/bin/sh"
+	node.Container.Environment["CI_CMDS"] = toScript(
+		node.Root().Path,
+		node.Commands,
+	)
+
+	return nil
+}
+
+func toScript(base string, commands []string) string {
+	var buf bytes.Buffer
+	for _, command := range commands {
+		buf.WriteString(fmt.Sprintf(
+			traceScript,
+			"<command>"+command+"</command>",
+			command,
+		))
+	}
+
+	script := fmt.Sprintf(
+		setupScript,
+		buf.String(),
+	)
+
+	return base64.StdEncoding.EncodeToString([]byte(script))
+}
+
+// setupScript is a helper script this is added to the build to ensure
+// a minimum set of environment variables are set correctly.
+const setupScript = `
+echo $DRONE_NETRC > $HOME/.netrc
+
+%s
+`
+
+// traceScript is a helper script that is added to the build script
+// to trace a command.
+const traceScript = `
+echo %q
+%s
+`

engine/compiler/builtin/shell_test.go

@@ -0,0 +1,44 @@
+package builtin
+
+import (
+	"testing"
+
+	"github.com/drone/drone/engine/compiler/parse"
+	"github.com/drone/drone/engine/runner"
+
+	"github.com/franela/goblin"
+)
+
+func Test_shell(t *testing.T) {
+
+	g := goblin.Goblin(t)
+	g.Describe("shell containers", func() {
+
+		g.It("should ignore plugin steps", func() {
+			root := parse.NewRootNode()
+			c := root.NewPluginNode()
+			c.Container = runner.Container{}
+			ops := NewShellOp(Linux_adm64)
+			ops.VisitContainer(c)
+
+			g.Assert(len(c.Container.Entrypoint)).Equal(0)
+			g.Assert(len(c.Container.Command)).Equal(0)
+			g.Assert(c.Container.Environment["CI_CMDS"]).Equal("")
+		})
+
+		g.It("should set entrypoint, command and environment variables", func() {
+			root := parse.NewRootNode()
+			root.Base = "/go"
+			root.Path = "/go/src/github.com/octocat/hello-world"
+
+			c := root.NewShellNode()
+			c.Commands = []string{"go build"}
+			ops := NewShellOp(Linux_adm64)
+			ops.VisitContainer(c)
+
+			g.Assert(c.Container.Entrypoint).Equal([]string{"/bin/sh", "-c"})
+			g.Assert(c.Container.Command).Equal([]string{"echo $CI_CMDS | base64 -d | /bin/sh -e"})
+			g.Assert(c.Container.Environment["CI_CMDS"] != "").IsTrue()
+		})
+	})
+}

engine/compiler/builtin/validate.go

@@ -0,0 +1,105 @@
+package builtin
+
+import (
+	"fmt"
+	"path/filepath"
+
+	"github.com/drone/drone/engine/compiler/parse"
+)
+
+type validateOp struct {
+	visitor
+	plugins []string
+	trusted bool
+}
+
+// NewValidateOp returns a linter that checks container configuration.
+func NewValidateOp(trusted bool, plugins []string) Visitor {
+	return &validateOp{
+		trusted: trusted,
+		plugins: plugins,
+	}
+}
+
+func (v *validateOp) VisitContainer(node *parse.ContainerNode) error {
+	switch node.NodeType {
+	case parse.NodePlugin, parse.NodeCache, parse.NodeClone:
+		if err := v.validatePlugins(node); err != nil {
+			return err
+		}
+	}
+	if node.NodeType == parse.NodePlugin {
+		if err := v.validatePluginConfig(node); err != nil {
+			return err
+		}
+	}
+	return v.validateConfig(node)
+}
+
+// validate the plugin image and return an error if the plugin
+// image does not match the whitelist.
+func (v *validateOp) validatePlugins(node *parse.ContainerNode) error {
+	match := false
+	for _, pattern := range v.plugins {
+		ok, err := filepath.Match(pattern, node.Container.Image)
+		if ok && err == nil {
+			match = true
+			break
+		}
+	}
+	if !match {
+		return fmt.Errorf(
+			"Plugin %s is not in the whitelist",
+			node.Container.Image,
+		)
+	}
+	return nil
+}
+
+// validate the plugin command and entrypoint and return an error
+// the user attempts to set or override these values.
+func (v *validateOp) validatePluginConfig(node *parse.ContainerNode) error {
+	if len(node.Container.Entrypoint) != 0 {
+		return fmt.Errorf("Cannot set plugin Entrypoint")
+	}
+	if len(node.Container.Command) != 0 {
+		return fmt.Errorf("Cannot set plugin Command")
+	}
+	return nil
+}
+
+// validate the container configuration and return an error if
+// restricted configurations are used.
+func (v *validateOp) validateConfig(node *parse.ContainerNode) error {
+	if v.trusted {
+		return nil
+	}
+	if node.Container.Privileged {
+		return fmt.Errorf("Insufficient privileges to use privileged mode")
+	}
+	if len(node.Container.DNS) != 0 {
+		return fmt.Errorf("Insufficient privileges to use custom dns")
+	}
+	if len(node.Container.DNSSearch) != 0 {
+		return fmt.Errorf("Insufficient privileges to use dns_search")
+	}
+	if len(node.Container.Devices) != 0 {
+		return fmt.Errorf("Insufficient privileges to use devices")
+	}
+	if len(node.Container.ExtraHosts) != 0 {
+		return fmt.Errorf("Insufficient privileges to use extra_hosts")
+	}
+	if len(node.Container.Network) != 0 {
+		return fmt.Errorf("Insufficient privileges to override the network")
+	}
+	if node.Container.OomKillDisable {
+		return fmt.Errorf("Insufficient privileges to disable oom_kill")
+	}
+	if len(node.Container.Volumes) != 0 && node.Type() != parse.NodeCache {
+		return fmt.Errorf("Insufficient privileges to use volumes")
+	}
+	if len(node.Container.VolumesFrom) != 0 {
+		return fmt.Errorf("Insufficient privileges to use volumes_from")
+	}
+	return nil
+}

engine/compiler/builtin/validate_test.go

@@ -0,0 +1,199 @@
+package builtin
+
+import (
+	"testing"
+
+	"github.com/drone/drone/engine/compiler/parse"
+	"github.com/drone/drone/engine/runner"
+
+	"github.com/franela/goblin"
+)
+
+func Test_validate(t *testing.T) {
+	root := parse.NewRootNode()
+
+	g := goblin.Goblin(t)
+	g.Describe("validating", func() {
+
+		g.Describe("privileged attributes", func() {
+
+			g.It("should not error when trusted build", func() {
+				c := root.NewContainerNode()
+				c.Container = runner.Container{}
+				ops := NewValidateOp(true, []string{"plugins/*"})
+				err := ops.VisitContainer(c)
+
+				g.Assert(err == nil).IsTrue("error should be nil")
+			})
+
+			g.It("should error when privleged mode", func() {
+				c := root.NewContainerNode()
+				c.Container = runner.Container{}
+				c.Container.Privileged = true
+				ops := NewValidateOp(false, []string{"plugins/*"})
+				err := ops.VisitContainer(c)
+
+				g.Assert(err != nil).IsTrue("error should not be nil")
+				g.Assert(err.Error()).Equal("Insufficient privileges to use privileged mode")
+			})
+
+			g.It("should error when dns configured", func() {
+				c := root.NewContainerNode()
+				c.Container = runner.Container{}
+				c.Container.DNS = []string{"8.8.8.8"}
+				ops := NewValidateOp(false, []string{"plugins/*"})
+				err := ops.VisitContainer(c)
+
+				g.Assert(err != nil).IsTrue("error should not be nil")
+				g.Assert(err.Error()).Equal("Insufficient privileges to use custom dns")
+			})
+
+			g.It("should error when dns_search configured", func() {
+				c := root.NewContainerNode()
+				c.Container = runner.Container{}
+				c.Container.DNSSearch = []string{"8.8.8.8"}
+				ops := NewValidateOp(false, []string{"plugins/*"})
+				err := ops.VisitContainer(c)
+
+				g.Assert(err != nil).IsTrue("error should not be nil")
+				g.Assert(err.Error()).Equal("Insufficient privileges to use dns_search")
+			})
+
+			g.It("should error when devices configured", func() {
+				c := root.NewContainerNode()
+				c.Container = runner.Container{}
+				c.Container.Devices = []string{"/dev/foo"}
+				ops := NewValidateOp(false, []string{"plugins/*"})
+				err := ops.VisitContainer(c)
+
+				g.Assert(err != nil).IsTrue("error should not be nil")
+				g.Assert(err.Error()).Equal("Insufficient privileges to use devices")
+			})
+
+			g.It("should error when extra_hosts configured", func() {
+				c := root.NewContainerNode()
+				c.Container = runner.Container{}
+				c.Container.ExtraHosts = []string{"1.2.3.4 foo.com"}
+				ops := NewValidateOp(false, []string{"plugins/*"})
+				err := ops.VisitContainer(c)
+
+				g.Assert(err != nil).IsTrue("error should not be nil")
+				g.Assert(err.Error()).Equal("Insufficient privileges to use extra_hosts")
+			})
+
+			g.It("should error when network configured", func() {
+				c := root.NewContainerNode()
+				c.Container = runner.Container{}
+				c.Container.Network = "host"
+				ops := NewValidateOp(false, []string{"plugins/*"})
+				err := ops.VisitContainer(c)
+
+				g.Assert(err != nil).IsTrue("error should not be nil")
+				g.Assert(err.Error()).Equal("Insufficient privileges to override the network")
+			})
+
+			g.It("should error when oom_kill_disabled configured", func() {
+				c := root.NewContainerNode()
+				c.Container = runner.Container{}
+				c.Container.OomKillDisable = true
+				ops := NewValidateOp(false, []string{"plugins/*"})
+				err := ops.VisitContainer(c)
+
+				g.Assert(err != nil).IsTrue("error should not be nil")
+				g.Assert(err.Error()).Equal("Insufficient privileges to disable oom_kill")
+			})
+
+			g.It("should error when volumes configured", func() {
+				c := root.NewContainerNode()
+				c.Container = runner.Container{}
+				c.Container.Volumes = []string{"/:/tmp"}
+				ops := NewValidateOp(false, []string{"plugins/*"})
+				err := ops.VisitContainer(c)
+
+				g.Assert(err != nil).IsTrue("error should not be nil")
+				g.Assert(err.Error()).Equal("Insufficient privileges to use volumes")
+			})
+
+			g.It("should error when volumes_from configured", func() {
+				c := root.NewContainerNode()
+				c.Container = runner.Container{}
+				c.Container.VolumesFrom = []string{"drone"}
+				ops := NewValidateOp(false, []string{"plugins/*"})
+				err := ops.VisitContainer(c)
+
+				g.Assert(err != nil).IsTrue("error should not be nil")
+				g.Assert(err.Error()).Equal("Insufficient privileges to use volumes_from")
+			})
+		})
+
+		g.Describe("plugin configuration", func() {
+			g.It("should error when entrypoint is configured", func() {
+				c := root.NewPluginNode()
+				c.Container = runner.Container{Image: "plugins/git"}
+				c.Container.Entrypoint = []string{"/bin/sh"}
+				ops := NewValidateOp(false, []string{"plugins/*"})
+				err := ops.VisitContainer(c)
+
+				g.Assert(err != nil).IsTrue("error should not be nil")
+				g.Assert(err.Error()).Equal("Cannot set plugin Entrypoint")
+			})
+
+			g.It("should error when command is configured", func() {
+				c := root.NewPluginNode()
+				c.Container = runner.Container{Image: "plugins/git"}
+				c.Container.Command = []string{"cat", "/proc/1/status"}
+				ops := NewValidateOp(false, []string{"plugins/*"})
+				err := ops.VisitContainer(c)
+
+				g.Assert(err != nil).IsTrue("error should not be nil")
+				g.Assert(err.Error()).Equal("Cannot set plugin Command")
+			})
+
+			g.It("should not error when empty entrypoint, command", func() {
+				c := root.NewPluginNode()
+				c.Container = runner.Container{Image: "plugins/git"}
+				ops := NewValidateOp(false, []string{"plugins/*"})
+				err := ops.VisitContainer(c)
+
+				g.Assert(err == nil).IsTrue("error should be nil")
+			})
+		})
+
+		g.Describe("plugin whitelist", func() {
+
+			g.It("should error when no match found", func() {
+				c := root.NewPluginNode()
+				c.Container = runner.Container{}
+				c.Container.Image = "custom/git"
+
+				ops := NewValidateOp(false, []string{"plugins/*"})
+				err := ops.VisitContainer(c)
+
+				g.Assert(err != nil).IsTrue("error should be nil")
+				g.Assert(err.Error()).Equal("Plugin custom/git is not in the whitelist")
+			})
+
+			g.It("should not error when match found", func() {
+				c := root.NewPluginNode()
+				c.Container = runner.Container{}
+				c.Container.Image = "plugins/git"
+
+				ops := NewValidateOp(false, []string{"plugins/*"})
+				err := ops.VisitContainer(c)
+
+				g.Assert(err == nil).IsTrue("error should be nil")
+			})
+
+			g.It("should ignore build images", func() {
+				c := root.NewShellNode()
+				c.Container = runner.Container{}
+				c.Container.Image = "google/golang"
+
+				ops := NewValidateOp(false, []string{"plugins/*"})
+				err := ops.VisitContainer(c)
+
+				g.Assert(err == nil).IsTrue("error should be nil")
+			})
+		})
+	})
+}

engine/compiler/builtin/visitor.go

@@ -0,0 +1,23 @@
+package builtin
+
+import "github.com/drone/drone/engine/compiler/parse"
+
+// Visitor interface for walking the Yaml file.
+type Visitor interface {
+	VisitRoot(*parse.RootNode) error
+	VisitVolume(*parse.VolumeNode) error
+	VisitNetwork(*parse.NetworkNode) error
+	VisitBuild(*parse.BuildNode) error
+	VisitContainer(*parse.ContainerNode) error
+}
+
+// visitor provides an easy default implementation of a Visitor interface with
+// stubbed methods. This can be embedded in transforms to meet the basic
+// requirements.
+type visitor struct{}
+
+func (visitor) VisitRoot(*parse.RootNode) error           { return nil }
+func (visitor) VisitVolume(*parse.VolumeNode) error       { return nil }
+func (visitor) VisitNetwork(*parse.NetworkNode) error     { return nil }
+func (visitor) VisitBuild(*parse.BuildNode) error         { return nil }
+func (visitor) VisitContainer(*parse.ContainerNode) error { return nil }

engine/compiler/builtin/workspace.go

@@ -0,0 +1,50 @@
+package builtin
+
+import (
+	"path/filepath"
+
+	"github.com/drone/drone/engine/compiler/parse"
+)
+
+type workspaceOp struct {
+	visitor
+	base string
+	path string
+}
+
+// NewWorkspaceOp returns a transformer that provides a default workspace paths,
+// including the base path (mounted as a volume) and absolute path where the
+// code is cloned.
+func NewWorkspaceOp(base, path string) Visitor {
+	return &workspaceOp{
+		base: base,
+		path: path,
+	}
+}
+
+func (v *workspaceOp) VisitRoot(node *parse.RootNode) error {
+	if node.Base == "" {
+		node.Base = v.base
+	}
+	if node.Path == "" {
+		node.Path = v.path
+	}
+	if !filepath.IsAbs(node.Path) {
+		node.Path = filepath.Join(
+			node.Base,
+			node.Path,
+		)
+	}
+	return nil
+}
+
+func (v *workspaceOp) VisitContainer(node *parse.ContainerNode) error {
+	if node.NodeType == parse.NodeService {
+		// we must not override the default working
+		// directory of service containers. All other
+		// container should launch in the workspace
+		return nil
+	}
+	node.Container.WorkingDir = node.Root().Path
+	return nil
+}

engine/compiler/builtin/workspace_test.go

@@ -0,0 +1,89 @@
+package builtin
+
+import (
+	"testing"
+
+	"github.com/franela/goblin"
+	"github.com/drone/drone/engine/compiler/parse"
+)
+
+func Test_workspace(t *testing.T) {
+	g := goblin.Goblin(t)
+
+	g.Describe("workspace", func() {
+
+		var defaultBase = "/go"
+		var defaultPath = "src/github.com/octocat/hello-world"
+
+		g.It("should not override user paths", func() {
+			var base = "/drone"
+			var path = "/drone/src/github.com/octocat/hello-world"
+
+			op := NewWorkspaceOp(defaultBase, defaultPath)
+			root := parse.NewRootNode()
+			root.Base = base
+			root.Path = path
+
+			op.VisitRoot(root)
+			g.Assert(root.Base).Equal(base)
+			g.Assert(root.Path).Equal(path)
+		})
+
+		g.It("should convert user paths to absolute", func() {
+			var base = "/drone"
+			var path = "src/github.com/octocat/hello-world"
+			var abs = "/drone/src/github.com/octocat/hello-world"
+
+			op := NewWorkspaceOp(defaultBase, defaultPath)
+			root := parse.NewRootNode()
+			root.Base = base
+			root.Path = path
+
+			op.VisitRoot(root)
+			g.Assert(root.Base).Equal(base)
+			g.Assert(root.Path).Equal(abs)
+		})
+
+		g.It("should set the default path", func() {
+			var base = "/go"
+			var path = "/go/src/github.com/octocat/hello-world"
+
+			op := NewWorkspaceOp(defaultBase, defaultPath)
+			root := parse.NewRootNode()
+
+			op.VisitRoot(root)
+			g.Assert(root.Base).Equal(base)
+			g.Assert(root.Path).Equal(path)
+		})
+
+		g.It("should use workspace as working_dir", func() {
+			var base = "/drone"
+			var path = "/drone/src/github.com/octocat/hello-world"
+
+			root := parse.NewRootNode()
+			root.Base = base
+			root.Path = path
+
+			c := root.NewContainerNode()
+
+			op := NewWorkspaceOp(defaultBase, defaultPath)
+			op.VisitContainer(c)
+			g.Assert(c.Container.WorkingDir).Equal(root.Path)
+		})
+
+		g.It("should not use workspace as working_dir for services", func() {
+			var base = "/drone"
+			var path = "/drone/src/github.com/octocat/hello-world"
+
+			root := parse.NewRootNode()
+			root.Base = base
+			root.Path = path
+
+			c := root.NewServiceNode()
+
+			op := NewWorkspaceOp(defaultBase, defaultPath)
+			op.VisitContainer(c)
+			g.Assert(c.Container.WorkingDir).Equal("")
+		})
+	})
+}

engine/compiler/compile.go

@@ -0,0 +1,146 @@
+package libyaml
+
+import (
+	"github.com/drone/drone/engine/runner"
+	"github.com/drone/drone/engine/runner/parse"
+
+	yaml "github.com/drone/drone/engine/compiler/parse"
+)
+
+// Compiler compiles the Yaml file to the intermediate representation.
+type Compiler struct {
+	trans []Transform
+}
+
+func New() *Compiler {
+	return &Compiler{}
+}
+
+// Transforms sets the compiler transforms use to transform the intermediate
+// representation during compilation.
+func (c *Compiler) Transforms(trans []Transform) *Compiler {
+	c.trans = append(c.trans, trans...)
+	return c
+}
+
+// CompileString compiles the Yaml configuration string and returns
+// the intermediate representation for the interpreter.
+func (c *Compiler) CompileString(in string) (*runner.Spec, error) {
+	return c.Compile([]byte(in))
+}
+
+// CompileString compiles the Yaml configuration file and returns
+// the intermediate representation for the interpreter.
+func (c *Compiler) Compile(in []byte) (*runner.Spec, error) {
+	root, err := yaml.Parse(in)
+	if err != nil {
+		return nil, err
+	}
+	if err := root.Walk(c.walk); err != nil {
+		return nil, err
+	}
+
+	config := &runner.Spec{}
+	tree := parse.NewTree()
+
+	// pod section
+	if root.Pod != nil {
+		node, ok := root.Pod.(*yaml.ContainerNode)
+		if ok {
+			config.Containers = append(config.Containers, &node.Container)
+			tree.Append(parse.NewRunNode().SetName(node.Container.Name).SetDetach(true))
+		}
+	}
+
+	// cache section
+	if root.Cache != nil {
+		node, ok := root.Cache.(*yaml.ContainerNode)
+		if ok && !node.Disabled {
+			config.Containers = append(config.Containers, &node.Container)
+			tree.Append(parse.NewRunNode().SetName(node.Container.Name))
+		}
+	}
+
+	// clone section
+	if root.Clone != nil {
+		node, ok := root.Clone.(*yaml.ContainerNode)
+		if ok && !node.Disabled {
+			config.Containers = append(config.Containers, &node.Container)
+			tree.Append(parse.NewRunNode().SetName(node.Container.Name))
+		}
+	}
+
+	// services section
+	for _, container := range root.Services {
+		node, ok := container.(*yaml.ContainerNode)
+		if !ok || node.Disabled {
+			continue
+		}
+
+		config.Containers = append(config.Containers, &node.Container)
+		tree.Append(parse.NewRunNode().SetName(node.Container.Name).SetDetach(true))
+	}
+
+	// pipeline section
+	for i, container := range root.Script {
+		node, ok := container.(*yaml.ContainerNode)
+		if !ok || node.Disabled {
+			continue
+		}
+
+		config.Containers = append(config.Containers, &node.Container)
+
+		// step 1: lookahead to see if any status=failure exist
+		list := parse.NewListNode()
+		for ii, next := range root.Script {
+			if i >= ii {
+				continue
+			}
+			node, ok := next.(*yaml.ContainerNode)
+			if !ok || node.Disabled || !node.OnFailure() {
+				continue
+			}
+
+			list.Append(
+				parse.NewRecoverNode().SetBody(
+					parse.NewRunNode().SetName(
+						node.Container.Name,
+					),
+				),
+			)
+		}
+		// step 2: if yes, collect these and append to "error" node
+		if len(list.Body) == 0 {
+			tree.Append(parse.NewRunNode().SetName(node.Container.Name))
+		} else {
+			errorNode := parse.NewErrorNode()
+			errorNode.SetBody(parse.NewRunNode().SetName(node.Container.Name))
+			errorNode.SetDefer(list)
+			tree.Append(errorNode)
+		}
+	}
+
+	config.Nodes = tree
+	return config, nil
+}
+
+func (c *Compiler) walk(node yaml.Node) (err error) {
+	for _, trans := range c.trans {
+		switch v := node.(type) {
+		case *yaml.BuildNode:
+			err = trans.VisitBuild(v)
+		case *yaml.ContainerNode:
+			err = trans.VisitContainer(v)
+		case *yaml.NetworkNode:
+			err = trans.VisitNetwork(v)
+		case *yaml.VolumeNode:
+			err = trans.VisitVolume(v)
+		case *yaml.RootNode:
+			err = trans.VisitRoot(v)
+		}
+		if err != nil {
+			break
+		}
+	}
+	return err
+}

engine/compiler/compile_test.go

@@ -0,0 +1 @@
+package libyaml

engine/compiler/parse/node.go

@@ -0,0 +1,34 @@
+package parse
+
+const (
+	NodeBuild     = "build"
+	NodeCache     = "cache"
+	NodeClone     = "clone"
+	NodeContainer = "container"
+	NodeNetwork   = "network"
+	NodePlugin    = "plugin"
+	NodeRoot      = "root"
+	NodeService   = "service"
+	NodeShell     = "shell"
+	NodeVolume    = "volume"
+)
+
+// NodeType identifies the type of parse tree node.
+type NodeType string
+
+// Type returns itself an provides an easy default implementation.
+// for embedding in a Node. Embedded in all non-trivial Nodes.
+func (t NodeType) Type() NodeType {
+	return t
+}
+
+// String returns the string value of the Node type.
+func (t NodeType) String() string {
+	return string(t)
+}
+
+// A Node is an element in the parse tree.
+type Node interface {
+	Type() NodeType
+	Root() *RootNode
+}

engine/compiler/parse/node_build.go

@@ -0,0 +1,42 @@
+package parse
+
+// BuildNode represents Docker image build instructions.
+type BuildNode struct {
+	NodeType
+
+	Context    string
+	Dockerfile string
+	Args       map[string]string
+
+	root *RootNode
+}
+
+// Root returns the root node.
+func (n *BuildNode) Root() *RootNode { return n.root }
+
+//
+// intermediate types for yaml decoding.
+//
+
+type build struct {
+	Context    string
+	Dockerfile string
+	Args       map[string]string
+}
+
+func (b *build) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	err := unmarshal(&b.Context)
+	if err == nil {
+		return nil
+	}
+	out := struct {
+		Context    string
+		Dockerfile string
+		Args       map[string]string
+	}{}
+	err = unmarshal(&out)
+	b.Context = out.Context
+	b.Args = out.Args
+	b.Dockerfile = out.Dockerfile
+	return err
+}

engine/compiler/parse/node_build_test.go

@@ -0,0 +1,38 @@
+package parse
+
+import (
+	"testing"
+
+	"github.com/franela/goblin"
+	"gopkg.in/yaml.v2"
+)
+
+func TestBuildNode(t *testing.T) {
+	g := goblin.Goblin(t)
+
+	g.Describe("Build", func() {
+		g.Describe("given a yaml file", func() {
+
+			g.It("should unmarshal", func() {
+				in := []byte(".")
+				out := build{}
+				err := yaml.Unmarshal(in, &out)
+				if err != nil {
+					g.Fail(err)
+				}
+				g.Assert(out.Context).Equal(".")
+			})
+
+			g.It("should unmarshal shorthand", func() {
+				in := []byte("{ context: ., dockerfile: Dockerfile }")
+				out := build{}
+				err := yaml.Unmarshal(in, &out)
+				if err != nil {
+					g.Fail(err)
+				}
+				g.Assert(out.Context).Equal(".")
+				g.Assert(out.Dockerfile).Equal("Dockerfile")
+			})
+		})
+	})
+}

engine/compiler/parse/node_container.go

@@ -0,0 +1,180 @@
+package parse
+
+import (
+	"fmt"
+
+	"github.com/drone/drone/engine/runner"
+
+	"gopkg.in/yaml.v2"
+)
+
+type Conditions struct {
+	Platform    []string
+	Environment []string
+	Event       []string
+	Branch      []string
+	Status      []string
+	Matrix      map[string]string
+}
+
+// ContainerNode represents a Docker container.
+type ContainerNode struct {
+	NodeType
+
+	// Container represents the container configuration.
+	Container  runner.Container
+	Conditions Conditions
+	Disabled   bool
+	Commands   []string
+	Vargs      map[string]interface{}
+
+	root *RootNode
+}
+
+// Root returns the root node.
+func (n *ContainerNode) Root() *RootNode { return n.root }
+
+// OnSuccess returns true if the container should be executed
+// when the exit code of the previous step is 0.
+func (n *ContainerNode) OnSuccess() bool {
+	for _, status := range n.Conditions.Status {
+		if status == "success" {
+			return true
+		}
+	}
+	return false
+}
+
+// OnFailure returns true if the container should be executed
+// even when the exit code of the previous step != 0.
+func (n *ContainerNode) OnFailure() bool {
+	for _, status := range n.Conditions.Status {
+		if status == "failure" {
+			return true
+		}
+	}
+	return false
+}
+
+//
+// intermediate types for yaml decoding.
+//
+
+type container struct {
+	Name           string        `yaml:"name"`
+	Image          string        `yaml:"image"`
+	Build          string        `yaml:"build"`
+	Pull           bool          `yaml:"pull"`
+	Privileged     bool          `yaml:"privileged"`
+	Environment    mapEqualSlice `yaml:"environment"`
+	Entrypoint     stringOrSlice `yaml:"entrypoint"`
+	Command        stringOrSlice `yaml:"command"`
+	Commands       stringOrSlice `yaml:"commands"`
+	ExtraHosts     stringOrSlice `yaml:"extra_hosts"`
+	Volumes        stringOrSlice `yaml:"volumes"`
+	VolumesFrom    stringOrSlice `yaml:"volumes_from"`
+	Devices        stringOrSlice `yaml:"devices"`
+	Network        string        `yaml:"network_mode"`
+	DNS            stringOrSlice `yaml:"dns"`
+	DNSSearch      stringOrSlice `yaml:"dns_search"`
+	MemSwapLimit   int64         `yaml:"memswap_limit"`
+	MemLimit       int64         `yaml:"mem_limit"`
+	CPUQuota       int64         `yaml:"cpu_quota"`
+	CPUShares      int64         `yaml:"cpu_shares"`
+	CPUSet         string        `yaml:"cpuset"`
+	OomKillDisable bool          `yaml:"oom_kill_disable"`
+
+	AuthConfig struct {
+		Username string `yaml:"username"`
+		Password string `yaml:"password"`
+		Email    string `yaml:"email"`
+		Token    string `yaml:"registry_token"`
+	} `yaml:"auth_config"`
+
+	Conditions struct {
+		Platform    stringOrSlice     `yaml:"platform"`
+		Environment stringOrSlice     `yaml:"environment"`
+		Event       stringOrSlice     `yaml:"event"`
+		Branch      stringOrSlice     `yaml:"branch"`
+		Status      stringOrSlice     `yaml:"status"`
+		Matrix      map[string]string `yaml:"matrix"`
+	} `yaml:"when"`
+
+	Vargs map[string]interface{} `yaml:",inline"`
+}
+
+func (c *container) ToContainer() runner.Container {
+	return runner.Container{
+		Name:           c.Name,
+		Image:          c.Image,
+		Pull:           c.Pull,
+		Privileged:     c.Privileged,
+		Environment:    c.Environment.parts,
+		Entrypoint:     c.Entrypoint.parts,
+		Command:        c.Command.parts,
+		ExtraHosts:     c.ExtraHosts.parts,
+		Volumes:        c.Volumes.parts,
+		VolumesFrom:    c.VolumesFrom.parts,
+		Devices:        c.Devices.parts,
+		Network:        c.Network,
+		DNS:            c.DNS.parts,
+		DNSSearch:      c.DNSSearch.parts,
+		MemSwapLimit:   c.MemSwapLimit,
+		MemLimit:       c.MemLimit,
+		CPUQuota:       c.CPUQuota,
+		CPUShares:      c.CPUShares,
+		CPUSet:         c.CPUSet,
+		OomKillDisable: c.OomKillDisable,
+		AuthConfig: runner.Auth{
+			Username: c.AuthConfig.Username,
+			Password: c.AuthConfig.Password,
+			Email:    c.AuthConfig.Email,
+			Token:    c.AuthConfig.Token,
+		},
+	}
+}
+
+func (c *container) ToConditions() Conditions {
+	return Conditions{
+		Platform:    c.Conditions.Platform.parts,
+		Environment: c.Conditions.Environment.parts,
+		Event:       c.Conditions.Event.parts,
+		Branch:      c.Conditions.Branch.parts,
+		Status:      c.Conditions.Status.parts,
+		Matrix:      c.Conditions.Matrix,
+	}
+}
+
+type containerList struct {
+	containers []*container
+}
+
+func (c *containerList) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	slice := yaml.MapSlice{}
+	err := unmarshal(&slice)
+	if err != nil {
+		return err
+	}
+
+	for _, s := range slice {
+		cc := container{}
+
+		out, err := yaml.Marshal(s.Value)
+		if err != nil {
+			return err
+		}
+
+		err = yaml.Unmarshal(out, &cc)
+		if err != nil {
+			return err
+		}
+		if cc.Name == "" {
+			cc.Name = fmt.Sprintf("%v", s.Key)
+		}
+		if cc.Image == "" {
+			cc.Image = fmt.Sprintf("%v", s.Key)
+		}
+		c.containers = append(c.containers, &cc)
+	}
+	return err
+}

engine/compiler/parse/node_container_test.go

@@ -0,0 +1,97 @@
+package parse
+
+import (
+	"testing"
+
+	"github.com/franela/goblin"
+	"gopkg.in/yaml.v2"
+)
+
+func TestContainerNode(t *testing.T) {
+	g := goblin.Goblin(t)
+
+	g.Describe("Containers", func() {
+		g.Describe("given a yaml file", func() {
+
+			g.It("should unmarshal", func() {
+				in := []byte(sampleContainer)
+				out := containerList{}
+				err := yaml.Unmarshal(in, &out)
+				if err != nil {
+					g.Fail(err)
+				}
+				g.Assert(len(out.containers)).Equal(1)
+
+				c := out.containers[0]
+				g.Assert(c.Name).Equal("foo")
+				g.Assert(c.Image).Equal("golang")
+				g.Assert(c.Build).Equal(".")
+				g.Assert(c.Pull).Equal(true)
+				g.Assert(c.Privileged).Equal(true)
+				g.Assert(c.Entrypoint.parts).Equal([]string{"/bin/sh"})
+				g.Assert(c.Command.parts).Equal([]string{"yes"})
+				g.Assert(c.Commands.parts).Equal([]string{"whoami"})
+				g.Assert(c.ExtraHosts.parts).Equal([]string{"foo.com"})
+				g.Assert(c.Volumes.parts).Equal([]string{"/foo:/bar"})
+				g.Assert(c.VolumesFrom.parts).Equal([]string{"foo"})
+				g.Assert(c.Devices.parts).Equal([]string{"/dev/tty0"})
+				g.Assert(c.Network).Equal("bridge")
+				g.Assert(c.DNS.parts).Equal([]string{"8.8.8.8"})
+				g.Assert(c.MemSwapLimit).Equal(int64(1))
+				g.Assert(c.MemLimit).Equal(int64(2))
+				g.Assert(c.CPUQuota).Equal(int64(3))
+				g.Assert(c.CPUSet).Equal("1,2")
+				g.Assert(c.OomKillDisable).Equal(true)
+				g.Assert(c.AuthConfig.Username).Equal("octocat")
+				g.Assert(c.AuthConfig.Password).Equal("password")
+				g.Assert(c.AuthConfig.Email).Equal("octocat@github.com")
+				g.Assert(c.Vargs["access_key"]).Equal("970d28f4dd477bc184fbd10b376de753")
+				g.Assert(c.Vargs["secret_key"]).Equal("9c5785d3ece6a9cdefa42eb99b58986f9095ff1c")
+			})
+
+			g.It("should unmarshal named", func() {
+				in := []byte("foo: { name: bar }")
+				out := containerList{}
+				err := yaml.Unmarshal(in, &out)
+				if err != nil {
+					g.Fail(err)
+				}
+				g.Assert(len(out.containers)).Equal(1)
+				g.Assert(out.containers[0].Name).Equal("bar")
+			})
+
+		})
+	})
+}
+
+var sampleContainer = `
+foo:
+  image: golang
+  build: .
+  pull: true
+  privileged: true
+  environment:
+    FOO: BAR
+  entrypoint: /bin/sh
+  command: "yes"
+  commands: whoami
+  extra_hosts: foo.com
+  volumes: /foo:/bar
+  volumes_from: foo
+  devices: /dev/tty0
+  network_mode: bridge
+  dns: 8.8.8.8
+  memswap_limit: 1
+  mem_limit: 2
+  cpu_quota: 3
+  cpuset: 1,2
+  oom_kill_disable: true
+
+  auth_config:
+    username: octocat
+    password: password
+    email: octocat@github.com
+
+  access_key: 970d28f4dd477bc184fbd10b376de753
+  secret_key: 9c5785d3ece6a9cdefa42eb99b58986f9095ff1c
+`

engine/compiler/parse/node_network.go

@@ -0,0 +1,68 @@
+package parse
+
+import (
+	"fmt"
+
+	"gopkg.in/yaml.v2"
+)
+
+// NetworkNode represents a Docker network.
+type NetworkNode struct {
+	NodeType
+	root *RootNode
+
+	Name       string
+	Driver     string
+	DriverOpts map[string]string
+}
+
+// Root returns the root node.
+func (n *NetworkNode) Root() *RootNode { return n.root }
+
+//
+// intermediate types for yaml decoding.
+//
+
+// network is an intermediate type used for decoding a networks in a format
+// compatible with docker-compose.yml
+type network struct {
+	Name       string
+	Driver     string
+	DriverOpts map[string]string `yaml:"driver_opts"`
+}
+
+// networkList is an intermediate type used for decoding a slice of networks
+// in a format compatible with docker-compose.yml
+type networkList struct {
+	networks []*network
+}
+
+func (n *networkList) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	slice := yaml.MapSlice{}
+	err := unmarshal(&slice)
+	if err != nil {
+		return err
+	}
+
+	for _, s := range slice {
+		nn := network{}
+
+		out, err := yaml.Marshal(s.Value)
+		if err != nil {
+			return err
+		}
+
+		err = yaml.Unmarshal(out, &nn)
+		if err != nil {
+			return err
+		}
+		if nn.Name == "" {
+			nn.Name = fmt.Sprintf("%v", s.Key)
+		}
+		if nn.Driver == "" {
+			nn.Driver = "bridge"
+		}
+		n.networks = append(n.networks, &nn)
+	}
+	return err
+}

engine/compiler/parse/node_network_test.go

@@ -0,0 +1,51 @@
+package parse
+
+import (
+	"testing"
+
+	"github.com/franela/goblin"
+	"gopkg.in/yaml.v2"
+)
+
+func TestNetworkNode(t *testing.T) {
+	g := goblin.Goblin(t)
+
+	g.Describe("Networks", func() {
+		g.Describe("given a yaml file", func() {
+
+			g.It("should unmarshal", func() {
+				in := []byte("foo: { driver: overlay }")
+				out := networkList{}
+				err := yaml.Unmarshal(in, &out)
+				if err != nil {
+					g.Fail(err)
+				}
+				g.Assert(len(out.networks)).Equal(1)
+				g.Assert(out.networks[0].Name).Equal("foo")
+				g.Assert(out.networks[0].Driver).Equal("overlay")
+			})
+
+			g.It("should unmarshal named", func() {
+				in := []byte("foo: { name: bar }")
+				out := networkList{}
+				err := yaml.Unmarshal(in, &out)
+				if err != nil {
+					g.Fail(err)
+				}
+				g.Assert(len(out.networks)).Equal(1)
+				g.Assert(out.networks[0].Name).Equal("bar")
+			})
+
+			g.It("should unmarshal and use default driver", func() {
+				in := []byte("foo: { name: bar }")
+				out := volumeList{}
+				err := yaml.Unmarshal(in, &out)
+				if err != nil {
+					g.Fail(err)
+				}
+				g.Assert(len(out.volumes)).Equal(1)
+				g.Assert(out.volumes[0].Driver).Equal("local")
+			})
+		})
+	})
+}

engine/compiler/parse/node_root.go

@@ -0,0 +1,148 @@
+package parse
+
+// RootNode is the root node in the parsed Yaml file.
+type RootNode struct {
+	NodeType
+
+	Platform string
+	Base     string
+	Path     string
+	Image    string
+
+	Pod      Node
+	Build    Node
+	Cache    Node
+	Clone    Node
+	Script   []Node
+	Volumes  []Node
+	Networks []Node
+	Services []Node
+}
+
+// NewRootNode returns a new root node.
+func NewRootNode() *RootNode {
+	return &RootNode{
+		NodeType: NodeRoot,
+	}
+}
+
+// Root returns the root node.
+func (n *RootNode) Root() *RootNode { return n }
+
+// Returns a new Volume Node.
+func (n *RootNode) NewVolumeNode(name string) *VolumeNode {
+	return &VolumeNode{
+		NodeType: NodeVolume,
+		Name:     name,
+		root:     n,
+	}
+}
+
+// Returns a new Network Node.
+func (n *RootNode) NewNetworkNode(name string) *NetworkNode {
+	return &NetworkNode{
+		NodeType: NodeNetwork,
+		Name:     name,
+		root:     n,
+	}
+}
+
+// Returns a new Network Node.
+func (n *RootNode) NewBuildNode(context string) *BuildNode {
+	return &BuildNode{
+		NodeType: NodeBuild,
+		Context:  context,
+		root:     n,
+	}
+}
+
+// Returns a new Container Plugin Node.
+func (n *RootNode) NewPluginNode() *ContainerNode {
+	return &ContainerNode{
+		NodeType: NodePlugin,
+		root:     n,
+	}
+}
+
+// Returns a new Container Shell Node.
+func (n *RootNode) NewShellNode() *ContainerNode {
+	return &ContainerNode{
+		NodeType: NodeShell,
+		root:     n,
+	}
+}
+
+// Returns a new Container Service Node.
+func (n *RootNode) NewServiceNode() *ContainerNode {
+	return &ContainerNode{
+		NodeType: NodeService,
+		root:     n,
+	}
+}
+
+// Returns a new Container Clone Node.
+func (n *RootNode) NewCloneNode() *ContainerNode {
+	return &ContainerNode{
+		NodeType: NodeClone,
+		root:     n,
+	}
+}
+
+// Returns a new Container Cache Node.
+func (n *RootNode) NewCacheNode() *ContainerNode {
+	return &ContainerNode{
+		NodeType: NodeCache,
+		root:     n,
+	}
+}
+
+// Returns a new Container Node.
+func (n *RootNode) NewContainerNode() *ContainerNode {
+	return &ContainerNode{
+		NodeType: NodeContainer,
+		root:     n,
+	}
+}
+
+// Walk is a function that walk through all child nodes of the RootNode
+// and invokes the Walk callback function for each Node.
+func (n *RootNode) Walk(fn WalkFunc) (err error) {
+	var nodes []Node
+	nodes = append(nodes, n)
+	nodes = append(nodes, n.Build)
+	nodes = append(nodes, n.Cache)
+	nodes = append(nodes, n.Clone)
+	nodes = append(nodes, n.Script...)
+	nodes = append(nodes, n.Volumes...)
+	nodes = append(nodes, n.Networks...)
+	nodes = append(nodes, n.Services...)
+	for _, node := range nodes {
+		err = fn(node)
+		if err != nil {
+			return
+		}
+	}
+	return
+}
+
+type WalkFunc func(Node) error
+
+//
+// intermediate types for yaml decoding.
+//
+
+type root struct {
+	Workspace struct {
+		Path string
+		Base string
+	}
+	Image    string
+	Platform string
+	Volumes  volumeList
+	Networks networkList
+	Services containerList
+	Script   containerList
+	Cache    container
+	Clone    container
+	Build    build
+}

engine/compiler/parse/node_root_test.go

@@ -0,0 +1,85 @@
+package parse
+
+import (
+	"testing"
+
+	"github.com/franela/goblin"
+)
+
+func TestRootNode(t *testing.T) {
+	g := goblin.Goblin(t)
+	r := &RootNode{}
+
+	g.Describe("Root Node", func() {
+
+		g.It("should return self as root", func() {
+			g.Assert(r).Equal(r.Root())
+		})
+
+		g.It("should create a Volume Node", func() {
+			n := r.NewVolumeNode("foo")
+			g.Assert(n.Root()).Equal(r)
+			g.Assert(n.Name).Equal("foo")
+			g.Assert(n.String()).Equal(NodeVolume)
+			g.Assert(n.Type()).Equal(NodeType(NodeVolume))
+		})
+
+		g.It("should create a Network Node", func() {
+			n := r.NewNetworkNode("foo")
+			g.Assert(n.Root()).Equal(r)
+			g.Assert(n.Name).Equal("foo")
+			g.Assert(n.String()).Equal(NodeNetwork)
+			g.Assert(n.Type()).Equal(NodeType(NodeNetwork))
+		})
+
+		g.It("should create a Plugin Node", func() {
+			n := r.NewPluginNode()
+			g.Assert(n.Root()).Equal(r)
+			g.Assert(n.String()).Equal(NodePlugin)
+			g.Assert(n.Type()).Equal(NodeType(NodePlugin))
+		})
+
+		g.It("should create a Shell Node", func() {
+			n := r.NewShellNode()
+			g.Assert(n.Root()).Equal(r)
+			g.Assert(n.String()).Equal(NodeShell)
+			g.Assert(n.Type()).Equal(NodeType(NodeShell))
+		})
+
+		g.It("should create a Service Node", func() {
+			n := r.NewServiceNode()
+			g.Assert(n.Root()).Equal(r)
+			g.Assert(n.String()).Equal(NodeService)
+			g.Assert(n.Type()).Equal(NodeType(NodeService))
+		})
+
+		g.It("should create a Build Node", func() {
+			n := r.NewBuildNode(".")
+			g.Assert(n.Root()).Equal(r)
+			g.Assert(n.Context).Equal(".")
+			g.Assert(n.String()).Equal(NodeBuild)
+			g.Assert(n.Type()).Equal(NodeType(NodeBuild))
+		})
+
+		g.It("should create a Cache Node", func() {
+			n := r.NewCacheNode()
+			g.Assert(n.Root()).Equal(r)
+			g.Assert(n.String()).Equal(NodeCache)
+			g.Assert(n.Type()).Equal(NodeType(NodeCache))
+		})
+
+		g.It("should create a Clone Node", func() {
+			n := r.NewCloneNode()
+			g.Assert(n.Root()).Equal(r)
+			g.Assert(n.String()).Equal(NodeClone)
+			g.Assert(n.Type()).Equal(NodeType(NodeClone))
+		})
+
+		g.It("should create a Container Node", func() {
+			n := r.NewContainerNode()
+			g.Assert(n.Root()).Equal(r)
+			g.Assert(n.String()).Equal(NodeContainer)
+			g.Assert(n.Type()).Equal(NodeType(NodeContainer))
+		})
+	})
+}

engine/compiler/parse/node_volume.go

@@ -0,0 +1,69 @@
+package parse
+
+import (
+	"fmt"
+
+	"gopkg.in/yaml.v2"
+)
+
+// VolumeNode represents a Docker volume.
+type VolumeNode struct {
+	NodeType
+	root *RootNode
+
+	Name       string
+	Driver     string
+	DriverOpts map[string]string
+	External   bool
+}
+
+// Root returns the root node.
+func (n *VolumeNode) Root() *RootNode { return n.root }
+
+//
+// intermediate types for yaml decoding.
+//
+
+// volume is an intermediate type used for decoding a volumes in a format
+// compatible with docker-compose.yml
+type volume struct {
+	Name       string
+	Driver     string
+	DriverOpts map[string]string `yaml:"driver_opts"`
+}
+
+// volumeList is an intermediate type used for decoding a slice of volumes
+// in a format compatible with docker-compose.yml
+type volumeList struct {
+	volumes []*volume
+}
+
+func (v *volumeList) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	slice := yaml.MapSlice{}
+	err := unmarshal(&slice)
+	if err != nil {
+		return err
+	}
+
+	for _, s := range slice {
+		vv := volume{}
+
+		out, err := yaml.Marshal(s.Value)
+		if err != nil {
+			return err
+		}
+
+		err = yaml.Unmarshal(out, &vv)
+		if err != nil {
+			return err
+		}
+		if vv.Name == "" {
+			vv.Name = fmt.Sprintf("%v", s.Key)
+		}
+		if vv.Driver == "" {
+			vv.Driver = "local"
+		}
+		v.volumes = append(v.volumes, &vv)
+	}
+	return err
+}

engine/compiler/parse/node_volume_test.go

@@ -0,0 +1,51 @@
+package parse
+
+import (
+	"testing"
+
+	"github.com/franela/goblin"
+	"gopkg.in/yaml.v2"
+)
+
+func TestVolumeNode(t *testing.T) {
+	g := goblin.Goblin(t)
+
+	g.Describe("Volumes", func() {
+		g.Describe("given a yaml file", func() {
+
+			g.It("should unmarshal", func() {
+				in := []byte("foo: { driver: blockbridge }")
+				out := volumeList{}
+				err := yaml.Unmarshal(in, &out)
+				if err != nil {
+					g.Fail(err)
+				}
+				g.Assert(len(out.volumes)).Equal(1)
+				g.Assert(out.volumes[0].Name).Equal("foo")
+				g.Assert(out.volumes[0].Driver).Equal("blockbridge")
+			})
+
+			g.It("should unmarshal named", func() {
+				in := []byte("foo: { name: bar }")
+				out := volumeList{}
+				err := yaml.Unmarshal(in, &out)
+				if err != nil {
+					g.Fail(err)
+				}
+				g.Assert(len(out.volumes)).Equal(1)
+				g.Assert(out.volumes[0].Name).Equal("bar")
+			})
+
+			g.It("should unmarshal and use default driver", func() {
+				in := []byte("foo: { name: bar }")
+				out := volumeList{}
+				err := yaml.Unmarshal(in, &out)
+				if err != nil {
+					g.Fail(err)
+				}
+				g.Assert(len(out.volumes)).Equal(1)
+				g.Assert(out.volumes[0].Driver).Equal("local")
+			})
+		})
+	})
+}

engine/compiler/parse/parse.go

@@ -0,0 +1,100 @@
+package parse
+
+import (
+	"gopkg.in/yaml.v2"
+)
+
+// Parse parses a Yaml file and returns a Tree structure.
+func Parse(in []byte) (*RootNode, error) {
+	out := root{}
+	err := yaml.Unmarshal(in, &out)
+	if err != nil {
+		return nil, err
+	}
+
+	root := NewRootNode()
+	root.Platform = out.Platform
+	root.Path = out.Workspace.Path
+	root.Base = out.Workspace.Base
+	root.Image = out.Image
+
+	// append volume nodes to tree
+	for _, v := range out.Volumes.volumes {
+		vv := root.NewVolumeNode(v.Name)
+		vv.Driver = v.Driver
+		vv.DriverOpts = v.DriverOpts
+		root.Volumes = append(root.Volumes, vv)
+	}
+
+	// append network nodes to tree
+	for _, n := range out.Networks.networks {
+		nn := root.NewNetworkNode(n.Name)
+		nn.Driver = n.Driver
+		nn.DriverOpts = n.DriverOpts
+		root.Networks = append(root.Networks, nn)
+	}
+
+	// add the build section
+	if out.Build.Context != "" {
+		root.Build = &BuildNode{
+			NodeType:   NodeBuild,
+			Context:    out.Build.Context,
+			Dockerfile: out.Build.Dockerfile,
+			Args:       out.Build.Args,
+			root:       root,
+		}
+	}
+
+	// add the cache section
+	{
+		cc := root.NewCacheNode()
+		cc.Container = out.Cache.ToContainer()
+		cc.Conditions = out.Cache.ToConditions()
+		cc.Container.Name = "cache"
+		cc.Vargs = out.Cache.Vargs
+		root.Cache = cc
+	}
+
+	// add the clone section
+	{
+		cc := root.NewCloneNode()
+		cc.Conditions = out.Clone.ToConditions()
+		cc.Container = out.Clone.ToContainer()
+		cc.Container.Name = "clone"
+		cc.Vargs = out.Clone.Vargs
+		root.Clone = cc
+	}
+
+	// append services
+	for _, c := range out.Services.containers {
+		if c.Build != "" {
+			continue
+		}
+		cc := root.NewServiceNode()
+		cc.Conditions = c.ToConditions()
+		cc.Container = c.ToContainer()
+		root.Services = append(root.Services, cc)
+	}
+
+	// append scripts
+	for _, c := range out.Script.containers {
+		var cc *ContainerNode
+		if len(c.Commands.parts) == 0 {
+			cc = root.NewPluginNode()
+		} else {
+			cc = root.NewShellNode()
+		}
+		cc.Commands = c.Commands.parts
+		cc.Vargs = c.Vargs
+		cc.Container = c.ToContainer()
+		cc.Conditions = c.ToConditions()
+		root.Script = append(root.Script, cc)
+	}
+
+	return root, nil
+}
+
+// ParseString parses a Yaml string and returns a Tree structure.
+func ParseString(in string) (*RootNode, error) {
+	return Parse([]byte(in))
+}

engine/compiler/parse/parse_test.go

@@ -0,0 +1,96 @@
+package parse
+
+import (
+	"testing"
+
+	"github.com/franela/goblin"
+)
+
+func TestParse(t *testing.T) {
+	g := goblin.Goblin(t)
+
+	g.Describe("Parser", func() {
+		g.Describe("given a yaml file", func() {
+
+			g.It("should unmarshal a string", func() {
+				out, err := ParseString(sampleYaml)
+				if err != nil {
+					g.Fail(err)
+				}
+				g.Assert(out.Image).Equal("hello-world")
+				g.Assert(out.Base).Equal("/go")
+				g.Assert(out.Path).Equal("src/github.com/octocat/hello-world")
+				g.Assert(out.Build.(*BuildNode).Context).Equal(".")
+				g.Assert(out.Build.(*BuildNode).Dockerfile).Equal("Dockerfile")
+				g.Assert(out.Cache.(*ContainerNode).Vargs["mount"]).Equal("node_modules")
+				g.Assert(out.Clone.(*ContainerNode).Container.Image).Equal("git")
+				g.Assert(out.Clone.(*ContainerNode).Vargs["depth"]).Equal(1)
+				g.Assert(out.Volumes[0].(*VolumeNode).Name).Equal("custom")
+				g.Assert(out.Volumes[0].(*VolumeNode).Driver).Equal("blockbridge")
+				g.Assert(out.Networks[0].(*NetworkNode).Name).Equal("custom")
+				g.Assert(out.Networks[0].(*NetworkNode).Driver).Equal("overlay")
+				g.Assert(out.Services[0].(*ContainerNode).Container.Name).Equal("database")
+				g.Assert(out.Services[0].(*ContainerNode).Container.Image).Equal("mysql")
+				g.Assert(out.Script[0].(*ContainerNode).Container.Name).Equal("test")
+				g.Assert(out.Script[0].(*ContainerNode).Container.Image).Equal("golang")
+				g.Assert(out.Script[0].(*ContainerNode).Commands).Equal([]string{"go install", "go test"})
+				g.Assert(out.Script[0].(*ContainerNode).String()).Equal(NodeShell)
+				g.Assert(out.Script[1].(*ContainerNode).Container.Name).Equal("build")
+				g.Assert(out.Script[1].(*ContainerNode).Container.Image).Equal("golang")
+				g.Assert(out.Script[1].(*ContainerNode).Commands).Equal([]string{"go build"})
+				g.Assert(out.Script[1].(*ContainerNode).String()).Equal(NodeShell)
+				g.Assert(out.Script[2].(*ContainerNode).Container.Name).Equal("notify")
+				g.Assert(out.Script[2].(*ContainerNode).Container.Image).Equal("slack")
+				g.Assert(out.Script[2].(*ContainerNode).String()).Equal(NodePlugin)
+			})
+		})
+	})
+}
+
+var sampleYaml = `
+image: hello-world
+build:
+  context: .
+  dockerfile: Dockerfile
+
+workspace:
+  path: src/github.com/octocat/hello-world
+  base: /go
+
+clone:
+  image: git
+  depth: 1
+
+cache:
+  mount: node_modules
+
+script:
+  test:
+    image: golang
+    commands:
+      - go install
+      - go test
+  build:
+    image: golang
+    commands:
+      - go build
+    when:
+      event: push
+  notify:
+    image: slack
+    channel: dev
+    when:
+      event: failure
+
+services:
+  database:
+    image: mysql
+
+networks:
+  custom:
+    driver: overlay
+
+volumes:
+  custom:
+    driver: blockbridge
+`

engine/compiler/parse/types.go

@@ -0,0 +1,55 @@
+package parse
+
+import "strings"
+
+// mapEqualSlice represents a map[string]string or a slice of
+// strings in key=value format.
+type mapEqualSlice struct {
+	parts map[string]string
+}
+
+func (s *mapEqualSlice) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	s.parts = map[string]string{}
+	err := unmarshal(&s.parts)
+	if err == nil {
+		return nil
+	}
+
+	var slice []string
+	err = unmarshal(&slice)
+	if err != nil {
+		return err
+	}
+	for _, v := range slice {
+		parts := strings.SplitN(v, "=", 2)
+		if len(parts) == 2 {
+			key := parts[0]
+			val := parts[1]
+			s.parts[key] = val
+		}
+	}
+	return nil
+}
+
+// stringOrSlice represents a string or an array of strings.
+type stringOrSlice struct {
+	parts []string
+}
+
+func (s *stringOrSlice) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	var sliceType []string
+	err := unmarshal(&sliceType)
+	if err == nil {
+		s.parts = sliceType
+		return nil
+	}
+
+	var stringType string
+	err = unmarshal(&stringType)
+	if err == nil {
+		sliceType = make([]string, 0, 1)
+		s.parts = append(sliceType, string(stringType))
+		return nil
+	}
+	return err
+}

engine/compiler/parse/types_test.go

@@ -0,0 +1,75 @@
+package parse
+
+import (
+	"testing"
+
+	"github.com/franela/goblin"
+	"gopkg.in/yaml.v2"
+)
+
+func TestTypes(t *testing.T) {
+	g := goblin.Goblin(t)
+
+	g.Describe("Yaml types", func() {
+		g.Describe("given a yaml file", func() {
+
+			g.It("should unmarshal a string", func() {
+				in := []byte("foo")
+				out := stringOrSlice{}
+				err := yaml.Unmarshal(in, &out)
+				if err != nil {
+					g.Fail(err)
+				}
+				g.Assert(len(out.parts)).Equal(1)
+				g.Assert(out.parts[0]).Equal("foo")
+			})
+
+			g.It("should unmarshal a string slice", func() {
+				in := []byte("[ foo ]")
+				out := stringOrSlice{}
+				err := yaml.Unmarshal(in, &out)
+				if err != nil {
+					g.Fail(err)
+				}
+				g.Assert(len(out.parts)).Equal(1)
+				g.Assert(out.parts[0]).Equal("foo")
+			})
+
+			g.It("should throw error when invalid string slice", func() {
+				in := []byte("{ }") // string value should fail parse
+				out := stringOrSlice{}
+				err := yaml.Unmarshal(in, &out)
+				g.Assert(err != nil).IsTrue("expects error")
+			})
+
+			g.It("should unmarshal a map", func() {
+				in := []byte("foo: bar")
+				out := mapEqualSlice{}
+				err := yaml.Unmarshal(in, &out)
+				if err != nil {
+					g.Fail(err)
+				}
+				g.Assert(len(out.parts)).Equal(1)
+				g.Assert(out.parts["foo"]).Equal("bar")
+			})
+
+			g.It("should unmarshal a map equal slice", func() {
+				in := []byte("[ foo=bar ]")
+				out := mapEqualSlice{}
+				err := yaml.Unmarshal(in, &out)
+				if err != nil {
+					g.Fail(err)
+				}
+				g.Assert(len(out.parts)).Equal(1)
+				g.Assert(out.parts["foo"]).Equal("bar")
+			})
+
+			g.It("should throw error when invalid map equal slice", func() {
+				in := []byte("foo") // string value should fail parse
+				out := mapEqualSlice{}
+				err := yaml.Unmarshal(in, &out)
+				g.Assert(err != nil).IsTrue("expects error")
+			})
+		})
+	})
+}

engine/compiler/transform.go

@@ -0,0 +1,13 @@
+package libyaml
+
+import "github.com/drone/drone/engine/compiler/parse"
+
+// Transform is used to transform nodes from the parsed Yaml file during the
+// compilation process. A Transform may be used to add, disable or alter nodes.
+type Transform interface {
+	VisitRoot(*parse.RootNode) error
+	VisitVolume(*parse.VolumeNode) error
+	VisitNetwork(*parse.NetworkNode) error
+	VisitBuild(*parse.BuildNode) error
+	VisitContainer(*parse.ContainerNode) error
+}

engine/context.go

@@ -0,0 +1,23 @@
+package engine
+
+import (
+	"golang.org/x/net/context"
+)
+
+const key = "engine"
+
+// Setter defines a context that enables setting values.
+type Setter interface {
+	Set(string, interface{})
+}
+
+// FromContext returns the Engine associated with this context.
+func FromContext(c context.Context) Engine {
+	return c.Value(key).(Engine)
+}
+
+// ToContext adds the Engine to this context if it supports
+// the Setter interface.
+func ToContext(c Setter, engine Engine) {
+	c.Set(key, engine)
+}

engine/engine.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	"os"
 	"runtime"
 	"time"
 
@@ -15,7 +16,6 @@ import (
 	"github.com/docker/docker/pkg/stdcopy"
 	"github.com/drone/drone/model"
 	"github.com/drone/drone/shared/docker"
-	"github.com/drone/drone/shared/envconfig"
 	"github.com/drone/drone/store"
 	"github.com/samalba/dockerclient"
 	"golang.org/x/net/context"
@@ -60,7 +60,7 @@ type engine struct {
 // Load creates a new build engine, loaded with registered nodes from the
 // database. The registered nodes are added to the pool of nodes to immediately
 // start accepting workloads.
-func Load(env envconfig.Env, s store.Store) Engine {
+func Load(s store.Store) Engine {
 	engine := &engine{}
 	engine.bus = newEventbus()
 	engine.pool = newPool()
@@ -70,7 +70,7 @@ func Load(env envconfig.Env, s store.Store) Engine {
 	// throughout the build environment.
 	var proxyVars = []string{"HTTP_PROXY", "http_proxy", "HTTPS_PROXY", "https_proxy", "NO_PROXY", "no_proxy"}
 	for _, proxyVar := range proxyVars {
-		proxyVal := env.Get(proxyVar)
+		proxyVal := os.Getenv(proxyVar)
 		if len(proxyVal) != 0 {
 			engine.envs = append(engine.envs, proxyVar+"="+proxyVal)
 		}

engine/expander/expand.go

@@ -0,0 +1,33 @@
+package expander
+
+import "sort"
+
+// Expand expands variables into the Yaml configuration using a
+// ${key} template parameter with limited support for bash string functions.
+func Expand(config []byte, envs map[string]string) []byte {
+	return []byte(
+		ExpandString(string(config), envs),
+	)
+}
+
+// ExpandString injects the variables into the Yaml configuration string using
+// a ${key} template parameter with limited support for bash string functions.
+func ExpandString(config string, envs map[string]string) string {
+	if envs == nil || len(envs) == 0 {
+		return config
+	}
+	keys := []string{}
+	for k := range envs {
+		keys = append(keys, k)
+	}
+	sort.Sort(sort.Reverse(sort.StringSlice(keys)))
+	expanded := config
+	for _, k := range keys {
+		v := envs[k]
+
+		for _, substitute := range substitutors {
+			expanded = substitute(expanded, k, v)
+		}
+	}
+	return expanded
+}

engine/expander/expand_test.go

@@ -0,0 +1,48 @@
+package expander
+
+import (
+	"testing"
+
+	"github.com/franela/goblin"
+)
+
+func TestExpand(t *testing.T) {
+
+	g := goblin.Goblin(t)
+	g.Describe("Expand params", func() {
+
+		g.It("Should replace vars with ${key}", func() {
+			s := "echo ${FOO} $BAR"
+			m := map[string]string{}
+			m["FOO"] = "BAZ"
+			g.Assert("echo BAZ $BAR").Equal(ExpandString(s, m))
+		})
+
+		g.It("Should not replace vars in nil map", func() {
+			s := "echo ${FOO} $BAR"
+			g.Assert(s).Equal(ExpandString(s, nil))
+		})
+
+		g.It("Should escape quoted variables", func() {
+			s := `echo "${FOO}"`
+			m := map[string]string{}
+			m["FOO"] = "hello\nworld"
+			g.Assert(`echo "hello\nworld"`).Equal(ExpandString(s, m))
+		})
+
+		g.It("Should replace variable prefix", func() {
+			s := `tag: ${TAG=${SHA:8}}`
+			m := map[string]string{}
+			m["TAG"] = ""
+			m["SHA"] = "f36cbf54ee1a1eeab264c8e388f386218ab1701b"
+			g.Assert("tag: f36cbf54").Equal(ExpandString(s, m))
+		})
+
+		g.It("Should handle nested substitution operations", func() {
+			s := `echo "${TAG##v}"`
+			m := map[string]string{}
+			m["TAG"] = "v1.0.0"
+			g.Assert(`echo "1.0.0"`).Equal(ExpandString(s, m))
+		})
+	})
+}

engine/expander/func.go

@@ -0,0 +1,172 @@
+package expander
+
+import (
+	"fmt"
+	"regexp"
+	"strconv"
+	"strings"
+)
+
+// these are helper functions that bring bash-substitution to the drone yaml file.
+// see http://tldp.org/LDP/abs/html/parameter-substitution.html
+
+type substituteFunc func(str, key, val string) string
+
+var substitutors = []substituteFunc{
+	substituteQ,
+	substitute,
+	substitutePrefix,
+	substituteSuffix,
+	substituteDefault,
+	substituteReplace,
+	substituteLeft,
+	substituteSubstr,
+}
+
+// substitute is a helper function that substitutes a simple parameter using
+// ${parameter} notation.
+func substitute(str, key, val string) string {
+	key = fmt.Sprintf("${%s}", key)
+	return strings.Replace(str, key, val, -1)
+}
+
+// substituteQ is a helper function that substitutes a simple parameter using
+// "${parameter}" notation with the escaped value, using %q.
+func substituteQ(str, key, val string) string {
+	key = fmt.Sprintf(`"${%s}"`, key)
+	val = fmt.Sprintf("%q", val)
+	return strings.Replace(str, key, val, -1)
+}
+
+// substitutePrefix is a helper function that substitutes paramters using
+// ${parameter##prefix} notation with the parameter value minus the trimmed prefix.
+func substitutePrefix(str, key, val string) string {
+	key = fmt.Sprintf("\\${%s##(.+)}", key)
+	reg, err := regexp.Compile(key)
+	if err != nil {
+		return str
+	}
+	for _, match := range reg.FindAllStringSubmatch(str, -1) {
+		if len(match) != 2 {
+			continue
+		}
+		val_ := strings.TrimPrefix(val, match[1])
+		str = strings.Replace(str, match[0], val_, -1)
+	}
+	return str
+}
+
+// substituteSuffix is a helper function that substitutes paramters using
+// ${parameter%%suffix} notation with the parameter value minus the trimmed suffix.
+func substituteSuffix(str, key, val string) string {
+	key = fmt.Sprintf("\\${%s%%%%(.+)}", key)
+	reg, err := regexp.Compile(key)
+	if err != nil {
+		return str
+	}
+	for _, match := range reg.FindAllStringSubmatch(str, -1) {
+		if len(match) != 2 {
+			continue
+		}
+		val_ := strings.TrimSuffix(val, match[1])
+		str = strings.Replace(str, match[0], val_, -1)
+	}
+	return str
+}
+
+// substituteDefault is a helper function that substitutes paramters using
+// ${parameter=default} notation with the parameter value. When empty the
+// default value is used.
+func substituteDefault(str, key, val string) string {
+	key = fmt.Sprintf("\\${%s=(.+)}", key)
+	reg, err := regexp.Compile(key)
+	if err != nil {
+		return str
+	}
+	for _, match := range reg.FindAllStringSubmatch(str, -1) {
+		if len(match) != 2 {
+			continue
+		}
+		if len(val) == 0 {
+			str = strings.Replace(str, match[0], match[1], -1)
+		} else {
+			str = strings.Replace(str, match[0], val, -1)
+		}
+	}
+	return str
+}
+
+// substituteReplace is a helper function that substitutes paramters using
+// ${parameter/old/new} notation with the parameter value. A find and replace
+// is performed before injecting the strings, replacing the old pattern with
+// the new value.
+func substituteReplace(str, key, val string) string {
+	key = fmt.Sprintf("\\${%s/(.+)/(.+)}", key)
+	reg, err := regexp.Compile(key)
+	if err != nil {
+		return str
+	}
+	for _, match := range reg.FindAllStringSubmatch(str, -1) {
+		if len(match) != 3 {
+			continue
+		}
+		with := strings.Replace(val, match[1], match[2], -1)
+		str = strings.Replace(str, match[0], with, -1)
+	}
+	return str
+}
+
+// substituteLeft is a helper function that substitutes paramters using
+// ${parameter:pos} notation with the parameter value, sliced up to the
+// specified position.
+func substituteLeft(str, key, val string) string {
+	key = fmt.Sprintf("\\${%s:([0-9]*)}", key)
+	reg, err := regexp.Compile(key)
+	if err != nil {
+		return str
+	}
+	for _, match := range reg.FindAllStringSubmatch(str, -1) {
+		if len(match) != 2 {
+			continue
+		}
+		index, err := strconv.Atoi(match[1])
+		if err != nil {
+			continue // skip
+		}
+		if index > len(val)-1 {
+			continue // skip
+		}
+
+		str = strings.Replace(str, match[0], val[:index], -1)
+	}
+	return str
+}
+
+// substituteLeft is a helper function that substitutes paramters using
+// ${parameter:pos:len} notation with the parameter value as a substring,
+// starting at the specified position for the specified length.
+func substituteSubstr(str, key, val string) string {
+	key = fmt.Sprintf("\\${%s:([0-9]*):([0-9]*)}", key)
+	reg, err := regexp.Compile(key)
+	if err != nil {
+		return str
+	}
+	for _, match := range reg.FindAllStringSubmatch(str, -1) {
+		if len(match) != 3 {
+			continue
+		}
+		pos, err := strconv.Atoi(match[1])
+		if err != nil {
+			continue // skip
+		}
+		length, err := strconv.Atoi(match[2])
+		if err != nil {
+			continue // skip
+		}
+		if pos+length > len(val)-1 {
+			continue // skip
+		}
+		str = strings.Replace(str, match[0], val[pos:pos+length], -1)
+	}
+	return str
+}

engine/expander/func_test.go

@@ -0,0 +1,68 @@
+package expander
+
+import (
+	"testing"
+
+	"github.com/franela/goblin"
+)
+
+func TestSubstitution(t *testing.T) {
+
+	g := goblin.Goblin(t)
+	g.Describe("Parameter Substitution", func() {
+
+		g.It("Should substitute simple parameters", func() {
+			before := "echo ${GREETING} WORLD"
+			after := "echo HELLO WORLD"
+			g.Assert(substitute(before, "GREETING", "HELLO")).Equal(after)
+		})
+
+		g.It("Should substitute quoted parameters", func() {
+			before := "echo \"${GREETING}\" WORLD"
+			after := "echo \"HELLO\" WORLD"
+			g.Assert(substituteQ(before, "GREETING", "HELLO")).Equal(after)
+		})
+
+		g.It("Should substitute parameters and trim prefix", func() {
+			before := "echo ${GREETING##asdf} WORLD"
+			after := "echo HELLO WORLD"
+			g.Assert(substitutePrefix(before, "GREETING", "asdfHELLO")).Equal(after)
+		})
+
+		g.It("Should substitute parameters and trim suffix", func() {
+			before := "echo ${GREETING%%asdf} WORLD"
+			after := "echo HELLO WORLD"
+			g.Assert(substituteSuffix(before, "GREETING", "HELLOasdf")).Equal(after)
+		})
+
+		g.It("Should substitute parameters without using the default", func() {
+			before := "echo ${GREETING=HOLA} WORLD"
+			after := "echo HELLO WORLD"
+			g.Assert(substituteDefault(before, "GREETING", "HELLO")).Equal(after)
+		})
+
+		g.It("Should substitute parameters using the a default", func() {
+			before := "echo ${GREETING=HOLA} WORLD"
+			after := "echo HOLA WORLD"
+			g.Assert(substituteDefault(before, "GREETING", "")).Equal(after)
+		})
+
+		g.It("Should substitute parameters with replacement", func() {
+			before := "echo ${GREETING/HE/A} MONDE"
+			after := "echo ALLO MONDE"
+			g.Assert(substituteReplace(before, "GREETING", "HELLO")).Equal(after)
+		})
+
+		g.It("Should substitute parameters with left substr", func() {
+			before := "echo ${FOO:4} IS COOL"
+			after := "echo THIS IS COOL"
+			g.Assert(substituteLeft(before, "FOO", "THIS IS A REALLY LONG STRING")).Equal(after)
+		})
+
+		g.It("Should substitute parameters with substr", func() {
+			before := "echo ${FOO:8:5} IS COOL"
+			after := "echo DRONE IS COOL"
+			g.Assert(substituteSubstr(before, "FOO", "THIS IS DRONE CI")).Equal(after)
+		})
+	})
+}

engine/parser/branch.go

@@ -0,0 +1,77 @@
+package parser
+
+import (
+	"path/filepath"
+
+	"gopkg.in/yaml.v2"
+)
+
+type Branch struct {
+	Include []string `yaml:"include"`
+	Exclude []string `yaml:"exclude"`
+}
+
+// ParseBranch parses the branch section of the Yaml document.
+func ParseBranch(in []byte) *Branch {
+	return parseBranch(in)
+}
+
+// ParseBranchString parses the branch section of the Yaml document.
+func ParseBranchString(in string) *Branch {
+	return ParseBranch([]byte(in))
+}
+
+// Matches returns true if the branch matches the include patterns and
+// does not match any of the exclude patterns.
+func (b *Branch) Matches(branch string) bool {
+	// when no includes or excludes automatically match
+	if len(b.Include) == 0 && len(b.Exclude) == 0 {
+		return true
+	}
+
+	// exclusions are processed first. So we can include everything and
+	// then selectively exclude certain sub-patterns.
+	for _, pattern := range b.Exclude {
+		if pattern == branch {
+			return false
+		}
+		if ok, _ := filepath.Match(pattern, branch); ok {
+			return false
+		}
+	}
+
+	for _, pattern := range b.Include {
+		if pattern == branch {
+			return true
+		}
+		if ok, _ := filepath.Match(pattern, branch); ok {
+			return true
+		}
+	}
+
+	return false
+}
+
+func parseBranch(in []byte) *Branch {
+	out1 := struct {
+		Branch struct {
+			Include stringOrSlice `yaml:"include"`
+			Exclude stringOrSlice `yaml:"exclude"`
+		} `yaml:"branches"`
+	}{}
+
+	out2 := struct {
+		Include stringOrSlice `yaml:"branches"`
+	}{}
+
+	yaml.Unmarshal(in, &out1)
+	yaml.Unmarshal(in, &out2)
+
+	return &Branch{
+		Exclude: out1.Branch.Exclude.Slice(),
+		Include: append(
+			out1.Branch.Include.Slice(),
+			out2.Include.Slice()...,
+		),
+	}
+}

engine/parser/branch_test.go

@@ -0,0 +1,74 @@
+package parser
+
+import (
+	"testing"
+
+	"github.com/franela/goblin"
+)
+
+func TestBranch(t *testing.T) {
+
+	g := goblin.Goblin(t)
+	g.Describe("Branch filter", func() {
+
+		g.It("Should parse and match emtpy", func() {
+			branch := ParseBranchString("")
+			g.Assert(branch.Matches("master")).IsTrue()
+		})
+
+		g.It("Should parse and match", func() {
+			branch := ParseBranchString("branches: { include: [ master, develop ] }")
+			g.Assert(branch.Matches("master")).IsTrue()
+		})
+
+		g.It("Should parse and match shortand", func() {
+			branch := ParseBranchString("branches: [ master, develop ]")
+			g.Assert(branch.Matches("master")).IsTrue()
+		})
+
+		g.It("Should parse and match shortand string", func() {
+			branch := ParseBranchString("branches: master")
+			g.Assert(branch.Matches("master")).IsTrue()
+		})
+
+		g.It("Should parse and match exclude", func() {
+			branch := ParseBranchString("branches: { exclude: [ master, develop ] }")
+			g.Assert(branch.Matches("master")).IsFalse()
+		})
+
+		g.It("Should parse and match exclude shorthand", func() {
+			branch := ParseBranchString("branches: { exclude: master }")
+			g.Assert(branch.Matches("master")).IsFalse()
+		})
+
+		g.It("Should match include", func() {
+			b := Branch{}
+			b.Include = []string{"master"}
+			g.Assert(b.Matches("master")).IsTrue()
+		})
+
+		g.It("Should match include pattern", func() {
+			b := Branch{}
+			b.Include = []string{"feature/*"}
+			g.Assert(b.Matches("feature/foo")).IsTrue()
+		})
+
+		g.It("Should fail to match include pattern", func() {
+			b := Branch{}
+			b.Include = []string{"feature/*"}
+			g.Assert(b.Matches("master")).IsFalse()
+		})
+
+		g.It("Should match exclude", func() {
+			b := Branch{}
+			b.Exclude = []string{"master"}
+			g.Assert(b.Matches("master")).IsFalse()
+		})
+
+		g.It("Should match exclude pattern", func() {
+			b := Branch{}
+			b.Exclude = []string{"feature/*"}
+			g.Assert(b.Matches("feature/foo")).IsFalse()
+		})
+	})
+}

engine/parser/matrix.go

@@ -0,0 +1,100 @@
+package parser
+
+import (
+	"strings"
+
+	"gopkg.in/yaml.v2"
+)
+
+const (
+	limitTags = 10
+	limitAxis = 25
+)
+
+// Matrix represents the build matrix.
+type Matrix map[string][]string
+
+// Axis represents a single permutation of entries from the build matrix.
+type Axis map[string]string
+
+// String returns a string representation of an Axis as a comma-separated list
+// of environment variables.
+func (a Axis) String() string {
+	var envs []string
+	for k, v := range a {
+		envs = append(envs, k+"="+v)
+	}
+	return strings.Join(envs, " ")
+}
+
+// ParseMatrix parses the Yaml matrix definition.
+func ParseMatrix(data []byte) ([]Axis, error) {
+	matrix, err := parseMatrix(data)
+	if err != nil {
+		return nil, err
+	}
+
+	// if not a matrix build return an array with just the single axis.
+	if len(matrix) == 0 {
+		return nil, nil
+	}
+
+	return calcMatrix(matrix), nil
+}
+
+// ParseMatrixString parses the Yaml string matrix definition.
+func ParseMatrixString(data string) ([]Axis, error) {
+	return ParseMatrix([]byte(data))
+}
+
+func calcMatrix(matrix Matrix) []Axis {
+	// calculate number of permutations and extract the list of tags
+	// (ie go_version, redis_version, etc)
+	var perm int
+	var tags []string
+	for k, v := range matrix {
+		perm *= len(v)
+		if perm == 0 {
+			perm = len(v)
+		}
+		tags = append(tags, k)
+	}
+
+	// structure to hold the transformed result set
+	axisList := []Axis{}
+
+	// for each axis calculate the uniqe set of values that should be used.
+	for p := 0; p < perm; p++ {
+		axis := map[string]string{}
+		decr := perm
+		for i, tag := range tags {
+			elems := matrix[tag]
+			decr = decr / len(elems)
+			elem := p / decr % len(elems)
+			axis[tag] = elems[elem]
+
+			// enforce a maximum number of tags in the build matrix.
+			if i > limitTags {
+				break
+			}
+		}
+
+		// append to the list of axis.
+		axisList = append(axisList, axis)
+
+		// enforce a maximum number of axis that should be calculated.
+		if p > limitAxis {
+			break
+		}
+	}
+
+	return axisList
+}
+
+func parseMatrix(raw []byte) (Matrix, error) {
+	data := struct {
+		Matrix map[string][]string
+	}{}
+	err := yaml.Unmarshal(raw, &data)
+	return data.Matrix, err
+}

engine/parser/matrix_test.go

@@ -1,4 +1,4 @@
-package matrix
+package parser
 
 import (
 	"testing"
@@ -6,12 +6,12 @@ import (
 	"github.com/franela/goblin"
 )
 
-func Test_Matrix(t *testing.T) {
+func TestMatrix(t *testing.T) {
 
 	g := goblin.Goblin(t)
 	g.Describe("Calculate matrix", func() {
 
-		axis, _ := Parse(fakeMatrix)
+		axis, _ := ParseMatrixString(fakeMatrix)
 
 		g.It("Should calculate permutations", func() {
 			g.Assert(len(axis)).Equal(24)
@@ -26,7 +26,7 @@ func Test_Matrix(t *testing.T) {
 		})
 
 		g.It("Should return nil if no matrix", func() {
-			axis, err := Parse("")
+			axis, err := ParseMatrixString("")
 			g.Assert(err == nil).IsTrue()
 			g.Assert(axis == nil).IsTrue()
 		})

engine/parser/types.go

@@ -0,0 +1,28 @@
+package parser
+
+// stringOrSlice represents a string or an array of strings.
+type stringOrSlice struct {
+	parts []string
+}
+
+func (s *stringOrSlice) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	var sliceType []string
+	err := unmarshal(&sliceType)
+	if err == nil {
+		s.parts = sliceType
+		return nil
+	}
+
+	var stringType string
+	err = unmarshal(&stringType)
+	if err == nil {
+		sliceType = make([]string, 0, 1)
+		s.parts = append(sliceType, string(stringType))
+		return nil
+	}
+	return err
+}
+
+func (s stringOrSlice) Slice() []string {
+	return s.parts
+}

engine/parser/types_test.go

@@ -0,0 +1,46 @@
+package parser
+
+import (
+	"testing"
+
+	"github.com/franela/goblin"
+	"gopkg.in/yaml.v2"
+)
+
+func TestTypes(t *testing.T) {
+	g := goblin.Goblin(t)
+
+	g.Describe("Yaml types", func() {
+		g.Describe("given a yaml file", func() {
+
+			g.It("should unmarshal a string", func() {
+				in := []byte("foo")
+				out := stringOrSlice{}
+				err := yaml.Unmarshal(in, &out)
+				if err != nil {
+					g.Fail(err)
+				}
+				g.Assert(len(out.parts)).Equal(1)
+				g.Assert(out.parts[0]).Equal("foo")
+			})
+
+			g.It("should unmarshal a string slice", func() {
+				in := []byte("[ foo ]")
+				out := stringOrSlice{}
+				err := yaml.Unmarshal(in, &out)
+				if err != nil {
+					g.Fail(err)
+				}
+				g.Assert(len(out.parts)).Equal(1)
+				g.Assert(out.parts[0]).Equal("foo")
+			})
+
+			g.It("should throw error when invalid string slice", func() {
+				in := []byte("{ }") // string value should fail parse
+				out := stringOrSlice{}
+				err := yaml.Unmarshal(in, &out)
+				g.Assert(err != nil).IsTrue("expects error")
+			})
+		})
+	})
+}

engine/runner/container.go

@@ -0,0 +1,72 @@
+package runner
+
+import "fmt"
+
+// Container defines the container configuration.
+type Container struct {
+	Name           string            `json:"name"`
+	Alias          string            `json:"alias"`
+	Image          string            `json:"image"`
+	Pull           bool              `json:"pull,omitempty"`
+	AuthConfig     Auth              `json:"auth_config,omitempty"`
+	Privileged     bool              `json:"privileged,omitempty"`
+	WorkingDir     string            `json:"working_dir,omitempty"`
+	Environment    map[string]string `json:"environment,omitempty"`
+	Entrypoint     []string          `json:"entrypoint,omitempty"`
+	Command        []string          `json:"command,omitempty"`
+	ExtraHosts     []string          `json:"extra_hosts,omitempty"`
+	Volumes        []string          `json:"volumes,omitempty"`
+	VolumesFrom    []string          `json:"volumes_from,omitempty"`
+	Devices        []string          `json:"devices,omitempty"`
+	Network        string            `json:"network_mode,omitempty"`
+	DNS            []string          `json:"dns,omitempty"`
+	DNSSearch      []string          `json:"dns_search,omitempty"`
+	MemSwapLimit   int64             `json:"memswap_limit,omitempty"`
+	MemLimit       int64             `json:"mem_limit,omitempty"`
+	CPUQuota       int64             `json:"cpu_quota,omitempty"`
+	CPUShares      int64             `json:"cpu_shares,omitempty"`
+	CPUSet         string            `json:"cpuset,omitempty"`
+	OomKillDisable bool              `json:"oom_kill_disable,omitempty"`
+}
+
+// Validate validates the container configuration details and returns an error
+// if the validation fails.
+func (c *Container) Validate() error {
+	switch {
+
+	case c.Name == "":
+		return fmt.Errorf("Missing container name")
+	case c.Image == "":
+		return fmt.Errorf("Missing container image")
+	default:
+		return nil
+	}
+
+}
+
+// Auth provides authentication parameters to authenticate to a remote
+// container registry for image download.
+type Auth struct {
+	Username string `json:"username,omitempty"`
+	Password string `json:"password,omitempty"`
+	Email    string `json:"email,omitempty"`
+	Token    string `json:"registry_token,omitempty"`
+}
+
+// Volume defines a container volume.
+type Volume struct {
+	Name       string            `json:"name,omitempty"`
+	Alias      string            `json:"alias,omitempty"`
+	Driver     string            `json:"driver,omitempty"`
+	DriverOpts map[string]string `json:"driver_opts,omitempty"`
+	External   bool              `json:"external,omitempty"`
+}
+
+// Network defines a container network.
+type Network struct {
+	Name       string            `json:"name,omitempty"`
+	Alias      string            `json:"alias,omitempty"`
+	Driver     string            `json:"driver,omitempty"`
+	DriverOpts map[string]string `json:"driver_opts,omitempty"`
+	External   bool              `json:"external,omitempty"`
+}

engine/runner/container_test.go

@@ -0,0 +1,40 @@
+package runner
+
+import (
+	"testing"
+
+	"github.com/franela/goblin"
+)
+
+func TestContainer(t *testing.T) {
+	g := goblin.Goblin(t)
+
+	g.Describe("Container validation", func() {
+
+		g.It("fails with an invalid name", func() {
+			c := Container{
+				Image: "golang:1.5",
+			}
+			err := c.Validate()
+			g.Assert(err != nil).IsTrue()
+			g.Assert(err.Error()).Equal("Missing container name")
+		})
+
+		g.It("fails with an invalid image", func() {
+			c := Container{
+				Name: "container_0",
+			}
+			err := c.Validate()
+			g.Assert(err != nil).IsTrue()
+			g.Assert(err.Error()).Equal("Missing container image")
+		})
+
+		g.It("passes with valid attributes", func() {
+			c := Container{
+				Name:  "container_0",
+				Image: "golang:1.5",
+			}
+			g.Assert(c.Validate() == nil).IsTrue()
+		})
+	})
+}

engine/runner/engine.go

@@ -0,0 +1,22 @@
+package runner
+
+//go:generate mockery -name Engine -output mock -case=underscore
+
+import "io"
+
+// Engine defines the container runtime engine.
+type Engine interface {
+	// VolumeCreate(*Volume) (string, error)
+	// VolumeRemove(string) error
+	ContainerStart(*Container) (string, error)
+	ContainerStop(string) error
+	ContainerRemove(string) error
+	ContainerWait(string) (*State, error)
+	ContainerLogs(string) (io.ReadCloser, error)
+}
+
+// State defines the state of the container.
+type State struct {
+	ExitCode  int  // container exit code
+	OOMKilled bool // container exited due to oom error
+}

engine/runner/error.go

@@ -0,0 +1,37 @@
+package runner
+
+import (
+	"errors"
+	"fmt"
+)
+
+var (
+	// ErrSkip is used as a return value when container execution should be
+	// skipped at runtime. It is not returned as an error by any function.
+	ErrSkip = errors.New("Skip")
+
+	// ErrTerm is used as a return value when the runner should terminate
+	// execution and exit. It is not returned as an error by any function.
+	ErrTerm = errors.New("Terminate")
+)
+
+// An ExitError reports an unsuccessful exit.
+type ExitError struct {
+	Name string
+	Code int
+}
+
+// Error reteurns the error message in string format.
+func (e *ExitError) Error() string {
+	return fmt.Sprintf("%s : exit code %d", e.Name, e.Code)
+}
+
+// An OomError reports the process received an OOMKill from the kernel.
+type OomError struct {
+	Name string
+}
+
+// Error reteurns the error message in string format.
+func (e *OomError) Error() string {
+	return fmt.Sprintf("%s : received oom kill", e.Name)
+}

engine/runner/error_test.go

@@ -0,0 +1,26 @@
+package runner
+
+import (
+	"testing"
+
+	"github.com/franela/goblin"
+)
+
+func TestErrors(t *testing.T) {
+	g := goblin.Goblin(t)
+
+	g.Describe("Error messages", func() {
+
+		g.It("should include OOM details", func() {
+			err := OomError{Name: "golang"}
+			got, want := err.Error(), "golang : received oom kill"
+			g.Assert(got).Equal(want)
+		})
+
+		g.It("should include Exit code", func() {
+			err := ExitError{Name: "golang", Code: 255}
+			got, want := err.Error(), "golang : exit code 255"
+			g.Assert(got).Equal(want)
+		})
+	})
+}

engine/runner/helper.go

@@ -0,0 +1,24 @@
+package runner
+
+import (
+	"encoding/json"
+	"io/ioutil"
+)
+
+// Parse parses a raw file containing a JSON encoded format of an intermediate
+// representation of the pipeline.
+func Parse(data []byte) (*Spec, error) {
+	v := &Spec{}
+	err := json.Unmarshal(data, v)
+	return v, err
+}
+
+// ParseFile parses a file containing a JSON encoded format of an intermediate
+// representation of the pipeline.
+func ParseFile(filename string) (*Spec, error) {
+	out, err := ioutil.ReadFile(filename)
+	if err != nil {
+		return nil, err
+	}
+	return Parse(out)
+}

engine/runner/helper_test.go

@@ -0,0 +1,97 @@
+package runner
+
+import (
+	"io/ioutil"
+	"os"
+	"testing"
+
+	"github.com/franela/goblin"
+)
+
+func TestHelper(t *testing.T) {
+	g := goblin.Goblin(t)
+
+	g.Describe("Parsing", func() {
+
+		g.It("should unmarhsal file []byte", func() {
+			res, err := Parse(sample)
+			if err != nil {
+				t.Error(err)
+				return
+			}
+			g.Assert(err == nil).IsTrue("expect file parsed")
+			g.Assert(len(res.Containers)).Equal(2)
+			g.Assert(len(res.Volumes)).Equal(1)
+		})
+
+		g.It("should unmarshal from file", func() {
+			temp, _ := ioutil.TempFile("", "spec_")
+			defer os.Remove(temp.Name())
+
+			ioutil.WriteFile(temp.Name(), sample, 0700)
+
+			_, err := ParseFile(temp.Name())
+			if err != nil {
+				t.Error(err)
+				return
+			}
+			g.Assert(err == nil).IsTrue("expect file parsed")
+		})
+
+		g.It("should error when file not found", func() {
+			_, err := ParseFile("/tmp/foo/bar/dummy/file.json")
+			g.Assert(err == nil).IsFalse("expect file not found error")
+		})
+	})
+}
+
+// invalid json representation, simulate parsing error
+var invalid = []byte(`[]`)
+
+// valid json representation, verify parsing
+var sample = []byte(`{
+	"containers": [
+		{
+			"name": "container_0",
+			"image": "node:latest" 
+		},
+		{
+			"name": "container_1",
+			"image": "golang:latest" 
+		}
+	],
+	"volumes": [
+		{
+			"name": "volume_0"
+		}
+	],
+	"program": {
+		"type": "list",
+		"body": [
+			{
+				"type": "defer",
+				"body": {
+					"type": "recover",
+					"body": {
+						"type": "run",
+						"name": "container_0"
+					}
+				},
+				"defer": {
+					"type": "parallel",
+					"body": [
+						{
+							"type": "run",
+							"name": "container_1"
+						},
+						{
+							"type": "run",
+							"name": "container_1"
+						}
+					],
+					"limit": 2
+				}
+			}
+		]
+	}
+}`)

engine/runner/parse/node.go

@@ -0,0 +1,30 @@
+package parse
+
+const (
+	NodeList     = "list"
+	NodeDefer    = "defer"
+	NodeError    = "error"
+	NodeRecover  = "recover"
+	NodeParallel = "parallel"
+	NodeRun      = "run"
+)
+
+// NodeType identifies the type of a parse tree node.
+type NodeType string
+
+// Type returns itself and provides an easy default implementation
+// for embedding in a Node. Embedded in all non-trivial Nodes.
+func (t NodeType) Type() NodeType {
+	return t
+}
+
+// String returns the string value of the Node type.
+func (t NodeType) String() string {
+	return string(t)
+}
+
+// A Node is an element in the parse tree.
+type Node interface {
+	Type() NodeType
+	Validate() error
+}

engine/runner/parse/node_defer.go

@@ -0,0 +1,40 @@
+package parse
+
+import "fmt"
+
+// DeferNode executes the child node, and then executes the deffered node.
+// The deffered node is guaranteed to execute, even when the child node fails.
+type DeferNode struct {
+	NodeType `json:"type"`
+
+	Body  Node `json:"body"`  // evaluate node
+	Defer Node `json:"defer"` // defer evaluation of node.
+}
+
+// NewDeferNode returns a new DeferNode.
+func NewDeferNode() *DeferNode {
+	return &DeferNode{NodeType: NodeDefer}
+}
+
+func (n *DeferNode) SetBody(node Node) *DeferNode {
+	n.Body = node
+	return n
+}
+
+func (n *DeferNode) SetDefer(node Node) *DeferNode {
+	n.Defer = node
+	return n
+}
+
+func (n *DeferNode) Validate() error {
+	switch {
+	case n.NodeType != NodeDefer:
+		return fmt.Errorf("Defer Node uses an invalid type")
+	case n.Body == nil:
+		return fmt.Errorf("Defer Node body is empty")
+	case n.Defer == nil:
+		return fmt.Errorf("Defer Node defer is empty")
+	default:
+		return nil
+	}
+}

engine/runner/parse/node_defer_test.go

@@ -0,0 +1,56 @@
+package parse
+
+import (
+	"testing"
+
+	"github.com/franela/goblin"
+)
+
+func TestDeferNode(t *testing.T) {
+	g := goblin.Goblin(t)
+
+	g.Describe("DeferNode", func() {
+		g.It("should set body and defer node", func() {
+			node0 := NewRunNode()
+			node1 := NewRunNode()
+
+			defer0 := NewDeferNode()
+			defer1 := defer0.SetBody(node0)
+			defer2 := defer0.SetDefer(node1)
+			g.Assert(defer0.Type().String()).Equal(NodeDefer)
+			g.Assert(defer0.Body).Equal(node0)
+			g.Assert(defer0.Defer).Equal(node1)
+			g.Assert(defer0).Equal(defer1)
+			g.Assert(defer0).Equal(defer2)
+		})
+
+		g.It("should fail validation when invalid type", func() {
+			defer0 := DeferNode{}
+			err := defer0.Validate()
+			g.Assert(err == nil).IsFalse()
+			g.Assert(err.Error()).Equal("Defer Node uses an invalid type")
+		})
+
+		g.It("should fail validation when empty body", func() {
+			defer0 := NewDeferNode()
+			err := defer0.Validate()
+			g.Assert(err == nil).IsFalse()
+			g.Assert(err.Error()).Equal("Defer Node body is empty")
+		})
+
+		g.It("should fail validation when empty defer", func() {
+			defer0 := NewDeferNode()
+			defer0.SetBody(NewRunNode())
+			err := defer0.Validate()
+			g.Assert(err == nil).IsFalse()
+			g.Assert(err.Error()).Equal("Defer Node defer is empty")
+		})
+
+		g.It("should pass validation", func() {
+			defer0 := NewDeferNode()
+			defer0.SetBody(NewRunNode())
+			defer0.SetDefer(NewRunNode())
+			g.Assert(defer0.Validate() == nil).IsTrue()
+		})
+	})
+}

engine/runner/parse/node_error.go

@@ -0,0 +1,40 @@
+package parse
+
+import "fmt"
+
+// ErrorNode executes the body node, and then executes the error node if
+// the body node errors. This is similar to defer but only executes on error.
+type ErrorNode struct {
+	NodeType `json:"type"`
+
+	Body  Node `json:"body"`  // evaluate node
+	Defer Node `json:"defer"` // defer evaluation of node on error.
+}
+
+// NewErrorNode returns a new ErrorNode.
+func NewErrorNode() *ErrorNode {
+	return &ErrorNode{NodeType: NodeError}
+}
+
+func (n *ErrorNode) SetBody(node Node) *ErrorNode {
+	n.Body = node
+	return n
+}
+
+func (n *ErrorNode) SetDefer(node Node) *ErrorNode {
+	n.Defer = node
+	return n
+}
+
+func (n *ErrorNode) Validate() error {
+	switch {
+	case n.NodeType != NodeError:
+		return fmt.Errorf("Error Node uses an invalid type")
+	case n.Body == nil:
+		return fmt.Errorf("Error Node body is empty")
+	case n.Defer == nil:
+		return fmt.Errorf("Error Node defer is empty")
+	default:
+		return nil
+	}
+}

engine/runner/parse/node_error_test.go

@@ -0,0 +1,56 @@
+package parse
+
+import (
+	"testing"
+
+	"github.com/franela/goblin"
+)
+
+func TestErrorNode(t *testing.T) {
+	g := goblin.Goblin(t)
+
+	g.Describe("ErrorNode", func() {
+		g.It("should set body and error node", func() {
+			node0 := NewRunNode()
+			node1 := NewRunNode()
+
+			error0 := NewErrorNode()
+			error1 := error0.SetBody(node0)
+			error2 := error0.SetDefer(node1)
+			g.Assert(error0.Type().String()).Equal(NodeError)
+			g.Assert(error0.Body).Equal(node0)
+			g.Assert(error0.Defer).Equal(node1)
+			g.Assert(error0).Equal(error1)
+			g.Assert(error0).Equal(error2)
+		})
+
+		g.It("should fail validation when invalid type", func() {
+			error0 := ErrorNode{}
+			err := error0.Validate()
+			g.Assert(err == nil).IsFalse()
+			g.Assert(err.Error()).Equal("Error Node uses an invalid type")
+		})
+
+		g.It("should fail validation when empty body", func() {
+			error0 := NewErrorNode()
+			err := error0.Validate()
+			g.Assert(err == nil).IsFalse()
+			g.Assert(err.Error()).Equal("Error Node body is empty")
+		})
+
+		g.It("should fail validation when empty error", func() {
+			error0 := NewErrorNode()
+			error0.SetBody(NewRunNode())
+			err := error0.Validate()
+			g.Assert(err == nil).IsFalse()
+			g.Assert(err.Error()).Equal("Error Node defer is empty")
+		})
+
+		g.It("should pass validation", func() {
+			error0 := NewErrorNode()
+			error0.SetBody(NewRunNode())
+			error0.SetDefer(NewRunNode())
+			g.Assert(error0.Validate() == nil).IsTrue()
+		})
+	})
+}

engine/runner/parse/node_list.go

@@ -0,0 +1,33 @@
+package parse
+
+import "fmt"
+
+// ListNode serially executes a list of child nodes.
+type ListNode struct {
+	NodeType `json:"type"`
+
+	// Body is the list of child nodes
+	Body []Node `json:"body"`
+}
+
+// NewListNode returns a new ListNode.
+func NewListNode() *ListNode {
+	return &ListNode{NodeType: NodeList}
+}
+
+// Append appens a child node to the list.
+func (n *ListNode) Append(node Node) *ListNode {
+	n.Body = append(n.Body, node)
+	return n
+}
+
+func (n *ListNode) Validate() error {
+	switch {
+	case n.NodeType != NodeList:
+		return fmt.Errorf("List Node uses an invalid type")
+	case len(n.Body) == 0:
+		return fmt.Errorf("List Node body is empty")
+	default:
+		return nil
+	}
+}

engine/runner/parse/node_list_test.go

@@ -0,0 +1,44 @@
+package parse
+
+import (
+	"testing"
+
+	"github.com/franela/goblin"
+)
+
+func TestListNode(t *testing.T) {
+	g := goblin.Goblin(t)
+
+	g.Describe("ListNode", func() {
+		g.It("should append nodes", func() {
+			node := NewRunNode()
+
+			list0 := NewListNode()
+			list1 := list0.Append(node)
+			g.Assert(list0.Type().String()).Equal(NodeList)
+			g.Assert(list0.Body[0]).Equal(node)
+			g.Assert(list0).Equal(list1)
+		})
+
+		g.It("should fail validation when invalid type", func() {
+			list := ListNode{}
+			err := list.Validate()
+			g.Assert(err == nil).IsFalse()
+			g.Assert(err.Error()).Equal("List Node uses an invalid type")
+		})
+
+		g.It("should fail validation when empty body", func() {
+			list := NewListNode()
+			err := list.Validate()
+			g.Assert(err == nil).IsFalse()
+			g.Assert(err.Error()).Equal("List Node body is empty")
+		})
+
+		g.It("should pass validation", func() {
+			node := NewRunNode()
+			list := NewListNode()
+			list.Append(node)
+			g.Assert(list.Validate() == nil).IsTrue()
+		})
+	})
+}

engine/runner/parse/node_parallel.go

@@ -0,0 +1,36 @@
+package parse
+
+import "fmt"
+
+// ParallelNode executes a list of child nodes in parallel.
+type ParallelNode struct {
+	NodeType `json:"type"`
+
+	Body  []Node `json:"body"`  // nodes for parallel evaluation.
+	Limit int    `json:"limit"` // limit for parallel evaluation.
+}
+
+func NewParallelNode() *ParallelNode {
+	return &ParallelNode{NodeType: NodeParallel}
+}
+
+func (n *ParallelNode) Append(node Node) *ParallelNode {
+	n.Body = append(n.Body, node)
+	return n
+}
+
+func (n *ParallelNode) SetLimit(limit int) *ParallelNode {
+	n.Limit = limit
+	return n
+}
+
+func (n *ParallelNode) Validate() error {
+	switch {
+	case n.NodeType != NodeParallel:
+		return fmt.Errorf("Parallel Node uses an invalid type")
+	case len(n.Body) == 0:
+		return fmt.Errorf("Parallel Node body is empty")
+	default:
+		return nil
+	}
+}

engine/runner/parse/node_parallel_test.go

@@ -0,0 +1,42 @@
+package parse
+
+import (
+	"testing"
+
+	"github.com/franela/goblin"
+)
+
+func TestParallelNode(t *testing.T) {
+	g := goblin.Goblin(t)
+
+	g.Describe("ParallelNode", func() {
+		g.It("should append nodes", func() {
+			node := NewRunNode()
+
+			parallel0 := NewParallelNode()
+			parallel1 := parallel0.Append(node)
+			g.Assert(parallel0.Type().String()).Equal(NodeParallel)
+			g.Assert(parallel0.Body[0]).Equal(node)
+			g.Assert(parallel0).Equal(parallel1)
+		})
+
+		g.It("should fail validation when invalid type", func() {
+			node := ParallelNode{}
+			err := node.Validate()
+			g.Assert(err == nil).IsFalse()
+			g.Assert(err.Error()).Equal("Parallel Node uses an invalid type")
+		})
+
+		g.It("should fail validation when empty body", func() {
+			node := NewParallelNode()
+			err := node.Validate()
+			g.Assert(err == nil).IsFalse()
+			g.Assert(err.Error()).Equal("Parallel Node body is empty")
+		})
+
+		g.It("should pass validation", func() {
+			node := NewParallelNode().Append(NewRunNode())
+			g.Assert(node.Validate() == nil).IsTrue()
+		})
+	})
+}

engine/runner/parse/node_recover.go

@@ -0,0 +1,29 @@
+package parse
+
+import "fmt"
+
+type RecoverNode struct {
+	NodeType `json:"type"`
+
+	Body Node `json:"body"` // evaluate node and catch all errors.
+}
+
+func NewRecoverNode() *RecoverNode {
+	return &RecoverNode{NodeType: NodeRecover}
+}
+
+func (n *RecoverNode) SetBody(node Node) *RecoverNode {
+	n.Body = node
+	return n
+}
+
+func (n *RecoverNode) Validate() error {
+	switch {
+	case n.NodeType != NodeRecover:
+		return fmt.Errorf("Recover Node uses an invalid type")
+	case n.Body == nil:
+		return fmt.Errorf("Recover Node body is empty")
+	default:
+		return nil
+	}
+}

engine/runner/parse/node_recover_test.go

@@ -0,0 +1,43 @@
+package parse
+
+import (
+	"testing"
+
+	"github.com/franela/goblin"
+)
+
+func TestRecoverNode(t *testing.T) {
+	g := goblin.Goblin(t)
+
+	g.Describe("RecoverNode", func() {
+		g.It("should set body", func() {
+			node0 := NewRunNode()
+
+			recover0 := NewRecoverNode()
+			recover1 := recover0.SetBody(node0)
+			g.Assert(recover0.Type().String()).Equal(NodeRecover)
+			g.Assert(recover0.Body).Equal(node0)
+			g.Assert(recover0).Equal(recover1)
+		})
+
+		g.It("should fail validation when invalid type", func() {
+			recover0 := RecoverNode{}
+			err := recover0.Validate()
+			g.Assert(err == nil).IsFalse()
+			g.Assert(err.Error()).Equal("Recover Node uses an invalid type")
+		})
+
+		g.It("should fail validation when empty body", func() {
+			recover0 := NewRecoverNode()
+			err := recover0.Validate()
+			g.Assert(err == nil).IsFalse()
+			g.Assert(err.Error()).Equal("Recover Node body is empty")
+		})
+
+		g.It("should pass validation", func() {
+			recover0 := NewRecoverNode()
+			recover0.SetBody(NewRunNode())
+			g.Assert(recover0.Validate() == nil).IsTrue()
+		})
+	})
+}

engine/runner/parse/node_run.go

@@ -0,0 +1,41 @@
+package parse
+
+import "fmt"
+
+type RunNode struct {
+	NodeType `json:"type"`
+
+	Name   string `json:"name"`
+	Detach bool   `json:"detach,omitempty"`
+	Silent bool   `json:"silent,omitempty"`
+}
+
+func (n *RunNode) SetName(name string) *RunNode {
+	n.Name = name
+	return n
+}
+
+func (n *RunNode) SetDetach(detach bool) *RunNode {
+	n.Detach = detach
+	return n
+}
+
+func (n *RunNode) SetSilent(silent bool) *RunNode {
+	n.Silent = silent
+	return n
+}
+
+func NewRunNode() *RunNode {
+	return &RunNode{NodeType: NodeRun}
+}
+
+func (n *RunNode) Validate() error {
+	switch {
+	case n.NodeType != NodeRun:
+		return fmt.Errorf("Run Node uses an invalid type")
+	case n.Name == "":
+		return fmt.Errorf("Run Node has an invalid name")
+	default:
+		return nil
+	}
+}

engine/runner/parse/node_run_test.go

@@ -0,0 +1,41 @@
+package parse
+
+import (
+	"testing"
+
+	"github.com/franela/goblin"
+)
+
+func TestRunNode(t *testing.T) {
+	g := goblin.Goblin(t)
+
+	g.Describe("RunNode", func() {
+		g.It("should set container name for lookup", func() {
+			node0 := NewRunNode()
+			node1 := node0.SetName("foo")
+
+			g.Assert(node0.Type().String()).Equal(NodeRun)
+			g.Assert(node0.Name).Equal("foo")
+			g.Assert(node0).Equal(node1)
+		})
+
+		g.It("should fail validation when invalid type", func() {
+			node := RunNode{}
+			err := node.Validate()
+			g.Assert(err == nil).IsFalse()
+			g.Assert(err.Error()).Equal("Run Node uses an invalid type")
+		})
+
+		g.It("should fail validation when invalid name", func() {
+			node := NewRunNode()
+			err := node.Validate()
+			g.Assert(err == nil).IsFalse()
+			g.Assert(err.Error()).Equal("Run Node has an invalid name")
+		})
+
+		g.It("should pass validation", func() {
+			node := NewRunNode().SetName("foo")
+			g.Assert(node.Validate() == nil).IsTrue()
+		})
+	})
+}

engine/runner/parse/parse.go

@@ -0,0 +1,221 @@
+package parse
+
+import "encoding/json"
+
+// Tree is the intermediate representation of a pipeline.
+type Tree struct {
+	*ListNode // top-level Tree node
+}
+
+// New allocates a new Tree.
+func NewTree() *Tree {
+	return &Tree{
+		NewListNode(),
+	}
+}
+
+// Parse parses a JSON encoded Tree.
+func Parse(data []byte) (*Tree, error) {
+	tree := &Tree{}
+	err := tree.UnmarshalJSON(data)
+	return tree, err
+}
+
+// MarshalJSON implements the Marshaler interface and returns
+// a JSON encoded representation of the Tree.
+func (t *Tree) MarshalJSON() ([]byte, error) {
+	return json.Marshal(t.ListNode)
+}
+
+// UnmarshalJSON implements the Unmarshaler interface and returns
+// a Tree from a JSON representation.
+func (t *Tree) UnmarshalJSON(data []byte) error {
+	block, err := decodeList(data)
+	if err != nil {
+		return nil
+	}
+	t.ListNode = block.(*ListNode)
+	return nil
+}
+
+//
+// below are custom decoding functions. We cannot use the default json
+// decoder because the tree structure uses interfaces and the json decoder
+// has difficulty ascertaining the interface type when decoding.
+//
+
+func decodeNode(data []byte) (Node, error) {
+	node := &nodeType{}
+
+	err := json.Unmarshal(data, node)
+	if err != nil {
+		return nil, err
+	}
+	switch node.Type {
+	case NodeList:
+		return decodeList(data)
+	case NodeDefer:
+		return decodeDefer(data)
+	case NodeError:
+		return decodeError(data)
+	case NodeRecover:
+		return decodeRecover(data)
+	case NodeParallel:
+		return decodeParallel(data)
+	case NodeRun:
+		return decodeRun(data)
+	}
+	return nil, nil
+}
+
+func decodeNodes(data []json.RawMessage) ([]Node, error) {
+	var nodes []Node
+	for _, d := range data {
+		node, err := decodeNode(d)
+		if err != nil {
+			return nil, err
+		}
+		nodes = append(nodes, node)
+	}
+	return nodes, nil
+}
+
+func decodeList(data []byte) (Node, error) {
+	v := &nodeList{}
+	err := json.Unmarshal(data, v)
+	if err != nil {
+		return nil, err
+	}
+	b, err := decodeNodes(v.Body)
+	if err != nil {
+		return nil, err
+	}
+	n := NewListNode()
+	n.Body = b
+	return n, nil
+}
+
+func decodeDefer(data []byte) (Node, error) {
+	v := &nodeDefer{}
+	err := json.Unmarshal(data, v)
+	if err != nil {
+		return nil, err
+	}
+	b, err := decodeNode(v.Body)
+	if err != nil {
+		return nil, err
+	}
+	d, err := decodeNode(v.Defer)
+	if err != nil {
+		return nil, err
+	}
+	n := NewDeferNode()
+	n.Body = b
+	n.Defer = d
+	return n, nil
+}
+
+func decodeError(data []byte) (Node, error) {
+	v := &nodeError{}
+	err := json.Unmarshal(data, v)
+	if err != nil {
+		return nil, err
+	}
+	b, err := decodeNode(v.Body)
+	if err != nil {
+		return nil, err
+	}
+	d, err := decodeNode(v.Defer)
+	if err != nil {
+		return nil, err
+	}
+	n := NewErrorNode()
+	n.Body = b
+	n.Defer = d
+	return n, nil
+}
+
+func decodeRecover(data []byte) (Node, error) {
+	v := &nodeRecover{}
+	err := json.Unmarshal(data, v)
+	if err != nil {
+		return nil, err
+	}
+	b, err := decodeNode(v.Body)
+	if err != nil {
+		return nil, err
+	}
+	n := NewRecoverNode()
+	n.Body = b
+	return n, nil
+}
+
+func decodeParallel(data []byte) (Node, error) {
+	v := &nodeParallel{}
+	err := json.Unmarshal(data, v)
+	if err != nil {
+		return nil, err
+	}
+	b, err := decodeNodes(v.Body)
+	if err != nil {
+		return nil, err
+	}
+	n := NewParallelNode()
+	n.Body = b
+	n.Limit = v.Limit
+	return n, nil
+}
+
+func decodeRun(data []byte) (Node, error) {
+	v := &nodeRun{}
+	err := json.Unmarshal(data, v)
+	if err != nil {
+		return nil, err
+	}
+	return &RunNode{NodeRun, v.Name, v.Detach, v.Silent}, nil
+}
+
+//
+// below are intermediate representations of the node structures
+// since we cannot simply encode / decode using the built-in json
+// encoding and decoder.
+//
+
+type nodeType struct {
+	Type NodeType `json:"type"`
+}
+
+type nodeDefer struct {
+	Type  NodeType        `json:"type"`
+	Body  json.RawMessage `json:"body"`
+	Defer json.RawMessage `json:"defer"`
+}
+
+type nodeError struct {
+	Type  NodeType        `json:"type"`
+	Body  json.RawMessage `json:"body"`
+	Defer json.RawMessage `json:"defer"`
+}
+
+type nodeList struct {
+	Type NodeType          `json:"type"`
+	Body []json.RawMessage `json:"body"`
+}
+
+type nodeRecover struct {
+	Type NodeType        `json:"type"`
+	Body json.RawMessage `json:"body"`
+}
+
+type nodeParallel struct {
+	Type  NodeType          `json:"type"`
+	Body  []json.RawMessage `json:"body"`
+	Limit int               `json:"limit"`
+}
+
+type nodeRun struct {
+	Type   NodeType `json:"type"`
+	Name   string   `json:"name"`
+	Detach bool     `json:"detach,omitempty"`
+	Silent bool     `json:"silent,omitempty"`
+}

engine/runner/parse/parse_test.go

@@ -0,0 +1,80 @@
+package parse
+
+import (
+	"bytes"
+	"encoding/json"
+	"reflect"
+	"testing"
+)
+
+func TestUnmarshal(t *testing.T) {
+
+	node1 := NewRunNode().SetName("foo")
+	node2 := NewRecoverNode().SetBody(node1)
+
+	node3 := NewRunNode().SetName("bar")
+	node4 := NewRunNode().SetName("bar")
+
+	node5 := NewParallelNode().
+		Append(node3).
+		Append(node4).
+		SetLimit(2)
+
+	node6 := NewDeferNode().
+		SetBody(node2).
+		SetDefer(node5)
+
+	tree := NewTree()
+	tree.Append(node6)
+
+	encoded, err := json.MarshalIndent(tree, "", "\t")
+	if err != nil {
+		t.Error(err)
+	}
+
+	if !bytes.Equal(encoded, sample) {
+		t.Errorf("Want to marshal Tree to %s, got %s",
+			string(sample),
+			string(encoded),
+		)
+	}
+
+	parsed, err := Parse(encoded)
+	if err != nil {
+		t.Error(err)
+	}
+
+	if !reflect.DeepEqual(tree, parsed) {
+		t.Errorf("Want to marsnal and then unmarshal Tree")
+	}
+}
+
+var sample = []byte(`{
+	"type": "list",
+	"body": [
+		{
+			"type": "defer",
+			"body": {
+				"type": "recover",
+				"body": {
+					"type": "run",
+					"name": "foo"
+				}
+			},
+			"defer": {
+				"type": "parallel",
+				"body": [
+					{
+						"type": "run",
+						"name": "bar"
+					},
+					{
+						"type": "run",
+						"name": "bar"
+					}
+				],
+				"limit": 2
+			}
+		}
+	]
+}`)

engine/runner/pipe.go

@@ -0,0 +1,49 @@
+package runner
+
+import "fmt"
+
+// Pipe returns a buffered pipe that is connected to the console output.
+type Pipe struct {
+	lines chan *Line
+	eof   chan bool
+}
+
+// Next returns the next Line of console output.
+func (p *Pipe) Next() *Line {
+	select {
+	case line := <-p.lines:
+		return line
+	case <-p.eof:
+		return nil
+	}
+}
+
+// Close closes the pipe of console output.
+func (p *Pipe) Close() {
+	go func() {
+		p.eof <- true
+	}()
+}
+
+func newPipe(buffer int) *Pipe {
+	return &Pipe{
+		lines: make(chan *Line, buffer),
+		eof:   make(chan bool),
+	}
+}
+
+// Line is a line of console output.
+type Line struct {
+	Proc string `json:"proc,omitempty"`
+	Time int64  `json:"time,omitempty"`
+	Type int    `json:"type,omitempty"`
+	Pos  int    `json:"pos,omityempty"`
+	Out  string `json:"out,omitempty"`
+}
+
+func (l *Line) String() string {
+	return fmt.Sprintf("[%s:L%v:%vs] %s", l.Proc, l.Pos, l.Time, l.Out)
+}
+
+// TODO(bradrydzewski) consider an alternate buffer impelmentation based on the
+// x.crypto ssh buffer https://github.com/golang/crypto/blob/master/ssh/buffer.go

engine/runner/pipe_test.go

@@ -0,0 +1,54 @@
+package runner
+
+import (
+	"sync"
+	"testing"
+
+	"github.com/franela/goblin"
+)
+
+func TestPipe(t *testing.T) {
+	g := goblin.Goblin(t)
+
+	g.Describe("Pipe", func() {
+		g.It("should get next line from buffer", func() {
+			line := &Line{
+				Proc: "redis",
+				Pos:  1,
+				Out:  "starting redis server",
+			}
+			pipe := newPipe(10)
+			pipe.lines <- line
+			next := pipe.Next()
+			g.Assert(next).Equal(line)
+		})
+
+		g.It("should get null line on buffer closed", func() {
+			pipe := newPipe(10)
+
+			var wg sync.WaitGroup
+			wg.Add(1)
+
+			go func() {
+				next := pipe.Next()
+				g.Assert(next == nil).IsTrue("line should be nil")
+				wg.Done()
+			}()
+
+			pipe.Close()
+			wg.Wait()
+		})
+
+		g.Describe("Line output", func() {
+			g.It("should prefix string() with metadata", func() {
+				line := Line{
+					Proc: "redis",
+					Time: 60,
+					Pos:  1,
+					Out:  "starting redis server",
+				}
+				g.Assert(line.String()).Equal("[redis:L1:60s] starting redis server")
+			})
+		})
+	})
+}

engine/runner/runner.go

@@ -0,0 +1,247 @@
+package runner
+
+import (
+	"bufio"
+	"time"
+	"fmt"
+
+	"github.com/drone/drone/engine/runner/parse"
+
+	"golang.org/x/net/context"
+)
+
+// NoContext is the default context you should supply if not using your own
+// context.Context
+var NoContext = context.TODO()
+
+// Tracer defines a tracing function that is invoked prior to creating and
+// running the container.
+type Tracer func(c *Container) error
+
+// Config defines the configuration for creating the Runner.
+type Config struct {
+	Tracer Tracer
+	Engine Engine
+
+	// Buffer defines the size of the buffer for the channel to which the
+	// console output is streamed.
+	Buffer uint
+}
+
+// Runner creates a build Runner using the specific configuration for the given
+// Context and Specification.
+func (c *Config) Runner(ctx context.Context, spec *Spec) *Runner {
+
+	// TODO(bradyrdzewski) we should make a copy of the configuration parameters
+	// instead of a direct reference. This helps avoid any race conditions or
+	//unexpected behavior if the Config changes.
+	return &Runner{
+		ctx:  ctx,
+		conf: c,
+		spec: spec,
+		errc: make(chan error),
+		pipe: newPipe(int(c.Buffer) + 1),
+	}
+}
+
+type Runner struct {
+	ctx  context.Context
+	conf *Config
+	spec *Spec
+	pipe *Pipe
+	errc chan (error)
+
+	containers []string
+	volumes    []string
+	networks   []string
+}
+
+// Run starts the build runner but does not wait for it to complete. The Wait
+// method will return the exit code and release associated resources once the
+// running containers exit.
+func (r *Runner) Run() error {
+
+	go func() {
+		r.setup()
+		err := r.exec(r.spec.Nodes.ListNode)
+		r.pipe.Close()
+		r.cancel()
+		r.teardown()
+		r.errc <- err
+	}()
+
+	go func() {
+		<-r.ctx.Done()
+		r.cancel()
+	}()
+
+	return nil
+}
+
+// Wait waits for the runner to exit.
+func (r *Runner) Wait() error {
+	return <-r.errc
+}
+
+// Pipe returns a Pipe that is connected to the console output stream.
+func (r *Runner) Pipe() *Pipe {
+	return r.pipe
+}
+
+func (r *Runner) exec(node parse.Node) error {
+	switch v := node.(type) {
+	case *parse.ListNode:
+		return r.execList(v)
+	case *parse.DeferNode:
+		return r.execDefer(v)
+	case *parse.ErrorNode:
+		return r.execError(v)
+	case *parse.RecoverNode:
+		return r.execRecover(v)
+	case *parse.ParallelNode:
+		return r.execParallel(v)
+	case *parse.RunNode:
+		return r.execRun(v)
+	}
+	return fmt.Errorf("runner: unexepected node %s", node)
+}
+
+func (r *Runner) execList(node *parse.ListNode) error {
+	for _, n := range node.Body {
+		err := r.exec(n)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (r *Runner) execDefer(node *parse.DeferNode) error {
+	err1 := r.exec(node.Body)
+	err2 := r.exec(node.Defer)
+	if err1 != nil {
+		return err1
+	}
+	return err2
+}
+
+func (r *Runner) execError(node *parse.ErrorNode) error {
+	err := r.exec(node.Body)
+	if err != nil {
+		r.exec(node.Defer)
+	}
+	return err
+}
+
+func (r *Runner) execRecover(node *parse.RecoverNode) error {
+	r.exec(node.Body)
+	return nil
+}
+
+func (r *Runner) execParallel(node *parse.ParallelNode) error {
+	errc := make(chan error)
+
+	for _, n := range node.Body {
+		go func(node parse.Node) {
+			errc <- r.exec(node)
+		}(n)
+	}
+
+	var err error
+	for i := 0; i < len(node.Body); i++ {
+		select {
+		case cerr := <-errc:
+			if cerr != nil {
+				err = cerr
+			}
+		}
+	}
+
+	return err
+}
+
+func (r *Runner) execRun(node *parse.RunNode) error {
+	container, err := r.spec.lookupContainer(node.Name)
+	if err != nil {
+		return err
+	}
+	if r.conf.Tracer != nil {
+		err := r.conf.Tracer(container)
+		switch {
+		case err == ErrSkip:
+			return nil
+		case err != nil:
+			return err
+		}
+	}
+	// TODO(bradrydzewski) there is potential here for a race condition where
+	// the context is cancelled just after this line, resulting in the container
+	// still being started.
+	if r.ctx.Err() != nil {
+		return err
+	}
+
+	name, err := r.conf.Engine.ContainerStart(container)
+	if err != nil {
+		return err
+	}
+	r.containers = append(r.containers, name)
+
+	go func() {
+		if node.Silent {
+			return
+		}
+		rc, err := r.conf.Engine.ContainerLogs(name)
+		if err != nil {
+			return
+		}
+		defer rc.Close()
+
+		num := 0
+		now := time.Now().UTC()
+		scanner := bufio.NewScanner(rc)
+		for scanner.Scan() {
+			r.pipe.lines <- &Line{
+				Proc: container.Alias,
+				Time: int64(time.Since(now).Seconds()),
+				Pos:  num,
+				Out:  scanner.Text(),
+			}
+			num++
+		}
+	}()
+
+	// exit when running container in detached mode in background
+	if node.Detach {
+		return nil
+	}
+
+	state, err := r.conf.Engine.ContainerWait(name)
+	if err != nil {
+		return err
+	}
+	if state.OOMKilled {
+		return &OomError{name}
+	} else if state.ExitCode != 0 {
+		return &ExitError{name, state.ExitCode}
+	}
+	return nil
+}
+
+func (r *Runner) setup() {
+	// this is where we will setup network and volumes
+}
+
+func (r *Runner) teardown() {
+	// TODO(bradrydzewski) this is not yet thread safe.
+	for _, container := range r.containers {
+		r.conf.Engine.ContainerRemove(container)
+	}
+}
+
+func (r *Runner) cancel() {
+	// TODO(bradrydzewski) this is not yet thread safe.
+	for _, container := range r.containers {
+		r.conf.Engine.ContainerStop(container)
+	}
+}

engine/runner/runner_test.go

@@ -0,0 +1,7 @@
+package runner
+
+import "testing"
+
+func TestRunner(t *testing.T) {
+	t.Skip()
+}

engine/runner/spec.go

@@ -0,0 +1,33 @@
+package runner
+
+import (
+	"fmt"
+
+	"github.com/drone/drone/engine/runner/parse"
+)
+
+// Spec defines the pipeline configuration and exeuction.
+type Spec struct {
+	// Volumes defines a list of all container volumes.
+	Volumes []*Volume `json:"volumes,omitempty"`
+
+	// Networks defines a list of all container networks.
+	Networks []*Network `json:"networks,omitempty"`
+
+	// Containers defines a list of all containers in the pipeline.
+	Containers []*Container `json:"containers,omitempty"`
+
+	// Nodes defines the container execution tree.
+	Nodes *parse.Tree `json:"program,omitempty"`
+}
+
+// lookupContainer is a helper funciton that returns the named container from
+// the slice of containers.
+func (s *Spec) lookupContainer(name string) (*Container, error) {
+	for _, container := range s.Containers {
+		if container.Name == name {
+			return container, nil
+		}
+	}
+	return nil, fmt.Errorf("runner: unknown container %s", name)
+}

engine/runner/spec_test.go

@@ -0,0 +1,35 @@
+package runner
+
+import (
+	"testing"
+
+	"github.com/franela/goblin"
+)
+
+func TestSpec(t *testing.T) {
+	g := goblin.Goblin(t)
+
+	g.Describe("Spec file", func() {
+
+		g.Describe("when looking up a container", func() {
+
+			spec := Spec{}
+			spec.Containers = append(spec.Containers, &Container{
+				Name: "golang",
+			})
+
+			g.It("should find and return the container", func() {
+				c, err := spec.lookupContainer("golang")
+				g.Assert(err == nil).IsTrue("error should be nil")
+				g.Assert(c).Equal(spec.Containers[0])
+			})
+
+			g.It("should return an error when not found", func() {
+				c, err := spec.lookupContainer("node")
+				g.Assert(err == nil).IsFalse("should return error")
+				g.Assert(c == nil).IsTrue("should return nil container")
+			})
+
+		})
+	})
+}

engine/shasum/shasum.go

@@ -0,0 +1,72 @@
+package shasum
+
+import (
+	"crypto/sha1"
+	"crypto/sha256"
+	"crypto/sha512"
+	"fmt"
+	"io"
+	"strings"
+)
+
+// Check is a calculates and verifies a file checksum.
+// This supports the sha1, sha256 and sha512 values.
+func Check(in, checksum string) bool {
+	hash, size, _ := split(checksum)
+
+	// if a byte size is provided for the
+	// Yaml file it must match.
+	if size > 0 && int64(len(in)) != size {
+		return false
+	}
+
+	switch len(hash) {
+	case 64:
+		return sha256sum(in) == hash
+	case 128:
+		return sha512sum(in) == hash
+	case 40:
+		return sha1sum(in) == hash
+	case 0:
+		return true // if no checksum assume valid
+	}
+
+	return false
+}
+
+func sha1sum(in string) string {
+	h := sha1.New()
+	io.WriteString(h, in)
+	return fmt.Sprintf("%x", h.Sum(nil))
+}
+
+func sha256sum(in string) string {
+	h := sha256.New()
+	io.WriteString(h, in)
+	return fmt.Sprintf("%x", h.Sum(nil))
+}
+
+func sha512sum(in string) string {
+	h := sha512.New()
+	io.WriteString(h, in)
+	return fmt.Sprintf("%x", h.Sum(nil))
+}
+
+func split(in string) (string, int64, string) {
+	var hash string
+	var name string
+	var size int64
+
+	// the checksum might be split into multiple
+	// sections including the file size and name.
+	switch strings.Count(in, " ") {
+	case 1:
+		fmt.Sscanf(in, "%s %s", &hash, &name)
+	case 2:
+		fmt.Sscanf(in, "%s %d %s", &hash, &size, &name)
+	default:
+		hash = in
+	}
+
+	return hash, size, name
+}

engine/shasum/shasum_test.go

@@ -0,0 +1,97 @@
+package shasum
+
+import (
+	"testing"
+
+	"github.com/franela/goblin"
+)
+
+func TestParse(t *testing.T) {
+
+	g := goblin.Goblin(t)
+	g.Describe("Shasum", func() {
+
+		g.It("Should parse the shasum string", func() {
+			hash, _, _ := split("f1d2d2f924e986ac86fdf7b36c94bcdf32beec15")
+			g.Assert(hash).Equal("f1d2d2f924e986ac86fdf7b36c94bcdf32beec15")
+		})
+
+		g.It("Should parse a two-part shasum string", func() {
+			hash, _, name := split("f1d2d2f924e986ac86fdf7b36c94bcdf32beec15 .drone.yml")
+			g.Assert(hash).Equal("f1d2d2f924e986ac86fdf7b36c94bcdf32beec15")
+			g.Assert(name).Equal(".drone.yml")
+		})
+
+		g.It("Should parse a three-part shasum string", func() {
+			hash, size, name := split("f1d2d2f924e986ac86fdf7b36c94bcdf32beec15 42 .drone.yml")
+			g.Assert(hash).Equal("f1d2d2f924e986ac86fdf7b36c94bcdf32beec15")
+			g.Assert(name).Equal(".drone.yml")
+			g.Assert(size).Equal(int64(42))
+		})
+
+		g.It("Should calc a sha1 sum", func() {
+			hash := sha1sum("foo\n")
+			g.Assert(hash).Equal("f1d2d2f924e986ac86fdf7b36c94bcdf32beec15")
+		})
+
+		g.It("Should calc a sha256 sum", func() {
+			hash := sha256sum("foo\n")
+			g.Assert(hash).Equal("b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c")
+		})
+
+		g.It("Should calc a sha512 sum", func() {
+			hash := sha512sum("foo\n")
+			g.Assert(hash).Equal("0cf9180a764aba863a67b6d72f0918bc131c6772642cb2dce5a34f0a702f9470ddc2bf125c12198b1995c233c34b4afd346c54a2334c350a948a51b6e8b4e6b6")
+		})
+
+		g.It("Should calc a sha1 sum", func() {
+			hash := sha1sum("foo\n")
+			g.Assert(hash).Equal("f1d2d2f924e986ac86fdf7b36c94bcdf32beec15")
+		})
+
+		g.It("Should validate sha1 sum with file size", func() {
+			ok := Check("foo\n", "f1d2d2f924e986ac86fdf7b36c94bcdf32beec15 4 -")
+			g.Assert(ok).IsTrue()
+		})
+
+		g.It("Should validate sha256 sum with file size", func() {
+			ok := Check("foo\n", "b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c 4 -")
+			g.Assert(ok).IsTrue()
+		})
+
+		g.It("Should validate sha512 sum with file size", func() {
+			ok := Check("foo\n", "0cf9180a764aba863a67b6d72f0918bc131c6772642cb2dce5a34f0a702f9470ddc2bf125c12198b1995c233c34b4afd346c54a2334c350a948a51b6e8b4e6b6 4 -")
+			g.Assert(ok).IsTrue()
+		})
+
+		g.It("Should fail validation if incorrect sha1", func() {
+			ok := Check("bar\n", "f1d2d2f924e986ac86fdf7b36c94bcdf32beec15 4 -")
+			g.Assert(ok).IsFalse()
+		})
+
+		g.It("Should fail validation if incorrect sha256", func() {
+			ok := Check("bar\n", "b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c 4 -")
+			g.Assert(ok).IsFalse()
+		})
+
+		g.It("Should fail validation if incorrect sha512", func() {
+			ok := Check("bar\n", "0cf9180a764aba863a67b6d72f0918bc131c6772642cb2dce5a34f0a702f9470ddc2bf125c12198b1995c233c34b4afd346c54a2334c350a948a51b6e8b4e6b6 4 -")
+			g.Assert(ok).IsFalse()
+		})
+
+		g.It("Should return false if file size mismatch", func() {
+			ok := Check("foo\n", "f1d2d2f924e986ac86fdf7b36c94bcdf32beec15 12 -")
+			g.Assert(ok).IsFalse()
+		})
+
+		g.It("Should return false if invalid checksum string", func() {
+			ok := Check("foo\n", "f1d2d2f924e986ac86fdf7b36c94bcdf32beec15234")
+			g.Assert(ok).IsFalse()
+		})
+
+		g.It("Should return true if empty checksum", func() {
+			ok := Check("foo\n", "")
+			g.Assert(ok).IsTrue()
+		})
+	})
+}

queue/context.go

@@ -0,0 +1,23 @@
+package queue
+
+import (
+	"golang.org/x/net/context"
+)
+
+const key = "queue"
+
+// Setter defines a context that enables setting values.
+type Setter interface {
+	Set(string, interface{})
+}
+
+// FromContext returns the Queue associated with this context.
+func FromContext(c context.Context) Queue {
+	return c.Value(key).(Queue)
+}
+
+// ToContext adds the Queue to this context if it supports
+// the Setter interface.
+func ToContext(c Setter, q Queue) {
+	c.Set(key, q)
+}

queue/queue.go

@@ -0,0 +1,67 @@
+package queue
+
+//go:generate mockery -name Queue -output mock -case=underscore
+
+import (
+	"errors"
+
+	"golang.org/x/net/context"
+)
+
+// ErrNotFound indicates the requested work item does not
+// exist in the queue.
+var ErrNotFound = errors.New("queue item not found")
+
+type Queue interface {
+	// Publish inserts work at the tail of this queue, waiting for
+	// space to become available if the queue is full.
+	Publish(*Work) error
+
+	// Remove removes the specified work item from this queue,
+	// if it is present.
+	Remove(*Work) error
+
+	// PullClose retrieves and removes the head of this queue,
+	// waiting if necessary until work becomes available.
+	Pull() *Work
+
+	// PullClose retrieves and removes the head of this queue,
+	// waiting if necessary until work becomes available. The
+	// CloseNotifier should be provided to clone the channel
+	// if the subscribing client terminates its connection.
+	PullClose(CloseNotifier) *Work
+}
+
+// Publish inserts work at the tail of this queue, waiting for
+// space to become available if the queue is full.
+func Publish(c context.Context, w *Work) error {
+	return FromContext(c).Publish(w)
+}
+
+// Remove removes the specified work item from this queue,
+// if it is present.
+func Remove(c context.Context, w *Work) error {
+	return FromContext(c).Remove(w)
+}
+
+// PullClose retrieves and removes the head of this queue,
+// waiting if necessary until work becomes available.
+func Pull(c context.Context) *Work {
+	return FromContext(c).Pull()
+}
+
+// PullClose retrieves and removes the head of this queue,
+// waiting if necessary until work becomes available. The
+// CloseNotifier should be provided to clone the channel
+// if the subscribing client terminates its connection.
+func PullClose(c context.Context, cn CloseNotifier) *Work {
+	return FromContext(c).PullClose(cn)
+}
+
+// CloseNotifier defines a datastructure that is capable of notifying
+// a subscriber when its connection is closed.
+type CloseNotifier interface {
+	// CloseNotify returns a channel that receives a single value
+	// when the client connection has gone away.
+	CloseNotify() <-chan bool
+}

queue/queue_impl.go

@@ -0,0 +1,85 @@
+package queue
+
+import "sync"
+
+type queue struct {
+	sync.Mutex
+
+	items map[*Work]struct{}
+	itemc chan *Work
+}
+
+func New() Queue {
+	return newQueue()
+}
+
+func newQueue() *queue {
+	return &queue{
+		items: make(map[*Work]struct{}),
+		itemc: make(chan *Work, 999),
+	}
+}
+
+func (q *queue) Publish(work *Work) error {
+	q.Lock()
+	q.items[work] = struct{}{}
+	q.Unlock()
+	q.itemc <- work
+	return nil
+}
+
+func (q *queue) Remove(work *Work) error {
+	q.Lock()
+	defer q.Unlock()
+
+	_, ok := q.items[work]
+	if !ok {
+		return ErrNotFound
+	}
+	var items []*Work
+
+	// loop through and drain all items
+	// from the
+drain:
+	for {
+		select {
+		case item := <-q.itemc:
+			items = append(items, item)
+		default:
+			break drain
+		}
+	}
+
+	// re-add all items to the queue except
+	// the item we're trying to remove
+	for _, item := range items {
+		if item == work {
+			delete(q.items, work)
+			continue
+		}
+		q.itemc <- item
+	}
+	return nil
+}
+
+func (q *queue) Pull() *Work {
+	work := <-q.itemc
+	q.Lock()
+	delete(q.items, work)
+	q.Unlock()
+	return work
+}
+
+func (q *queue) PullClose(cn CloseNotifier) *Work {
+	for {
+		select {
+		case <-cn.CloseNotify():
+			return nil
+		case work := <-q.itemc:
+			q.Lock()
+			delete(q.items, work)
+			q.Unlock()
+			return work
+		}
+	}
+}

queue/queue_impl_test.go

@@ -0,0 +1,93 @@
+package queue
+
+import (
+	"sync"
+	"testing"
+
+	. "github.com/franela/goblin"
+	"github.com/gin-gonic/gin"
+)
+
+func TestBuild(t *testing.T) {
+	g := Goblin(t)
+	g.Describe("Queue", func() {
+
+		g.It("Should publish item", func() {
+			c := new(gin.Context)
+			q := newQueue()
+			ToContext(c, q)
+
+			w1 := &Work{}
+			w2 := &Work{}
+			Publish(c, w1)
+			Publish(c, w2)
+			g.Assert(len(q.items)).Equal(2)
+			g.Assert(len(q.itemc)).Equal(2)
+		})
+
+		g.It("Should remove item", func() {
+			c := new(gin.Context)
+			q := newQueue()
+			ToContext(c, q)
+
+			w1 := &Work{}
+			w2 := &Work{}
+			w3 := &Work{}
+			Publish(c, w1)
+			Publish(c, w2)
+			Publish(c, w3)
+			Remove(c, w2)
+			g.Assert(len(q.items)).Equal(2)
+			g.Assert(len(q.itemc)).Equal(2)
+
+			g.Assert(Pull(c)).Equal(w1)
+			g.Assert(Pull(c)).Equal(w3)
+			g.Assert(Remove(c, w2)).Equal(ErrNotFound)
+		})
+
+		g.It("Should pull item", func() {
+			c := new(gin.Context)
+			q := New()
+			ToContext(c, q)
+
+			cn := new(closeNotifier)
+			cn.closec = make(chan bool, 1)
+			w1 := &Work{}
+			w2 := &Work{}
+
+			Publish(c, w1)
+			g.Assert(Pull(c)).Equal(w1)
+
+			Publish(c, w2)
+			g.Assert(PullClose(c, cn)).Equal(w2)
+		})
+
+		g.It("Should cancel pulling item", func() {
+			c := new(gin.Context)
+			q := New()
+			ToContext(c, q)
+
+			cn := new(closeNotifier)
+			cn.closec = make(chan bool, 1)
+			var wg sync.WaitGroup
+			go func() {
+				wg.Add(1)
+				g.Assert(PullClose(c, cn) == nil).IsTrue()
+				wg.Done()
+			}()
+			go func() {
+				cn.closec <- true
+			}()
+			wg.Wait()
+
+		})
+	})
+}
+
+type closeNotifier struct {
+	closec chan bool
+}
+
+func (c *closeNotifier) CloseNotify() <-chan bool {
+	return c.closec
+}

queue/types.go

@@ -0,0 +1,18 @@
+package queue
+
+import "github.com/drone/drone/model"
+
+// Work represents an item for work to be
+// processed by a worker.
+type Work struct {
+	Yaml      string        `json:"config"`
+	YamlEnc   string        `json:"secret"`
+	Repo      *model.Repo   `json:"repo"`
+	Build     *model.Build  `json:"build"`
+	BuildLast *model.Build  `json:"build_last"`
+	Job       *model.Job    `json:"job"`
+	Netrc     *model.Netrc  `json:"netrc"`
+	Keys      *model.Key    `json:"keys"`
+	System    *model.System `json:"system"`
+	User      *model.User   `json:"user"`
+}

remote/bitbucket/bitbucket.go

@@ -9,7 +9,6 @@ import (
 	"strconv"
 
 	"github.com/drone/drone/model"
-	"github.com/drone/drone/shared/envconfig"
 	"github.com/drone/drone/shared/httputil"
 
 	log "github.com/Sirupsen/logrus"
@@ -24,8 +23,7 @@ type Bitbucket struct {
 	Open   bool
 }
 
-func Load(env envconfig.Env) *Bitbucket {
-	config := env.String("REMOTE_CONFIG", "")
+func Load(config string) *Bitbucket {
 
 	// parse the remote DSN configuration string
 	url_, err := url.Parse(config)
@@ -416,10 +414,10 @@ func (bb *Bitbucket) pushHook(r *http.Request) (*model.Repo, *model.Build, error
 		// we only support tag and branch pushes for now
 		buildEventType := model.EventPush
 		buildRef := fmt.Sprintf("refs/heads/%s", change.New.Name)
-		if change.New.Type == "tag" || change.New.Type == "annotated_tag" {
+		if change.New.Type == "tag" || change.New.Type == "annotated_tag" || change.New.Type == "bookmark" {
 			buildEventType = model.EventTag
 			buildRef = fmt.Sprintf("refs/tags/%s", change.New.Name)
-		} else if change.New.Type != "branch" {
+		} else if change.New.Type != "branch" && change.New.Type != "named_branch" {
 			continue
 		}