Merge pull request #1582 from bradrydzewski/master

enable server and agent

.drone.sec

@@ -1 +1 @@
-eyJhbGciOiJSU0EtT0FFUCIsImVuYyI6IkExMjhHQ00ifQ.eZfdbd1tRr10nL0gMJ2f4I4p7IBTRsllJywHAFiV0rreyA7zTgnp5Ot6htiFTJhMyVgpEif5x0zsDW52BVQBIjugObo3IwNa3_Cj6Uox5ws-wDZMJS2v4DSRR0CMNHHfphMj5XMF0wCPV3HSAfdPbDeo_n1fDR8IjWZKMzUxeM6r3AZMDO1eXnzVzoz7BzuTmxaZXSC138njTdp_BnB8mlbgQ0S2NqXm5rz-08FteHRGS5JPIWG8N3Tkz4wdDvm7tCul0z7_dMaSFBxeRW0zpIDqYbRM1nmlvVvOZKrsU3yZs9-YsBM19xIbXEH-QVgt3-7eCK52e5_UKwQHz2IXVA.--C0VUR9-XPypgGI.RF_5cehGQl9sqCs8JBs9sEriYqBlrcErzL9jkMIqEtPbBZNrymy4TTmU1JWU79j1QJ3UImTwPdm04tFhrSpc1phn3CdQ4Ikhtw1QcNxOyjCwhbmpQxZ8DEd0fNudyh8xHfIcWJdLrcM4MoHThVT2o9xLII-G-Ub9URJwK6okzkLF0MFz7WRKQYkKwm4qHKn00IJ8W9U3zd53Nhhu0njvz56z77lzYbhmMBt9aAgV6IenvTEL7NwKCMtLpmPN2gT-MDPqcEU6tQLajI08RV7CnboqwUMlfSZDYh0eRiM-w4-lNPe2-YsWPPn0pZaNPLYQVoKekEST4k0ERF8HLIlQd83yHnAL6uGIwPjkSUXbBXPgdtRhCfKMZYCXRq2hhPHiv6AWwQ.5rLNsafk0QBM6kOpCLE0Sg
+eyJhbGciOiJSU0EtT0FFUCIsImVuYyI6IkExMjhHQ00ifQ.r4ScD0KSIWeQR-1OfO7OkmSOk12y1hmqCt2keJ1i3VZD-9wYtTV2PnkpUMkgAp0teUVpmR1n6ndbLAYsUljro2KYMH1PrWI9pRA9aZNWZr-aVTjHMDJFXWgMr6ecxN9bmYMwzv0lAJWRcrvPdyhr3yaiGsnjYH7c9FZGUMDwh191sK7iRdFfDNbYF6AMY2_2pkrUIkyMdPPK1FF0ILsSHHXHgUPSMCfURvaKF2MrVSccHP5fQk-9q5mdob79UkGAqtMHAyFq0dPmBQvZrKSVkDBEYp7exwGLs8bJqx3QFCmN7sXN6DXtJlNdCe616dzzfLDMZ4-qG5UEv74VR6lX8w.mDyt_QqhOtpAfO8t.b277f0m6GO1B3pWE6jctayN2_HdmfhPG4nXjkD__p9Y5ktRfQK4QMiOj4sQPny1yxxBvvyKOlVB7tQ9fSivWnACVnJFZieZ6FmoEtHsf63tZadS5idEkF5S0zY8OSaBZ6SuHb3omrOI1xmz53TXLcI5Ox-7xVMelWT53FTkFZ-s7fBc3p2zYIvDHv17rhEk6ocGDI_euHnbjT-r3nGroESVnMPBynTw8PhhR3EOENgoTH4rcH3xLK3yOjHKNl5RPkfrIUawpeSNBQMCsFVZIgvGwGFnmoJ4Y4ASRQWnL0iupycP45k_NYD_-d3UDJD02B_2GPNY6OHzp-OFbBDXlov5tmVzA0WzPSxbbSWhMdgEzJthhpPuZ_77lZ4cwx01UgCAiJA.akTuVfyRAJ8CYP-aCoOVIg

.drone.yml

@@ -2,13 +2,21 @@ clone:
   path: github.com/drone/drone
 
 build:
-  image: drone/golang:1.5
+  test:
-  environment:
+    image: drone/golang:1.5
-    - GO15VENDOREXPERIMENT=1
+    environment:
-  commands:
+      - GO15VENDOREXPERIMENT=1
-    - make deps gen
+    commands:
-    - make test test_postgres test_mysql
+      - make deps gen
-    - make build build_static deb docs
+      - make test test_postgres test_mysql
+  dist:
+    image: drone/golang:1.5
+    environment:
+      - GO15VENDOREXPERIMENT=1
+    commands:
+      - make build
+    when:
+      event: push
 
 compose:
   postgres:
@@ -27,10 +35,7 @@ publish:
     password: $$DOCKER_PASS
     email: $$DOCKER_EMAIL
     repo: drone/drone
-    tag:
+    tag: [ "latest" ]
-      - "latest" 
-      - "0.4.1"
-      - "0.4"
     when:
       repo: drone/drone
       branch: master

.gitignore

@@ -1,5 +1,4 @@
-drone
+drone/drone
-drone_*
 *.sqlite
 *_gen.go
 *.html

Dockerfile

@@ -19,11 +19,12 @@ ADD contrib/docker/etc/nsswitch.conf /etc/
 ENV DATABASE_DRIVER=sqlite3
 ENV DATABASE_CONFIG=/var/lib/drone/drone.sqlite
 
-ADD drone_static /drone_static
+ADD drone/drone /drone
 
 # Alpine Linux doesn't use pam, which means that there is no /etc/nsswitch.conf,
 # but Go and CGO rely on /etc/nsswitch.conf to check the order of DNS resolving.
 # To fix this we just create /etc/nsswitch.conf and add the following line:
 #RUN echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf
 
-ENTRYPOINT ["/drone_static"]
+ENTRYPOINT ["/drone"]
+CMD ["serve"]

Makefile

@@ -24,11 +24,10 @@ gen_template:
 gen_migrations:
 	go generate github.com/drone/drone/store/datastore/ddl
 
-build:
+build: build_static
-	go build
 
 build_static:
-	go build --ldflags '-extldflags "-static" -X github.com/drone/drone/version.VersionDev=$(CI_BUILD_NUMBER)' -o drone_static
+	cd drone && go build --ldflags '-extldflags "-static" -X github.com/drone/drone/version.VersionDev=$(CI_BUILD_NUMBER)' -o drone
 
 test:
 	go test -cover $(PACKAGES)

api/build.go

@@ -12,6 +12,7 @@ import (
 
 	log "github.com/Sirupsen/logrus"
 	"github.com/drone/drone/engine"
+	"github.com/drone/drone/queue"
 	"github.com/drone/drone/remote"
 	"github.com/drone/drone/shared/httputil"
 	"github.com/drone/drone/store"
@@ -280,6 +281,36 @@ func PostBuild(c *gin.Context) {
 	// on status change notifications
 	last, _ := store.GetBuildLastBefore(c, repo, build.Branch, build.ID)
 
+
+	// IMPORTANT. PLEASE READ
+	//
+	// The below code uses a feature flag to switch between the current
+	// build engine and the exerimental 0.5 build engine. This can be
+	// enabled using with the environment variable CANARY=true
+
+	if os.Getenv("CANARY") == "true" {
+		for _, job := range jobs {
+			queue.Publish(c, &queue.Work{
+				User:      user,
+				Repo:      repo,
+				Build:     build,
+				BuildLast: last,
+				Job:       job,
+				Keys:      key,
+				Netrc:     netrc,
+				Yaml:      string(raw),
+				YamlEnc:   string(sec),
+				System: &model.System{
+					Link:      httputil.GetURL(c.Request),
+					Plugins:   strings.Split(os.Getenv("PLUGIN_FILTER"), " "),
+					Globals:   strings.Split(os.Getenv("PLUGIN_PARAMS"), " "),
+					Escalates: strings.Split(os.Getenv("ESCALATE_FILTER"), " "),
+				},
+			})
+		}
+		return // EXIT NOT TO AVOID THE 0.4 ENGINE CODE BELOW
+	}
+
 	engine_ := engine.FromContext(c)
 	go engine_.Schedule(c.Copy(), &engine.Task{
 		User:      user,

api/queue.go

@@ -0,0 +1,155 @@
+package api
+
+import (
+	"fmt"
+	"io"
+	"strconv"
+
+	"github.com/Sirupsen/logrus"
+	"github.com/drone/drone/bus"
+	"github.com/drone/drone/model"
+	"github.com/drone/drone/queue"
+	"github.com/drone/drone/remote"
+	"github.com/drone/drone/store"
+	"github.com/drone/drone/stream"
+	"github.com/gin-gonic/gin"
+)
+
+// Pull is a long request that polls and attemts to pull work off the queue stack.
+func Pull(c *gin.Context) {
+	logrus.Debugf("Agent %s connected.", c.ClientIP())
+
+	w := queue.PullClose(c, c.Writer)
+	if w == nil {
+		logrus.Debugf("Agent %s could not pull work.", c.ClientIP())
+	} else {
+		c.JSON(202, w)
+
+		logrus.Debugf("Agent %s assigned work. %s/%s#%d.%d",
+			c.ClientIP(),
+			w.Repo.Owner,
+			w.Repo.Name,
+			w.Build.Number,
+			w.Job.Number,
+		)
+	}
+}
+
+// Wait is a long request that polls and waits for cancelled build requests.
+func Wait(c *gin.Context) {
+	id, err := strconv.ParseInt(c.Param("id"), 10, 64)
+	if err != nil {
+		c.String(500, "Invalid input. %s", err)
+		return
+	}
+
+	eventc := make(chan *bus.Event, 1)
+
+	bus.Subscribe(c, eventc)
+	defer bus.Unsubscribe(c, eventc)
+
+	for {
+		select {
+		case event := <-eventc:
+			if event.Job.ID == id &&
+				event.Job.Status != model.StatusPending &&
+				event.Job.Status != model.StatusRunning {
+				c.JSON(200, event.Job)
+				return
+			}
+		case <-c.Writer.CloseNotify():
+			return
+		}
+	}
+}
+
+// Update handles build updates from the agent  and persists to the database.
+func Update(c *gin.Context) {
+	work := &queue.Work{}
+	if err := c.BindJSON(work); err != nil {
+		logrus.Errorf("Invalid input. %s", err)
+		return
+	}
+
+	// TODO(bradrydzewski) it is really annoying that we have to do this lookup
+	// and I'd prefer not to. The reason we do this is because the Build and Job
+	// have fields that aren't serialized to json and would be reset to their
+	// empty values if we just saved what was coming in the http.Request body.
+	build, err := store.GetBuild(c, work.Build.ID)
+	if err != nil {
+		c.String(404, "Unable to find build. %s", err)
+		return
+	}
+	job, err := store.GetJob(c, work.Job.ID)
+	if err != nil {
+		c.String(404, "Unable to find job. %s", err)
+		return
+	}
+	build.Started = work.Build.Started
+	build.Finished = work.Build.Finished
+	build.Status = work.Build.Status
+	job.Started = work.Job.Started
+	job.Finished = work.Job.Finished
+	job.Status = work.Job.Status
+	job.ExitCode = work.Job.ExitCode
+
+	ok, err := store.UpdateBuildJob(c, build, job)
+	if err != nil {
+		c.String(500, "Unable to update job. %s", err)
+		return
+	}
+
+	if ok {
+		// get the user because we transfer the user form the server to agent
+		// and back we lose the token which does not get serialized to json.
+		user, err := store.GetUser(c, work.User.ID)
+		if err != nil {
+			c.String(500, "Unable to find user. %s", err)
+			return
+		}
+		bus.Publish(c, &bus.Event{})
+		remote.Status(c, user, work.Repo, build,
+			fmt.Sprintf("%s/%s/%d", work.System.Link, work.Repo.FullName, work.Build.Number))
+	}
+	c.JSON(200, work)
+}
+
+// Stream streams the logs to disk or memory for broadcasing to listeners. Once
+// the stream is closed it is moved to permanent storage in the database.
+func Stream(c *gin.Context) {
+	id, err := strconv.ParseInt(c.Param("id"), 10, 64)
+	if err != nil {
+		c.String(500, "Invalid input. %s", err)
+		return
+	}
+
+	key := stream.ToKey(id)
+	rc, wc, err := stream.Create(c, key)
+	if err != nil {
+		logrus.Errorf("Agent %s failed to create stream. %s.", c.ClientIP(), err)
+		return
+	}
+
+	defer func() {
+		wc.Close()
+		rc.Close()
+		stream.Remove(c, key)
+	}()
+
+	io.Copy(wc, c.Request.Body)
+	wc.Close()
+
+	rcc, _, err := stream.Open(c, key)
+	if err != nil {
+		logrus.Errorf("Agent %s failed to read cache. %s.", c.ClientIP(), err)
+		return
+	}
+	defer func() {
+		rcc.Close()
+	}()
+
+	store.WriteLog(c, &model.Job{ID: id}, rcc)
+	c.String(200, "")
+
+	logrus.Debugf("Agent %s wrote stream to database", c.ClientIP())
+}

client/client.go

@@ -0,0 +1,235 @@
+package client
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"strconv"
+
+	"github.com/drone/drone/model"
+	"github.com/drone/drone/queue"
+	"golang.org/x/net/context"
+	"golang.org/x/net/context/ctxhttp"
+	"golang.org/x/oauth2"
+)
+
+const (
+	pathPull   = "%s/api/queue/pull"
+	pathWait   = "%s/api/queue/wait/%d"
+	pathStream = "%s/api/queue/stream/%d"
+	pathPush   = "%s/api/queue/status/%d"
+)
+
+type client struct {
+	client *http.Client
+	base   string // base url
+}
+
+// NewClient returns a client at the specified url.
+func NewClient(uri string) Client {
+	return &client{http.DefaultClient, uri}
+}
+
+// NewClientToken returns a client at the specified url that
+// authenticates all outbound requests with the given token.
+func NewClientToken(uri, token string) Client {
+	config := new(oauth2.Config)
+	auther := config.Client(oauth2.NoContext, &oauth2.Token{AccessToken: token})
+	return &client{auther, uri}
+}
+
+// Pull pulls work from the server queue.
+func (c *client) Pull() (*queue.Work, error) {
+	out := new(queue.Work)
+	uri := fmt.Sprintf(pathPull, c.base)
+	err := c.post(uri, nil, out)
+	return out, err
+}
+
+// Push pushes an update to the server.
+func (c *client) Push(p *queue.Work) error {
+	uri := fmt.Sprintf(pathPush, c.base, p.Job.ID)
+	err := c.post(uri, p, nil)
+	return err
+}
+
+// Stream streams the build logs to the server.
+func (c *client) Stream(id int64, rc io.ReadCloser) error {
+	uri := fmt.Sprintf(pathStream, c.base, id)
+	err := c.post(uri, rc, nil)
+	return err
+}
+
+// Wait watches and waits for the build to cancel or finish.
+func (c *client) Wait(id int64) *Wait {
+	ctx, cancel := context.WithCancel(context.Background())
+	return &Wait{id, c, ctx, cancel}
+}
+
+////////
+
+type CancelNotifier interface {
+	Canecel()
+	CancelNotify() bool
+	IsCancelled() bool
+}
+
+////////
+
+type Wait struct {
+	id     int64
+	client *client
+
+	ctx    context.Context
+	cancel context.CancelFunc
+}
+
+func (w *Wait) Done() (*model.Job, error) {
+	uri := fmt.Sprintf(pathWait, w.client.base, w.id)
+	req, err := w.client.createRequest(uri, "POST", nil)
+	if err != nil {
+		return nil, err
+	}
+
+	res, err := ctxhttp.Do(w.ctx, w.client.client, req)
+	if err != nil {
+		return nil, err
+	}
+	defer res.Body.Close()
+	job := &model.Job{}
+	err = json.NewDecoder(res.Body).Decode(&job)
+	if err != nil {
+		return nil, err
+	}
+	return job, nil
+}
+
+func (w *Wait) Cancel() {
+	w.cancel()
+}
+
+//
+// http request helper functions
+//
+
+// helper function for making an http GET request.
+func (c *client) get(rawurl string, out interface{}) error {
+	return c.do(rawurl, "GET", nil, out)
+}
+
+// helper function for making an http POST request.
+func (c *client) post(rawurl string, in, out interface{}) error {
+	return c.do(rawurl, "POST", in, out)
+}
+
+// helper function for making an http PUT request.
+func (c *client) put(rawurl string, in, out interface{}) error {
+	return c.do(rawurl, "PUT", in, out)
+}
+
+// helper function for making an http PATCH request.
+func (c *client) patch(rawurl string, in, out interface{}) error {
+	return c.do(rawurl, "PATCH", in, out)
+}
+
+// helper function for making an http DELETE request.
+func (c *client) delete(rawurl string) error {
+	return c.do(rawurl, "DELETE", nil, nil)
+}
+
+// helper function to make an http request
+func (c *client) do(rawurl, method string, in, out interface{}) error {
+	// executes the http request and returns the body as
+	// and io.ReadCloser
+	body, err := c.open(rawurl, method, in, out)
+	if err != nil {
+		return err
+	}
+	defer body.Close()
+
+	// if a json response is expected, parse and return
+	// the json response.
+	if out != nil {
+		return json.NewDecoder(body).Decode(out)
+	}
+	return nil
+}
+
+// helper function to open an http request
+func (c *client) open(rawurl, method string, in, out interface{}) (io.ReadCloser, error) {
+	uri, err := url.Parse(rawurl)
+	if err != nil {
+		return nil, err
+	}
+
+	// creates a new http request to bitbucket.
+	req, err := http.NewRequest(method, uri.String(), nil)
+	if err != nil {
+		return nil, err
+	}
+
+	// if we are posting or putting data, we need to
+	// write it to the body of the request.
+	if in != nil {
+		rc, ok := in.(io.ReadCloser)
+		if ok {
+			req.Body = rc
+			req.Header.Set("Content-Type", "plain/text")
+		} else {
+			inJson, err := json.Marshal(in)
+			if err != nil {
+				return nil, err
+			}
+
+			buf := bytes.NewBuffer(inJson)
+			req.Body = ioutil.NopCloser(buf)
+
+			req.ContentLength = int64(len(inJson))
+			req.Header.Set("Content-Length", strconv.Itoa(len(inJson)))
+			req.Header.Set("Content-Type", "application/json")
+		}
+	}
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	if resp.StatusCode > http.StatusPartialContent {
+		defer resp.Body.Close()
+		out, _ := ioutil.ReadAll(resp.Body)
+		return nil, fmt.Errorf(string(out))
+	}
+	return resp.Body, nil
+}
+
+// createRequest is a helper function that builds an http.Request.
+func (c *client) createRequest(rawurl, method string, in interface{}) (*http.Request, error) {
+	uri, err := url.Parse(rawurl)
+	if err != nil {
+		return nil, err
+	}
+
+	// if we are posting or putting data, we need to
+	// write it to the body of the request.
+	var buf io.ReadWriter
+	if in != nil {
+		buf = new(bytes.Buffer)
+		err := json.NewEncoder(buf).Encode(in)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	// creates a new http request to bitbucket.
+	req, err := http.NewRequest(method, uri.String(), buf)
+	if err != nil {
+		return nil, err
+	}
+	if in != nil {
+		req.Header.Set("Content-Type", "application/json")
+	}
+	return req, nil
+}

client/interface.go

@@ -0,0 +1,22 @@
+package client
+
+import (
+	"io"
+
+	"github.com/drone/drone/queue"
+)
+
+// Client is used to communicate with a Drone server.
+type Client interface {
+	// Pull pulls work from the server queue.
+	Pull() (*queue.Work, error)
+
+	// Push pushes an update to the server.
+	Push(*queue.Work) error
+
+	// Stream streams the build logs to the server.
+	Stream(int64, io.ReadCloser) error
+
+	// Wait waits for the job to the complete.
+	Wait(int64) *Wait
+}

drone/agent/agent.go

@@ -0,0 +1,110 @@
+package agent
+
+import (
+	"sync"
+	"time"
+
+	"github.com/drone/drone/client"
+	"github.com/samalba/dockerclient"
+
+	"github.com/Sirupsen/logrus"
+	"github.com/codegangsta/cli"
+)
+
+// AgentCmd is the exported command for starting the drone agent.
+var AgentCmd = cli.Command{
+	Name:   "agent",
+	Usage:  "starts the drone agent",
+	Action: start,
+	Flags: []cli.Flag{
+		cli.StringFlag{
+			EnvVar: "DOCKER_HOST",
+			Name:   "docker-host",
+			Usage:  "docker deamon address",
+			Value:  "unix:///var/run/docker.sock",
+		},
+		cli.BoolFlag{
+			EnvVar: "DOCKER_TLS_VERIFY",
+			Name:   "docker-tls-verify",
+			Usage:  "docker daemon supports tlsverify",
+		},
+		cli.StringFlag{
+			EnvVar: "DOCKER_CERT_PATH",
+			Name:   "docker-cert-path",
+			Usage:  "docker certificate directory",
+			Value:  "",
+		},
+		cli.IntFlag{
+			EnvVar: "DOCKER_MAX_PROCS",
+			Name:   "docker-max-procs",
+			Usage:  "limit number of running docker processes",
+			Value:  2,
+		},
+		cli.StringFlag{
+			EnvVar: "DRONE_SERVER",
+			Name:   "drone-server",
+			Usage:  "drone server address",
+			Value:  "http://localhost:8000",
+		},
+		cli.StringFlag{
+			EnvVar: "DRONE_TOKEN",
+			Name:   "drone-token",
+			Usage:  "drone authorization token",
+		},
+		cli.DurationFlag{
+			EnvVar: "BACKOFF",
+			Name:   "drone-backoff",
+			Usage:  "drone server backoff interval",
+			Value:  time.Second * 15,
+		},
+		cli.BoolFlag{
+			EnvVar: "DEBUG",
+			Name:   "debug",
+			Usage:  "start the agent in debug mode",
+		},
+		cli.BoolFlag{
+			EnvVar: "EXPERIMENTAL",
+			Name:   "experimental",
+			Usage:  "start the agent with experimental features",
+		},
+	},
+}
+
+func start(c *cli.Context) {
+
+	// debug level if requested by user
+	if c.Bool("debug") {
+		logrus.SetLevel(logrus.DebugLevel)
+	} else {
+		logrus.SetLevel(logrus.WarnLevel)
+	}
+
+	client := client.NewClientToken(
+		c.String("drone-server"),
+		c.String("drone-token"),
+	)
+
+	tls, _ := dockerclient.TLSConfigFromCertPath(c.String("docker-cert-path"))
+	if c.Bool("docker-host") {
+		tls.InsecureSkipVerify = true
+	}
+	docker, err := dockerclient.NewDockerClient(c.String("docker-host"), tls)
+	if err != nil {
+		logrus.Fatal(err)
+	}
+
+	var wg sync.WaitGroup
+	for i := 0; i < c.Int("docker-max-procs"); i++ {
+		wg.Add(1)
+		go func() {
+			for {
+				if err := recoverExec(client, docker); err != nil {
+					dur := c.Duration("drone-backoff")
+					logrus.Debugf("Attempting to reconnect in %v", dur)
+					time.Sleep(dur)
+				}
+			}
+		}()
+	}
+	wg.Wait()
+}

drone/agent/exec.go

@@ -0,0 +1,186 @@
+package agent
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"time"
+
+	"github.com/Sirupsen/logrus"
+	"github.com/dchest/uniuri"
+	"github.com/drone/drone/client"
+	"github.com/drone/drone/engine/compiler"
+	"github.com/drone/drone/engine/compiler/builtin"
+	"github.com/drone/drone/engine/runner"
+	engine "github.com/drone/drone/engine/runner/docker"
+	"github.com/drone/drone/model"
+
+	"github.com/samalba/dockerclient"
+	"golang.org/x/net/context"
+)
+
+func recoverExec(client client.Client, docker dockerclient.Client) error {
+	defer func() {
+		recover()
+	}()
+	return exec(client, docker)
+}
+
+func exec(client client.Client, docker dockerclient.Client) error {
+	w, err := client.Pull()
+	if err != nil {
+		return err
+	}
+
+	logrus.Infof("Starting build %s/%s#%d.%d",
+		w.Repo.Owner, w.Repo.Name, w.Build.Number, w.Job.Number)
+
+	w.Job.Status = model.StatusRunning
+	w.Job.Started = time.Now().Unix()
+
+	prefix := fmt.Sprintf("drone_%s", uniuri.New())
+
+	trans := []compiler.Transform{
+		builtin.NewCloneOp("plugins/git:latest", true),
+		builtin.NewCacheOp(
+			"plugins/cache:latest",
+			"/var/lib/drone/cache/"+w.Repo.FullName,
+			false,
+		),
+		builtin.NewNormalizeOp("plugins"),
+		builtin.NewWorkspaceOp("/drone", "drone/src/github.com/"+w.Repo.FullName),
+		builtin.NewEnvOp(map[string]string{
+			"CI":                "drone",
+			"CI_REPO":           w.Repo.FullName,
+			"CI_REPO_OWNER":     w.Repo.Owner,
+			"CI_REPO_NAME":      w.Repo.Name,
+			"CI_REPO_LINK":      w.Repo.Link,
+			"CI_REPO_AVATAR":    w.Repo.Avatar,
+			"CI_REPO_BRANCH":    w.Repo.Branch,
+			"CI_REPO_PRIVATE":   fmt.Sprintf("%v", w.Repo.IsPrivate),
+			"CI_REMOTE_URL":     w.Repo.Clone,
+			"CI_COMMIT_SHA":     w.Build.Commit,
+			"CI_COMMIT_REF":     w.Build.Ref,
+			"CI_COMMIT_BRANCH":  w.Build.Branch,
+			"CI_COMMIT_LINK":    w.Build.Link,
+			"CI_COMMIT_MESSAGE": w.Build.Message,
+			"CI_AUTHOR":         w.Build.Author,
+			"CI_AUTHOR_EMAIL":   w.Build.Email,
+			"CI_AUTHOR_AVATAR":  w.Build.Avatar,
+			"CI_BUILD_NUMBER":   fmt.Sprintf("%v", w.Build.Number),
+			"CI_BUILD_EVENT":    w.Build.Event,
+			// "CI_NETRC_USERNAME":    w.Netrc.Login,
+			// "CI_NETRC_PASSWORD":    w.Netrc.Password,
+			// "CI_NETRC_MACHINE":     w.Netrc.Machine,
+			// "CI_PREV_BUILD_STATUS": w.BuildLast.Status,
+			// "CI_PREV_BUILD_NUMBER": fmt.Sprintf("%v", w.BuildLast.Number),
+			// "CI_PREV_COMMIT_SHA":   w.BuildLast.Commit,
+		}),
+		builtin.NewValidateOp(
+			w.Repo.IsTrusted,
+			[]string{"plugins/*"},
+		),
+		builtin.NewShellOp(builtin.Linux_adm64),
+		builtin.NewArgsOp(),
+		builtin.NewPodOp(prefix),
+		builtin.NewAliasOp(prefix),
+		builtin.NewPullOp(false),
+		builtin.NewFilterOp(
+			model.StatusSuccess, // w.BuildLast.Status,
+			w.Build.Branch,
+			w.Build.Event,
+			w.Build.Deploy,
+			map[string]string{},
+		),
+	}
+
+	compile := compiler.New()
+	compile.Transforms(trans)
+	spec, err := compile.CompileString(w.Yaml)
+	if err != nil {
+		// TODO handle error
+		logrus.Infof("Error compiling Yaml %s/%s#%d %s",
+			w.Repo.Owner, w.Repo.Name, w.Build.Number, err.Error())
+		return err
+	}
+
+	if err := client.Push(w); err != nil {
+		logrus.Errorf("Error persisting update %s/%s#%d.%d. %s",
+			w.Repo.Owner, w.Repo.Name, w.Build.Number, w.Job.Number, err)
+		return err
+	}
+
+	conf := runner.Config{
+		Engine: engine.New(docker),
+	}
+
+	ctx := context.TODO()
+	ctx, cancel := context.WithCancel(ctx)
+
+	run := conf.Runner(ctx, spec)
+	run.Run()
+	defer cancel()
+
+	wait := client.Wait(w.Job.ID)
+	if err != nil {
+		return err
+	}
+	go func() {
+		_, werr := wait.Done()
+		if werr == nil {
+			logrus.Infof("Cancel build %s/%s#%d.%d",
+				w.Repo.Owner, w.Repo.Name, w.Build.Number, w.Job.Number)
+			cancel()
+		}
+	}()
+	defer wait.Cancel()
+
+	rc, wc := io.Pipe()
+	go func() {
+		err := client.Stream(w.Job.ID, rc)
+		if err != nil && err != io.ErrClosedPipe {
+			logrus.Errorf("Error streaming build logs. %s", err)
+		}
+	}()
+
+	pipe := run.Pipe()
+	for {
+		line := pipe.Next()
+		if line == nil {
+			break
+		}
+		linejson, _ := json.Marshal(line)
+		wc.Write(linejson)
+		wc.Write([]byte{'\n'})
+	}
+
+	err = run.Wait()
+
+	pipe.Close()
+	wc.Close()
+	rc.Close()
+
+	// catch the build result
+	if err != nil {
+		w.Job.ExitCode = 255
+	}
+	if exitErr, ok := err.(*runner.ExitError); ok {
+		w.Job.ExitCode = exitErr.Code
+	}
+
+	w.Job.Finished = time.Now().Unix()
+
+	switch w.Job.ExitCode {
+	case 128, 130:
+		w.Job.Status = model.StatusKilled
+	case 0:
+		w.Job.Status = model.StatusSuccess
+	default:
+		w.Job.Status = model.StatusFailure
+	}
+
+	logrus.Infof("Finished build %s/%s#%d.%d",
+		w.Repo.Owner, w.Repo.Name, w.Build.Number, w.Job.Number)
+
+	return client.Push(w)
+}

drone/drone.go

@@ -0,0 +1,29 @@
+package main
+
+import (
+	"os"
+
+	"github.com/drone/drone/drone/agent"
+	"github.com/drone/drone/drone/server"
+	"github.com/drone/drone/version"
+
+	"github.com/codegangsta/cli"
+	"github.com/ianschenck/envflag"
+	_ "github.com/joho/godotenv/autoload"
+)
+
+func main2() {
+	envflag.Parse()
+
+	app := cli.NewApp()
+	app.Name = "drone"
+	app.Version = version.Version
+	app.Usage = "command line utility"
+
+	app.Commands = []cli.Command{
+		agent.AgentCmd,
+		server.ServeCmd,
+	}
+
+	app.Run(os.Args)
+}

drone/main.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"net/http"
+	"os"
 	"time"
 
 	"github.com/drone/drone/router"
@@ -22,6 +23,11 @@ var (
 )
 
 func main() {
+	if os.Getenv("CANARY") == "true" {
+		main2()
+		return
+	}
+
 	envflag.Parse()
 
 	// debug level if requested by user
@@ -35,6 +41,9 @@ func main() {
 	handler := router.Load(
 		ginrus.Ginrus(logrus.StandardLogger(), time.RFC3339, true),
 		middleware.Version,
+		middleware.Queue(),
+		middleware.Stream(),
+		middleware.Bus(),
 		middleware.Cache(),
 		middleware.Store(),
 		middleware.Remote(),

drone/server/server.go

@@ -0,0 +1,89 @@
+package server
+
+import (
+	"net/http"
+	"time"
+
+	"github.com/drone/drone/router"
+	"github.com/drone/drone/router/middleware"
+
+	"github.com/Sirupsen/logrus"
+	"github.com/codegangsta/cli"
+	"github.com/gin-gonic/contrib/ginrus"
+)
+
+// ServeCmd is the exported command for starting the drone server.
+var ServeCmd = cli.Command{
+	Name:  "serve",
+	Usage: "starts the drone server",
+	Action: func(c *cli.Context) {
+		if err := start(c); err != nil {
+			logrus.Fatal(err)
+		}
+	},
+	Flags: []cli.Flag{
+		cli.StringFlag{
+			EnvVar: "SERVER_ADDR",
+			Name:   "server-addr",
+			Usage:  "server address",
+			Value:  ":8000",
+		},
+		cli.StringFlag{
+			EnvVar: "SERVER_CERT",
+			Name:   "server-cert",
+			Usage:  "server ssl cert",
+		},
+		cli.StringFlag{
+			EnvVar: "SERVER_KEY",
+			Name:   "server-key",
+			Usage:  "server ssl key",
+		},
+		cli.BoolFlag{
+			EnvVar: "DEBUG",
+			Name:   "debug",
+			Usage:  "start the server in debug mode",
+		},
+		cli.BoolFlag{
+			EnvVar: "EXPERIMENTAL",
+			Name:   "experimental",
+			Usage:  "start the server with experimental features",
+		},
+	},
+}
+
+func start(c *cli.Context) error {
+
+	// debug level if requested by user
+	if c.Bool("debug") {
+		logrus.SetLevel(logrus.DebugLevel)
+	} else {
+		logrus.SetLevel(logrus.WarnLevel)
+	}
+
+	// setup the server and start the listener
+	handler := router.Load(
+		ginrus.Ginrus(logrus.StandardLogger(), time.RFC3339, true),
+		middleware.Version,
+		middleware.Queue(),
+		middleware.Stream(),
+		middleware.Bus(),
+		middleware.Cache(),
+		middleware.Store(),
+		middleware.Remote(),
+		middleware.Engine(),
+	)
+
+	if c.String("server-cert") != "" {
+		return http.ListenAndServeTLS(
+			c.String("server-addr"),
+			c.String("server-cert"),
+			c.String("server-key"),
+			handler,
+		)
+	}
+
+	return http.ListenAndServe(
+		c.String("server-addr"),
+		handler,
+	)
+}

engine/compiler/compile.go

@@ -1,4 +1,4 @@
-package libyaml
+package compiler
 
 import (
 	"github.com/drone/drone/engine/runner"
@@ -143,4 +143,4 @@ func (c *Compiler) walk(node yaml.Node) (err error) {
 		}
 	}
 	return err
-}
+}

engine/compiler/compile_test.go

@@ -1 +1 @@
-package libyaml
+package compiler

engine/compiler/transform.go

@@ -1,4 +1,4 @@
-package libyaml
+package compiler
 
 import "github.com/drone/drone/engine/compiler/parse"
 

engine/runner/docker/context.go

@@ -0,0 +1,24 @@
+package docker
+
+import (
+	"github.com/drone/drone/engine/runner"
+	"golang.org/x/net/context"
+)
+
+const key = "docker"
+
+// Setter defines a context that enables setting values.
+type Setter interface {
+	Set(string, interface{})
+}
+
+// FromContext returns the Engine associated with this context.
+func FromContext(c context.Context) runner.Engine {
+	return c.Value(key).(runner.Engine)
+}
+
+// ToContext adds the Engine to this context if it supports the
+// Setter interface.
+func ToContext(c Setter, d runner.Engine) {
+	c.Set(key, d)
+}

engine/runner/docker/docker.go

@@ -0,0 +1,111 @@
+package docker
+
+import (
+	"io"
+
+	"github.com/drone/drone/engine/runner"
+	"github.com/drone/drone/engine/runner/docker/internal"
+
+	"github.com/samalba/dockerclient"
+)
+
+type dockerEngine struct {
+	client dockerclient.Client
+}
+
+func (e *dockerEngine) ContainerStart(container *runner.Container) (string, error) {
+	conf := toContainerConfig(container)
+	auth := toAuthConfig(container)
+
+	// pull the image if it does not exists or if the Container
+	// is configured to always pull a new image.
+	_, err := e.client.InspectImage(container.Image)
+	if err != nil || container.Pull {
+		e.client.PullImage(container.Image, auth)
+	}
+
+	// create and start the container and return the Container ID.
+	id, err := e.client.CreateContainer(conf, container.Name, auth)
+	if err != nil {
+		return id, err
+	}
+	err = e.client.StartContainer(id, &conf.HostConfig)
+	if err != nil {
+
+		// remove the container if it cannot be started
+		e.client.RemoveContainer(id, true, true)
+		return id, err
+	}
+	return id, nil
+}
+
+func (e *dockerEngine) ContainerStop(id string) error {
+	e.client.StopContainer(id, 1)
+	e.client.KillContainer(id, "9")
+	return nil
+}
+
+func (e *dockerEngine) ContainerRemove(id string) error {
+	e.client.StopContainer(id, 1)
+	e.client.KillContainer(id, "9")
+	e.client.RemoveContainer(id, true, true)
+	return nil
+}
+
+func (e *dockerEngine) ContainerWait(id string) (*runner.State, error) {
+	// wait for the container to exit
+	//
+	// TODO(bradrydzewski) we should have a for loop here
+	// to re-connect and wait if this channel returns a
+	// result even though the container is still running.
+	//
+	<-e.client.Wait(id)
+	v, err := e.client.InspectContainer(id)
+	if err != nil {
+		return nil, err
+	}
+	return &runner.State{
+		ExitCode:  v.State.ExitCode,
+		OOMKilled: v.State.OOMKilled,
+	}, nil
+}
+
+func (e *dockerEngine) ContainerLogs(id string) (io.ReadCloser, error) {
+	opts := &dockerclient.LogOptions{
+		Follow: true,
+		Stdout: true,
+		Stderr: true,
+	}
+
+	piper, pipew := io.Pipe()
+	go func() {
+		defer pipew.Close()
+
+		// sometimes the docker logs fails due to parsing errors. this
+		// routine will check for such a failure and attempt to resume
+		// if necessary.
+		for i := 0; i < 5; i++ {
+			if i > 0 {
+				opts.Tail = 1
+			}
+
+			rc, err := e.client.ContainerLogs(id, opts)
+			if err != nil {
+				return
+			}
+			defer rc.Close()
+
+			// use Docker StdCopy
+			internal.StdCopy(pipew, pipew, rc)
+
+			// check to see if the container is still running. If not,
+			// we can safely exit and assume there are no more logs left
+			// to stream.
+			v, err := e.client.InspectContainer(id)
+			if err != nil || !v.State.Running {
+				return
+			}
+		}
+	}()
+	return piper, nil
+}

engine/runner/docker/docker_test.go

@@ -0,0 +1 @@
+package docker

engine/runner/docker/helper.go

@@ -0,0 +1,49 @@
+package docker
+
+import (
+	"os"
+
+	"github.com/drone/drone/engine/runner"
+	"github.com/samalba/dockerclient"
+)
+
+var (
+	dockerHost = os.Getenv("DOCKER_HOST")
+	dockerCert = os.Getenv("DOCKER_CERT_PATH")
+	dockerTLS  = os.Getenv("DOCKER_TLS_VERIFY")
+)
+
+func init() {
+	if dockerHost == "" {
+		dockerHost = "unix:///var/run/docker.sock"
+	}
+}
+
+// New returns a new Docker engine using the provided Docker client.
+func New(client dockerclient.Client) runner.Engine {
+	return &dockerEngine{client}
+}
+
+// NewEnv returns a new Docker engine from the DOCKER_HOST and DOCKER_CERT_PATH
+// environment variables.
+func NewEnv() (runner.Engine, error) {
+	config, err := dockerclient.TLSConfigFromCertPath(dockerCert)
+	if err == nil && dockerTLS != "1" {
+		config.InsecureSkipVerify = true
+	}
+	client, err := dockerclient.NewDockerClient(dockerHost, config)
+	if err != nil {
+		return nil, err
+	}
+	return New(client), nil
+}
+
+// MustEnv returns a new Docker engine from the DOCKER_HOST and DOCKER_CERT_PATH
+// environment variables. Errors creating the Docker engine will panic.
+func MustEnv() runner.Engine {
+	engine, err := NewEnv()
+	if err != nil {
+		panic(err)
+	}
+	return engine
+}

engine/runner/docker/helper_test.go

@@ -0,0 +1 @@
+package docker

engine/runner/docker/internal/README

@@ -0,0 +1 @@
+This is an internal copy of the Docker stdcopy package that removes the logrus debug logging. The original package is found at https://github.com/docker/docker/tree/master/pkg/stdcopy

engine/runner/docker/internal/stdcopy.go

@@ -0,0 +1,167 @@
+package internal
+
+import (
+	"encoding/binary"
+	"errors"
+	"fmt"
+	"io"
+)
+
+// StdType is the type of standard stream
+// a writer can multiplex to.
+type StdType byte
+
+const (
+	// Stdin represents standard input stream type.
+	Stdin StdType = iota
+	// Stdout represents standard output stream type.
+	Stdout
+	// Stderr represents standard error steam type.
+	Stderr
+
+	stdWriterPrefixLen = 8
+	stdWriterFdIndex   = 0
+	stdWriterSizeIndex = 4
+
+	startingBufLen = 32*1024 + stdWriterPrefixLen + 1
+)
+
+// stdWriter is wrapper of io.Writer with extra customized info.
+type stdWriter struct {
+	io.Writer
+	prefix byte
+}
+
+// Write sends the buffer to the underneath writer.
+// It insert the prefix header before the buffer,
+// so stdcopy.StdCopy knows where to multiplex the output.
+// It makes stdWriter to implement io.Writer.
+func (w *stdWriter) Write(buf []byte) (n int, err error) {
+	if w == nil || w.Writer == nil {
+		return 0, errors.New("Writer not instantiated")
+	}
+	if buf == nil {
+		return 0, nil
+	}
+
+	header := [stdWriterPrefixLen]byte{stdWriterFdIndex: w.prefix}
+	binary.BigEndian.PutUint32(header[stdWriterSizeIndex:], uint32(len(buf)))
+
+	line := append(header[:], buf...)
+
+	n, err = w.Writer.Write(line)
+	n -= stdWriterPrefixLen
+
+	if n < 0 {
+		n = 0
+	}
+	return
+}
+
+// NewStdWriter instantiates a new Writer.
+// Everything written to it will be encapsulated using a custom format,
+// and written to the underlying `w` stream.
+// This allows multiple write streams (e.g. stdout and stderr) to be muxed into a single connection.
+// `t` indicates the id of the stream to encapsulate.
+// It can be stdcopy.Stdin, stdcopy.Stdout, stdcopy.Stderr.
+func NewStdWriter(w io.Writer, t StdType) io.Writer {
+	return &stdWriter{
+		Writer: w,
+		prefix: byte(t),
+	}
+}
+
+// StdCopy is a modified version of io.Copy.
+//
+// StdCopy will demultiplex `src`, assuming that it contains two streams,
+// previously multiplexed together using a StdWriter instance.
+// As it reads from `src`, StdCopy will write to `dstout` and `dsterr`.
+//
+// StdCopy will read until it hits EOF on `src`. It will then return a nil error.
+// In other words: if `err` is non nil, it indicates a real underlying error.
+//
+// `written` will hold the total number of bytes written to `dstout` and `dsterr`.
+func StdCopy(dstout, dsterr io.Writer, src io.Reader) (written int64, err error) {
+	var (
+		buf       = make([]byte, startingBufLen)
+		bufLen    = len(buf)
+		nr, nw    int
+		er, ew    error
+		out       io.Writer
+		frameSize int
+	)
+
+	for {
+		// Make sure we have at least a full header
+		for nr < stdWriterPrefixLen {
+			var nr2 int
+			nr2, er = src.Read(buf[nr:])
+			nr += nr2
+			if er == io.EOF {
+				if nr < stdWriterPrefixLen {
+					return written, nil
+				}
+				break
+			}
+			if er != nil {
+				return 0, er
+			}
+		}
+
+		// Check the first byte to know where to write
+		switch StdType(buf[stdWriterFdIndex]) {
+		case Stdin:
+			fallthrough
+		case Stdout:
+			// Write on stdout
+			out = dstout
+		case Stderr:
+			// Write on stderr
+			out = dsterr
+		default:
+			return 0, fmt.Errorf("Unrecognized input header: %d", buf[stdWriterFdIndex])
+		}
+
+		// Retrieve the size of the frame
+		frameSize = int(binary.BigEndian.Uint32(buf[stdWriterSizeIndex : stdWriterSizeIndex+4]))
+
+		// Check if the buffer is big enough to read the frame.
+		// Extend it if necessary.
+		if frameSize+stdWriterPrefixLen > bufLen {
+			buf = append(buf, make([]byte, frameSize+stdWriterPrefixLen-bufLen+1)...)
+			bufLen = len(buf)
+		}
+
+		// While the amount of bytes read is less than the size of the frame + header, we keep reading
+		for nr < frameSize+stdWriterPrefixLen {
+			var nr2 int
+			nr2, er = src.Read(buf[nr:])
+			nr += nr2
+			if er == io.EOF {
+				if nr < frameSize+stdWriterPrefixLen {
+					return written, nil
+				}
+				break
+			}
+			if er != nil {
+				return 0, er
+			}
+		}
+
+		// Write the retrieved frame (without header)
+		nw, ew = out.Write(buf[stdWriterPrefixLen : frameSize+stdWriterPrefixLen])
+		if ew != nil {
+			return 0, ew
+		}
+		// If the frame has not been fully written: error
+		if nw != frameSize {
+			return 0, io.ErrShortWrite
+		}
+		written += int64(nw)
+
+		// Move the rest of the buffer to the beginning
+		copy(buf, buf[frameSize+stdWriterPrefixLen:])
+		// Move the index
+		nr -= frameSize + stdWriterPrefixLen
+	}
+}

engine/runner/docker/internal/stdcopy_test.go

@@ -0,0 +1,260 @@
+package internal
+
+import (
+	"bytes"
+	"errors"
+	"io"
+	"io/ioutil"
+	"strings"
+	"testing"
+)
+
+func TestNewStdWriter(t *testing.T) {
+	writer := NewStdWriter(ioutil.Discard, Stdout)
+	if writer == nil {
+		t.Fatalf("NewStdWriter with an invalid StdType should not return nil.")
+	}
+}
+
+func TestWriteWithUnitializedStdWriter(t *testing.T) {
+	writer := stdWriter{
+		Writer: nil,
+		prefix: byte(Stdout),
+	}
+	n, err := writer.Write([]byte("Something here"))
+	if n != 0 || err == nil {
+		t.Fatalf("Should fail when given an uncomplete or uninitialized StdWriter")
+	}
+}
+
+func TestWriteWithNilBytes(t *testing.T) {
+	writer := NewStdWriter(ioutil.Discard, Stdout)
+	n, err := writer.Write(nil)
+	if err != nil {
+		t.Fatalf("Shouldn't have fail when given no data")
+	}
+	if n > 0 {
+		t.Fatalf("Write should have written 0 byte, but has written %d", n)
+	}
+}
+
+func TestWrite(t *testing.T) {
+	writer := NewStdWriter(ioutil.Discard, Stdout)
+	data := []byte("Test StdWrite.Write")
+	n, err := writer.Write(data)
+	if err != nil {
+		t.Fatalf("Error while writing with StdWrite")
+	}
+	if n != len(data) {
+		t.Fatalf("Write should have written %d byte but wrote %d.", len(data), n)
+	}
+}
+
+type errWriter struct {
+	n   int
+	err error
+}
+
+func (f *errWriter) Write(buf []byte) (int, error) {
+	return f.n, f.err
+}
+
+func TestWriteWithWriterError(t *testing.T) {
+	expectedError := errors.New("expected")
+	expectedReturnedBytes := 10
+	writer := NewStdWriter(&errWriter{
+		n:   stdWriterPrefixLen + expectedReturnedBytes,
+		err: expectedError}, Stdout)
+	data := []byte("This won't get written, sigh")
+	n, err := writer.Write(data)
+	if err != expectedError {
+		t.Fatalf("Didn't get expected error.")
+	}
+	if n != expectedReturnedBytes {
+		t.Fatalf("Didn't get expected written bytes %d, got %d.",
+			expectedReturnedBytes, n)
+	}
+}
+
+func TestWriteDoesNotReturnNegativeWrittenBytes(t *testing.T) {
+	writer := NewStdWriter(&errWriter{n: -1}, Stdout)
+	data := []byte("This won't get written, sigh")
+	actual, _ := writer.Write(data)
+	if actual != 0 {
+		t.Fatalf("Expected returned written bytes equal to 0, got %d", actual)
+	}
+}
+
+func getSrcBuffer(stdOutBytes, stdErrBytes []byte) (buffer *bytes.Buffer, err error) {
+	buffer = new(bytes.Buffer)
+	dstOut := NewStdWriter(buffer, Stdout)
+	_, err = dstOut.Write(stdOutBytes)
+	if err != nil {
+		return
+	}
+	dstErr := NewStdWriter(buffer, Stderr)
+	_, err = dstErr.Write(stdErrBytes)
+	return
+}
+
+func TestStdCopyWriteAndRead(t *testing.T) {
+	stdOutBytes := []byte(strings.Repeat("o", startingBufLen))
+	stdErrBytes := []byte(strings.Repeat("e", startingBufLen))
+	buffer, err := getSrcBuffer(stdOutBytes, stdErrBytes)
+	if err != nil {
+		t.Fatal(err)
+	}
+	written, err := StdCopy(ioutil.Discard, ioutil.Discard, buffer)
+	if err != nil {
+		t.Fatal(err)
+	}
+	expectedTotalWritten := len(stdOutBytes) + len(stdErrBytes)
+	if written != int64(expectedTotalWritten) {
+		t.Fatalf("Expected to have total of %d bytes written, got %d", expectedTotalWritten, written)
+	}
+}
+
+type customReader struct {
+	n            int
+	err          error
+	totalCalls   int
+	correctCalls int
+	src          *bytes.Buffer
+}
+
+func (f *customReader) Read(buf []byte) (int, error) {
+	f.totalCalls++
+	if f.totalCalls <= f.correctCalls {
+		return f.src.Read(buf)
+	}
+	return f.n, f.err
+}
+
+func TestStdCopyReturnsErrorReadingHeader(t *testing.T) {
+	expectedError := errors.New("error")
+	reader := &customReader{
+		err: expectedError}
+	written, err := StdCopy(ioutil.Discard, ioutil.Discard, reader)
+	if written != 0 {
+		t.Fatalf("Expected 0 bytes read, got %d", written)
+	}
+	if err != expectedError {
+		t.Fatalf("Didn't get expected error")
+	}
+}
+
+func TestStdCopyReturnsErrorReadingFrame(t *testing.T) {
+	expectedError := errors.New("error")
+	stdOutBytes := []byte(strings.Repeat("o", startingBufLen))
+	stdErrBytes := []byte(strings.Repeat("e", startingBufLen))
+	buffer, err := getSrcBuffer(stdOutBytes, stdErrBytes)
+	if err != nil {
+		t.Fatal(err)
+	}
+	reader := &customReader{
+		correctCalls: 1,
+		n:            stdWriterPrefixLen + 1,
+		err:          expectedError,
+		src:          buffer}
+	written, err := StdCopy(ioutil.Discard, ioutil.Discard, reader)
+	if written != 0 {
+		t.Fatalf("Expected 0 bytes read, got %d", written)
+	}
+	if err != expectedError {
+		t.Fatalf("Didn't get expected error")
+	}
+}
+
+func TestStdCopyDetectsCorruptedFrame(t *testing.T) {
+	stdOutBytes := []byte(strings.Repeat("o", startingBufLen))
+	stdErrBytes := []byte(strings.Repeat("e", startingBufLen))
+	buffer, err := getSrcBuffer(stdOutBytes, stdErrBytes)
+	if err != nil {
+		t.Fatal(err)
+	}
+	reader := &customReader{
+		correctCalls: 1,
+		n:            stdWriterPrefixLen + 1,
+		err:          io.EOF,
+		src:          buffer}
+	written, err := StdCopy(ioutil.Discard, ioutil.Discard, reader)
+	if written != startingBufLen {
+		t.Fatalf("Expected %d bytes read, got %d", startingBufLen, written)
+	}
+	if err != nil {
+		t.Fatal("Didn't get nil error")
+	}
+}
+
+func TestStdCopyWithInvalidInputHeader(t *testing.T) {
+	dstOut := NewStdWriter(ioutil.Discard, Stdout)
+	dstErr := NewStdWriter(ioutil.Discard, Stderr)
+	src := strings.NewReader("Invalid input")
+	_, err := StdCopy(dstOut, dstErr, src)
+	if err == nil {
+		t.Fatal("StdCopy with invalid input header should fail.")
+	}
+}
+
+func TestStdCopyWithCorruptedPrefix(t *testing.T) {
+	data := []byte{0x01, 0x02, 0x03}
+	src := bytes.NewReader(data)
+	written, err := StdCopy(nil, nil, src)
+	if err != nil {
+		t.Fatalf("StdCopy should not return an error with corrupted prefix.")
+	}
+	if written != 0 {
+		t.Fatalf("StdCopy should have written 0, but has written %d", written)
+	}
+}
+
+func TestStdCopyReturnsWriteErrors(t *testing.T) {
+	stdOutBytes := []byte(strings.Repeat("o", startingBufLen))
+	stdErrBytes := []byte(strings.Repeat("e", startingBufLen))
+	buffer, err := getSrcBuffer(stdOutBytes, stdErrBytes)
+	if err != nil {
+		t.Fatal(err)
+	}
+	expectedError := errors.New("expected")
+
+	dstOut := &errWriter{err: expectedError}
+
+	written, err := StdCopy(dstOut, ioutil.Discard, buffer)
+	if written != 0 {
+		t.Fatalf("StdCopy should have written 0, but has written %d", written)
+	}
+	if err != expectedError {
+		t.Fatalf("Didn't get expected error, got %v", err)
+	}
+}
+
+func TestStdCopyDetectsNotFullyWrittenFrames(t *testing.T) {
+	stdOutBytes := []byte(strings.Repeat("o", startingBufLen))
+	stdErrBytes := []byte(strings.Repeat("e", startingBufLen))
+	buffer, err := getSrcBuffer(stdOutBytes, stdErrBytes)
+	if err != nil {
+		t.Fatal(err)
+	}
+	dstOut := &errWriter{n: startingBufLen - 10}
+
+	written, err := StdCopy(dstOut, ioutil.Discard, buffer)
+	if written != 0 {
+		t.Fatalf("StdCopy should have return 0 written bytes, but returned %d", written)
+	}
+	if err != io.ErrShortWrite {
+		t.Fatalf("Didn't get expected io.ErrShortWrite error")
+	}
+}
+
+func BenchmarkWrite(b *testing.B) {
+	w := NewStdWriter(ioutil.Discard, Stdout)
+	data := []byte("Test line for testing stdwriter performance\n")
+	data = bytes.Repeat(data, 100)
+	b.SetBytes(int64(len(data)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		if _, err := w.Write(data); err != nil {
+			b.Fatal(err)
+		}
+	}
+}

engine/runner/docker/util.go

@@ -0,0 +1,102 @@
+package docker
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/drone/drone/engine/runner"
+	"github.com/samalba/dockerclient"
+)
+
+// helper function that converts the Continer data structure to the exepcted
+// dockerclient.ContainerConfig.
+func toContainerConfig(c *runner.Container) *dockerclient.ContainerConfig {
+	config := &dockerclient.ContainerConfig{
+		Image:      c.Image,
+		Env:        toEnvironmentSlice(c.Environment),
+		Cmd:        c.Command,
+		Entrypoint: c.Entrypoint,
+		WorkingDir: c.WorkingDir,
+		HostConfig: dockerclient.HostConfig{
+			Privileged:       c.Privileged,
+			NetworkMode:      c.Network,
+			Memory:           c.MemLimit,
+			CpuShares:        c.CPUShares,
+			CpuQuota:         c.CPUQuota,
+			CpusetCpus:       c.CPUSet,
+			MemorySwappiness: -1,
+			OomKillDisable:   c.OomKillDisable,
+		},
+	}
+
+	if len(config.Entrypoint) == 0 {
+		config.Entrypoint = nil
+	}
+	if len(config.Cmd) == 0 {
+		config.Cmd = nil
+	}
+	if len(c.ExtraHosts) > 0 {
+		config.HostConfig.ExtraHosts = c.ExtraHosts
+	}
+	if len(c.DNS) != 0 {
+		config.HostConfig.Dns = c.DNS
+	}
+	if len(c.DNSSearch) != 0 {
+		config.HostConfig.DnsSearch = c.DNSSearch
+	}
+	if len(c.VolumesFrom) != 0 {
+		config.HostConfig.VolumesFrom = c.VolumesFrom
+	}
+
+	config.Volumes = map[string]struct{}{}
+	for _, path := range c.Volumes {
+		if strings.Index(path, ":") == -1 {
+			config.Volumes[path] = struct{}{}
+			continue
+		}
+		parts := strings.Split(path, ":")
+		config.Volumes[parts[1]] = struct{}{}
+		config.HostConfig.Binds = append(config.HostConfig.Binds, path)
+	}
+
+	for _, path := range c.Devices {
+		if strings.Index(path, ":") == -1 {
+			continue
+		}
+		parts := strings.Split(path, ":")
+		device := dockerclient.DeviceMapping{
+			PathOnHost:        parts[0],
+			PathInContainer:   parts[1],
+			CgroupPermissions: "rwm",
+		}
+		config.HostConfig.Devices = append(config.HostConfig.Devices, device)
+	}
+
+	return config
+}
+
+// helper function that converts the AuthConfig data structure to the exepcted
+// dockerclient.AuthConfig.
+func toAuthConfig(container *runner.Container) *dockerclient.AuthConfig {
+	if container.AuthConfig.Username == "" &&
+		container.AuthConfig.Password == "" &&
+		container.AuthConfig.Token == "" {
+		return nil
+	}
+	return &dockerclient.AuthConfig{
+		Email:         container.AuthConfig.Email,
+		Username:      container.AuthConfig.Username,
+		Password:      container.AuthConfig.Password,
+		RegistryToken: container.AuthConfig.Token,
+	}
+}
+
+// helper function that converts a key value map of environment variables to a
+// string slice in key=value format.
+func toEnvironmentSlice(env map[string]string) []string {
+	var envs []string
+	for k, v := range env {
+		envs = append(envs, fmt.Sprintf("%s=%s", k, v))
+	}
+	return envs
+}

engine/runner/docker/util_test.go

@@ -0,0 +1,24 @@
+package docker
+
+import (
+	"testing"
+)
+
+func Test_toContainerConfig(t *testing.T) {
+	t.Skip()
+}
+
+func Test_toAuthConfig(t *testing.T) {
+	t.Skip()
+}
+
+func Test_toEnvironmentSlice(t *testing.T) {
+	env := map[string]string{
+		"HOME": "/root",
+	}
+	envs := toEnvironmentSlice(env)
+	want, got := "HOME=/root", envs[0]
+	if want != got {
+		t.Errorf("Wanted envar %s got %s", want, got)
+	}
+}

engine/runner/runner.go

@@ -2,8 +2,8 @@ package runner
 
 import (
 	"bufio"
-	"time"
 	"fmt"
+	"time"
 
 	"github.com/drone/drone/engine/runner/parse"
 
@@ -59,7 +59,7 @@ type Runner struct {
 // Run starts the build runner but does not wait for it to complete. The Wait
 // method will return the exit code and release associated resources once the
 // running containers exit.
-func (r *Runner) Run() error {
+func (r *Runner) Run() {
 
 	go func() {
 		r.setup()
@@ -74,8 +74,6 @@ func (r *Runner) Run() error {
 		<-r.ctx.Done()
 		r.cancel()
 	}()
-
-	return nil
 }
 
 // Wait waits for the runner to exit.

model/repo.go

@@ -1,5 +1,7 @@
 package model
 
+import "strconv"
+
 type RepoLite struct {
 	Owner    string `json:"owner"`
 	Name     string `json:"name"`
@@ -31,3 +33,17 @@ type Repo struct {
 	AllowTag    bool   `json:"allow_tags"        meddler:"repo_allow_tags"`
 	Hash        string `json:"-"                 meddler:"repo_hash"`
 }
+
+// ToEnv returns environment variable valus for the repository.
+func (r *Repo) ToEnv(to map[string]string) {
+	to["CI_VCS"] = r.Kind
+	to["CI_REPO"] = r.FullName
+	to["CI_REPO_OWNER"] = r.Owner
+	to["CI_REPO_NAME"] = r.Name
+	to["CI_REPO_LINK"] = r.Link
+	to["CI_REPO_AVATAR"] = r.Avatar
+	to["CI_REPO_BRANCH"] = r.Branch
+	to["CI_REPO_PRIVATE"] = strconv.FormatBool(r.IsPrivate)
+	to["CI_REPO_TRUSTED"] = strconv.FormatBool(r.IsTrusted)
+	to["CI_REMOTE_URL"] = r.Clone
+}

router/middleware/bus.go

@@ -0,0 +1,14 @@
+package middleware
+
+import (
+	"github.com/drone/drone/bus"
+	"github.com/gin-gonic/gin"
+)
+
+func Bus() gin.HandlerFunc {
+	bus_ := bus.New()
+	return func(c *gin.Context) {
+		bus.ToContext(c, bus_)
+		c.Next()
+	}
+}

router/middleware/queue.go

@@ -0,0 +1,14 @@
+package middleware
+
+import (
+	"github.com/drone/drone/queue"
+	"github.com/gin-gonic/gin"
+)
+
+func Queue() gin.HandlerFunc {
+	queue_ := queue.New()
+	return func(c *gin.Context) {
+		queue.ToContext(c, queue_)
+		c.Next()
+	}
+}

router/middleware/stream.go

@@ -0,0 +1,14 @@
+package middleware
+
+import (
+	"github.com/drone/drone/stream"
+	"github.com/gin-gonic/gin"
+)
+
+func Stream() gin.HandlerFunc {
+	stream_ := stream.New()
+	return func(c *gin.Context) {
+		stream.ToContext(c, stream_)
+		c.Next()
+	}
+}

router/router.go

@@ -2,6 +2,7 @@ package router
 
 import (
 	"net/http"
+	"os"
 	"strings"
 
 	"github.com/gin-gonic/gin"
@@ -136,8 +137,14 @@ func Load(middleware ...gin.HandlerFunc) http.Handler {
 		stream.Use(session.SetRepo())
 		stream.Use(session.SetPerm())
 		stream.Use(session.MustPull)
-		stream.GET("/:owner/:name", web.GetRepoEvents)
+
-		stream.GET("/:owner/:name/:build/:number", web.GetStream)
+		if os.Getenv("CANARY") == "true" {
+			stream.GET("/:owner/:name", web.GetRepoEvents2)
+			stream.GET("/:owner/:name/:build/:number", web.GetStream2)
+		} else {
+			stream.GET("/:owner/:name", web.GetRepoEvents)
+			stream.GET("/:owner/:name/:build/:number", web.GetStream)
+		}
 	}
 
 	bots := e.Group("/bots")
@@ -154,6 +161,14 @@ func Load(middleware ...gin.HandlerFunc) http.Handler {
 		auth.POST("/token", web.GetLoginToken)
 	}
 
+	queue := e.Group("/api/queue")
+	{
+		queue.POST("/pull", api.Pull)
+		queue.POST("/wait/:id", api.Wait)
+		queue.POST("/stream/:id", api.Stream)
+		queue.POST("/status/:id", api.Update)
+	}
+
 	gitlab := e.Group("/gitlab/:owner/:name")
 	{
 		gitlab.Use(session.SetRepo())

store/store.go

@@ -275,6 +275,40 @@ func UpdateBuild(c context.Context, build *model.Build) error {
 	return FromContext(c).UpdateBuild(build)
 }
 
+func UpdateBuildJob(c context.Context, build *model.Build, job *model.Job) (bool, error) {
+	if err := UpdateJob(c, job); err != nil {
+		return false, err
+	}
+	jobs, err := GetJobList(c, build)
+	if err != nil {
+		return false, err
+	}
+	// check to see if all jobs are finished for this build. If yes, we need to
+	// calcualte the overall build status and finish time.
+	status := model.StatusSuccess
+	finish := job.Finished
+	for _, job := range jobs {
+		if job.Finished > finish {
+			finish = job.Finished
+		}
+		switch job.Status {
+		case model.StatusSuccess:
+			// no-op
+		case model.StatusRunning, model.StatusPending:
+			return false, nil
+		default:
+			status = job.Status
+		}
+	}
+
+	build.Status = status
+	build.Finished = finish
+	if err := FromContext(c).UpdateBuild(build); err != nil {
+		return false, err
+	}
+	return true, nil
+}
+
 func GetJob(c context.Context, id int64) (*model.Job, error) {
 	return FromContext(c).GetJob(id)
 }

vendor/github.com/codegangsta/cli/LICENSE

@@ -0,0 +1,21 @@
+Copyright (C) 2013 Jeremy Saenz
+All Rights Reserved.
+
+MIT LICENSE
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

vendor/github.com/codegangsta/cli/README.md

@@ -0,0 +1,315 @@
+[![Coverage](http://gocover.io/_badge/github.com/codegangsta/cli?0)](http://gocover.io/github.com/codegangsta/cli)
+[![Build Status](https://travis-ci.org/codegangsta/cli.png?branch=master)](https://travis-ci.org/codegangsta/cli)
+[![GoDoc](https://godoc.org/github.com/codegangsta/cli?status.svg)](https://godoc.org/github.com/codegangsta/cli)
+
+# cli.go
+`cli.go` is simple, fast, and fun package for building command line apps in Go. The goal is to enable developers to write fast and distributable command line applications in an expressive way.
+
+## Overview
+Command line apps are usually so tiny that there is absolutely no reason why your code should *not* be self-documenting. Things like generating help text and parsing command flags/options should not hinder productivity when writing a command line app.
+
+**This is where `cli.go` comes into play.** `cli.go` makes command line programming fun, organized, and expressive!
+
+## Installation
+Make sure you have a working Go environment (go 1.1+ is *required*). [See the install instructions](http://golang.org/doc/install.html).
+
+To install `cli.go`, simply run:
+```
+$ go get github.com/codegangsta/cli
+```
+
+Make sure your `PATH` includes to the `$GOPATH/bin` directory so your commands can be easily used:
+```
+export PATH=$PATH:$GOPATH/bin
+```
+
+## Getting Started
+One of the philosophies behind `cli.go` is that an API should be playful and full of discovery. So a `cli.go` app can be as little as one line of code in `main()`. 
+
+``` go
+package main
+
+import (
+  "os"
+  "github.com/codegangsta/cli"
+)
+
+func main() {
+  cli.NewApp().Run(os.Args)
+}
+```
+
+This app will run and show help text, but is not very useful. Let's give an action to execute and some help documentation:
+
+``` go
+package main
+
+import (
+  "os"
+  "github.com/codegangsta/cli"
+)
+
+func main() {
+  app := cli.NewApp()
+  app.Name = "boom"
+  app.Usage = "make an explosive entrance"
+  app.Action = func(c *cli.Context) {
+    println("boom! I say!")
+  }
+  
+  app.Run(os.Args)
+}
+```
+
+Running this already gives you a ton of functionality, plus support for things like subcommands and flags, which are covered below.
+
+## Example
+
+Being a programmer can be a lonely job. Thankfully by the power of automation that is not the case! Let's create a greeter app to fend off our demons of loneliness!
+
+Start by creating a directory named `greet`, and within it, add a file, `greet.go` with the following code in it:
+
+``` go
+package main
+
+import (
+  "os"
+  "github.com/codegangsta/cli"
+)
+
+func main() {
+  app := cli.NewApp()
+  app.Name = "greet"
+  app.Usage = "fight the loneliness!"
+  app.Action = func(c *cli.Context) {
+    println("Hello friend!")
+  }
+
+  app.Run(os.Args)
+}
+```
+
+Install our command to the `$GOPATH/bin` directory:
+
+```
+$ go install
+```
+
+Finally run our new command:
+
+```
+$ greet
+Hello friend!
+```
+
+`cli.go` also generates neat help text:
+
+```
+$ greet help
+NAME:
+    greet - fight the loneliness!
+
+USAGE:
+    greet [global options] command [command options] [arguments...]
+
+VERSION:
+    0.0.0
+
+COMMANDS:
+    help, h  Shows a list of commands or help for one command
+
+GLOBAL OPTIONS
+    --version	Shows version information
+```
+
+### Arguments
+You can lookup arguments by calling the `Args` function on `cli.Context`.
+
+``` go
+...
+app.Action = func(c *cli.Context) {
+  println("Hello", c.Args()[0])
+}
+...
+```
+
+### Flags
+Setting and querying flags is simple.
+``` go
+...
+app.Flags = []cli.Flag {
+  cli.StringFlag{
+    Name: "lang",
+    Value: "english",
+    Usage: "language for the greeting",
+  },
+}
+app.Action = func(c *cli.Context) {
+  name := "someone"
+  if len(c.Args()) > 0 {
+    name = c.Args()[0]
+  }
+  if c.String("lang") == "spanish" {
+    println("Hola", name)
+  } else {
+    println("Hello", name)
+  }
+}
+...
+```
+
+See full list of flags at http://godoc.org/github.com/codegangsta/cli
+
+#### Alternate Names
+
+You can set alternate (or short) names for flags by providing a comma-delimited list for the `Name`. e.g.
+
+``` go
+app.Flags = []cli.Flag {
+  cli.StringFlag{
+    Name: "lang, l",
+    Value: "english",
+    Usage: "language for the greeting",
+  },
+}
+```
+
+That flag can then be set with `--lang spanish` or `-l spanish`. Note that giving two different forms of the same flag in the same command invocation is an error.
+
+#### Values from the Environment
+
+You can also have the default value set from the environment via `EnvVar`.  e.g.
+
+``` go
+app.Flags = []cli.Flag {
+  cli.StringFlag{
+    Name: "lang, l",
+    Value: "english",
+    Usage: "language for the greeting",
+    EnvVar: "APP_LANG",
+  },
+}
+```
+
+The `EnvVar` may also be given as a comma-delimited "cascade", where the first environment variable that resolves is used as the default.
+
+``` go
+app.Flags = []cli.Flag {
+  cli.StringFlag{
+    Name: "lang, l",
+    Value: "english",
+    Usage: "language for the greeting",
+    EnvVar: "LEGACY_COMPAT_LANG,APP_LANG,LANG",
+  },
+}
+```
+
+### Subcommands
+
+Subcommands can be defined for a more git-like command line app.
+```go
+...
+app.Commands = []cli.Command{
+  {
+    Name:      "add",
+    Aliases:     []string{"a"},
+    Usage:     "add a task to the list",
+    Action: func(c *cli.Context) {
+      println("added task: ", c.Args().First())
+    },
+  },
+  {
+    Name:      "complete",
+    Aliases:     []string{"c"},
+    Usage:     "complete a task on the list",
+    Action: func(c *cli.Context) {
+      println("completed task: ", c.Args().First())
+    },
+  },
+  {
+    Name:      "template",
+    Aliases:     []string{"r"},
+    Usage:     "options for task templates",
+    Subcommands: []cli.Command{
+      {
+        Name:  "add",
+        Usage: "add a new template",
+        Action: func(c *cli.Context) {
+            println("new task template: ", c.Args().First())
+        },
+      },
+      {
+        Name:  "remove",
+        Usage: "remove an existing template",
+        Action: func(c *cli.Context) {
+          println("removed task template: ", c.Args().First())
+        },
+      },
+    },
+  },
+}
+...
+```
+
+### Bash Completion
+
+You can enable completion commands by setting the `EnableBashCompletion`
+flag on the `App` object.  By default, this setting will only auto-complete to
+show an app's subcommands, but you can write your own completion methods for
+the App or its subcommands.
+```go
+...
+var tasks = []string{"cook", "clean", "laundry", "eat", "sleep", "code"}
+app := cli.NewApp()
+app.EnableBashCompletion = true
+app.Commands = []cli.Command{
+  {
+    Name:  "complete",
+    Aliases: []string{"c"},
+    Usage: "complete a task on the list",
+    Action: func(c *cli.Context) {
+       println("completed task: ", c.Args().First())
+    },
+    BashComplete: func(c *cli.Context) {
+      // This will complete if no args are passed
+      if len(c.Args()) > 0 {
+        return
+      }
+      for _, t := range tasks {
+        fmt.Println(t)
+      }
+    },
+  }
+}
+...
+```
+
+#### To Enable
+
+Source the `autocomplete/bash_autocomplete` file in your `.bashrc` file while
+setting the `PROG` variable to the name of your program:
+
+`PROG=myprogram source /.../cli/autocomplete/bash_autocomplete`
+
+#### To Distribute
+
+Copy `autocomplete/bash_autocomplete` into `/etc/bash_completion.d/` and rename
+it to the name of the program you wish to add autocomplete support for (or
+automatically install it there if you are distributing a package). Don't forget
+to source the file to make it active in the current shell.
+
+```
+   sudo cp src/bash_autocomplete /etc/bash_completion.d/<myprogram>
+   source /etc/bash_completion.d/<myprogram>
+```
+
+Alternatively, you can just document that users should source the generic
+`autocomplete/bash_autocomplete` in their bash configuration with `$PROG` set
+to the name of their program (as above).
+
+## Contribution Guidelines
+Feel free to put up a pull request to fix a bug or maybe add a feature. I will give it a code review and make sure that it does not break backwards compatibility. If I or any other collaborators agree that it is in line with the vision of the project, we will work with you to get the code into a mergeable state and merge it into the master branch.
+
+If you have contributed something significant to the project, I will most likely add you as a collaborator. As a collaborator you are given the ability to merge others pull requests. It is very important that new code does not break existing code, so be careful about what code you do choose to merge. If you have any questions feel free to link @codegangsta to the issue in question and we can review it together.
+
+If you feel like you have contributed to the project but have not yet been added as a collaborator, I probably forgot to add you. Hit @codegangsta up over email and we will get it figured out.

vendor/github.com/codegangsta/cli/app.go

@@ -0,0 +1,333 @@
+package cli
+
+import (
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"time"
+)
+
+// App is the main structure of a cli application. It is recomended that
+// an app be created with the cli.NewApp() function
+type App struct {
+	// The name of the program. Defaults to os.Args[0]
+	Name string
+	// Full name of command for help, defaults to Name
+	HelpName string
+	// Description of the program.
+	Usage string
+	// Description of the program argument format.
+	ArgsUsage string
+	// Version of the program
+	Version string
+	// List of commands to execute
+	Commands []Command
+	// List of flags to parse
+	Flags []Flag
+	// Boolean to enable bash completion commands
+	EnableBashCompletion bool
+	// Boolean to hide built-in help command
+	HideHelp bool
+	// Boolean to hide built-in version flag
+	HideVersion bool
+	// An action to execute when the bash-completion flag is set
+	BashComplete func(context *Context)
+	// An action to execute before any subcommands are run, but after the context is ready
+	// If a non-nil error is returned, no subcommands are run
+	Before func(context *Context) error
+	// An action to execute after any subcommands are run, but after the subcommand has finished
+	// It is run even if Action() panics
+	After func(context *Context) error
+	// The action to execute when no subcommands are specified
+	Action func(context *Context)
+	// Execute this function if the proper command cannot be found
+	CommandNotFound func(context *Context, command string)
+	// Compilation date
+	Compiled time.Time
+	// List of all authors who contributed
+	Authors []Author
+	// Copyright of the binary if any
+	Copyright string
+	// Name of Author (Note: Use App.Authors, this is deprecated)
+	Author string
+	// Email of Author (Note: Use App.Authors, this is deprecated)
+	Email string
+	// Writer writer to write output to
+	Writer io.Writer
+}
+
+// Tries to find out when this binary was compiled.
+// Returns the current time if it fails to find it.
+func compileTime() time.Time {
+	info, err := os.Stat(os.Args[0])
+	if err != nil {
+		return time.Now()
+	}
+	return info.ModTime()
+}
+
+// Creates a new cli Application with some reasonable defaults for Name, Usage, Version and Action.
+func NewApp() *App {
+	return &App{
+		Name:         os.Args[0],
+		HelpName:     os.Args[0],
+		Usage:        "A new cli application",
+		Version:      "0.0.0",
+		BashComplete: DefaultAppComplete,
+		Action:       helpCommand.Action,
+		Compiled:     compileTime(),
+		Writer:       os.Stdout,
+	}
+}
+
+// Entry point to the cli app. Parses the arguments slice and routes to the proper flag/args combination
+func (a *App) Run(arguments []string) (err error) {
+	if a.Author != "" || a.Email != "" {
+		a.Authors = append(a.Authors, Author{Name: a.Author, Email: a.Email})
+	}
+
+	newCmds := []Command{}
+	for _, c := range a.Commands {
+		if c.HelpName == "" {
+			c.HelpName = fmt.Sprintf("%s %s", a.HelpName, c.Name)
+		}
+		newCmds = append(newCmds, c)
+	}
+	a.Commands = newCmds
+
+	// append help to commands
+	if a.Command(helpCommand.Name) == nil && !a.HideHelp {
+		a.Commands = append(a.Commands, helpCommand)
+		if (HelpFlag != BoolFlag{}) {
+			a.appendFlag(HelpFlag)
+		}
+	}
+
+	//append version/help flags
+	if a.EnableBashCompletion {
+		a.appendFlag(BashCompletionFlag)
+	}
+
+	if !a.HideVersion {
+		a.appendFlag(VersionFlag)
+	}
+
+	// parse flags
+	set := flagSet(a.Name, a.Flags)
+	set.SetOutput(ioutil.Discard)
+	err = set.Parse(arguments[1:])
+	nerr := normalizeFlags(a.Flags, set)
+	if nerr != nil {
+		fmt.Fprintln(a.Writer, nerr)
+		context := NewContext(a, set, nil)
+		ShowAppHelp(context)
+		return nerr
+	}
+	context := NewContext(a, set, nil)
+
+	if err != nil {
+		fmt.Fprintln(a.Writer, "Incorrect Usage.")
+		fmt.Fprintln(a.Writer)
+		ShowAppHelp(context)
+		return err
+	}
+
+	if checkCompletions(context) {
+		return nil
+	}
+
+	if !a.HideHelp && checkHelp(context) {
+		ShowAppHelp(context)
+		return nil
+	}
+
+	if !a.HideVersion && checkVersion(context) {
+		ShowVersion(context)
+		return nil
+	}
+
+	if a.After != nil {
+		defer func() {
+			afterErr := a.After(context)
+			if afterErr != nil {
+				if err != nil {
+					err = NewMultiError(err, afterErr)
+				} else {
+					err = afterErr
+				}
+			}
+		}()
+	}
+
+	if a.Before != nil {
+		err := a.Before(context)
+		if err != nil {
+			return err
+		}
+	}
+
+	args := context.Args()
+	if args.Present() {
+		name := args.First()
+		c := a.Command(name)
+		if c != nil {
+			return c.Run(context)
+		}
+	}
+
+	// Run default Action
+	a.Action(context)
+	return nil
+}
+
+// Another entry point to the cli app, takes care of passing arguments and error handling
+func (a *App) RunAndExitOnError() {
+	if err := a.Run(os.Args); err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+}
+
+// Invokes the subcommand given the context, parses ctx.Args() to generate command-specific flags
+func (a *App) RunAsSubcommand(ctx *Context) (err error) {
+	// append help to commands
+	if len(a.Commands) > 0 {
+		if a.Command(helpCommand.Name) == nil && !a.HideHelp {
+			a.Commands = append(a.Commands, helpCommand)
+			if (HelpFlag != BoolFlag{}) {
+				a.appendFlag(HelpFlag)
+			}
+		}
+	}
+
+	newCmds := []Command{}
+	for _, c := range a.Commands {
+		if c.HelpName == "" {
+			c.HelpName = fmt.Sprintf("%s %s", a.HelpName, c.Name)
+		}
+		newCmds = append(newCmds, c)
+	}
+	a.Commands = newCmds
+
+	// append flags
+	if a.EnableBashCompletion {
+		a.appendFlag(BashCompletionFlag)
+	}
+
+	// parse flags
+	set := flagSet(a.Name, a.Flags)
+	set.SetOutput(ioutil.Discard)
+	err = set.Parse(ctx.Args().Tail())
+	nerr := normalizeFlags(a.Flags, set)
+	context := NewContext(a, set, ctx)
+
+	if nerr != nil {
+		fmt.Fprintln(a.Writer, nerr)
+		fmt.Fprintln(a.Writer)
+		if len(a.Commands) > 0 {
+			ShowSubcommandHelp(context)
+		} else {
+			ShowCommandHelp(ctx, context.Args().First())
+		}
+		return nerr
+	}
+
+	if err != nil {
+		fmt.Fprintln(a.Writer, "Incorrect Usage.")
+		fmt.Fprintln(a.Writer)
+		ShowSubcommandHelp(context)
+		return err
+	}
+
+	if checkCompletions(context) {
+		return nil
+	}
+
+	if len(a.Commands) > 0 {
+		if checkSubcommandHelp(context) {
+			return nil
+		}
+	} else {
+		if checkCommandHelp(ctx, context.Args().First()) {
+			return nil
+		}
+	}
+
+	if a.After != nil {
+		defer func() {
+			afterErr := a.After(context)
+			if afterErr != nil {
+				if err != nil {
+					err = NewMultiError(err, afterErr)
+				} else {
+					err = afterErr
+				}
+			}
+		}()
+	}
+
+	if a.Before != nil {
+		err := a.Before(context)
+		if err != nil {
+			return err
+		}
+	}
+
+	args := context.Args()
+	if args.Present() {
+		name := args.First()
+		c := a.Command(name)
+		if c != nil {
+			return c.Run(context)
+		}
+	}
+
+	// Run default Action
+	a.Action(context)
+
+	return nil
+}
+
+// Returns the named command on App. Returns nil if the command does not exist
+func (a *App) Command(name string) *Command {
+	for _, c := range a.Commands {
+		if c.HasName(name) {
+			return &c
+		}
+	}
+
+	return nil
+}
+
+func (a *App) hasFlag(flag Flag) bool {
+	for _, f := range a.Flags {
+		if flag == f {
+			return true
+		}
+	}
+
+	return false
+}
+
+func (a *App) appendFlag(flag Flag) {
+	if !a.hasFlag(flag) {
+		a.Flags = append(a.Flags, flag)
+	}
+}
+
+// Author represents someone who has contributed to a cli project.
+type Author struct {
+	Name  string // The Authors name
+	Email string // The Authors email
+}
+
+// String makes Author comply to the Stringer interface, to allow an easy print in the templating process
+func (a Author) String() string {
+	e := ""
+	if a.Email != "" {
+		e = "<" + a.Email + "> "
+	}
+
+	return fmt.Sprintf("%v %v", a.Name, e)
+}

vendor/github.com/codegangsta/cli/cli.go

@@ -0,0 +1,40 @@
+// Package cli provides a minimal framework for creating and organizing command line
+// Go applications. cli is designed to be easy to understand and write, the most simple
+// cli application can be written as follows:
+//   func main() {
+//     cli.NewApp().Run(os.Args)
+//   }
+//
+// Of course this application does not do much, so let's make this an actual application:
+//   func main() {
+//     app := cli.NewApp()
+//     app.Name = "greet"
+//     app.Usage = "say a greeting"
+//     app.Action = func(c *cli.Context) {
+//       println("Greetings")
+//     }
+//
+//     app.Run(os.Args)
+//   }
+package cli
+
+import (
+	"strings"
+)
+
+type MultiError struct {
+	Errors []error
+}
+
+func NewMultiError(err ...error) MultiError {
+	return MultiError{Errors: err}
+}
+
+func (m MultiError) Error() string {
+	errs := make([]string, len(m.Errors))
+	for i, err := range m.Errors {
+		errs[i] = err.Error()
+	}
+
+	return strings.Join(errs, "\n")
+}

vendor/github.com/codegangsta/cli/command.go

@@ -0,0 +1,210 @@
+package cli
+
+import (
+	"fmt"
+	"io/ioutil"
+	"strings"
+)
+
+// Command is a subcommand for a cli.App.
+type Command struct {
+	// The name of the command
+	Name string
+	// short name of the command. Typically one character (deprecated, use `Aliases`)
+	ShortName string
+	// A list of aliases for the command
+	Aliases []string
+	// A short description of the usage of this command
+	Usage string
+	// A longer explanation of how the command works
+	Description string
+	// A short description of the arguments of this command
+	ArgsUsage string
+	// The function to call when checking for bash command completions
+	BashComplete func(context *Context)
+	// An action to execute before any sub-subcommands are run, but after the context is ready
+	// If a non-nil error is returned, no sub-subcommands are run
+	Before func(context *Context) error
+	// An action to execute after any subcommands are run, but after the subcommand has finished
+	// It is run even if Action() panics
+	After func(context *Context) error
+	// The function to call when this command is invoked
+	Action func(context *Context)
+	// List of child commands
+	Subcommands []Command
+	// List of flags to parse
+	Flags []Flag
+	// Treat all flags as normal arguments if true
+	SkipFlagParsing bool
+	// Boolean to hide built-in help command
+	HideHelp bool
+
+	// Full name of command for help, defaults to full command name, including parent commands.
+	HelpName        string
+	commandNamePath []string
+}
+
+// Returns the full name of the command.
+// For subcommands this ensures that parent commands are part of the command path
+func (c Command) FullName() string {
+	if c.commandNamePath == nil {
+		return c.Name
+	}
+	return strings.Join(c.commandNamePath, " ")
+}
+
+// Invokes the command given the context, parses ctx.Args() to generate command-specific flags
+func (c Command) Run(ctx *Context) error {
+	if len(c.Subcommands) > 0 || c.Before != nil || c.After != nil {
+		return c.startApp(ctx)
+	}
+
+	if !c.HideHelp && (HelpFlag != BoolFlag{}) {
+		// append help to flags
+		c.Flags = append(
+			c.Flags,
+			HelpFlag,
+		)
+	}
+
+	if ctx.App.EnableBashCompletion {
+		c.Flags = append(c.Flags, BashCompletionFlag)
+	}
+
+	set := flagSet(c.Name, c.Flags)
+	set.SetOutput(ioutil.Discard)
+
+	firstFlagIndex := -1
+	terminatorIndex := -1
+	for index, arg := range ctx.Args() {
+		if arg == "--" {
+			terminatorIndex = index
+			break
+		} else if strings.HasPrefix(arg, "-") && firstFlagIndex == -1 {
+			firstFlagIndex = index
+		}
+	}
+
+	var err error
+	if firstFlagIndex > -1 && !c.SkipFlagParsing {
+		args := ctx.Args()
+		regularArgs := make([]string, len(args[1:firstFlagIndex]))
+		copy(regularArgs, args[1:firstFlagIndex])
+
+		var flagArgs []string
+		if terminatorIndex > -1 {
+			flagArgs = args[firstFlagIndex:terminatorIndex]
+			regularArgs = append(regularArgs, args[terminatorIndex:]...)
+		} else {
+			flagArgs = args[firstFlagIndex:]
+		}
+
+		err = set.Parse(append(flagArgs, regularArgs...))
+	} else {
+		err = set.Parse(ctx.Args().Tail())
+	}
+
+	if err != nil {
+		fmt.Fprintln(ctx.App.Writer, "Incorrect Usage.")
+		fmt.Fprintln(ctx.App.Writer)
+		ShowCommandHelp(ctx, c.Name)
+		return err
+	}
+
+	nerr := normalizeFlags(c.Flags, set)
+	if nerr != nil {
+		fmt.Fprintln(ctx.App.Writer, nerr)
+		fmt.Fprintln(ctx.App.Writer)
+		ShowCommandHelp(ctx, c.Name)
+		return nerr
+	}
+	context := NewContext(ctx.App, set, ctx)
+
+	if checkCommandCompletions(context, c.Name) {
+		return nil
+	}
+
+	if checkCommandHelp(context, c.Name) {
+		return nil
+	}
+	context.Command = c
+	c.Action(context)
+	return nil
+}
+
+func (c Command) Names() []string {
+	names := []string{c.Name}
+
+	if c.ShortName != "" {
+		names = append(names, c.ShortName)
+	}
+
+	return append(names, c.Aliases...)
+}
+
+// Returns true if Command.Name or Command.ShortName matches given name
+func (c Command) HasName(name string) bool {
+	for _, n := range c.Names() {
+		if n == name {
+			return true
+		}
+	}
+	return false
+}
+
+func (c Command) startApp(ctx *Context) error {
+	app := NewApp()
+
+	// set the name and usage
+	app.Name = fmt.Sprintf("%s %s", ctx.App.Name, c.Name)
+	if c.HelpName == "" {
+		app.HelpName = c.HelpName
+	} else {
+		app.HelpName = fmt.Sprintf("%s %s", ctx.App.Name, c.Name)
+	}
+
+	if c.Description != "" {
+		app.Usage = c.Description
+	} else {
+		app.Usage = c.Usage
+	}
+
+	// set CommandNotFound
+	app.CommandNotFound = ctx.App.CommandNotFound
+
+	// set the flags and commands
+	app.Commands = c.Subcommands
+	app.Flags = c.Flags
+	app.HideHelp = c.HideHelp
+
+	app.Version = ctx.App.Version
+	app.HideVersion = ctx.App.HideVersion
+	app.Compiled = ctx.App.Compiled
+	app.Author = ctx.App.Author
+	app.Email = ctx.App.Email
+	app.Writer = ctx.App.Writer
+
+	// bash completion
+	app.EnableBashCompletion = ctx.App.EnableBashCompletion
+	if c.BashComplete != nil {
+		app.BashComplete = c.BashComplete
+	}
+
+	// set the actions
+	app.Before = c.Before
+	app.After = c.After
+	if c.Action != nil {
+		app.Action = c.Action
+	} else {
+		app.Action = helpSubcommand.Action
+	}
+
+	var newCmds []Command
+	for _, cc := range app.Commands {
+		cc.commandNamePath = []string{c.Name, cc.Name}
+		newCmds = append(newCmds, cc)
+	}
+	app.Commands = newCmds
+
+	return app.RunAsSubcommand(ctx)
+}

vendor/github.com/codegangsta/cli/context.go

@@ -0,0 +1,388 @@
+package cli
+
+import (
+	"errors"
+	"flag"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// Context is a type that is passed through to
+// each Handler action in a cli application. Context
+// can be used to retrieve context-specific Args and
+// parsed command-line options.
+type Context struct {
+	App            *App
+	Command        Command
+	flagSet        *flag.FlagSet
+	setFlags       map[string]bool
+	globalSetFlags map[string]bool
+	parentContext  *Context
+}
+
+// Creates a new context. For use in when invoking an App or Command action.
+func NewContext(app *App, set *flag.FlagSet, parentCtx *Context) *Context {
+	return &Context{App: app, flagSet: set, parentContext: parentCtx}
+}
+
+// Looks up the value of a local int flag, returns 0 if no int flag exists
+func (c *Context) Int(name string) int {
+	return lookupInt(name, c.flagSet)
+}
+
+// Looks up the value of a local time.Duration flag, returns 0 if no time.Duration flag exists
+func (c *Context) Duration(name string) time.Duration {
+	return lookupDuration(name, c.flagSet)
+}
+
+// Looks up the value of a local float64 flag, returns 0 if no float64 flag exists
+func (c *Context) Float64(name string) float64 {
+	return lookupFloat64(name, c.flagSet)
+}
+
+// Looks up the value of a local bool flag, returns false if no bool flag exists
+func (c *Context) Bool(name string) bool {
+	return lookupBool(name, c.flagSet)
+}
+
+// Looks up the value of a local boolT flag, returns false if no bool flag exists
+func (c *Context) BoolT(name string) bool {
+	return lookupBoolT(name, c.flagSet)
+}
+
+// Looks up the value of a local string flag, returns "" if no string flag exists
+func (c *Context) String(name string) string {
+	return lookupString(name, c.flagSet)
+}
+
+// Looks up the value of a local string slice flag, returns nil if no string slice flag exists
+func (c *Context) StringSlice(name string) []string {
+	return lookupStringSlice(name, c.flagSet)
+}
+
+// Looks up the value of a local int slice flag, returns nil if no int slice flag exists
+func (c *Context) IntSlice(name string) []int {
+	return lookupIntSlice(name, c.flagSet)
+}
+
+// Looks up the value of a local generic flag, returns nil if no generic flag exists
+func (c *Context) Generic(name string) interface{} {
+	return lookupGeneric(name, c.flagSet)
+}
+
+// Looks up the value of a global int flag, returns 0 if no int flag exists
+func (c *Context) GlobalInt(name string) int {
+	if fs := lookupGlobalFlagSet(name, c); fs != nil {
+		return lookupInt(name, fs)
+	}
+	return 0
+}
+
+// Looks up the value of a global time.Duration flag, returns 0 if no time.Duration flag exists
+func (c *Context) GlobalDuration(name string) time.Duration {
+	if fs := lookupGlobalFlagSet(name, c); fs != nil {
+		return lookupDuration(name, fs)
+	}
+	return 0
+}
+
+// Looks up the value of a global bool flag, returns false if no bool flag exists
+func (c *Context) GlobalBool(name string) bool {
+	if fs := lookupGlobalFlagSet(name, c); fs != nil {
+		return lookupBool(name, fs)
+	}
+	return false
+}
+
+// Looks up the value of a global string flag, returns "" if no string flag exists
+func (c *Context) GlobalString(name string) string {
+	if fs := lookupGlobalFlagSet(name, c); fs != nil {
+		return lookupString(name, fs)
+	}
+	return ""
+}
+
+// Looks up the value of a global string slice flag, returns nil if no string slice flag exists
+func (c *Context) GlobalStringSlice(name string) []string {
+	if fs := lookupGlobalFlagSet(name, c); fs != nil {
+		return lookupStringSlice(name, fs)
+	}
+	return nil
+}
+
+// Looks up the value of a global int slice flag, returns nil if no int slice flag exists
+func (c *Context) GlobalIntSlice(name string) []int {
+	if fs := lookupGlobalFlagSet(name, c); fs != nil {
+		return lookupIntSlice(name, fs)
+	}
+	return nil
+}
+
+// Looks up the value of a global generic flag, returns nil if no generic flag exists
+func (c *Context) GlobalGeneric(name string) interface{} {
+	if fs := lookupGlobalFlagSet(name, c); fs != nil {
+		return lookupGeneric(name, fs)
+	}
+	return nil
+}
+
+// Returns the number of flags set
+func (c *Context) NumFlags() int {
+	return c.flagSet.NFlag()
+}
+
+// Determines if the flag was actually set
+func (c *Context) IsSet(name string) bool {
+	if c.setFlags == nil {
+		c.setFlags = make(map[string]bool)
+		c.flagSet.Visit(func(f *flag.Flag) {
+			c.setFlags[f.Name] = true
+		})
+	}
+	return c.setFlags[name] == true
+}
+
+// Determines if the global flag was actually set
+func (c *Context) GlobalIsSet(name string) bool {
+	if c.globalSetFlags == nil {
+		c.globalSetFlags = make(map[string]bool)
+		ctx := c
+		if ctx.parentContext != nil {
+			ctx = ctx.parentContext
+		}
+		for ; ctx != nil && c.globalSetFlags[name] == false; ctx = ctx.parentContext {
+			ctx.flagSet.Visit(func(f *flag.Flag) {
+				c.globalSetFlags[f.Name] = true
+			})
+		}
+	}
+	return c.globalSetFlags[name]
+}
+
+// Returns a slice of flag names used in this context.
+func (c *Context) FlagNames() (names []string) {
+	for _, flag := range c.Command.Flags {
+		name := strings.Split(flag.getName(), ",")[0]
+		if name == "help" {
+			continue
+		}
+		names = append(names, name)
+	}
+	return
+}
+
+// Returns a slice of global flag names used by the app.
+func (c *Context) GlobalFlagNames() (names []string) {
+	for _, flag := range c.App.Flags {
+		name := strings.Split(flag.getName(), ",")[0]
+		if name == "help" || name == "version" {
+			continue
+		}
+		names = append(names, name)
+	}
+	return
+}
+
+// Returns the parent context, if any
+func (c *Context) Parent() *Context {
+	return c.parentContext
+}
+
+type Args []string
+
+// Returns the command line arguments associated with the context.
+func (c *Context) Args() Args {
+	args := Args(c.flagSet.Args())
+	return args
+}
+
+// Returns the nth argument, or else a blank string
+func (a Args) Get(n int) string {
+	if len(a) > n {
+		return a[n]
+	}
+	return ""
+}
+
+// Returns the first argument, or else a blank string
+func (a Args) First() string {
+	return a.Get(0)
+}
+
+// Return the rest of the arguments (not the first one)
+// or else an empty string slice
+func (a Args) Tail() []string {
+	if len(a) >= 2 {
+		return []string(a)[1:]
+	}
+	return []string{}
+}
+
+// Checks if there are any arguments present
+func (a Args) Present() bool {
+	return len(a) != 0
+}
+
+// Swaps arguments at the given indexes
+func (a Args) Swap(from, to int) error {
+	if from >= len(a) || to >= len(a) {
+		return errors.New("index out of range")
+	}
+	a[from], a[to] = a[to], a[from]
+	return nil
+}
+
+func lookupGlobalFlagSet(name string, ctx *Context) *flag.FlagSet {
+	if ctx.parentContext != nil {
+		ctx = ctx.parentContext
+	}
+	for ; ctx != nil; ctx = ctx.parentContext {
+		if f := ctx.flagSet.Lookup(name); f != nil {
+			return ctx.flagSet
+		}
+	}
+	return nil
+}
+
+func lookupInt(name string, set *flag.FlagSet) int {
+	f := set.Lookup(name)
+	if f != nil {
+		val, err := strconv.Atoi(f.Value.String())
+		if err != nil {
+			return 0
+		}
+		return val
+	}
+
+	return 0
+}
+
+func lookupDuration(name string, set *flag.FlagSet) time.Duration {
+	f := set.Lookup(name)
+	if f != nil {
+		val, err := time.ParseDuration(f.Value.String())
+		if err == nil {
+			return val
+		}
+	}
+
+	return 0
+}
+
+func lookupFloat64(name string, set *flag.FlagSet) float64 {
+	f := set.Lookup(name)
+	if f != nil {
+		val, err := strconv.ParseFloat(f.Value.String(), 64)
+		if err != nil {
+			return 0
+		}
+		return val
+	}
+
+	return 0
+}
+
+func lookupString(name string, set *flag.FlagSet) string {
+	f := set.Lookup(name)
+	if f != nil {
+		return f.Value.String()
+	}
+
+	return ""
+}
+
+func lookupStringSlice(name string, set *flag.FlagSet) []string {
+	f := set.Lookup(name)
+	if f != nil {
+		return (f.Value.(*StringSlice)).Value()
+
+	}
+
+	return nil
+}
+
+func lookupIntSlice(name string, set *flag.FlagSet) []int {
+	f := set.Lookup(name)
+	if f != nil {
+		return (f.Value.(*IntSlice)).Value()
+
+	}
+
+	return nil
+}
+
+func lookupGeneric(name string, set *flag.FlagSet) interface{} {
+	f := set.Lookup(name)
+	if f != nil {
+		return f.Value
+	}
+	return nil
+}
+
+func lookupBool(name string, set *flag.FlagSet) bool {
+	f := set.Lookup(name)
+	if f != nil {
+		val, err := strconv.ParseBool(f.Value.String())
+		if err != nil {
+			return false
+		}
+		return val
+	}
+
+	return false
+}
+
+func lookupBoolT(name string, set *flag.FlagSet) bool {
+	f := set.Lookup(name)
+	if f != nil {
+		val, err := strconv.ParseBool(f.Value.String())
+		if err != nil {
+			return true
+		}
+		return val
+	}
+
+	return false
+}
+
+func copyFlag(name string, ff *flag.Flag, set *flag.FlagSet) {
+	switch ff.Value.(type) {
+	case *StringSlice:
+	default:
+		set.Set(name, ff.Value.String())
+	}
+}
+
+func normalizeFlags(flags []Flag, set *flag.FlagSet) error {
+	visited := make(map[string]bool)
+	set.Visit(func(f *flag.Flag) {
+		visited[f.Name] = true
+	})
+	for _, f := range flags {
+		parts := strings.Split(f.getName(), ",")
+		if len(parts) == 1 {
+			continue
+		}
+		var ff *flag.Flag
+		for _, name := range parts {
+			name = strings.Trim(name, " ")
+			if visited[name] {
+				if ff != nil {
+					return errors.New("Cannot use two forms of the same flag: " + name + " " + ff.Name)
+				}
+				ff = set.Lookup(name)
+			}
+		}
+		if ff == nil {
+			continue
+		}
+		for _, name := range parts {
+			name = strings.Trim(name, " ")
+			if !visited[name] {
+				copyFlag(name, ff, set)
+			}
+		}
+	}
+	return nil
+}

vendor/github.com/codegangsta/cli/flag.go

@@ -0,0 +1,497 @@
+package cli
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// This flag enables bash-completion for all commands and subcommands
+var BashCompletionFlag = BoolFlag{
+	Name: "generate-bash-completion",
+}
+
+// This flag prints the version for the application
+var VersionFlag = BoolFlag{
+	Name:  "version, v",
+	Usage: "print the version",
+}
+
+// This flag prints the help for all commands and subcommands
+// Set to the zero value (BoolFlag{}) to disable flag -- keeps subcommand
+// unless HideHelp is set to true)
+var HelpFlag = BoolFlag{
+	Name:  "help, h",
+	Usage: "show help",
+}
+
+// Flag is a common interface related to parsing flags in cli.
+// For more advanced flag parsing techniques, it is recomended that
+// this interface be implemented.
+type Flag interface {
+	fmt.Stringer
+	// Apply Flag settings to the given flag set
+	Apply(*flag.FlagSet)
+	getName() string
+}
+
+func flagSet(name string, flags []Flag) *flag.FlagSet {
+	set := flag.NewFlagSet(name, flag.ContinueOnError)
+
+	for _, f := range flags {
+		f.Apply(set)
+	}
+	return set
+}
+
+func eachName(longName string, fn func(string)) {
+	parts := strings.Split(longName, ",")
+	for _, name := range parts {
+		name = strings.Trim(name, " ")
+		fn(name)
+	}
+}
+
+// Generic is a generic parseable type identified by a specific flag
+type Generic interface {
+	Set(value string) error
+	String() string
+}
+
+// GenericFlag is the flag type for types implementing Generic
+type GenericFlag struct {
+	Name   string
+	Value  Generic
+	Usage  string
+	EnvVar string
+}
+
+// String returns the string representation of the generic flag to display the
+// help text to the user (uses the String() method of the generic flag to show
+// the value)
+func (f GenericFlag) String() string {
+	return withEnvHint(f.EnvVar, fmt.Sprintf("%s%s \"%v\"\t%v", prefixFor(f.Name), f.Name, f.Value, f.Usage))
+}
+
+// Apply takes the flagset and calls Set on the generic flag with the value
+// provided by the user for parsing by the flag
+func (f GenericFlag) Apply(set *flag.FlagSet) {
+	val := f.Value
+	if f.EnvVar != "" {
+		for _, envVar := range strings.Split(f.EnvVar, ",") {
+			envVar = strings.TrimSpace(envVar)
+			if envVal := os.Getenv(envVar); envVal != "" {
+				val.Set(envVal)
+				break
+			}
+		}
+	}
+
+	eachName(f.Name, func(name string) {
+		set.Var(f.Value, name, f.Usage)
+	})
+}
+
+func (f GenericFlag) getName() string {
+	return f.Name
+}
+
+// StringSlice is an opaque type for []string to satisfy flag.Value
+type StringSlice []string
+
+// Set appends the string value to the list of values
+func (f *StringSlice) Set(value string) error {
+	*f = append(*f, value)
+	return nil
+}
+
+// String returns a readable representation of this value (for usage defaults)
+func (f *StringSlice) String() string {
+	return fmt.Sprintf("%s", *f)
+}
+
+// Value returns the slice of strings set by this flag
+func (f *StringSlice) Value() []string {
+	return *f
+}
+
+// StringSlice is a string flag that can be specified multiple times on the
+// command-line
+type StringSliceFlag struct {
+	Name   string
+	Value  *StringSlice
+	Usage  string
+	EnvVar string
+}
+
+// String returns the usage
+func (f StringSliceFlag) String() string {
+	firstName := strings.Trim(strings.Split(f.Name, ",")[0], " ")
+	pref := prefixFor(firstName)
+	return withEnvHint(f.EnvVar, fmt.Sprintf("%s [%v]\t%v", prefixedNames(f.Name), pref+firstName+" option "+pref+firstName+" option", f.Usage))
+}
+
+// Apply populates the flag given the flag set and environment
+func (f StringSliceFlag) Apply(set *flag.FlagSet) {
+	if f.EnvVar != "" {
+		for _, envVar := range strings.Split(f.EnvVar, ",") {
+			envVar = strings.TrimSpace(envVar)
+			if envVal := os.Getenv(envVar); envVal != "" {
+				newVal := &StringSlice{}
+				for _, s := range strings.Split(envVal, ",") {
+					s = strings.TrimSpace(s)
+					newVal.Set(s)
+				}
+				f.Value = newVal
+				break
+			}
+		}
+	}
+
+	eachName(f.Name, func(name string) {
+		if f.Value == nil {
+			f.Value = &StringSlice{}
+		}
+		set.Var(f.Value, name, f.Usage)
+	})
+}
+
+func (f StringSliceFlag) getName() string {
+	return f.Name
+}
+
+// StringSlice is an opaque type for []int to satisfy flag.Value
+type IntSlice []int
+
+// Set parses the value into an integer and appends it to the list of values
+func (f *IntSlice) Set(value string) error {
+	tmp, err := strconv.Atoi(value)
+	if err != nil {
+		return err
+	} else {
+		*f = append(*f, tmp)
+	}
+	return nil
+}
+
+// String returns a readable representation of this value (for usage defaults)
+func (f *IntSlice) String() string {
+	return fmt.Sprintf("%d", *f)
+}
+
+// Value returns the slice of ints set by this flag
+func (f *IntSlice) Value() []int {
+	return *f
+}
+
+// IntSliceFlag is an int flag that can be specified multiple times on the
+// command-line
+type IntSliceFlag struct {
+	Name   string
+	Value  *IntSlice
+	Usage  string
+	EnvVar string
+}
+
+// String returns the usage
+func (f IntSliceFlag) String() string {
+	firstName := strings.Trim(strings.Split(f.Name, ",")[0], " ")
+	pref := prefixFor(firstName)
+	return withEnvHint(f.EnvVar, fmt.Sprintf("%s [%v]\t%v", prefixedNames(f.Name), pref+firstName+" option "+pref+firstName+" option", f.Usage))
+}
+
+// Apply populates the flag given the flag set and environment
+func (f IntSliceFlag) Apply(set *flag.FlagSet) {
+	if f.EnvVar != "" {
+		for _, envVar := range strings.Split(f.EnvVar, ",") {
+			envVar = strings.TrimSpace(envVar)
+			if envVal := os.Getenv(envVar); envVal != "" {
+				newVal := &IntSlice{}
+				for _, s := range strings.Split(envVal, ",") {
+					s = strings.TrimSpace(s)
+					err := newVal.Set(s)
+					if err != nil {
+						fmt.Fprintf(os.Stderr, err.Error())
+					}
+				}
+				f.Value = newVal
+				break
+			}
+		}
+	}
+
+	eachName(f.Name, func(name string) {
+		if f.Value == nil {
+			f.Value = &IntSlice{}
+		}
+		set.Var(f.Value, name, f.Usage)
+	})
+}
+
+func (f IntSliceFlag) getName() string {
+	return f.Name
+}
+
+// BoolFlag is a switch that defaults to false
+type BoolFlag struct {
+	Name   string
+	Usage  string
+	EnvVar string
+}
+
+// String returns a readable representation of this value (for usage defaults)
+func (f BoolFlag) String() string {
+	return withEnvHint(f.EnvVar, fmt.Sprintf("%s\t%v", prefixedNames(f.Name), f.Usage))
+}
+
+// Apply populates the flag given the flag set and environment
+func (f BoolFlag) Apply(set *flag.FlagSet) {
+	val := false
+	if f.EnvVar != "" {
+		for _, envVar := range strings.Split(f.EnvVar, ",") {
+			envVar = strings.TrimSpace(envVar)
+			if envVal := os.Getenv(envVar); envVal != "" {
+				envValBool, err := strconv.ParseBool(envVal)
+				if err == nil {
+					val = envValBool
+				}
+				break
+			}
+		}
+	}
+
+	eachName(f.Name, func(name string) {
+		set.Bool(name, val, f.Usage)
+	})
+}
+
+func (f BoolFlag) getName() string {
+	return f.Name
+}
+
+// BoolTFlag this represents a boolean flag that is true by default, but can
+// still be set to false by --some-flag=false
+type BoolTFlag struct {
+	Name   string
+	Usage  string
+	EnvVar string
+}
+
+// String returns a readable representation of this value (for usage defaults)
+func (f BoolTFlag) String() string {
+	return withEnvHint(f.EnvVar, fmt.Sprintf("%s\t%v", prefixedNames(f.Name), f.Usage))
+}
+
+// Apply populates the flag given the flag set and environment
+func (f BoolTFlag) Apply(set *flag.FlagSet) {
+	val := true
+	if f.EnvVar != "" {
+		for _, envVar := range strings.Split(f.EnvVar, ",") {
+			envVar = strings.TrimSpace(envVar)
+			if envVal := os.Getenv(envVar); envVal != "" {
+				envValBool, err := strconv.ParseBool(envVal)
+				if err == nil {
+					val = envValBool
+					break
+				}
+			}
+		}
+	}
+
+	eachName(f.Name, func(name string) {
+		set.Bool(name, val, f.Usage)
+	})
+}
+
+func (f BoolTFlag) getName() string {
+	return f.Name
+}
+
+// StringFlag represents a flag that takes as string value
+type StringFlag struct {
+	Name   string
+	Value  string
+	Usage  string
+	EnvVar string
+}
+
+// String returns the usage
+func (f StringFlag) String() string {
+	var fmtString string
+	fmtString = "%s %v\t%v"
+
+	if len(f.Value) > 0 {
+		fmtString = "%s \"%v\"\t%v"
+	} else {
+		fmtString = "%s %v\t%v"
+	}
+
+	return withEnvHint(f.EnvVar, fmt.Sprintf(fmtString, prefixedNames(f.Name), f.Value, f.Usage))
+}
+
+// Apply populates the flag given the flag set and environment
+func (f StringFlag) Apply(set *flag.FlagSet) {
+	if f.EnvVar != "" {
+		for _, envVar := range strings.Split(f.EnvVar, ",") {
+			envVar = strings.TrimSpace(envVar)
+			if envVal := os.Getenv(envVar); envVal != "" {
+				f.Value = envVal
+				break
+			}
+		}
+	}
+
+	eachName(f.Name, func(name string) {
+		set.String(name, f.Value, f.Usage)
+	})
+}
+
+func (f StringFlag) getName() string {
+	return f.Name
+}
+
+// IntFlag is a flag that takes an integer
+// Errors if the value provided cannot be parsed
+type IntFlag struct {
+	Name   string
+	Value  int
+	Usage  string
+	EnvVar string
+}
+
+// String returns the usage
+func (f IntFlag) String() string {
+	return withEnvHint(f.EnvVar, fmt.Sprintf("%s \"%v\"\t%v", prefixedNames(f.Name), f.Value, f.Usage))
+}
+
+// Apply populates the flag given the flag set and environment
+func (f IntFlag) Apply(set *flag.FlagSet) {
+	if f.EnvVar != "" {
+		for _, envVar := range strings.Split(f.EnvVar, ",") {
+			envVar = strings.TrimSpace(envVar)
+			if envVal := os.Getenv(envVar); envVal != "" {
+				envValInt, err := strconv.ParseInt(envVal, 0, 64)
+				if err == nil {
+					f.Value = int(envValInt)
+					break
+				}
+			}
+		}
+	}
+
+	eachName(f.Name, func(name string) {
+		set.Int(name, f.Value, f.Usage)
+	})
+}
+
+func (f IntFlag) getName() string {
+	return f.Name
+}
+
+// DurationFlag is a flag that takes a duration specified in Go's duration
+// format: https://golang.org/pkg/time/#ParseDuration
+type DurationFlag struct {
+	Name   string
+	Value  time.Duration
+	Usage  string
+	EnvVar string
+}
+
+// String returns a readable representation of this value (for usage defaults)
+func (f DurationFlag) String() string {
+	return withEnvHint(f.EnvVar, fmt.Sprintf("%s \"%v\"\t%v", prefixedNames(f.Name), f.Value, f.Usage))
+}
+
+// Apply populates the flag given the flag set and environment
+func (f DurationFlag) Apply(set *flag.FlagSet) {
+	if f.EnvVar != "" {
+		for _, envVar := range strings.Split(f.EnvVar, ",") {
+			envVar = strings.TrimSpace(envVar)
+			if envVal := os.Getenv(envVar); envVal != "" {
+				envValDuration, err := time.ParseDuration(envVal)
+				if err == nil {
+					f.Value = envValDuration
+					break
+				}
+			}
+		}
+	}
+
+	eachName(f.Name, func(name string) {
+		set.Duration(name, f.Value, f.Usage)
+	})
+}
+
+func (f DurationFlag) getName() string {
+	return f.Name
+}
+
+// Float64Flag is a flag that takes an float value
+// Errors if the value provided cannot be parsed
+type Float64Flag struct {
+	Name   string
+	Value  float64
+	Usage  string
+	EnvVar string
+}
+
+// String returns the usage
+func (f Float64Flag) String() string {
+	return withEnvHint(f.EnvVar, fmt.Sprintf("%s \"%v\"\t%v", prefixedNames(f.Name), f.Value, f.Usage))
+}
+
+// Apply populates the flag given the flag set and environment
+func (f Float64Flag) Apply(set *flag.FlagSet) {
+	if f.EnvVar != "" {
+		for _, envVar := range strings.Split(f.EnvVar, ",") {
+			envVar = strings.TrimSpace(envVar)
+			if envVal := os.Getenv(envVar); envVal != "" {
+				envValFloat, err := strconv.ParseFloat(envVal, 10)
+				if err == nil {
+					f.Value = float64(envValFloat)
+				}
+			}
+		}
+	}
+
+	eachName(f.Name, func(name string) {
+		set.Float64(name, f.Value, f.Usage)
+	})
+}
+
+func (f Float64Flag) getName() string {
+	return f.Name
+}
+
+func prefixFor(name string) (prefix string) {
+	if len(name) == 1 {
+		prefix = "-"
+	} else {
+		prefix = "--"
+	}
+
+	return
+}
+
+func prefixedNames(fullName string) (prefixed string) {
+	parts := strings.Split(fullName, ",")
+	for i, name := range parts {
+		name = strings.Trim(name, " ")
+		prefixed += prefixFor(name) + name
+		if i < len(parts)-1 {
+			prefixed += ", "
+		}
+	}
+	return
+}
+
+func withEnvHint(envVar, str string) string {
+	envText := ""
+	if envVar != "" {
+		envText = fmt.Sprintf(" [$%s]", strings.Join(strings.Split(envVar, ","), ", $"))
+	}
+	return str + envText
+}

vendor/github.com/codegangsta/cli/help.go

@@ -0,0 +1,246 @@
+package cli
+
+import (
+	"fmt"
+	"io"
+	"strings"
+	"text/tabwriter"
+	"text/template"
+)
+
+// The text template for the Default help topic.
+// cli.go uses text/template to render templates. You can
+// render custom help text by setting this variable.
+var AppHelpTemplate = `NAME:
+   {{.Name}} - {{.Usage}}
+
+USAGE:
+   {{.HelpName}} {{if .Flags}}[global options]{{end}}{{if .Commands}} command [command options]{{end}} {{if .ArgsUsage}}{{.ArgsUsage}}{{else}}[arguments...]{{end}}
+   {{if .Version}}
+VERSION:
+   {{.Version}}
+   {{end}}{{if len .Authors}}
+AUTHOR(S):
+   {{range .Authors}}{{ . }}{{end}}
+   {{end}}{{if .Commands}}
+COMMANDS:
+   {{range .Commands}}{{join .Names ", "}}{{ "\t" }}{{.Usage}}
+   {{end}}{{end}}{{if .Flags}}
+GLOBAL OPTIONS:
+   {{range .Flags}}{{.}}
+   {{end}}{{end}}{{if .Copyright }}
+COPYRIGHT:
+   {{.Copyright}}
+   {{end}}
+`
+
+// The text template for the command help topic.
+// cli.go uses text/template to render templates. You can
+// render custom help text by setting this variable.
+var CommandHelpTemplate = `NAME:
+   {{.HelpName}} - {{.Usage}}
+
+USAGE:
+   {{.HelpName}}{{if .Flags}} [command options]{{end}} {{if .ArgsUsage}}{{.ArgsUsage}}{{else}}[arguments...]{{end}}{{if .Description}}
+
+DESCRIPTION:
+   {{.Description}}{{end}}{{if .Flags}}
+
+OPTIONS:
+   {{range .Flags}}{{.}}
+   {{end}}{{ end }}
+`
+
+// The text template for the subcommand help topic.
+// cli.go uses text/template to render templates. You can
+// render custom help text by setting this variable.
+var SubcommandHelpTemplate = `NAME:
+   {{.HelpName}} - {{.Usage}}
+
+USAGE:
+   {{.HelpName}} command{{if .Flags}} [command options]{{end}} {{if .ArgsUsage}}{{.ArgsUsage}}{{else}}[arguments...]{{end}}
+
+COMMANDS:
+   {{range .Commands}}{{join .Names ", "}}{{ "\t" }}{{.Usage}}
+   {{end}}{{if .Flags}}
+OPTIONS:
+   {{range .Flags}}{{.}}
+   {{end}}{{end}}
+`
+
+var helpCommand = Command{
+	Name:      "help",
+	Aliases:   []string{"h"},
+	Usage:     "Shows a list of commands or help for one command",
+	ArgsUsage: "[command]",
+	Action: func(c *Context) {
+		args := c.Args()
+		if args.Present() {
+			ShowCommandHelp(c, args.First())
+		} else {
+			ShowAppHelp(c)
+		}
+	},
+}
+
+var helpSubcommand = Command{
+	Name:      "help",
+	Aliases:   []string{"h"},
+	Usage:     "Shows a list of commands or help for one command",
+	ArgsUsage: "[command]",
+	Action: func(c *Context) {
+		args := c.Args()
+		if args.Present() {
+			ShowCommandHelp(c, args.First())
+		} else {
+			ShowSubcommandHelp(c)
+		}
+	},
+}
+
+// Prints help for the App or Command
+type helpPrinter func(w io.Writer, templ string, data interface{})
+
+var HelpPrinter helpPrinter = printHelp
+
+// Prints version for the App
+var VersionPrinter = printVersion
+
+func ShowAppHelp(c *Context) {
+	HelpPrinter(c.App.Writer, AppHelpTemplate, c.App)
+}
+
+// Prints the list of subcommands as the default app completion method
+func DefaultAppComplete(c *Context) {
+	for _, command := range c.App.Commands {
+		for _, name := range command.Names() {
+			fmt.Fprintln(c.App.Writer, name)
+		}
+	}
+}
+
+// Prints help for the given command
+func ShowCommandHelp(ctx *Context, command string) {
+	// show the subcommand help for a command with subcommands
+	if command == "" {
+		HelpPrinter(ctx.App.Writer, SubcommandHelpTemplate, ctx.App)
+		return
+	}
+
+	for _, c := range ctx.App.Commands {
+		if c.HasName(command) {
+			HelpPrinter(ctx.App.Writer, CommandHelpTemplate, c)
+			return
+		}
+	}
+
+	if ctx.App.CommandNotFound != nil {
+		ctx.App.CommandNotFound(ctx, command)
+	} else {
+		fmt.Fprintf(ctx.App.Writer, "No help topic for '%v'\n", command)
+	}
+}
+
+// Prints help for the given subcommand
+func ShowSubcommandHelp(c *Context) {
+	ShowCommandHelp(c, c.Command.Name)
+}
+
+// Prints the version number of the App
+func ShowVersion(c *Context) {
+	VersionPrinter(c)
+}
+
+func printVersion(c *Context) {
+	fmt.Fprintf(c.App.Writer, "%v version %v\n", c.App.Name, c.App.Version)
+}
+
+// Prints the lists of commands within a given context
+func ShowCompletions(c *Context) {
+	a := c.App
+	if a != nil && a.BashComplete != nil {
+		a.BashComplete(c)
+	}
+}
+
+// Prints the custom completions for a given command
+func ShowCommandCompletions(ctx *Context, command string) {
+	c := ctx.App.Command(command)
+	if c != nil && c.BashComplete != nil {
+		c.BashComplete(ctx)
+	}
+}
+
+func printHelp(out io.Writer, templ string, data interface{}) {
+	funcMap := template.FuncMap{
+		"join": strings.Join,
+	}
+
+	w := tabwriter.NewWriter(out, 0, 8, 1, '\t', 0)
+	t := template.Must(template.New("help").Funcs(funcMap).Parse(templ))
+	err := t.Execute(w, data)
+	if err != nil {
+		panic(err)
+	}
+	w.Flush()
+}
+
+func checkVersion(c *Context) bool {
+	found := false
+	if VersionFlag.Name != "" {
+		eachName(VersionFlag.Name, func(name string) {
+			if c.GlobalBool(name) || c.Bool(name) {
+				found = true
+			}
+		})
+	}
+	return found
+}
+
+func checkHelp(c *Context) bool {
+	found := false
+	if HelpFlag.Name != "" {
+		eachName(HelpFlag.Name, func(name string) {
+			if c.GlobalBool(name) || c.Bool(name) {
+				found = true
+			}
+		})
+	}
+	return found
+}
+
+func checkCommandHelp(c *Context, name string) bool {
+	if c.Bool("h") || c.Bool("help") {
+		ShowCommandHelp(c, name)
+		return true
+	}
+
+	return false
+}
+
+func checkSubcommandHelp(c *Context) bool {
+	if c.GlobalBool("h") || c.GlobalBool("help") {
+		ShowSubcommandHelp(c)
+		return true
+	}
+
+	return false
+}
+
+func checkCompletions(c *Context) bool {
+	if (c.GlobalBool(BashCompletionFlag.Name) || c.Bool(BashCompletionFlag.Name)) && c.App.EnableBashCompletion {
+		ShowCompletions(c)
+		return true
+	}
+
+	return false
+}
+
+func checkCommandCompletions(c *Context, name string) bool {
+	if c.Bool(BashCompletionFlag.Name) && c.App.EnableBashCompletion {
+		ShowCommandCompletions(c, name)
+		return true
+	}
+
+	return false
+}

vendor/github.com/dchest/uniuri/LICENSE

@@ -0,0 +1,19 @@
+Copyright (c) 2011 Dmitry Chestnykh
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

vendor/github.com/dchest/uniuri/README.md

@@ -0,0 +1,83 @@
+Package uniuri
+=====================
+
+[![Build Status](https://travis-ci.org/dchest/uniuri.png)](https://travis-ci.org/dchest/uniuri)
+
+```go
+import "github.com/dchest/uniuri"
+```
+
+Package uniuri generates random strings good for use in URIs to identify
+unique objects.
+
+Example usage:
+
+```go
+s := uniuri.New() // s is now "apHCJBl7L1OmC57n"
+```
+
+A standard string created by New() is 16 bytes in length and consists of
+Latin upper and lowercase letters, and numbers (from the set of 62 allowed
+characters), which means that it has ~95 bits of entropy. To get more
+entropy, you can use NewLen(UUIDLen), which returns 20-byte string, giving
+~119 bits of entropy, or any other desired length.
+
+Functions read from crypto/rand random source, and panic if they fail to
+read from it.
+
+
+Constants
+---------
+
+```go
+const (
+    // Standard length of uniuri string to achive ~95 bits of entropy.
+    StdLen = 16
+    // Length of uniurl string to achive ~119 bits of entropy, closest
+    // to what can be losslessly converted to UUIDv4 (122 bits).
+    UUIDLen = 20
+)
+```
+
+
+
+Variables
+---------
+
+```go
+var StdChars = []byte("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789")
+```
+
+Standard characters allowed in uniuri string.
+
+
+Functions
+---------
+
+### func New
+
+```go
+func New() string
+```
+
+New returns a new random string of the standard length, consisting of
+standard characters.
+
+### func NewLen
+
+```go
+func NewLen(length int) string
+```
+
+NewLen returns a new random string of the provided length, consisting of
+standard characters.
+
+### func NewLenChars
+
+```go
+func NewLenChars(length int, chars []byte) string
+```
+
+NewLenChars returns a new random string of the provided length, consisting
+of the provided byte slice of allowed characters (maximum 256).
+

vendor/github.com/dchest/uniuri/uniuri.go

@@ -0,0 +1,71 @@
+// Package uniuri generates random strings good for use in URIs to identify
+// unique objects.
+//
+// Example usage:
+//
+//	s := uniuri.New() // s is now "apHCJBl7L1OmC57n"
+//
+// A standard string created by New() is 16 bytes in length and consists of
+// Latin upper and lowercase letters, and numbers (from the set of 62 allowed
+// characters), which means that it has ~95 bits of entropy. To get more
+// entropy, you can use NewLen(UUIDLen), which returns 20-byte string, giving
+// ~119 bits of entropy, or any other desired length.
+//
+// Functions read from crypto/rand random source, and panic if they fail to
+// read from it.
+package uniuri
+
+import (
+	"crypto/rand"
+	"io"
+)
+
+const (
+	// Standard length of uniuri string to achive ~95 bits of entropy.
+	StdLen = 16
+	// Length of uniurl string to achive ~119 bits of entropy, closest
+	// to what can be losslessly converted to UUIDv4 (122 bits).
+	UUIDLen = 20
+)
+
+// Standard characters allowed in uniuri string.
+var StdChars = []byte("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789")
+
+// New returns a new random string of the standard length, consisting of
+// standard characters.
+func New() string {
+	return NewLenChars(StdLen, StdChars)
+}
+
+// NewLen returns a new random string of the provided length, consisting of
+// standard characters.
+func NewLen(length int) string {
+	return NewLenChars(length, StdChars)
+}
+
+// NewLenChars returns a new random string of the provided length, consisting
+// of the provided byte slice of allowed characters (maximum 256).
+func NewLenChars(length int, chars []byte) string {
+	b := make([]byte, length)
+	r := make([]byte, length+(length/4)) // storage for random bytes.
+	clen := byte(len(chars))
+	maxrb := byte(256 - (256 % len(chars)))
+	i := 0
+	for {
+		if _, err := io.ReadFull(rand.Reader, r); err != nil {
+			panic("error reading from random source: " + err.Error())
+		}
+		for _, c := range r {
+			if c >= maxrb {
+				// Skip this number to avoid modulo bias.
+				continue
+			}
+			b[i] = chars[c%clen]
+			i++
+			if i == length {
+				return string(b)
+			}
+		}
+	}
+	panic("unreachable")
+}

vendor/golang.org/x/net/context/ctxhttp/cancelreq.go

@@ -0,0 +1,19 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build go1.5
+
+package ctxhttp
+
+import "net/http"
+
+func canceler(client *http.Client, req *http.Request) func() {
+	// TODO(djd): Respect any existing value of req.Cancel.
+	ch := make(chan struct{})
+	req.Cancel = ch
+
+	return func() {
+		close(ch)
+	}
+}

vendor/golang.org/x/net/context/ctxhttp/cancelreq_go14.go

@@ -0,0 +1,23 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build !go1.5
+
+package ctxhttp
+
+import "net/http"
+
+type requestCanceler interface {
+	CancelRequest(*http.Request)
+}
+
+func canceler(client *http.Client, req *http.Request) func() {
+	rc, ok := client.Transport.(requestCanceler)
+	if !ok {
+		return func() {}
+	}
+	return func() {
+		rc.CancelRequest(req)
+	}
+}

vendor/golang.org/x/net/context/ctxhttp/ctxhttp.go

@@ -0,0 +1,145 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package ctxhttp provides helper functions for performing context-aware HTTP requests.
+package ctxhttp // import "golang.org/x/net/context/ctxhttp"
+
+import (
+	"io"
+	"net/http"
+	"net/url"
+	"strings"
+
+	"golang.org/x/net/context"
+)
+
+func nop() {}
+
+var (
+	testHookContextDoneBeforeHeaders = nop
+	testHookDoReturned               = nop
+	testHookDidBodyClose             = nop
+)
+
+// Do sends an HTTP request with the provided http.Client and returns an HTTP response.
+// If the client is nil, http.DefaultClient is used.
+// If the context is canceled or times out, ctx.Err() will be returned.
+func Do(ctx context.Context, client *http.Client, req *http.Request) (*http.Response, error) {
+	if client == nil {
+		client = http.DefaultClient
+	}
+
+	// Request cancelation changed in Go 1.5, see cancelreq.go and cancelreq_go14.go.
+	cancel := canceler(client, req)
+
+	type responseAndError struct {
+		resp *http.Response
+		err  error
+	}
+	result := make(chan responseAndError, 1)
+
+	// Make local copies of test hooks closed over by goroutines below.
+	// Prevents data races in tests.
+	testHookDoReturned := testHookDoReturned
+	testHookDidBodyClose := testHookDidBodyClose
+
+	go func() {
+		resp, err := client.Do(req)
+		testHookDoReturned()
+		result <- responseAndError{resp, err}
+	}()
+
+	var resp *http.Response
+
+	select {
+	case <-ctx.Done():
+		testHookContextDoneBeforeHeaders()
+		cancel()
+		// Clean up after the goroutine calling client.Do:
+		go func() {
+			if r := <-result; r.resp != nil {
+				testHookDidBodyClose()
+				r.resp.Body.Close()
+			}
+		}()
+		return nil, ctx.Err()
+	case r := <-result:
+		var err error
+		resp, err = r.resp, r.err
+		if err != nil {
+			return resp, err
+		}
+	}
+
+	c := make(chan struct{})
+	go func() {
+		select {
+		case <-ctx.Done():
+			cancel()
+		case <-c:
+			// The response's Body is closed.
+		}
+	}()
+	resp.Body = &notifyingReader{resp.Body, c}
+
+	return resp, nil
+}
+
+// Get issues a GET request via the Do function.
+func Get(ctx context.Context, client *http.Client, url string) (*http.Response, error) {
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		return nil, err
+	}
+	return Do(ctx, client, req)
+}
+
+// Head issues a HEAD request via the Do function.
+func Head(ctx context.Context, client *http.Client, url string) (*http.Response, error) {
+	req, err := http.NewRequest("HEAD", url, nil)
+	if err != nil {
+		return nil, err
+	}
+	return Do(ctx, client, req)
+}
+
+// Post issues a POST request via the Do function.
+func Post(ctx context.Context, client *http.Client, url string, bodyType string, body io.Reader) (*http.Response, error) {
+	req, err := http.NewRequest("POST", url, body)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("Content-Type", bodyType)
+	return Do(ctx, client, req)
+}
+
+// PostForm issues a POST request via the Do function.
+func PostForm(ctx context.Context, client *http.Client, url string, data url.Values) (*http.Response, error) {
+	return Post(ctx, client, url, "application/x-www-form-urlencoded", strings.NewReader(data.Encode()))
+}
+
+// notifyingReader is an io.ReadCloser that closes the notify channel after
+// Close is called or a Read fails on the underlying ReadCloser.
+type notifyingReader struct {
+	io.ReadCloser
+	notify chan<- struct{}
+}
+
+func (r *notifyingReader) Read(p []byte) (int, error) {
+	n, err := r.ReadCloser.Read(p)
+	if err != nil && r.notify != nil {
+		close(r.notify)
+		r.notify = nil
+	}
+	return n, err
+}
+
+func (r *notifyingReader) Close() error {
+	err := r.ReadCloser.Close()
+	if r.notify != nil {
+		close(r.notify)
+		r.notify = nil
+	}
+	return err
+}

vendor/vendor.json

@@ -17,12 +17,22 @@
 			"revision": "5cef21e2e4f0fd147973b558d4db7395176bcd95",
 			"revisionTime": "2016-03-22T13:50:45-07:00"
 		},
+		{
+			"path": "github.com/codegangsta/cli",
+			"revision": "70e3fa51ebed95df8c0fbe1519c1c1f9bc16bb13",
+			"revisionTime": "2015-10-29T20:11:27-07:00"
+		},
 		{
 			"origin": "github.com/stretchr/testify/vendor/github.com/davecgh/go-spew/spew",
 			"path": "github.com/davecgh/go-spew/spew",
 			"revision": "9f9027faeb0dad515336ed2f28317f9f8f527ab4",
 			"revisionTime": "2016-01-29T19:31:06Z"
 		},
+		{
+			"path": "github.com/dchest/uniuri",
+			"revision": "bc4af7603a3e0ce9d58009f82fca481555182e1c",
+			"revisionTime": "2013-12-23T09:21:49+01:00"
+		},
 		{
 			"path": "github.com/dgrijalva/jwt-go",
 			"revision": "c1da56349675b292d3200463e2c88b9aa5e02391",
@@ -218,6 +228,11 @@
 			"revision": "6acef71eb69611914f7a30939ea9f6e194c78172",
 			"revisionTime": "2016-02-25T17:50:37Z"
 		},
+		{
+			"path": "golang.org/x/net/context/ctxhttp",
+			"revision": "fb93926129b8ec0056f2f458b1f519654814edf0",
+			"revisionTime": "2016-04-13T08:48:50+10:00"
+		},
 		{
 			"path": "golang.org/x/oauth2",
 			"revision": "8a57ed94ffd43444c0879fe75701732a38afc985",

web/hook.go

@@ -11,12 +11,13 @@ import (
 
 	log "github.com/Sirupsen/logrus"
 	"github.com/drone/drone/engine"
-	"github.com/drone/drone/engine/parser"
 	"github.com/drone/drone/model"
+	"github.com/drone/drone/queue"
 	"github.com/drone/drone/remote"
 	"github.com/drone/drone/shared/httputil"
 	"github.com/drone/drone/shared/token"
 	"github.com/drone/drone/store"
+	"github.com/drone/drone/yaml"
 )
 
 var (
@@ -147,13 +148,13 @@ func PostHook(c *gin.Context) {
 		// NOTE we don't exit on failure. The sec file is optional
 	}
 
-	axes, err := parser.ParseMatrix(raw)
+	axes, err := yaml.ParseMatrix(raw)
 	if err != nil {
 		c.String(500, "Failed to parse yaml file or calculate matrix. %s", err)
 		return
 	}
 	if len(axes) == 0 {
-		axes = append(axes, parser.Axis{})
+		axes = append(axes, yaml.Axis{})
 	}
 
 	netrc, err := remote_.Netrc(user, repo)
@@ -165,7 +166,7 @@ func PostHook(c *gin.Context) {
 	key, _ := store.GetKey(c, repo)
 
 	// verify the branches can be built vs skipped
-	branches := parser.ParseBranch(raw)
+	branches := yaml.ParseBranch(raw)
 	if !branches.Matches(build.Branch) {
 		c.String(200, "Branch does not match restrictions defined in yaml")
 		return
@@ -204,6 +205,35 @@ func PostHook(c *gin.Context) {
 	// on status change notifications
 	last, _ := store.GetBuildLastBefore(c, repo, build.Branch, build.ID)
 
+	// IMPORTANT. PLEASE READ
+	//
+	// The below code uses a feature flag to switch between the current
+	// build engine and the exerimental 0.5 build engine. This can be
+	// enabled using with the environment variable CANARY=true
+
+	if os.Getenv("CANARY") == "true" {
+		for _, job := range jobs {
+			queue.Publish(c, &queue.Work{
+				User:      user,
+				Repo:      repo,
+				Build:     build,
+				BuildLast: last,
+				Job:       job,
+				Keys:      key,
+				Netrc:     netrc,
+				Yaml:      string(raw),
+				YamlEnc:   string(sec),
+				System: &model.System{
+					Link:      httputil.GetURL(c.Request),
+					Plugins:   strings.Split(os.Getenv("PLUGIN_FILTER"), " "),
+					Globals:   strings.Split(os.Getenv("PLUGIN_PARAMS"), " "),
+					Escalates: strings.Split(os.Getenv("ESCALATE_FILTER"), " "),
+				},
+			})
+		}
+		return // EXIT NOT TO AVOID THE 0.4 ENGINE CODE BELOW
+	}
+
 	engine_ := engine.FromContext(c)
 	go engine_.Schedule(c.Copy(), &engine.Task{
 		User:      user,
@@ -222,5 +252,4 @@ func PostHook(c *gin.Context) {
 			Escalates: strings.Split(os.Getenv("ESCALATE_FILTER"), " "),
 		},
 	})
-
 }

web/stream2.go

@@ -0,0 +1,126 @@
+package web
+
+import (
+	"bufio"
+	"encoding/json"
+	"io"
+	"strconv"
+
+	"github.com/gin-gonic/gin"
+
+	"github.com/drone/drone/bus"
+	"github.com/drone/drone/model"
+	"github.com/drone/drone/router/middleware/session"
+	"github.com/drone/drone/store"
+	"github.com/drone/drone/stream"
+
+	log "github.com/Sirupsen/logrus"
+
+	"github.com/manucorporat/sse"
+)
+
+// IMPORTANT. PLEASE READ
+//
+// This file containers experimental streaming features for the 0.5
+// release. These can be enabled with the feature flag CANARY=true
+
+// GetRepoEvents will upgrade the connection to a Websocket and will stream
+// event updates to the browser.
+func GetRepoEvents2(c *gin.Context) {
+	repo := session.Repo(c)
+	c.Writer.Header().Set("Content-Type", "text/event-stream")
+
+	eventc := make(chan *bus.Event, 1)
+	bus.Subscribe(c, eventc)
+	defer func() {
+		bus.Unsubscribe(c, eventc)
+		close(eventc)
+		log.Infof("closed event stream")
+	}()
+
+	c.Stream(func(w io.Writer) bool {
+		select {
+		case event := <-eventc:
+			if event == nil {
+				log.Infof("nil event received")
+				return false
+			}
+
+			if event.Repo.FullName == repo.FullName {
+
+				var payload = struct {
+					model.Build
+					Jobs []*model.Job `json:"jobs"`
+				}{}
+				payload.Build = event.Build
+				payload.Jobs, _ = store.GetJobList(c, &event.Build)
+				data, _ := json.Marshal(&payload)
+
+				sse.Encode(w, sse.Event{
+					Event: "message",
+					Data:  string(data),
+				})
+			}
+		case <-c.Writer.CloseNotify():
+			return false
+		}
+		return true
+	})
+}
+
+func GetStream2(c *gin.Context) {
+
+	repo := session.Repo(c)
+	buildn, _ := strconv.Atoi(c.Param("build"))
+	jobn, _ := strconv.Atoi(c.Param("number"))
+
+	c.Writer.Header().Set("Content-Type", "text/event-stream")
+
+	build, err := store.GetBuildNumber(c, repo, buildn)
+	if err != nil {
+		log.Debugln("stream cannot get build number.", err)
+		c.AbortWithError(404, err)
+		return
+	}
+	job, err := store.GetJobNumber(c, build, jobn)
+	if err != nil {
+		log.Debugln("stream cannot get job number.", err)
+		c.AbortWithError(404, err)
+		return
+	}
+
+	rc, wc, err := stream.Open(c, stream.ToKey(job.ID))
+	if err != nil {
+		c.AbortWithError(404, err)
+		return
+	}
+
+	defer func() {
+		if wc != nil {
+			wc.Close()
+		}
+		if rc != nil {
+			rc.Close()
+		}
+	}()
+
+	go func() {
+		<-c.Writer.CloseNotify()
+		rc.Close()
+	}()
+
+	var line int
+	var scanner = bufio.NewScanner(rc)
+	for scanner.Scan() {
+		line++
+		var err = sse.Encode(c.Writer, sse.Event{
+			Id:    strconv.Itoa(line),
+			Event: "message",
+			Data:  scanner.Text(),
+		})
+		if err != nil {
+			break
+		}
+		c.Writer.Flush()
+	}
+}

yaml/branch.go

@@ -1,4 +1,4 @@
-package parser
+package yaml
 
 import (
 	"path/filepath"

yaml/branch_test.go

@@ -1,4 +1,4 @@
-package parser
+package yaml
 
 import (
 	"testing"

yaml/checksum/checksum.go

@@ -1,4 +1,4 @@
-package shasum
+package checksum
 
 import (
 	"crypto/sha1"

yaml/checksum/checksum_test.go

@@ -1,4 +1,4 @@
-package shasum
+package checksum
 
 import (
 	"testing"

yaml/matrix.go

@@ -1,4 +1,4 @@
-package parser
+package yaml
 
 import (
 	"strings"

yaml/matrix_test.go

@@ -1,4 +1,4 @@
-package parser
+package yaml
 
 import (
 	"testing"

yaml/types.go

@@ -1,4 +1,4 @@
-package parser
+package yaml
 
 // stringOrSlice represents a string or an array of strings.
 type stringOrSlice struct {

yaml/types_test.go

@@ -1,4 +1,4 @@
-package parser
+package yaml
 
 import (
 	"testing"