drone exec and drone agent now share code

agent/agent.go

@@ -62,6 +62,7 @@ func (a *Agent) Run(payload *queue.Work, cancel <-chan bool) error {
 		a.Update(payload)
 		return err
 	}
+	a.Update(payload)
 	err = a.exec(spec, payload, cancel)
 
 	if err != nil {

agent/updater.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	"sync"
 	"time"
 
 	"github.com/Sirupsen/logrus"
@@ -41,10 +42,21 @@ func NewClientUpdater(client client.Client) UpdateFunc {
 	}
 }
 
-func NewClientLogger(w io.Writer) LoggerFunc {
+func NewClientLogger(client client.Client, id int64, rc io.ReadCloser, wc io.WriteCloser) LoggerFunc {
+	var once sync.Once
 	return func(line *build.Line) {
+		// annoying hack to only start streaming once the first line is written
+		once.Do(func() {
+			go func() {
+				err := client.Stream(id, rc)
+				if err != nil && err != io.ErrClosedPipe {
+					logrus.Errorf("Error streaming build logs. %s", err)
+				}
+			}()
+		})
+
 		linejson, _ := json.Marshal(line)
-		w.Write(linejson)
+		wc.Write(linejson)
-		w.Write([]byte{'\n'})
+		wc.Write([]byte{'\n'})
 	}
 }

drone/agent/agent.go

@@ -141,6 +141,10 @@ func start(c *cli.Context) {
 	} else {
 		logrus.SetLevel(logrus.WarnLevel)
 	}
+	logrus.Infof("Connecting to %s with token %s",
+		c.String("drone-server"),
+		c.String("drone-token"),
+	)
 
 	client := client.NewClientToken(
 		c.String("drone-server"),

drone/agent/exec.go

@@ -1,27 +1,15 @@
 package agent
 
 import (
-	"encoding/json"
-	"fmt"
 	"io"
-	"regexp"
-	"strings"
 	"time"
 
 	"github.com/Sirupsen/logrus"
-	"github.com/dchest/uniuri"
+	"github.com/drone/drone/agent"
+	"github.com/drone/drone/build/docker"
 	"github.com/drone/drone/client"
-	"github.com/drone/drone/engine/compiler"
-	"github.com/drone/drone/engine/compiler/builtin"
-	"github.com/drone/drone/engine/runner"
-	"github.com/drone/drone/engine/runner/docker"
-	"github.com/drone/drone/model"
-	"github.com/drone/drone/queue"
-	"github.com/drone/drone/version"
-	"github.com/drone/drone/yaml/expander"
 
 	"github.com/samalba/dockerclient"
-	"golang.org/x/net/context"
 )
 
 type config struct {
@@ -48,233 +36,45 @@ func (r *pipeline) run() error {
 	logrus.Infof("Starting build %s/%s#%d.%d",
 		w.Repo.Owner, w.Repo.Name, w.Build.Number, w.Job.Number)
 
-	w.Job.Status = model.StatusRunning
+	cancel := make(chan bool, 1)
-	w.Job.Started = time.Now().Unix()
+	engine := docker.NewClient(r.docker)
 
-	prefix := fmt.Sprintf("drone_%s", uniuri.New())
+	// streaming the logs
+	rc, wc := io.Pipe()
+	defer func() {
+		wc.Close()
+		rc.Close()
+	}()
 
-	envs := toEnv(w)
+	a := agent.Agent{
-	w.Yaml = expander.ExpandString(w.Yaml, envs)
+		Update:    agent.NewClientUpdater(r.drone),
-
+		Logger:    agent.NewClientLogger(r.drone, w.Job.ID, rc, wc),
-	// inject the netrc file into the clone plugin if the repositroy is
+		Engine:    engine,
-	// private and requires authentication.
+		Timeout:   time.Minute * 15,
-	var secrets []*model.Secret
+		Platform:  r.config.platform,
-	if w.Verified {
+		Namespace: r.config.namespace,
-		secrets = append(secrets, w.Secrets...)
+		Escalate:  r.config.privileged,
+		Pull:      r.config.pull,
 	}
 
-	if w.Repo.IsPrivate {
+	// signal for canceling the build.
-		secrets = append(secrets, &model.Secret{
-			Name:   "DRONE_NETRC_USERNAME",
-			Value:  w.Netrc.Login,
-			Images: []string{"*"},
-			Events: []string{"*"},
-		})
-		secrets = append(secrets, &model.Secret{
-			Name:   "DRONE_NETRC_PASSWORD",
-			Value:  w.Netrc.Password,
-			Images: []string{"*"},
-			Events: []string{"*"},
-		})
-		secrets = append(secrets, &model.Secret{
-			Name:   "DRONE_NETRC_MACHINE",
-			Value:  w.Netrc.Machine,
-			Images: []string{"*"},
-			Events: []string{"*"},
-		})
-	}
-
-	var lastStatus string
-	if w.BuildLast != nil {
-		lastStatus = w.BuildLast.Status
-	}
-
-	trans := []compiler.Transform{
-		builtin.NewCloneOp(w.Repo.Kind, true),
-		builtin.NewSecretOp(w.Build.Event, secrets),
-		builtin.NewNormalizeOp(r.config.namespace),
-		builtin.NewWorkspaceOp("/drone", "/drone/src/github.com/"+w.Repo.FullName),
-		builtin.NewValidateOp(
-			w.Repo.IsTrusted,
-			r.config.whitelist,
-		),
-		builtin.NewEnvOp(envs),
-		builtin.NewShellOp(builtin.Linux_adm64),
-		builtin.NewArgsOp(),
-		builtin.NewEscalateOp(r.config.privileged),
-		builtin.NewPodOp(prefix),
-		builtin.NewAliasOp(prefix),
-		builtin.NewPullOp(r.config.pull),
-		builtin.NewFilterOp(
-			lastStatus,
-			w.Build.Branch,
-			w.Build.Event,
-			w.Build.Deploy,
-			w.Job.Environment,
-		),
-	}
-
-	compile := compiler.New()
-	compile.Transforms(trans)
-	spec, err := compile.CompileString(w.Yaml)
-	if err != nil {
-		w.Job.Error = err.Error()
-		w.Job.ExitCode = 255
-		w.Job.Finished = w.Job.Started
-		w.Job.Status = model.StatusError
-		pushRetry(r.drone, w)
-		return nil
-	}
-
-	pushRetry(r.drone, w)
-
-	conf := runner.Config{
-		Engine: docker.New(r.docker),
-	}
-
-	c := context.TODO()
-	c, timout := context.WithTimeout(c, time.Minute*time.Duration(w.Repo.Timeout))
-	c, cancel := context.WithCancel(c)
-	defer cancel()
-	defer timout()
-
-	run := conf.Runner(c, spec)
-	run.Run()
-
 	wait := r.drone.Wait(w.Job.ID)
 	defer wait.Cancel()
 	go func() {
 		if _, err := wait.Done(); err == nil {
+			cancel <- true
 			logrus.Infof("Cancel build %s/%s#%d.%d",
 				w.Repo.Owner, w.Repo.Name, w.Build.Number, w.Job.Number)
-			cancel()
 		}
 	}()
 
-	rc, wc := io.Pipe()
+	a.Run(w, cancel)
-	go func() {
-		// TODO(bradrydzewski) figure out how to resume upload on failure
-		err := r.drone.Stream(w.Job.ID, rc)
-		if err != nil && err != io.ErrClosedPipe {
-			logrus.Errorf("Error streaming build logs. %s", err)
-		}
-	}()
 
-	pipe := run.Pipe()
-	for {
-		line := pipe.Next()
-		if line == nil {
-			break
-		}
-		linejson, _ := json.Marshal(line)
-		wc.Write(linejson)
-		wc.Write([]byte{'\n'})
-	}
-
-	err = run.Wait()
-
-	pipe.Close()
 	wc.Close()
 	rc.Close()
 
-	// catch the build result
-	if err != nil {
-		w.Job.ExitCode = 255
-	}
-	if exitErr, ok := err.(*runner.ExitError); ok {
-		w.Job.ExitCode = exitErr.Code
-	}
-
-	w.Job.Finished = time.Now().Unix()
-
-	switch w.Job.ExitCode {
-	case 128, 130, 137:
-		w.Job.Status = model.StatusKilled
-	case 0:
-		w.Job.Status = model.StatusSuccess
-	default:
-		w.Job.Status = model.StatusFailure
-	}
-
-	pushRetry(r.drone, w)
-
 	logrus.Infof("Finished build %s/%s#%d.%d",
 		w.Repo.Owner, w.Repo.Name, w.Build.Number, w.Job.Number)
 
 	return nil
 }
-
-func pushRetry(client client.Client, w *queue.Work) {
-	for {
-		err := client.Push(w)
-		if err == nil {
-			return
-		}
-		logrus.Errorf("Error updating %s/%s#%d.%d. Retry in 30s. %s",
-			w.Repo.Owner, w.Repo.Name, w.Build.Number, w.Job.Number, err)
-		logrus.Infof("Retry update in 30s")
-		time.Sleep(time.Second * 30)
-	}
-}
-
-func toEnv(w *queue.Work) map[string]string {
-	envs := map[string]string{
-		"CI":                         "drone",
-		"DRONE":                      "true",
-		"DRONE_ARCH":                 "linux_amd64",
-		"DRONE_REPO":                 w.Repo.FullName,
-		"DRONE_REPO_SCM":             w.Repo.Kind,
-		"DRONE_REPO_OWNER":           w.Repo.Owner,
-		"DRONE_REPO_NAME":            w.Repo.Name,
-		"DRONE_REPO_LINK":            w.Repo.Link,
-		"DRONE_REPO_AVATAR":          w.Repo.Avatar,
-		"DRONE_REPO_BRANCH":          w.Repo.Branch,
-		"DRONE_REPO_PRIVATE":         fmt.Sprintf("%v", w.Repo.IsPrivate),
-		"DRONE_REPO_TRUSTED":         fmt.Sprintf("%v", w.Repo.IsTrusted),
-		"DRONE_REMOTE_URL":           w.Repo.Clone,
-		"DRONE_COMMIT_SHA":           w.Build.Commit,
-		"DRONE_COMMIT_REF":           w.Build.Ref,
-		"DRONE_COMMIT_BRANCH":        w.Build.Branch,
-		"DRONE_COMMIT_LINK":          w.Build.Link,
-		"DRONE_COMMIT_MESSAGE":       w.Build.Message,
-		"DRONE_COMMIT_AUTHOR":        w.Build.Author,
-		"DRONE_COMMIT_AUTHOR_EMAIL":  w.Build.Email,
-		"DRONE_COMMIT_AUTHOR_AVATAR": w.Build.Avatar,
-		"DRONE_BUILD_NUMBER":         fmt.Sprintf("%d", w.Build.Number),
-		"DRONE_BUILD_EVENT":          w.Build.Event,
-		"DRONE_BUILD_STATUS":         w.Build.Status,
-		"DRONE_BUILD_LINK":           fmt.Sprintf("%s/%s/%d", w.System.Link, w.Repo.FullName, w.Build.Number),
-		"DRONE_BUILD_CREATED":        fmt.Sprintf("%d", w.Build.Created),
-		"DRONE_BUILD_STARTED":        fmt.Sprintf("%d", w.Build.Started),
-		"DRONE_BUILD_FINISHED":       fmt.Sprintf("%d", w.Build.Finished),
-		"DRONE_YAML_VERIFIED":        fmt.Sprintf("%v", w.Verified),
-		"DRONE_YAML_SIGNED":          fmt.Sprintf("%v", w.Signed),
-		"DRONE_BRANCH":               w.Build.Branch,
-		"DRONE_COMMIT":               w.Build.Commit,
-		"DRONE_VERSION":              version.Version,
-	}
-
-	if w.Build.Event == model.EventTag {
-		envs["DRONE_TAG"] = strings.TrimPrefix(w.Build.Ref, "refs/tags/")
-	}
-	if w.Build.Event == model.EventPull {
-		envs["DRONE_PULL_REQUEST"] = pullRegexp.FindString(w.Build.Ref)
-	}
-	if w.Build.Event == model.EventDeploy {
-		envs["DRONE_DEPLOY_TO"] = w.Build.Deploy
-	}
-
-	if w.BuildLast != nil {
-		envs["DRONE_PREV_BUILD_STATUS"] = w.BuildLast.Status
-		envs["DRONE_PREV_BUILD_NUMBER"] = fmt.Sprintf("%v", w.BuildLast.Number)
-		envs["DRONE_PREV_COMMIT_SHA"] = w.BuildLast.Commit
-	}
-
-	// inject matrix values as environment variables
-	for key, val := range w.Job.Environment {
-		envs[key] = val
-	}
-	return envs
-}
-
-var pullRegexp = regexp.MustCompile("\\d+")

model/job.go

@@ -6,7 +6,7 @@ type Job struct {
 	BuildID  int64  `json:"-"            meddler:"job_build_id"`
 	NodeID   int64  `json:"-"            meddler:"job_node_id"`
 	Number   int    `json:"number"       meddler:"job_number"`
-	Error    string `json:"error"        meddler:"-"`
+	Error    string `json:"error"        meddler:"job_error"`
 	Status   string `json:"status"       meddler:"job_status"`
 	ExitCode int    `json:"exit_code"    meddler:"job_exit_code"`
 	Enqueued int64  `json:"enqueued_at"  meddler:"job_enqueued"`

server/queue.go

@@ -91,6 +91,7 @@ func Update(c *gin.Context) {
 	job.Finished = work.Job.Finished
 	job.Status = work.Job.Status
 	job.ExitCode = work.Job.ExitCode
+	job.Error = work.Job.Error
 
 	if build.Status == model.StatusPending {
 		build.Status = model.StatusRunning

store/datastore/ddl/mysql/4.sql

@@ -0,0 +1,9 @@
+-- +migrate Up
+
+ALTER TABLE jobs ADD COLUMN job_error VARCHAR(500);
+
+UPDATE jobs SET job_error = '' job_error = null;
+
+-- +migrate Down
+
+ALTER TABLE jobs DROP COLUMN job_error;

store/datastore/ddl/postgres/4.sql

@@ -0,0 +1,9 @@
+-- +migrate Up
+
+ALTER TABLE jobs ADD COLUMN job_error VARCHAR(500);
+
+UPDATE jobs SET job_error = '';
+
+-- +migrate Down
+
+ALTER TABLE jobs DROP COLUMN job_error;

store/datastore/ddl/sqlite3/4.sql

@@ -0,0 +1,9 @@
+-- +migrate Up
+
+ALTER TABLE jobs ADD COLUMN job_error TEXT;
+
+UPDATE jobs SET job_error = '';
+
+-- +migrate Down
+
+ALTER TABLE jobs DROP COLUMN job_error;

template/amber/build.amber

@@ -75,6 +75,9 @@ block content
                     button.btn.btn-info.hidden#cancel cancel
 
             div.col-md-8
+                if Job.Error != ""
+                    div.alert.alert-danger #{Job.Error}
+                else
                     pre#output
                     button.tail#tail
                         i.material-icons expand_more
@@ -88,4 +91,3 @@ block append scripts
         var status = #{json(Job.Status)};
 
         var view = new JobViewModel(repo, build, job, status);
-