From bcc768b15748dcf375d63c3a4ec954cfb2a88222 Mon Sep 17 00:00:00 2001 From: Brad Rydzewski Date: Thu, 21 Apr 2016 00:25:30 -0700 Subject: [PATCH] fully functioning build using 0.5 agents --- api/build.go | 11 ++- api/queue.go | 19 +++-- bus/types.go | 8 ++ client/client.go | 10 --- drone/agent/exec.go | 111 ++++++++++++++++++++-------- engine/compiler/builtin/secrets.go | 33 +++++++++ engine/compiler/builtin/validate.go | 15 ++++ model/repo.go | 16 ---- model/secret.go | 26 +++++++ model/secret_test.go | 40 ++++++++++ queue/types.go | 21 +++--- store/store.go | 7 ++ web/hook.go | 4 + web/stream2.go | 2 + 14 files changed, 250 insertions(+), 73 deletions(-) create mode 100644 engine/compiler/builtin/secrets.go create mode 100644 model/secret_test.go diff --git a/api/build.go b/api/build.go index 496662a1f..81ecd3488 100644 --- a/api/build.go +++ b/api/build.go @@ -11,6 +11,7 @@ import ( "time" log "github.com/Sirupsen/logrus" + "github.com/drone/drone/bus" "github.com/drone/drone/engine" "github.com/drone/drone/queue" "github.com/drone/drone/remote" @@ -149,6 +150,12 @@ func DeleteBuild(c *gin.Context) { c.AbortWithError(404, err) return } + + if os.Getenv("CANARY") == "true" { + bus.Publish(c, bus.NewEvent(bus.Cancelled, repo, build, job)) + return + } + node, err := store.GetNode(c, job.NodeID) if err != nil { c.AbortWithError(404, err) @@ -280,7 +287,7 @@ func PostBuild(c *gin.Context) { // get the previous build so that we can send // on status change notifications last, _ := store.GetBuildLastBefore(c, repo, build.Branch, build.ID) - + secs, _ := store.GetSecretList(c, repo) // IMPORTANT. PLEASE READ // @@ -289,6 +296,7 @@ func PostBuild(c *gin.Context) { // enabled using with the environment variable CANARY=true if os.Getenv("CANARY") == "true" { + bus.Publish(c, bus.NewBuildEvent(bus.Enqueued, repo, build)) for _, job := range jobs { queue.Publish(c, &queue.Work{ User: user, @@ -300,6 +308,7 @@ func PostBuild(c *gin.Context) { Netrc: netrc, Yaml: string(raw), YamlEnc: string(sec), + Secrets: secs, System: &model.System{ Link: httputil.GetURL(c.Request), Plugins: strings.Split(os.Getenv("PLUGIN_FILTER"), " "), diff --git a/api/queue.go b/api/queue.go index e7176ed10..4147b58b3 100644 --- a/api/queue.go +++ b/api/queue.go @@ -51,9 +51,7 @@ func Wait(c *gin.Context) { for { select { case event := <-eventc: - if event.Job.ID == id && - event.Job.Status != model.StatusPending && - event.Job.Status != model.StatusRunning { + if event.Job.ID == id && event.Type == bus.Cancelled { c.JSON(200, event.Job) return } @@ -93,13 +91,18 @@ func Update(c *gin.Context) { job.Status = work.Job.Status job.ExitCode = work.Job.ExitCode + if build.Status == model.StatusPending { + build.Status = model.StatusRunning + store.UpdateBuild(c, build) + } + ok, err := store.UpdateBuildJob(c, build, job) if err != nil { c.String(500, "Unable to update job. %s", err) return } - if ok { + if ok && build.Status != model.StatusRunning { // get the user because we transfer the user form the server to agent // and back we lose the token which does not get serialized to json. user, err := store.GetUser(c, work.User.ID) @@ -107,10 +110,16 @@ func Update(c *gin.Context) { c.String(500, "Unable to find user. %s", err) return } - bus.Publish(c, &bus.Event{}) remote.Status(c, user, work.Repo, build, fmt.Sprintf("%s/%s/%d", work.System.Link, work.Repo.FullName, work.Build.Number)) } + + if build.Status == model.StatusRunning { + bus.Publish(c, bus.NewEvent(bus.Started, work.Repo, build, job)) + } else { + bus.Publish(c, bus.NewEvent(bus.Finished, work.Repo, build, job)) + } + c.JSON(200, work) } diff --git a/bus/types.go b/bus/types.go index 3dded50b3..6be95a3d7 100644 --- a/bus/types.go +++ b/bus/types.go @@ -30,3 +30,11 @@ func NewEvent(t EventType, r *model.Repo, b *model.Build, j *model.Job) *Event { Job: *j, } } + +func NewBuildEvent(t EventType, r *model.Repo, b *model.Build) *Event { + return &Event{ + Type: t, + Repo: *r, + Build: *b, + } +} diff --git a/client/client.go b/client/client.go index db029df99..8ab26ead2 100644 --- a/client/client.go +++ b/client/client.go @@ -70,16 +70,6 @@ func (c *client) Wait(id int64) *Wait { return &Wait{id, c, ctx, cancel} } -//////// - -type CancelNotifier interface { - Canecel() - CancelNotify() bool - IsCancelled() bool -} - -//////// - type Wait struct { id int64 client *client diff --git a/drone/agent/exec.go b/drone/agent/exec.go index 0db150f3b..60d66e5d2 100644 --- a/drone/agent/exec.go +++ b/drone/agent/exec.go @@ -4,6 +4,8 @@ import ( "encoding/json" "fmt" "io" + "regexp" + "strings" "time" "github.com/Sirupsen/logrus" @@ -14,6 +16,8 @@ import ( "github.com/drone/drone/engine/runner" engine "github.com/drone/drone/engine/runner/docker" "github.com/drone/drone/model" + "github.com/drone/drone/queue" + "github.com/drone/drone/yaml/expander" "github.com/samalba/dockerclient" "golang.org/x/net/context" @@ -40,57 +44,37 @@ func exec(client client.Client, docker dockerclient.Client) error { prefix := fmt.Sprintf("drone_%s", uniuri.New()) + envs := toEnv(w) + w.Yaml = expander.ExpandString(w.Yaml, envs) + + w.Secrets = append(w.Secrets, &model.Secret{Name: "HEROKU_TOKEN", Value: "GODZILLA", Images: []string{"golang:1.4.2"}, Events: []string{w.Build.Event}}) + trans := []compiler.Transform{ - builtin.NewCloneOp("plugins/git:latest", true), + builtin.NewCloneOp("plugins/"+w.Repo.Kind+":latest", true), builtin.NewCacheOp( "plugins/cache:latest", "/var/lib/drone/cache/"+w.Repo.FullName, false, ), + builtin.NewSecretOp(w.Build.Event, w.Secrets), builtin.NewNormalizeOp("plugins"), builtin.NewWorkspaceOp("/drone", "drone/src/github.com/"+w.Repo.FullName), - builtin.NewEnvOp(map[string]string{ - "CI": "drone", - "CI_REPO": w.Repo.FullName, - "CI_REPO_OWNER": w.Repo.Owner, - "CI_REPO_NAME": w.Repo.Name, - "CI_REPO_LINK": w.Repo.Link, - "CI_REPO_AVATAR": w.Repo.Avatar, - "CI_REPO_BRANCH": w.Repo.Branch, - "CI_REPO_PRIVATE": fmt.Sprintf("%v", w.Repo.IsPrivate), - "CI_REMOTE_URL": w.Repo.Clone, - "CI_COMMIT_SHA": w.Build.Commit, - "CI_COMMIT_REF": w.Build.Ref, - "CI_COMMIT_BRANCH": w.Build.Branch, - "CI_COMMIT_LINK": w.Build.Link, - "CI_COMMIT_MESSAGE": w.Build.Message, - "CI_AUTHOR": w.Build.Author, - "CI_AUTHOR_EMAIL": w.Build.Email, - "CI_AUTHOR_AVATAR": w.Build.Avatar, - "CI_BUILD_NUMBER": fmt.Sprintf("%v", w.Build.Number), - "CI_BUILD_EVENT": w.Build.Event, - // "CI_NETRC_USERNAME": w.Netrc.Login, - // "CI_NETRC_PASSWORD": w.Netrc.Password, - // "CI_NETRC_MACHINE": w.Netrc.Machine, - // "CI_PREV_BUILD_STATUS": w.BuildLast.Status, - // "CI_PREV_BUILD_NUMBER": fmt.Sprintf("%v", w.BuildLast.Number), - // "CI_PREV_COMMIT_SHA": w.BuildLast.Commit, - }), builtin.NewValidateOp( w.Repo.IsTrusted, []string{"plugins/*"}, ), + builtin.NewEnvOp(envs), builtin.NewShellOp(builtin.Linux_adm64), builtin.NewArgsOp(), builtin.NewPodOp(prefix), builtin.NewAliasOp(prefix), builtin.NewPullOp(false), builtin.NewFilterOp( - model.StatusSuccess, // w.BuildLast.Status, + model.StatusSuccess, // TODO(bradrydzewski) please add the last build status here w.Build.Branch, w.Build.Event, w.Build.Deploy, - map[string]string{}, + w.Job.Environment, ), } @@ -171,7 +155,7 @@ func exec(client client.Client, docker dockerclient.Client) error { w.Job.Finished = time.Now().Unix() switch w.Job.ExitCode { - case 128, 130: + case 128, 130, 137: w.Job.Status = model.StatusKilled case 0: w.Job.Status = model.StatusSuccess @@ -184,3 +168,68 @@ func exec(client client.Client, docker dockerclient.Client) error { return client.Push(w) } + +func toEnv(w *queue.Work) map[string]string { + envs := map[string]string{ + "CI": "drone", + "DRONE": "true", + "DRONE_ARCH": "linux_amd64", + "DRONE_REPO": w.Repo.FullName, + "DRONE_REPO_SCM": w.Repo.Kind, + "DRONE_REPO_OWNER": w.Repo.Owner, + "DRONE_REPO_NAME": w.Repo.Name, + "DRONE_REPO_LINK": w.Repo.Link, + "DRONE_REPO_AVATAR": w.Repo.Avatar, + "DRONE_REPO_BRANCH": w.Repo.Branch, + "DRONE_REPO_PRIVATE": fmt.Sprintf("%v", w.Repo.IsPrivate), + "DRONE_REPO_TRUSTED": fmt.Sprintf("%v", w.Repo.IsTrusted), + "DRONE_REMOTE_URL": w.Repo.Clone, + "DRONE_COMMIT_SHA": w.Build.Commit, + "DRONE_COMMIT_REF": w.Build.Ref, + "DRONE_COMMIT_BRANCH": w.Build.Branch, + "DRONE_COMMIT_LINK": w.Build.Link, + "DRONE_COMMIT_MESSAGE": w.Build.Message, + "DRONE_AUTHOR": w.Build.Author, + "DRONE_AUTHOR_EMAIL": w.Build.Email, + "DRONE_AUTHOR_AVATAR": w.Build.Avatar, + "DRONE_BUILD_NUMBER": fmt.Sprintf("%d", w.Build.Number), + "DRONE_BUILD_EVENT": w.Build.Event, + "DRONE_BUILD_CREATED": fmt.Sprintf("%d", w.Build.Created), + "DRONE_BUILD_STARTED": fmt.Sprintf("%d", w.Build.Started), + "DRONE_BUILD_FINISHED": fmt.Sprintf("%d", w.Build.Finished), + "DRONE_BUILD_VERIFIED": fmt.Sprintf("%v", false), + + // SHORTER ALIASES + "DRONE_BRANCH": w.Build.Branch, + "DRONE_COMMIT": w.Build.Commit, + + // TODO(bradrydzewski) netrc should only be injected via secrets + // "DRONE_NETRC_USERNAME": w.Netrc.Login, + // "DRONE_NETRC_PASSWORD": w.Netrc.Password, + // "DRONE_NETRC_MACHINE": w.Netrc.Machine, + } + + if w.Build.Event == model.EventTag { + envs["DRONE_TAG"] = strings.TrimPrefix(w.Build.Ref, "refs/tags/") + } + if w.Build.Event == model.EventPull { + envs["DRONE_PULL_REQUEST"] = pullRegexp.FindString(w.Build.Ref) + } + if w.Build.Event == model.EventDeploy { + envs["DRONE_DEPLOY_TO"] = w.Build.Deploy + } + + if w.BuildLast != nil { + envs["DRONE_PREV_BUILD_STATUS"] = w.BuildLast.Status + envs["DRONE_PREV_BUILD_NUMBER"] = fmt.Sprintf("%v", w.BuildLast.Number) + envs["DRONE_PREV_COMMIT_SHA"] = w.BuildLast.Commit + } + + // inject matrix values as environment variables + for key, val := range w.Job.Environment { + envs[key] = val + } + return envs +} + +var pullRegexp = regexp.MustCompile("\\d+") diff --git a/engine/compiler/builtin/secrets.go b/engine/compiler/builtin/secrets.go new file mode 100644 index 000000000..bd4729502 --- /dev/null +++ b/engine/compiler/builtin/secrets.go @@ -0,0 +1,33 @@ +package builtin + +import ( + "github.com/drone/drone/engine/compiler/parse" + "github.com/drone/drone/model" +) + +type secretOp struct { + visitor + event string + secrets []*model.Secret +} + +// NewSecretOp returns a transformer that configures plugin secrets. +func NewSecretOp(event string, secrets []*model.Secret) Visitor { + return &secretOp{ + event: event, + secrets: secrets, + } +} + +func (v *secretOp) VisitContainer(node *parse.ContainerNode) error { + for _, secret := range v.secrets { + if !secret.Match(node.Container.Image, v.event) { + continue + } + if node.Container.Environment == nil { + node.Container.Environment = map[string]string{} + } + node.Container.Environment[secret.Name] = secret.Value + } + return nil +} diff --git a/engine/compiler/builtin/validate.go b/engine/compiler/builtin/validate.go index b19678e8c..ec88953c5 100644 --- a/engine/compiler/builtin/validate.go +++ b/engine/compiler/builtin/validate.go @@ -103,3 +103,18 @@ func (v *validateOp) validateConfig(node *parse.ContainerNode) error { } return nil } + +// validate the environment configuration and return an error if +// an attempt is made to override system environment variables. +// func (v *validateOp) validateEnvironment(node *parse.ContainerNode) error { +// for key := range node.Container.Environment { +// upper := strings.ToUpper(key) +// switch { +// case strings.HasPrefix(upper, "DRONE_"): +// return fmt.Errorf("Cannot set or override DRONE_ environment variables") +// case strings.HasPrefix(upper, "PLUGIN_"): +// return fmt.Errorf("Cannot set or override PLUGIN_ environment variables") +// } +// } +// return nil +// } diff --git a/model/repo.go b/model/repo.go index 271017abe..062f53131 100644 --- a/model/repo.go +++ b/model/repo.go @@ -1,7 +1,5 @@ package model -import "strconv" - type RepoLite struct { Owner string `json:"owner"` Name string `json:"name"` @@ -33,17 +31,3 @@ type Repo struct { AllowTag bool `json:"allow_tags" meddler:"repo_allow_tags"` Hash string `json:"-" meddler:"repo_hash"` } - -// ToEnv returns environment variable valus for the repository. -func (r *Repo) ToEnv(to map[string]string) { - to["CI_VCS"] = r.Kind - to["CI_REPO"] = r.FullName - to["CI_REPO_OWNER"] = r.Owner - to["CI_REPO_NAME"] = r.Name - to["CI_REPO_LINK"] = r.Link - to["CI_REPO_AVATAR"] = r.Avatar - to["CI_REPO_BRANCH"] = r.Branch - to["CI_REPO_PRIVATE"] = strconv.FormatBool(r.IsPrivate) - to["CI_REPO_TRUSTED"] = strconv.FormatBool(r.IsTrusted) - to["CI_REMOTE_URL"] = r.Clone -} diff --git a/model/secret.go b/model/secret.go index 6fe7d625e..919dda5e7 100644 --- a/model/secret.go +++ b/model/secret.go @@ -22,6 +22,32 @@ type Secret struct { Events []string `json:"event,omitempty" meddler:"secret_events,json"` } +// Match returns true if an image and event match the restricted list. +func (s *Secret) Match(image, event string) bool { + return s.MatchImage(image) && s.MatchEvent(event) +} + +// MatchImage returns true if an image matches the restricted list. +func (s *Secret) MatchImage(want string) bool { + for _, got := range s.Images { + if want == got { + return true + } + } + return false +} + +// MatchEvent returns true if an event matches the restricted list. +func (s *Secret) MatchEvent(want string) bool { + for _, got := range s.Events { + if want == got { + return true + } + } + return false +} + +// Validate validates the required fields and formats. func (s *Secret) Validate() error { return nil } diff --git a/model/secret_test.go b/model/secret_test.go new file mode 100644 index 000000000..4e159d094 --- /dev/null +++ b/model/secret_test.go @@ -0,0 +1,40 @@ +package model + +import ( + "testing" + + "github.com/franela/goblin" +) + +func TestSecret(t *testing.T) { + + g := goblin.Goblin(t) + g.Describe("Secret", func() { + + g.It("should match image", func() { + secret := Secret{} + secret.Images = []string{"golang"} + g.Assert(secret.MatchImage("golang")).IsTrue() + }) + g.It("should match event", func() { + secret := Secret{} + secret.Events = []string{"pull_request"} + g.Assert(secret.MatchEvent("pull_request")).IsTrue() + }) + g.It("should not match image", func() { + secret := Secret{} + secret.Images = []string{"golang"} + g.Assert(secret.MatchImage("node")).IsFalse() + }) + g.It("should not match event", func() { + secret := Secret{} + secret.Events = []string{"pull_request"} + g.Assert(secret.MatchEvent("push")).IsFalse() + }) + g.It("should pass validation") + g.Describe("should fail validation", func() { + g.It("when no image") + g.It("when no event") + }) + }) +} diff --git a/queue/types.go b/queue/types.go index 1740c7fe9..1408a8ca8 100644 --- a/queue/types.go +++ b/queue/types.go @@ -5,14 +5,15 @@ import "github.com/drone/drone/model" // Work represents an item for work to be // processed by a worker. type Work struct { - Yaml string `json:"config"` - YamlEnc string `json:"secret"` - Repo *model.Repo `json:"repo"` - Build *model.Build `json:"build"` - BuildLast *model.Build `json:"build_last"` - Job *model.Job `json:"job"` - Netrc *model.Netrc `json:"netrc"` - Keys *model.Key `json:"keys"` - System *model.System `json:"system"` - User *model.User `json:"user"` + Yaml string `json:"config"` + YamlEnc string `json:"secret"` + Repo *model.Repo `json:"repo"` + Build *model.Build `json:"build"` + BuildLast *model.Build `json:"build_last"` + Job *model.Job `json:"job"` + Netrc *model.Netrc `json:"netrc"` + Keys *model.Key `json:"keys"` + System *model.System `json:"system"` + Secrets []*model.Secret `json:"secret"` + User *model.User `json:"user"` } diff --git a/store/store.go b/store/store.go index 5698460a5..f0fd97968 100644 --- a/store/store.go +++ b/store/store.go @@ -279,6 +279,13 @@ func UpdateBuildJob(c context.Context, build *model.Build, job *model.Job) (bool if err := UpdateJob(c, job); err != nil { return false, err } + + // if the job is running or started we don't need to update the build + // status since. + if job.Status == model.StatusRunning || job.Status == model.StatusPending { + return false, nil + } + jobs, err := GetJobList(c, build) if err != nil { return false, err diff --git a/web/hook.go b/web/hook.go index f8a645c5d..8c3cdca19 100644 --- a/web/hook.go +++ b/web/hook.go @@ -10,6 +10,7 @@ import ( "github.com/gin-gonic/gin" log "github.com/Sirupsen/logrus" + "github.com/drone/drone/bus" "github.com/drone/drone/engine" "github.com/drone/drone/model" "github.com/drone/drone/queue" @@ -204,6 +205,7 @@ func PostHook(c *gin.Context) { // get the previous build so that we can send // on status change notifications last, _ := store.GetBuildLastBefore(c, repo, build.Branch, build.ID) + secs, _ := store.GetSecretList(c, repo) // IMPORTANT. PLEASE READ // @@ -212,6 +214,7 @@ func PostHook(c *gin.Context) { // enabled using with the environment variable CANARY=true if os.Getenv("CANARY") == "true" { + bus.Publish(c, bus.NewBuildEvent(bus.Enqueued, repo, build)) for _, job := range jobs { queue.Publish(c, &queue.Work{ User: user, @@ -223,6 +226,7 @@ func PostHook(c *gin.Context) { Netrc: netrc, Yaml: string(raw), YamlEnc: string(sec), + Secrets: secs, System: &model.System{ Link: httputil.GetURL(c.Request), Plugins: strings.Split(os.Getenv("PLUGIN_FILTER"), " "), diff --git a/web/stream2.go b/web/stream2.go index ec245f598..039b5dfff 100644 --- a/web/stream2.go +++ b/web/stream2.go @@ -46,6 +46,8 @@ func GetRepoEvents2(c *gin.Context) { return false } + // TODO(bradrydzewski) This is a super hacky workaround until we improve + // the actual bus. Having a per-call database event is just plain stupid. if event.Repo.FullName == repo.FullName { var payload = struct {