Merge pull request #1586 from bradrydzewski/master

fully functioning builds using 0.5 agents, secrets and yaml (behind feature flag)
This commit is contained in:
Brad Rydzewski 2016-04-21 00:30:41 -07:00
commit a0f8457e87
14 changed files with 250 additions and 73 deletions

View file

@ -11,6 +11,7 @@ import (
"time" "time"
log "github.com/Sirupsen/logrus" log "github.com/Sirupsen/logrus"
"github.com/drone/drone/bus"
"github.com/drone/drone/engine" "github.com/drone/drone/engine"
"github.com/drone/drone/queue" "github.com/drone/drone/queue"
"github.com/drone/drone/remote" "github.com/drone/drone/remote"
@ -149,6 +150,12 @@ func DeleteBuild(c *gin.Context) {
c.AbortWithError(404, err) c.AbortWithError(404, err)
return return
} }
if os.Getenv("CANARY") == "true" {
bus.Publish(c, bus.NewEvent(bus.Cancelled, repo, build, job))
return
}
node, err := store.GetNode(c, job.NodeID) node, err := store.GetNode(c, job.NodeID)
if err != nil { if err != nil {
c.AbortWithError(404, err) c.AbortWithError(404, err)
@ -280,7 +287,7 @@ func PostBuild(c *gin.Context) {
// get the previous build so that we can send // get the previous build so that we can send
// on status change notifications // on status change notifications
last, _ := store.GetBuildLastBefore(c, repo, build.Branch, build.ID) last, _ := store.GetBuildLastBefore(c, repo, build.Branch, build.ID)
secs, _ := store.GetSecretList(c, repo)
// IMPORTANT. PLEASE READ // IMPORTANT. PLEASE READ
// //
@ -289,6 +296,7 @@ func PostBuild(c *gin.Context) {
// enabled using with the environment variable CANARY=true // enabled using with the environment variable CANARY=true
if os.Getenv("CANARY") == "true" { if os.Getenv("CANARY") == "true" {
bus.Publish(c, bus.NewBuildEvent(bus.Enqueued, repo, build))
for _, job := range jobs { for _, job := range jobs {
queue.Publish(c, &queue.Work{ queue.Publish(c, &queue.Work{
User: user, User: user,
@ -300,6 +308,7 @@ func PostBuild(c *gin.Context) {
Netrc: netrc, Netrc: netrc,
Yaml: string(raw), Yaml: string(raw),
YamlEnc: string(sec), YamlEnc: string(sec),
Secrets: secs,
System: &model.System{ System: &model.System{
Link: httputil.GetURL(c.Request), Link: httputil.GetURL(c.Request),
Plugins: strings.Split(os.Getenv("PLUGIN_FILTER"), " "), Plugins: strings.Split(os.Getenv("PLUGIN_FILTER"), " "),

View file

@ -51,9 +51,7 @@ func Wait(c *gin.Context) {
for { for {
select { select {
case event := <-eventc: case event := <-eventc:
if event.Job.ID == id && if event.Job.ID == id && event.Type == bus.Cancelled {
event.Job.Status != model.StatusPending &&
event.Job.Status != model.StatusRunning {
c.JSON(200, event.Job) c.JSON(200, event.Job)
return return
} }
@ -93,13 +91,18 @@ func Update(c *gin.Context) {
job.Status = work.Job.Status job.Status = work.Job.Status
job.ExitCode = work.Job.ExitCode job.ExitCode = work.Job.ExitCode
if build.Status == model.StatusPending {
build.Status = model.StatusRunning
store.UpdateBuild(c, build)
}
ok, err := store.UpdateBuildJob(c, build, job) ok, err := store.UpdateBuildJob(c, build, job)
if err != nil { if err != nil {
c.String(500, "Unable to update job. %s", err) c.String(500, "Unable to update job. %s", err)
return return
} }
if ok { if ok && build.Status != model.StatusRunning {
// get the user because we transfer the user form the server to agent // get the user because we transfer the user form the server to agent
// and back we lose the token which does not get serialized to json. // and back we lose the token which does not get serialized to json.
user, err := store.GetUser(c, work.User.ID) user, err := store.GetUser(c, work.User.ID)
@ -107,10 +110,16 @@ func Update(c *gin.Context) {
c.String(500, "Unable to find user. %s", err) c.String(500, "Unable to find user. %s", err)
return return
} }
bus.Publish(c, &bus.Event{})
remote.Status(c, user, work.Repo, build, remote.Status(c, user, work.Repo, build,
fmt.Sprintf("%s/%s/%d", work.System.Link, work.Repo.FullName, work.Build.Number)) fmt.Sprintf("%s/%s/%d", work.System.Link, work.Repo.FullName, work.Build.Number))
} }
if build.Status == model.StatusRunning {
bus.Publish(c, bus.NewEvent(bus.Started, work.Repo, build, job))
} else {
bus.Publish(c, bus.NewEvent(bus.Finished, work.Repo, build, job))
}
c.JSON(200, work) c.JSON(200, work)
} }

View file

@ -30,3 +30,11 @@ func NewEvent(t EventType, r *model.Repo, b *model.Build, j *model.Job) *Event {
Job: *j, Job: *j,
} }
} }
func NewBuildEvent(t EventType, r *model.Repo, b *model.Build) *Event {
return &Event{
Type: t,
Repo: *r,
Build: *b,
}
}

View file

@ -70,16 +70,6 @@ func (c *client) Wait(id int64) *Wait {
return &Wait{id, c, ctx, cancel} return &Wait{id, c, ctx, cancel}
} }
////////
type CancelNotifier interface {
Canecel()
CancelNotify() bool
IsCancelled() bool
}
////////
type Wait struct { type Wait struct {
id int64 id int64
client *client client *client

View file

@ -4,6 +4,8 @@ import (
"encoding/json" "encoding/json"
"fmt" "fmt"
"io" "io"
"regexp"
"strings"
"time" "time"
"github.com/Sirupsen/logrus" "github.com/Sirupsen/logrus"
@ -14,6 +16,8 @@ import (
"github.com/drone/drone/engine/runner" "github.com/drone/drone/engine/runner"
engine "github.com/drone/drone/engine/runner/docker" engine "github.com/drone/drone/engine/runner/docker"
"github.com/drone/drone/model" "github.com/drone/drone/model"
"github.com/drone/drone/queue"
"github.com/drone/drone/yaml/expander"
"github.com/samalba/dockerclient" "github.com/samalba/dockerclient"
"golang.org/x/net/context" "golang.org/x/net/context"
@ -40,57 +44,37 @@ func exec(client client.Client, docker dockerclient.Client) error {
prefix := fmt.Sprintf("drone_%s", uniuri.New()) prefix := fmt.Sprintf("drone_%s", uniuri.New())
envs := toEnv(w)
w.Yaml = expander.ExpandString(w.Yaml, envs)
w.Secrets = append(w.Secrets, &model.Secret{Name: "HEROKU_TOKEN", Value: "GODZILLA", Images: []string{"golang:1.4.2"}, Events: []string{w.Build.Event}})
trans := []compiler.Transform{ trans := []compiler.Transform{
builtin.NewCloneOp("plugins/git:latest", true), builtin.NewCloneOp("plugins/"+w.Repo.Kind+":latest", true),
builtin.NewCacheOp( builtin.NewCacheOp(
"plugins/cache:latest", "plugins/cache:latest",
"/var/lib/drone/cache/"+w.Repo.FullName, "/var/lib/drone/cache/"+w.Repo.FullName,
false, false,
), ),
builtin.NewSecretOp(w.Build.Event, w.Secrets),
builtin.NewNormalizeOp("plugins"), builtin.NewNormalizeOp("plugins"),
builtin.NewWorkspaceOp("/drone", "drone/src/github.com/"+w.Repo.FullName), builtin.NewWorkspaceOp("/drone", "drone/src/github.com/"+w.Repo.FullName),
builtin.NewEnvOp(map[string]string{
"CI": "drone",
"CI_REPO": w.Repo.FullName,
"CI_REPO_OWNER": w.Repo.Owner,
"CI_REPO_NAME": w.Repo.Name,
"CI_REPO_LINK": w.Repo.Link,
"CI_REPO_AVATAR": w.Repo.Avatar,
"CI_REPO_BRANCH": w.Repo.Branch,
"CI_REPO_PRIVATE": fmt.Sprintf("%v", w.Repo.IsPrivate),
"CI_REMOTE_URL": w.Repo.Clone,
"CI_COMMIT_SHA": w.Build.Commit,
"CI_COMMIT_REF": w.Build.Ref,
"CI_COMMIT_BRANCH": w.Build.Branch,
"CI_COMMIT_LINK": w.Build.Link,
"CI_COMMIT_MESSAGE": w.Build.Message,
"CI_AUTHOR": w.Build.Author,
"CI_AUTHOR_EMAIL": w.Build.Email,
"CI_AUTHOR_AVATAR": w.Build.Avatar,
"CI_BUILD_NUMBER": fmt.Sprintf("%v", w.Build.Number),
"CI_BUILD_EVENT": w.Build.Event,
// "CI_NETRC_USERNAME": w.Netrc.Login,
// "CI_NETRC_PASSWORD": w.Netrc.Password,
// "CI_NETRC_MACHINE": w.Netrc.Machine,
// "CI_PREV_BUILD_STATUS": w.BuildLast.Status,
// "CI_PREV_BUILD_NUMBER": fmt.Sprintf("%v", w.BuildLast.Number),
// "CI_PREV_COMMIT_SHA": w.BuildLast.Commit,
}),
builtin.NewValidateOp( builtin.NewValidateOp(
w.Repo.IsTrusted, w.Repo.IsTrusted,
[]string{"plugins/*"}, []string{"plugins/*"},
), ),
builtin.NewEnvOp(envs),
builtin.NewShellOp(builtin.Linux_adm64), builtin.NewShellOp(builtin.Linux_adm64),
builtin.NewArgsOp(), builtin.NewArgsOp(),
builtin.NewPodOp(prefix), builtin.NewPodOp(prefix),
builtin.NewAliasOp(prefix), builtin.NewAliasOp(prefix),
builtin.NewPullOp(false), builtin.NewPullOp(false),
builtin.NewFilterOp( builtin.NewFilterOp(
model.StatusSuccess, // w.BuildLast.Status, model.StatusSuccess, // TODO(bradrydzewski) please add the last build status here
w.Build.Branch, w.Build.Branch,
w.Build.Event, w.Build.Event,
w.Build.Deploy, w.Build.Deploy,
map[string]string{}, w.Job.Environment,
), ),
} }
@ -171,7 +155,7 @@ func exec(client client.Client, docker dockerclient.Client) error {
w.Job.Finished = time.Now().Unix() w.Job.Finished = time.Now().Unix()
switch w.Job.ExitCode { switch w.Job.ExitCode {
case 128, 130: case 128, 130, 137:
w.Job.Status = model.StatusKilled w.Job.Status = model.StatusKilled
case 0: case 0:
w.Job.Status = model.StatusSuccess w.Job.Status = model.StatusSuccess
@ -184,3 +168,68 @@ func exec(client client.Client, docker dockerclient.Client) error {
return client.Push(w) return client.Push(w)
} }
func toEnv(w *queue.Work) map[string]string {
envs := map[string]string{
"CI": "drone",
"DRONE": "true",
"DRONE_ARCH": "linux_amd64",
"DRONE_REPO": w.Repo.FullName,
"DRONE_REPO_SCM": w.Repo.Kind,
"DRONE_REPO_OWNER": w.Repo.Owner,
"DRONE_REPO_NAME": w.Repo.Name,
"DRONE_REPO_LINK": w.Repo.Link,
"DRONE_REPO_AVATAR": w.Repo.Avatar,
"DRONE_REPO_BRANCH": w.Repo.Branch,
"DRONE_REPO_PRIVATE": fmt.Sprintf("%v", w.Repo.IsPrivate),
"DRONE_REPO_TRUSTED": fmt.Sprintf("%v", w.Repo.IsTrusted),
"DRONE_REMOTE_URL": w.Repo.Clone,
"DRONE_COMMIT_SHA": w.Build.Commit,
"DRONE_COMMIT_REF": w.Build.Ref,
"DRONE_COMMIT_BRANCH": w.Build.Branch,
"DRONE_COMMIT_LINK": w.Build.Link,
"DRONE_COMMIT_MESSAGE": w.Build.Message,
"DRONE_AUTHOR": w.Build.Author,
"DRONE_AUTHOR_EMAIL": w.Build.Email,
"DRONE_AUTHOR_AVATAR": w.Build.Avatar,
"DRONE_BUILD_NUMBER": fmt.Sprintf("%d", w.Build.Number),
"DRONE_BUILD_EVENT": w.Build.Event,
"DRONE_BUILD_CREATED": fmt.Sprintf("%d", w.Build.Created),
"DRONE_BUILD_STARTED": fmt.Sprintf("%d", w.Build.Started),
"DRONE_BUILD_FINISHED": fmt.Sprintf("%d", w.Build.Finished),
"DRONE_BUILD_VERIFIED": fmt.Sprintf("%v", false),
// SHORTER ALIASES
"DRONE_BRANCH": w.Build.Branch,
"DRONE_COMMIT": w.Build.Commit,
// TODO(bradrydzewski) netrc should only be injected via secrets
// "DRONE_NETRC_USERNAME": w.Netrc.Login,
// "DRONE_NETRC_PASSWORD": w.Netrc.Password,
// "DRONE_NETRC_MACHINE": w.Netrc.Machine,
}
if w.Build.Event == model.EventTag {
envs["DRONE_TAG"] = strings.TrimPrefix(w.Build.Ref, "refs/tags/")
}
if w.Build.Event == model.EventPull {
envs["DRONE_PULL_REQUEST"] = pullRegexp.FindString(w.Build.Ref)
}
if w.Build.Event == model.EventDeploy {
envs["DRONE_DEPLOY_TO"] = w.Build.Deploy
}
if w.BuildLast != nil {
envs["DRONE_PREV_BUILD_STATUS"] = w.BuildLast.Status
envs["DRONE_PREV_BUILD_NUMBER"] = fmt.Sprintf("%v", w.BuildLast.Number)
envs["DRONE_PREV_COMMIT_SHA"] = w.BuildLast.Commit
}
// inject matrix values as environment variables
for key, val := range w.Job.Environment {
envs[key] = val
}
return envs
}
var pullRegexp = regexp.MustCompile("\\d+")

View file

@ -0,0 +1,33 @@
package builtin
import (
"github.com/drone/drone/engine/compiler/parse"
"github.com/drone/drone/model"
)
type secretOp struct {
visitor
event string
secrets []*model.Secret
}
// NewSecretOp returns a transformer that configures plugin secrets.
func NewSecretOp(event string, secrets []*model.Secret) Visitor {
return &secretOp{
event: event,
secrets: secrets,
}
}
func (v *secretOp) VisitContainer(node *parse.ContainerNode) error {
for _, secret := range v.secrets {
if !secret.Match(node.Container.Image, v.event) {
continue
}
if node.Container.Environment == nil {
node.Container.Environment = map[string]string{}
}
node.Container.Environment[secret.Name] = secret.Value
}
return nil
}

View file

@ -103,3 +103,18 @@ func (v *validateOp) validateConfig(node *parse.ContainerNode) error {
} }
return nil return nil
} }
// validate the environment configuration and return an error if
// an attempt is made to override system environment variables.
// func (v *validateOp) validateEnvironment(node *parse.ContainerNode) error {
// for key := range node.Container.Environment {
// upper := strings.ToUpper(key)
// switch {
// case strings.HasPrefix(upper, "DRONE_"):
// return fmt.Errorf("Cannot set or override DRONE_ environment variables")
// case strings.HasPrefix(upper, "PLUGIN_"):
// return fmt.Errorf("Cannot set or override PLUGIN_ environment variables")
// }
// }
// return nil
// }

View file

@ -1,7 +1,5 @@
package model package model
import "strconv"
type RepoLite struct { type RepoLite struct {
Owner string `json:"owner"` Owner string `json:"owner"`
Name string `json:"name"` Name string `json:"name"`
@ -33,17 +31,3 @@ type Repo struct {
AllowTag bool `json:"allow_tags" meddler:"repo_allow_tags"` AllowTag bool `json:"allow_tags" meddler:"repo_allow_tags"`
Hash string `json:"-" meddler:"repo_hash"` Hash string `json:"-" meddler:"repo_hash"`
} }
// ToEnv returns environment variable valus for the repository.
func (r *Repo) ToEnv(to map[string]string) {
to["CI_VCS"] = r.Kind
to["CI_REPO"] = r.FullName
to["CI_REPO_OWNER"] = r.Owner
to["CI_REPO_NAME"] = r.Name
to["CI_REPO_LINK"] = r.Link
to["CI_REPO_AVATAR"] = r.Avatar
to["CI_REPO_BRANCH"] = r.Branch
to["CI_REPO_PRIVATE"] = strconv.FormatBool(r.IsPrivate)
to["CI_REPO_TRUSTED"] = strconv.FormatBool(r.IsTrusted)
to["CI_REMOTE_URL"] = r.Clone
}

View file

@ -22,6 +22,32 @@ type Secret struct {
Events []string `json:"event,omitempty" meddler:"secret_events,json"` Events []string `json:"event,omitempty" meddler:"secret_events,json"`
} }
// Match returns true if an image and event match the restricted list.
func (s *Secret) Match(image, event string) bool {
return s.MatchImage(image) && s.MatchEvent(event)
}
// MatchImage returns true if an image matches the restricted list.
func (s *Secret) MatchImage(want string) bool {
for _, got := range s.Images {
if want == got {
return true
}
}
return false
}
// MatchEvent returns true if an event matches the restricted list.
func (s *Secret) MatchEvent(want string) bool {
for _, got := range s.Events {
if want == got {
return true
}
}
return false
}
// Validate validates the required fields and formats.
func (s *Secret) Validate() error { func (s *Secret) Validate() error {
return nil return nil
} }

40
model/secret_test.go Normal file
View file

@ -0,0 +1,40 @@
package model
import (
"testing"
"github.com/franela/goblin"
)
func TestSecret(t *testing.T) {
g := goblin.Goblin(t)
g.Describe("Secret", func() {
g.It("should match image", func() {
secret := Secret{}
secret.Images = []string{"golang"}
g.Assert(secret.MatchImage("golang")).IsTrue()
})
g.It("should match event", func() {
secret := Secret{}
secret.Events = []string{"pull_request"}
g.Assert(secret.MatchEvent("pull_request")).IsTrue()
})
g.It("should not match image", func() {
secret := Secret{}
secret.Images = []string{"golang"}
g.Assert(secret.MatchImage("node")).IsFalse()
})
g.It("should not match event", func() {
secret := Secret{}
secret.Events = []string{"pull_request"}
g.Assert(secret.MatchEvent("push")).IsFalse()
})
g.It("should pass validation")
g.Describe("should fail validation", func() {
g.It("when no image")
g.It("when no event")
})
})
}

View file

@ -14,5 +14,6 @@ type Work struct {
Netrc *model.Netrc `json:"netrc"` Netrc *model.Netrc `json:"netrc"`
Keys *model.Key `json:"keys"` Keys *model.Key `json:"keys"`
System *model.System `json:"system"` System *model.System `json:"system"`
Secrets []*model.Secret `json:"secret"`
User *model.User `json:"user"` User *model.User `json:"user"`
} }

View file

@ -279,6 +279,13 @@ func UpdateBuildJob(c context.Context, build *model.Build, job *model.Job) (bool
if err := UpdateJob(c, job); err != nil { if err := UpdateJob(c, job); err != nil {
return false, err return false, err
} }
// if the job is running or started we don't need to update the build
// status since.
if job.Status == model.StatusRunning || job.Status == model.StatusPending {
return false, nil
}
jobs, err := GetJobList(c, build) jobs, err := GetJobList(c, build)
if err != nil { if err != nil {
return false, err return false, err

View file

@ -10,6 +10,7 @@ import (
"github.com/gin-gonic/gin" "github.com/gin-gonic/gin"
log "github.com/Sirupsen/logrus" log "github.com/Sirupsen/logrus"
"github.com/drone/drone/bus"
"github.com/drone/drone/engine" "github.com/drone/drone/engine"
"github.com/drone/drone/model" "github.com/drone/drone/model"
"github.com/drone/drone/queue" "github.com/drone/drone/queue"
@ -204,6 +205,7 @@ func PostHook(c *gin.Context) {
// get the previous build so that we can send // get the previous build so that we can send
// on status change notifications // on status change notifications
last, _ := store.GetBuildLastBefore(c, repo, build.Branch, build.ID) last, _ := store.GetBuildLastBefore(c, repo, build.Branch, build.ID)
secs, _ := store.GetSecretList(c, repo)
// IMPORTANT. PLEASE READ // IMPORTANT. PLEASE READ
// //
@ -212,6 +214,7 @@ func PostHook(c *gin.Context) {
// enabled using with the environment variable CANARY=true // enabled using with the environment variable CANARY=true
if os.Getenv("CANARY") == "true" { if os.Getenv("CANARY") == "true" {
bus.Publish(c, bus.NewBuildEvent(bus.Enqueued, repo, build))
for _, job := range jobs { for _, job := range jobs {
queue.Publish(c, &queue.Work{ queue.Publish(c, &queue.Work{
User: user, User: user,
@ -223,6 +226,7 @@ func PostHook(c *gin.Context) {
Netrc: netrc, Netrc: netrc,
Yaml: string(raw), Yaml: string(raw),
YamlEnc: string(sec), YamlEnc: string(sec),
Secrets: secs,
System: &model.System{ System: &model.System{
Link: httputil.GetURL(c.Request), Link: httputil.GetURL(c.Request),
Plugins: strings.Split(os.Getenv("PLUGIN_FILTER"), " "), Plugins: strings.Split(os.Getenv("PLUGIN_FILTER"), " "),

View file

@ -46,6 +46,8 @@ func GetRepoEvents2(c *gin.Context) {
return false return false
} }
// TODO(bradrydzewski) This is a super hacky workaround until we improve
// the actual bus. Having a per-call database event is just plain stupid.
if event.Repo.FullName == repo.FullName { if event.Repo.FullName == repo.FullName {
var payload = struct { var payload = struct {