From faf7ff675d72751753755971b5f6beebf8173578 Mon Sep 17 00:00:00 2001 From: Brad Rydzewski Date: Thu, 21 Apr 2016 17:10:19 -0700 Subject: [PATCH] use new .drone.sig signature file --- api/build.go | 28 ++++++--- client/client.go | 6 +- client/interface.go | 2 +- drone/agent/agent.go | 59 +++++++++++++++++-- drone/agent/exec.go | 69 +++++++++++++++-------- engine/compiler/builtin/escalate.go | 30 ++++++++++ engine/compiler/builtin/escalate_test.go | 54 ++++++++++++++++++ engine/compiler/builtin/normalize.go | 3 + engine/compiler/builtin/normalize_test.go | 9 +++ queue/types.go | 2 + router/middleware/agent.go | 45 +++++++++++++++ router/middleware/session/user.go | 14 ++--- router/router.go | 7 ++- shared/token/token.go | 9 +-- web/hook.go | 28 ++++++--- 15 files changed, 301 insertions(+), 64 deletions(-) create mode 100644 engine/compiler/builtin/escalate.go create mode 100644 engine/compiler/builtin/escalate_test.go create mode 100644 router/middleware/agent.go diff --git a/api/build.go b/api/build.go index 81ecd3488..225837359 100644 --- a/api/build.go +++ b/api/build.go @@ -18,6 +18,7 @@ import ( "github.com/drone/drone/shared/httputil" "github.com/drone/drone/store" "github.com/gin-gonic/gin" + "github.com/square/go-jose" "github.com/drone/drone/model" "github.com/drone/drone/router/middleware/session" @@ -33,6 +34,9 @@ func init() { droneYml = ".drone.yml" } droneSec = fmt.Sprintf("%s.sec", strings.TrimSuffix(droneYml, filepath.Ext(droneYml))) + if os.Getenv("CANARY") == "true" { + droneSec = fmt.Sprintf("%s.sig", strings.TrimSuffix(droneYml, filepath.Ext(droneYml))) + } } func GetBuilds(c *gin.Context) { @@ -296,25 +300,33 @@ func PostBuild(c *gin.Context) { // enabled using with the environment variable CANARY=true if os.Getenv("CANARY") == "true" { + + var signed bool + var verified bool + + signature, err := jose.ParseSigned(string(sec)) + if err == nil && len(sec) != 0 { + signed = true + output, err := signature.Verify(repo.Hash) + if err == nil && string(output) == string(raw) { + verified = true + } + } + bus.Publish(c, bus.NewBuildEvent(bus.Enqueued, repo, build)) for _, job := range jobs { queue.Publish(c, &queue.Work{ + Signed: signed, + Verified: verified, User: user, Repo: repo, Build: build, BuildLast: last, Job: job, - Keys: key, Netrc: netrc, Yaml: string(raw), - YamlEnc: string(sec), Secrets: secs, - System: &model.System{ - Link: httputil.GetURL(c.Request), - Plugins: strings.Split(os.Getenv("PLUGIN_FILTER"), " "), - Globals: strings.Split(os.Getenv("PLUGIN_PARAMS"), " "), - Escalates: strings.Split(os.Getenv("ESCALATE_FILTER"), " "), - }, + System: &model.System{Link: httputil.GetURL(c.Request)}, }) } return // EXIT NOT TO AVOID THE 0.4 ENGINE CODE BELOW diff --git a/client/client.go b/client/client.go index 8ab26ead2..307a4e74e 100644 --- a/client/client.go +++ b/client/client.go @@ -18,7 +18,7 @@ import ( ) const ( - pathPull = "%s/api/queue/pull" + pathPull = "%s/api/queue/pull/%s/%s" pathWait = "%s/api/queue/wait/%d" pathStream = "%s/api/queue/stream/%d" pathPush = "%s/api/queue/status/%d" @@ -43,9 +43,9 @@ func NewClientToken(uri, token string) Client { } // Pull pulls work from the server queue. -func (c *client) Pull() (*queue.Work, error) { +func (c *client) Pull(os, arch string) (*queue.Work, error) { out := new(queue.Work) - uri := fmt.Sprintf(pathPull, c.base) + uri := fmt.Sprintf(pathPull, c.base, os, arch) err := c.post(uri, nil, out) return out, err } diff --git a/client/interface.go b/client/interface.go index f307d3146..783c2c936 100644 --- a/client/interface.go +++ b/client/interface.go @@ -9,7 +9,7 @@ import ( // Client is used to communicate with a Drone server. type Client interface { // Pull pulls work from the server queue. - Pull() (*queue.Work, error) + Pull(os, arch string) (*queue.Work, error) // Push pushes an update to the server. Push(*queue.Work) error diff --git a/drone/agent/agent.go b/drone/agent/agent.go index 2aeb43336..cfe7ae19e 100644 --- a/drone/agent/agent.go +++ b/drone/agent/agent.go @@ -40,6 +40,18 @@ var AgentCmd = cli.Command{ Usage: "limit number of running docker processes", Value: 2, }, + cli.StringFlag{ + EnvVar: "DOCKER_OS", + Name: "docker-os", + Usage: "docker operating system", + Value: "linux", + }, + cli.StringFlag{ + EnvVar: "DOCKER_ARCH", + Name: "docker-arch", + Usage: "docker architecture system", + Value: "amd64", + }, cli.StringFlag{ EnvVar: "DRONE_SERVER", Name: "drone-server", @@ -68,16 +80,40 @@ var AgentCmd = cli.Command{ Usage: "start the agent with experimental features", }, cli.StringSliceFlag{ - EnvVar: "DRONE_NETRC_PLUGIN", + EnvVar: "DRONE_PLUGIN_NETRC", Name: "netrc-plugin", Usage: "plugins that receive the netrc file", Value: &cli.StringSlice{"git", "hg"}, }, cli.StringSliceFlag{ - EnvVar: "DRONE_PRIVILEGED_PLUGIN", - Name: "privileged-plugin", + EnvVar: "DRONE_PLUGIN_PRIVILEGED", + Name: "privileged", Usage: "plugins that require privileged mode", - Value: &cli.StringSlice{"docker", "gcr", "ecr"}, + Value: &cli.StringSlice{ + "plugins/docker", + "plugins/docker:*", + "plguins/gcr", + "plguins/gcr:*", + "plugins/ecr", + "plugins/ecr:*", + }, + }, + cli.BoolFlag{ + EnvVar: "DRONE_PLUGIN_PULL", + Name: "pull", + Usage: "always pull latest plugin images", + }, + cli.StringFlag{ + EnvVar: "DRONE_PLUGIN_NAMESPACE", + Name: "namespace", + Value: "plugins", + Usage: "default plugin image namespace", + }, + cli.StringSliceFlag{ + EnvVar: "DRONE_PLUGIN_WHITELIST", + Name: "whitelist", + Usage: "plugins that are permitted to run on the host", + Value: &cli.StringSlice{"plugins/*"}, }, }, } @@ -109,10 +145,21 @@ func start(c *cli.Context) { for i := 0; i < c.Int("docker-max-procs"); i++ { wg.Add(1) go func() { + r := pipeline{ + drone: client, + docker: docker, + config: config{ + whitelist: c.StringSlice("whitelist"), + namespace: c.String("namespace"), + privileged: c.StringSlice("privileged"), + netrc: c.StringSlice("netrc-plugin"), + pull: c.Bool("pull"), + }, + } for { - if err := recoverExec(client, docker); err != nil { + if err := r.run(); err != nil { dur := c.Duration("backoff") - logrus.Debugf("Attempting to reconnect in %v", dur) + logrus.Warnf("Attempting to reconnect in %v", dur) time.Sleep(dur) } } diff --git a/drone/agent/exec.go b/drone/agent/exec.go index 16b5c98dd..f26d94a72 100644 --- a/drone/agent/exec.go +++ b/drone/agent/exec.go @@ -14,7 +14,7 @@ import ( "github.com/drone/drone/engine/compiler" "github.com/drone/drone/engine/compiler/builtin" "github.com/drone/drone/engine/runner" - engine "github.com/drone/drone/engine/runner/docker" + "github.com/drone/drone/engine/runner/docker" "github.com/drone/drone/model" "github.com/drone/drone/queue" "github.com/drone/drone/yaml/expander" @@ -23,15 +23,23 @@ import ( "golang.org/x/net/context" ) -func recoverExec(client client.Client, docker dockerclient.Client) error { - defer func() { - recover() - }() - return exec(client, docker) +type config struct { + platform string + namespace string + whitelist []string + privileged []string + netrc []string + pull bool } -func exec(client client.Client, docker dockerclient.Client) error { - w, err := client.Pull() +type pipeline struct { + drone client.Client + docker dockerclient.Client + config config +} + +func (r *pipeline) run() error { + w, err := r.drone.Pull("linux", "amd64") if err != nil { return err } @@ -46,23 +54,34 @@ func exec(client client.Client, docker dockerclient.Client) error { envs := toEnv(w) w.Yaml = expander.ExpandString(w.Yaml, envs) + if w.Verified { + + } + if w.Signed { + + } // inject the netrc file into the clone plugin if the repositroy is // private and requires authentication. + var secrets []*model.Secret + if w.Verified { + secrets = append(secrets, w.Secrets...) + } + if w.Repo.IsPrivate { - w.Secrets = append(w.Secrets, &model.Secret{ + secrets = append(secrets, &model.Secret{ Name: "DRONE_NETRC_USERNAME", Value: w.Netrc.Login, Images: []string{"git", "hg"}, // TODO(bradrydzewski) use the command line parameters here Events: []string{model.EventDeploy, model.EventPull, model.EventPush, model.EventTag}, }) - w.Secrets = append(w.Secrets, &model.Secret{ + secrets = append(secrets, &model.Secret{ Name: "DRONE_NETRC_PASSWORD", Value: w.Netrc.Password, - Images: []string{w.Repo.Kind}, + Images: []string{"git", "hg"}, Events: []string{model.EventDeploy, model.EventPull, model.EventPush, model.EventTag}, }) - w.Secrets = append(w.Secrets, &model.Secret{ + secrets = append(secrets, &model.Secret{ Name: "DRONE_NETRC_MACHINE", Value: w.Netrc.Machine, Images: []string{"git", "hg"}, @@ -71,25 +90,26 @@ func exec(client client.Client, docker dockerclient.Client) error { } trans := []compiler.Transform{ - builtin.NewCloneOp("plugins/"+w.Repo.Kind+":latest", true), + builtin.NewCloneOp("plugins/git:latest", true), builtin.NewCacheOp( "plugins/cache:latest", "/var/lib/drone/cache/"+w.Repo.FullName, false, ), - builtin.NewSecretOp(w.Build.Event, w.Secrets), - builtin.NewNormalizeOp("plugins"), - builtin.NewWorkspaceOp("/drone", "drone/src/github.com/"+w.Repo.FullName), + builtin.NewSecretOp(w.Build.Event, secrets), + builtin.NewNormalizeOp(r.config.namespace), + builtin.NewWorkspaceOp("/drone", "/drone/src/github.com/"+w.Repo.FullName), builtin.NewValidateOp( w.Repo.IsTrusted, - []string{"plugins/*"}, + r.config.whitelist, ), builtin.NewEnvOp(envs), builtin.NewShellOp(builtin.Linux_adm64), builtin.NewArgsOp(), + builtin.NewEscalateOp(r.config.privileged), builtin.NewPodOp(prefix), builtin.NewAliasOp(prefix), - builtin.NewPullOp(false), + builtin.NewPullOp(r.config.pull), builtin.NewFilterOp( model.StatusSuccess, // TODO(bradrydzewski) please add the last build status here w.Build.Branch, @@ -109,14 +129,14 @@ func exec(client client.Client, docker dockerclient.Client) error { return err } - if err := client.Push(w); err != nil { + if err := r.drone.Push(w); err != nil { logrus.Errorf("Error persisting update %s/%s#%d.%d. %s", w.Repo.Owner, w.Repo.Name, w.Build.Number, w.Job.Number, err) return err } conf := runner.Config{ - Engine: engine.New(docker), + Engine: docker.New(r.docker), } ctx := context.TODO() @@ -126,7 +146,7 @@ func exec(client client.Client, docker dockerclient.Client) error { run.Run() defer cancel() - wait := client.Wait(w.Job.ID) + wait := r.drone.Wait(w.Job.ID) if err != nil { return err } @@ -142,7 +162,7 @@ func exec(client client.Client, docker dockerclient.Client) error { rc, wc := io.Pipe() go func() { - err := client.Stream(w.Job.ID, rc) + err := r.drone.Stream(w.Job.ID, rc) if err != nil && err != io.ErrClosedPipe { logrus.Errorf("Error streaming build logs. %s", err) } @@ -187,7 +207,7 @@ func exec(client client.Client, docker dockerclient.Client) error { logrus.Infof("Finished build %s/%s#%d.%d", w.Repo.Owner, w.Repo.Name, w.Build.Number, w.Job.Number) - return client.Push(w) + return r.drone.Push(w) } func toEnv(w *queue.Work) map[string]string { @@ -218,7 +238,8 @@ func toEnv(w *queue.Work) map[string]string { "DRONE_BUILD_CREATED": fmt.Sprintf("%d", w.Build.Created), "DRONE_BUILD_STARTED": fmt.Sprintf("%d", w.Build.Started), "DRONE_BUILD_FINISHED": fmt.Sprintf("%d", w.Build.Finished), - "DRONE_BUILD_VERIFIED": fmt.Sprintf("%v", false), + "DRONE_YAML_VERIFIED": fmt.Sprintf("%v", w.Verified), + "DRONE_YAML_SIGNED": fmt.Sprintf("%v", w.Signed), // SHORTER ALIASES "DRONE_BRANCH": w.Build.Branch, diff --git a/engine/compiler/builtin/escalate.go b/engine/compiler/builtin/escalate.go new file mode 100644 index 000000000..78a0c1893 --- /dev/null +++ b/engine/compiler/builtin/escalate.go @@ -0,0 +1,30 @@ +package builtin + +import ( + "path/filepath" + + "github.com/drone/drone/engine/compiler/parse" +) + +type escalateOp struct { + visitor + plugins []string +} + +// NewEscalateOp returns a transformer that configures plugins to automatically +// execute in privileged mode. This is intended for plugins running dind. +func NewEscalateOp(plugins []string) Visitor { + return &escalateOp{ + plugins: plugins, + } +} + +func (v *escalateOp) VisitContainer(node *parse.ContainerNode) error { + for _, pattern := range v.plugins { + ok, _ := filepath.Match(pattern, node.Container.Image) + if ok { + node.Container.Privileged = true + } + } + return nil +} diff --git a/engine/compiler/builtin/escalate_test.go b/engine/compiler/builtin/escalate_test.go new file mode 100644 index 000000000..e1374bedb --- /dev/null +++ b/engine/compiler/builtin/escalate_test.go @@ -0,0 +1,54 @@ +package builtin + +import ( + "testing" + + "github.com/drone/drone/engine/compiler/parse" + "github.com/drone/drone/engine/runner" + + "github.com/franela/goblin" +) + +func Test_escalate(t *testing.T) { + root := parse.NewRootNode() + + g := goblin.Goblin(t) + g.Describe("privileged transform", func() { + + g.It("should handle matches", func() { + c := root.NewPluginNode() + c.Container = runner.Container{Image: "plugins/docker"} + op := NewEscalateOp([]string{"plugins/docker"}) + + op.VisitContainer(c) + g.Assert(c.Container.Privileged).IsTrue() + }) + + g.It("should handle glob matches", func() { + c := root.NewPluginNode() + c.Container = runner.Container{Image: "plugins/docker"} + op := NewEscalateOp([]string{"plugins/*"}) + + op.VisitContainer(c) + g.Assert(c.Container.Privileged).IsTrue() + }) + + g.It("should handle non matches", func() { + c := root.NewPluginNode() + c.Container = runner.Container{Image: "plugins/git"} + op := NewEscalateOp([]string{"plugins/docker"}) + + op.VisitContainer(c) + g.Assert(c.Container.Privileged).IsFalse() + }) + + g.It("should handle non glob matches", func() { + c := root.NewPluginNode() + c.Container = runner.Container{Image: "plugins/docker:develop"} + op := NewEscalateOp([]string{"plugins/docker"}) + + op.VisitContainer(c) + g.Assert(c.Container.Privileged).IsFalse() + }) + }) +} diff --git a/engine/compiler/builtin/normalize.go b/engine/compiler/builtin/normalize.go index 90c404189..4de12720d 100644 --- a/engine/compiler/builtin/normalize.go +++ b/engine/compiler/builtin/normalize.go @@ -43,6 +43,9 @@ func (v *normalizeOp) normalizePlugin(node *parse.ContainerNode) { if strings.Contains(node.Container.Image, "/") { return } + if strings.Contains(node.Container.Image, "_") { + node.Container.Image = strings.Replace(node.Container.Image, "_", "-", -1) + } node.Container.Image = filepath.Join(v.namespace, node.Container.Image) } diff --git a/engine/compiler/builtin/normalize_test.go b/engine/compiler/builtin/normalize_test.go index ecf6e4ecb..dbb24f2f6 100644 --- a/engine/compiler/builtin/normalize_test.go +++ b/engine/compiler/builtin/normalize_test.go @@ -56,6 +56,15 @@ func Test_normalize(t *testing.T) { g.Assert(c.Container.Image).Equal("index.docker.io/drone/git:latest") }) + g.It("should replace underscores with dashes", func() { + c := root.NewPluginNode() + c.Container = runner.Container{Image: "gh_pages"} + op := NewNormalizeOp("plugins") + + op.VisitContainer(c) + g.Assert(c.Container.Image).Equal("plugins/gh-pages:latest") + }) + g.It("should ignore shell or service types", func() { c := root.NewShellNode() c.Container = runner.Container{Image: "golang"} diff --git a/queue/types.go b/queue/types.go index 1408a8ca8..85b87b7bc 100644 --- a/queue/types.go +++ b/queue/types.go @@ -5,6 +5,8 @@ import "github.com/drone/drone/model" // Work represents an item for work to be // processed by a worker. type Work struct { + Signed bool `json:"signed"` + Verified bool `json:"verified"` Yaml string `json:"config"` YamlEnc string `json:"secret"` Repo *model.Repo `json:"repo"` diff --git a/router/middleware/agent.go b/router/middleware/agent.go new file mode 100644 index 000000000..e227089a9 --- /dev/null +++ b/router/middleware/agent.go @@ -0,0 +1,45 @@ +package middleware + +import ( + "github.com/drone/drone/shared/token" + + "github.com/Sirupsen/logrus" + "github.com/gin-gonic/gin" + "github.com/ianschenck/envflag" +) + +var ( + secret = envflag.String("AGENT_SECRET", "", "") + noauth = envflag.Bool("AGENT_NO_AUTH", false, "") +) + +// Agent is a middleware function that initializes the authorization middleware +// for agents to connect to the queue. +func AgentMust() gin.HandlerFunc { + + if *secret == "" { + logrus.Fatalf("please provide the agent secret to authenticate agent requests") + } + + t := token.New(token.AgentToken, "") + s, err := t.Sign(*secret) + if err != nil { + logrus.Fatalf("invalid agent secret. %s", err) + } + + logrus.Infof("using agent secret %s", *secret) + logrus.Warnf("agents can connect with token %s", s) + + return func(c *gin.Context) { + parsed, err := token.ParseRequest(c.Request, func(t *token.Token) (string, error) { + return *secret, nil + }) + if err != nil { + c.AbortWithError(403, err) + } else if parsed.Kind != token.AgentToken { + c.AbortWithStatus(403) + } else { + c.Next() + } + } +} diff --git a/router/middleware/session/user.go b/router/middleware/session/user.go index a828b662d..8453d208b 100644 --- a/router/middleware/session/user.go +++ b/router/middleware/session/user.go @@ -70,15 +70,14 @@ func MustAdmin() gin.HandlerFunc { user := User(c) switch { case user == nil: - c.AbortWithStatus(http.StatusUnauthorized) - // c.HTML(http.StatusUnauthorized, "401.html", gin.H{}) + c.String(401, "User not authorized") + c.Abort() case user.Admin == false: - c.AbortWithStatus(http.StatusForbidden) - // c.HTML(http.StatusForbidden, "401.html", gin.H{}) + c.String(413, "User not authorized") + c.Abort() default: c.Next() } - } } @@ -87,11 +86,10 @@ func MustUser() gin.HandlerFunc { user := User(c) switch { case user == nil: - c.AbortWithStatus(http.StatusUnauthorized) - // c.HTML(http.StatusUnauthorized, "401.html", gin.H{}) + c.String(401, "User not authorized") + c.Abort() default: c.Next() } - } } diff --git a/router/router.go b/router/router.go index 6bce12a18..efa06160e 100644 --- a/router/router.go +++ b/router/router.go @@ -8,6 +8,7 @@ import ( "github.com/gin-gonic/gin" "github.com/drone/drone/api" + "github.com/drone/drone/router/middleware" "github.com/drone/drone/router/middleware/header" "github.com/drone/drone/router/middleware/session" "github.com/drone/drone/router/middleware/token" @@ -16,7 +17,7 @@ import ( "github.com/drone/drone/web" ) -func Load(middleware ...gin.HandlerFunc) http.Handler { +func Load(middlewares ...gin.HandlerFunc) http.Handler { e := gin.New() e.Use(gin.Recovery()) @@ -26,7 +27,7 @@ func Load(middleware ...gin.HandlerFunc) http.Handler { e.Use(header.NoCache) e.Use(header.Options) e.Use(header.Secure) - e.Use(middleware...) + e.Use(middlewares...) e.Use(session.SetUser()) e.Use(token.Refresh) @@ -163,7 +164,9 @@ func Load(middleware ...gin.HandlerFunc) http.Handler { queue := e.Group("/api/queue") { + queue.Use(middleware.AgentMust()) queue.POST("/pull", api.Pull) + queue.POST("/pull/:os/:arch", api.Pull) queue.POST("/wait/:id", api.Wait) queue.POST("/stream/:id", api.Stream) queue.POST("/status/:id", api.Update) diff --git a/shared/token/token.go b/shared/token/token.go index 5741d9f65..d4f83a2b8 100644 --- a/shared/token/token.go +++ b/shared/token/token.go @@ -10,10 +10,11 @@ import ( type SecretFunc func(*Token) (string, error) const ( - UserToken = "user" - SessToken = "sess" - HookToken = "hook" - CsrfToken = "csrf" + UserToken = "user" + SessToken = "sess" + HookToken = "hook" + CsrfToken = "csrf" + AgentToken = "agent" ) // Default algorithm used to sign JWT tokens. diff --git a/web/hook.go b/web/hook.go index 8c3cdca19..1f9c896b4 100644 --- a/web/hook.go +++ b/web/hook.go @@ -8,6 +8,7 @@ import ( "strings" "github.com/gin-gonic/gin" + "github.com/square/go-jose" log "github.com/Sirupsen/logrus" "github.com/drone/drone/bus" @@ -31,6 +32,9 @@ func init() { droneYml = ".drone.yml" } droneSec = fmt.Sprintf("%s.sec", strings.TrimSuffix(droneYml, filepath.Ext(droneYml))) + if os.Getenv("CANARY") == "true" { + droneSec = fmt.Sprintf("%s.sig", strings.TrimSuffix(droneYml, filepath.Ext(droneYml))) + } } var skipRe = regexp.MustCompile(`\[(?i:ci *skip|skip *ci)\]`) @@ -214,25 +218,33 @@ func PostHook(c *gin.Context) { // enabled using with the environment variable CANARY=true if os.Getenv("CANARY") == "true" { + + var signed bool + var verified bool + + signature, err := jose.ParseSigned(string(sec)) + if err == nil && len(sec) != 0 { + signed = true + output, err := signature.Verify(repo.Hash) + if err == nil && string(output) == string(raw) { + verified = true + } + } + bus.Publish(c, bus.NewBuildEvent(bus.Enqueued, repo, build)) for _, job := range jobs { queue.Publish(c, &queue.Work{ + Signed: signed, + Verified: verified, User: user, Repo: repo, Build: build, BuildLast: last, Job: job, - Keys: key, Netrc: netrc, Yaml: string(raw), - YamlEnc: string(sec), Secrets: secs, - System: &model.System{ - Link: httputil.GetURL(c.Request), - Plugins: strings.Split(os.Getenv("PLUGIN_FILTER"), " "), - Globals: strings.Split(os.Getenv("PLUGIN_PARAMS"), " "), - Escalates: strings.Split(os.Getenv("ESCALATE_FILTER"), " "), - }, + System: &model.System{Link: httputil.GetURL(c.Request)}, }) } return // EXIT NOT TO AVOID THE 0.4 ENGINE CODE BELOW