Merge pull request #1994 from bradrydzewski/master

support docker-compose secret declaration

client/client.go

@@ -115,9 +115,6 @@ type Client interface {
 	// target environment.
 	Deploy(string, string, int, string, map[string]string) (*model.Build, error)
 
-	// AgentList returns a list of build agents.
-	AgentList() ([]*model.Agent, error)
-
 	// Registry returns a registry by hostname.
 	Registry(owner, name, hostname string) (*model.Registry, error)
 

client/client_impl.go

@@ -53,7 +53,6 @@ const (
 	pathUsers          = "%s/api/users"
 	pathUser           = "%s/api/users/%s"
 	pathBuildQueue     = "%s/api/builds"
-	pathAgent          = "%s/api/agents"
 )
 
 type client struct {
@@ -367,14 +366,6 @@ func (c *client) Sign(owner, name string, in []byte) ([]byte, error) {
 	return ioutil.ReadAll(rc)
 }
 
-// AgentList returns a list of build agents.
-func (c *client) AgentList() ([]*model.Agent, error) {
-	var out []*model.Agent
-	uri := fmt.Sprintf(pathAgent, c.base)
-	err := c.get(uri, &out)
-	return out, err
-}
-
 // Registry returns a registry by hostname.
 func (c *client) Registry(owner, name, hostname string) (*model.Registry, error) {
 	out := new(model.Registry)

drone/secret.go

@@ -120,8 +120,11 @@ func secretDisplayList(secrets []*model.Secret, c *cli.Context) error {
 // template for secret list items
 var tmplSecretList = "\x1b[33m{{ .Name }} \x1b[0m" + `
 Events: {{ list .Events }}
-SkipVerify: {{ .SkipVerify }}
-Conceal: {{ .Conceal }}
+{{- if .Images }}
+Images: {{ list .Images }}
+{{- else }}
+Images: <any>
+{{- end }}
 `
 
 var secretFuncMap = template.FuncMap{

model/agent.go

@@ -1,10 +0,0 @@
-package model
-
-type Agent struct {
-	ID       int64  `json:"id"         meddler:"agent_id,pk"`
-	Address  string `json:"address"    meddler:"agent_addr"`
-	Platform string `json:"platform"   meddler:"agent_platform"`
-	Capacity int    `json:"capacity"   meddler:"agent_capacity"`
-	Created  int64  `json:"created_at" meddler:"agent_created"`
-	Updated  int64  `json:"updated_at" meddler:"agent_updated"`
-}

model/approval.go

@@ -0,0 +1,17 @@
+package model
+
+type (
+	// ApprovalStore persists approved committers to storage.
+	ApprovalStore interface {
+		Approve(*Repo) error
+	}
+
+	// Approval represents an approved committer.
+	Approval struct {
+		Username  string `json:"username"   meddler:"approval_username"`
+		Approved  bool   `json:"approved"   meddler:"approval_approved"`
+		Comments  string `json:"comments"   meddler:"approval_comments"`
+		CreatedBy string `json:"created_by" meddler:"approval_created_by"`
+		CreatedAt int64  `json:"created_at" meddler:"approval_created_at"`
+	}
+)

model/gate.go

@@ -0,0 +1,23 @@
+package model
+
+// Gated indicates a build is gated and requires approval to proceed. It also
+// includes the reason the build was gated.
+type Gated struct {
+	Gated  bool   `json:"gated"`
+	Reason string `json:"reason"`
+}
+
+// GateService defines a service for gating builds.
+type GateService interface {
+	Gated(*User, *Repo, *Build) (*Gated, error)
+}
+
+type gateService struct{}
+
+func (s *gateService) Gated(owner *User, repo *Repo, build *Build) (*Gated, error) {
+	g := new(Gated)
+	if repo.IsPrivate && build.Event == EventPull && build.Sender != owner.Login {
+		g.Gated = true
+	}
+	return g, nil
+}

model/job.go

@@ -1,17 +0,0 @@
-package model
-
-// // swagger:model job
-// type Job struct {
-// 	ID       int64  `json:"id"           meddler:"job_id,pk"`
-// 	BuildID  int64  `json:"-"            meddler:"job_build_id"`
-// 	NodeID   int64  `json:"-"            meddler:"job_node_id"`
-// 	Number   int    `json:"number"       meddler:"job_number"`
-// 	Error    string `json:"error"        meddler:"job_error"`
-// 	Status   string `json:"status"       meddler:"job_status"`
-// 	ExitCode int    `json:"exit_code"    meddler:"job_exit_code"`
-// 	Enqueued int64  `json:"enqueued_at"  meddler:"job_enqueued"`
-// 	Started  int64  `json:"started_at"   meddler:"job_started"`
-// 	Finished int64  `json:"finished_at"  meddler:"job_finished"`
-//
-// 	Environment map[string]string `json:"environment" meddler:"job_environment,json"`
-// }

server/hook.go

@@ -1,7 +1,6 @@
 package server
 
 import (
-	"bytes"
 	"context"
 	"encoding/json"
 	"fmt"
@@ -164,67 +163,68 @@ func PostHook(c *gin.Context) {
 		logrus.Debugf("Error getting registry credentials for %s#%d. %s", repo.FullName, build.Number, err)
 	}
 
-	var mustApprove bool
-	if build.Event == model.EventPull {
-		for _, sec := range secs {
-			if sec.SkipVerify {
-				continue
-			}
-			if sec.MatchEvent(model.EventPull) {
-				mustApprove = true
-				break
-			}
-		}
-		if !mustApprove {
-			logrus.Debugf("no secrets exposed to pull_request: status: accepted")
-		}
-	}
+	// var mustApprove bool
+	// if build.Event == model.EventPull {
+	// 	for _, sec := range secs {
+	// 		if sec.SkipVerify {
+	// 			continue
+	// 		}
+	// 		if sec.MatchEvent(model.EventPull) {
+	// 			mustApprove = true
+	// 			break
+	// 		}
+	// 	}
+	// 	if !mustApprove {
+	// 		logrus.Debugf("no secrets exposed to pull_request: status: accepted")
+	// 	}
+	// }
 
-	if build.Event == model.EventPull && mustApprove {
-		old, ferr := remote_.FileRef(user, repo, build.Branch, repo.Config)
-		if ferr != nil {
-			build.Status = model.StatusBlocked
-			logrus.Debugf("cannot fetch base yaml: status: blocked")
-		} else if bytes.Equal(old, raw) {
-			build.Status = model.StatusPending
-			logrus.Debugf("base yaml matches head yaml: status: accepted")
-		} else {
-			// this block is executed if the target yaml file
-			// does not match the base yaml.
-
-			// TODO unfortunately we have no good way to get the
-			// sender repository permissions unless the user is
-			// a registered drone user.
-			sender, uerr := store.GetUserLogin(c, build.Sender)
-			if uerr != nil {
-				build.Status = model.StatusBlocked
-				logrus.Debugf("sender does not have a drone account: status: blocked")
-			} else {
-				if refresher, ok := remote_.(remote.Refresher); ok {
-					ok, _ := refresher.Refresh(sender)
-					if ok {
-						store.UpdateUser(c, sender)
-					}
-				}
-				// if the sender does not have push access to the
-				// repository the pull request should be blocked.
-				perm, perr := remote_.Perm(sender, repo.Owner, repo.Name)
-				if perr == nil && perm.Push == true {
-					build.Status = model.StatusPending
-					logrus.Debugf("sender %s has push access: status: accepted", sender.Login)
-				} else {
-					build.Status = model.StatusBlocked
-					logrus.Debugf("sender %s does not have push access: status: blocked", sender.Login)
-				}
-			}
-		}
-	} else {
-		build.Status = model.StatusPending
-	}
+	// if build.Event == model.EventPull && mustApprove {
+	// 	old, ferr := remote_.FileRef(user, repo, build.Branch, repo.Config)
+	// 	if ferr != nil {
+	// 		build.Status = model.StatusBlocked
+	// 		logrus.Debugf("cannot fetch base yaml: status: blocked")
+	// 	} else if bytes.Equal(old, raw) {
+	// 		build.Status = model.StatusPending
+	// 		logrus.Debugf("base yaml matches head yaml: status: accepted")
+	// 	} else {
+	// 		// this block is executed if the target yaml file
+	// 		// does not match the base yaml.
+	//
+	// 		// TODO unfortunately we have no good way to get the
+	// 		// sender repository permissions unless the user is
+	// 		// a registered drone user.
+	// 		sender, uerr := store.GetUserLogin(c, build.Sender)
+	// 		if uerr != nil {
+	// 			build.Status = model.StatusBlocked
+	// 			logrus.Debugf("sender does not have a drone account: status: blocked")
+	// 		} else {
+	// 			if refresher, ok := remote_.(remote.Refresher); ok {
+	// 				ok, _ := refresher.Refresh(sender)
+	// 				if ok {
+	// 					store.UpdateUser(c, sender)
+	// 				}
+	// 			}
+	// 			// if the sender does not have push access to the
+	// 			// repository the pull request should be blocked.
+	// 			perm, perr := remote_.Perm(sender, repo.Owner, repo.Name)
+	// 			if perr == nil && perm.Push == true {
+	// 				build.Status = model.StatusPending
+	// 				logrus.Debugf("sender %s has push access: status: accepted", sender.Login)
+	// 			} else {
+	// 				build.Status = model.StatusBlocked
+	// 				logrus.Debugf("sender %s does not have push access: status: blocked", sender.Login)
+	// 			}
+	// 		}
+	// 	}
+	// } else {
+	// 	build.Status = model.StatusPending
+	// }
 
 	// update some build fields
 	build.RepoID = repo.ID
 	build.Verified = true
+	build.Status = model.StatusPending
 
 	if err := store.CreateBuild(c, build, build.Procs...); err != nil {
 		logrus.Errorf("failure to save commit for %s. %s", repo.FullName, err)
@@ -234,9 +234,9 @@ func PostHook(c *gin.Context) {
 
 	c.JSON(200, build)
 
-	if build.Status == model.StatusBlocked {
-		return
-	}
+	// if build.Status == model.StatusBlocked {
+	// 	return
+	// }
 
 	// get the previous build so that we can send
 	// on status change notifications
@@ -454,29 +454,28 @@ func (b *builder) Build() ([]*buildItem, error) {
 		for k, v := range metadata.EnvironDrone() {
 			environ[k] = v
 		}
-
 		for k, v := range axis {
 			environ[k] = v
 		}
 
-		secrets := map[string]string{}
+		var secrets []compiler.Secret
 		for _, sec := range b.Secs {
 			if !sec.MatchEvent(b.Curr.Event) {
 				continue
 			}
 			if b.Curr.Verified || sec.SkipVerify {
-				secrets[sec.Name] = sec.Value
+				secrets = append(secrets, compiler.Secret{
+					Name:  sec.Name,
+					Value: sec.Value,
+					Match: sec.Images,
+				})
 			}
 		}
-		sub := func(name string) string {
-			if v, ok := environ[name]; ok {
-				return v
-			}
-			return secrets[name]
-		}
 
 		y := b.Yaml
-		s, err := envsubst.Eval(y, sub)
+		s, err := envsubst.Eval(y, func(name string) string {
+			return environ[name]
+		})
 		if err != nil {
 			return nil, err
 		}
@@ -521,6 +520,7 @@ func (b *builder) Build() ([]*buildItem, error) {
 				b.Repo.IsPrivate,
 			),
 			compiler.WithRegistry(registries...),
+			compiler.WithSecret(secrets...),
 			compiler.WithPrefix(
 				fmt.Sprintf(
 					"%d_%d",
@@ -536,18 +536,18 @@ func (b *builder) Build() ([]*buildItem, error) {
 			compiler.WithMetadata(metadata),
 		).Compile(parsed)
 
-		for _, sec := range b.Secs {
-			if !sec.MatchEvent(b.Curr.Event) {
-				continue
-			}
-			if b.Curr.Verified || sec.SkipVerify {
-				ir.Secrets = append(ir.Secrets, &backend.Secret{
-					Mask:  sec.Conceal,
-					Name:  sec.Name,
-					Value: sec.Value,
-				})
-			}
-		}
+		// for _, sec := range b.Secs {
+		// 	if !sec.MatchEvent(b.Curr.Event) {
+		// 		continue
+		// 	}
+		// 	if b.Curr.Verified || sec.SkipVerify {
+		// 		ir.Secrets = append(ir.Secrets, &backend.Secret{
+		// 			Mask:  sec.Conceal,
+		// 			Name:  sec.Name,
+		// 			Value: sec.Value,
+		// 		})
+		// 	}
+		// }
 
 		item := &buildItem{
 			Proc:     proc,

vendor/github.com/cncd/pipeline/pipeline/frontend/yaml/compiler/compiler.go

@@ -20,6 +20,12 @@ type Registry struct {
 	Token    string
 }
 
+type Secret struct {
+	Name  string
+	Value string
+	Match []string
+}
+
 // Compiler compiles the yaml
 type Compiler struct {
 	local      bool
@@ -31,13 +37,16 @@ type Compiler struct {
 	path       string
 	metadata   frontend.Metadata
 	registries []Registry
+	secrets    map[string]Secret
 	aliases    []string
 }
 
 // New creates a new Compiler with options.
 func New(opts ...Option) *Compiler {
-	compiler := new(Compiler)
-	compiler.env = map[string]string{}
+	compiler := &Compiler{
+		env:     map[string]string{},
+		secrets: map[string]Secret{},
+	}
 	for _, opt := range opts {
 		opt(compiler)
 	}

vendor/github.com/cncd/pipeline/pipeline/frontend/yaml/compiler/convert.go

@@ -3,6 +3,7 @@ package compiler
 import (
 	"fmt"
 	"path"
+	"strings"
 
 	"github.com/cncd/pipeline/pipeline/backend"
 	"github.com/cncd/pipeline/pipeline/frontend/yaml"
@@ -73,7 +74,7 @@ func (c *Compiler) createProcess(name string, container *yaml.Container) *backen
 	if isPlugin(container) {
 		paramsToEnv(container.Vargs, environment)
 
-		if imageMatches(container.Image, c.escalated) {
+		if matchImage(container.Image, c.escalated...) {
 			privileged = true
 			entrypoint = []string{}
 			command = []string{}
@@ -102,6 +103,13 @@ func (c *Compiler) createProcess(name string, container *yaml.Container) *backen
 		}
 	}
 
+	for _, requested := range container.Secrets.Secrets {
+		secret, ok := c.secrets[strings.ToLower(requested.Source)]
+		if ok && (len(secret.Match) == 0 || matchImage(image, secret.Match...)) {
+			environment[strings.ToUpper(requested.Target)] = secret.Value
+		}
+	}
+
 	return &backend.Step{
 		Name:         name,
 		Alias:        container.Name,
@@ -134,16 +142,6 @@ func (c *Compiler) createProcess(name string, container *yaml.Container) *backen
 	}
 }
 
-func imageMatches(image string, to []string) bool {
-	image = trimImage(image)
-	for _, i := range to {
-		if image == i {
-			return true
-		}
-	}
-	return false
-}
-
 func isPlugin(c *yaml.Container) bool {
 	return len(c.Vargs) != 0
 }

vendor/github.com/cncd/pipeline/pipeline/frontend/yaml/compiler/option.go

@@ -39,6 +39,16 @@ func WithRegistry(registries ...Registry) Option {
 	}
 }
 
+// WithSecret configures the compiler with external secrets
+// to be injected into the container at runtime.
+func WithSecret(secrets ...Secret) Option {
+	return func(compiler *Compiler) {
+		for _, secret := range secrets {
+			compiler.secrets[strings.ToLower(secret.Name)] = secret
+		}
+	}
+}
+
 // WithMetadata configutes the compiler with the repostiory, build
 // and system metadata. The metadata is used to remove steps from
 // the compiled pipeline configuration that should be skipped. The

vendor/github.com/cncd/pipeline/pipeline/frontend/yaml/container.go

@@ -52,6 +52,7 @@ type (
 		ShmSize       libcompose.MemStringorInt `yaml:"shm_size,omitempty"`
 		Ulimits       libcompose.Ulimits        `yaml:"ulimits,omitempty"`
 		Volumes       libcompose.Volumes        `yaml:"volumes,omitempty"`
+		Secrets       Secrets                   `yaml:"secrets,omitempty"`
 		Constraints   Constraints               `yaml:"when,omitempty"`
 		Vargs         map[string]interface{}    `yaml:",inline"`
 	}

vendor/github.com/cncd/pipeline/pipeline/frontend/yaml/secret.go

@@ -0,0 +1,30 @@
+package yaml
+
+type (
+	// Secrets defines a collection of secrets.
+	Secrets struct {
+		Secrets []*Secret
+	}
+
+	// Secret defines a container secret.
+	Secret struct {
+		Source string `yaml:"source"`
+		Target string `yaml:"target"`
+	}
+)
+
+// UnmarshalYAML implements the Unmarshaller interface.
+func (s *Secrets) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	var strslice []string
+	err := unmarshal(&strslice)
+	if err == nil {
+		for _, str := range strslice {
+			s.Secrets = append(s.Secrets, &Secret{
+				Source: str,
+				Target: str,
+			})
+		}
+		return nil
+	}
+	return unmarshal(&s.Secrets)
+}

vendor/vendor.json

@@ -28,68 +28,68 @@
 		{
 			"checksumSHA1": "W3AuK8ocqHwlUajGmQLFvnRhTZE=",
 			"path": "github.com/cncd/pipeline/pipeline",
-			"revision": "087d10834b19bbb8d1665152696ca63883610021",
-			"revisionTime": "2017-04-06T15:46:03Z"
+			"revision": "94d637b94d0439ed4853e8089d8a1b1820b67c65",
+			"revisionTime": "2017-04-09T09:45:58Z"
 		},
 		{
 			"checksumSHA1": "Qu2FreqaMr8Yx2bW9O0cxAGgjr0=",
 			"path": "github.com/cncd/pipeline/pipeline/backend",
-			"revision": "087d10834b19bbb8d1665152696ca63883610021",
-			"revisionTime": "2017-04-06T15:46:03Z"
+			"revision": "94d637b94d0439ed4853e8089d8a1b1820b67c65",
+			"revisionTime": "2017-04-09T09:45:58Z"
 		},
 		{
 			"checksumSHA1": "0CGXRaYwZhJxGIrGhn8WGpkFqPo=",
 			"path": "github.com/cncd/pipeline/pipeline/backend/docker",
-			"revision": "087d10834b19bbb8d1665152696ca63883610021",
-			"revisionTime": "2017-04-06T15:46:03Z"
+			"revision": "94d637b94d0439ed4853e8089d8a1b1820b67c65",
+			"revisionTime": "2017-04-09T09:45:58Z"
 		},
 		{
 			"checksumSHA1": "/8wE+cVb7T4PQZgpLNu0DHzKGuE=",
 			"path": "github.com/cncd/pipeline/pipeline/frontend",
-			"revision": "087d10834b19bbb8d1665152696ca63883610021",
-			"revisionTime": "2017-04-06T15:46:03Z"
+			"revision": "94d637b94d0439ed4853e8089d8a1b1820b67c65",
+			"revisionTime": "2017-04-09T09:45:58Z"
 		},
 		{
-			"checksumSHA1": "O0sulBQAHJeNLg3lO38Cq5uf/eg=",
+			"checksumSHA1": "QWs+L3emrt5DDTWvqD6rbMtLKMw=",
 			"path": "github.com/cncd/pipeline/pipeline/frontend/yaml",
-			"revision": "087d10834b19bbb8d1665152696ca63883610021",
-			"revisionTime": "2017-04-06T15:46:03Z"
+			"revision": "94d637b94d0439ed4853e8089d8a1b1820b67c65",
+			"revisionTime": "2017-04-09T09:45:58Z"
 		},
 		{
-			"checksumSHA1": "4gmWpW2MkXgWGSSvSoRFu1YjGbQ=",
+			"checksumSHA1": "Bsp5Fq7oc6ZDDX5COo//pajb0kk=",
 			"path": "github.com/cncd/pipeline/pipeline/frontend/yaml/compiler",
-			"revision": "087d10834b19bbb8d1665152696ca63883610021",
-			"revisionTime": "2017-04-06T15:46:03Z"
+			"revision": "94d637b94d0439ed4853e8089d8a1b1820b67c65",
+			"revisionTime": "2017-04-09T09:45:58Z"
 		},
 		{
 			"checksumSHA1": "Q0GkNUFamVYIA1Fd8r0A5M6Gx54=",
 			"path": "github.com/cncd/pipeline/pipeline/frontend/yaml/linter",
-			"revision": "087d10834b19bbb8d1665152696ca63883610021",
-			"revisionTime": "2017-04-06T15:46:03Z"
+			"revision": "94d637b94d0439ed4853e8089d8a1b1820b67c65",
+			"revisionTime": "2017-04-09T09:45:58Z"
 		},
 		{
 			"checksumSHA1": "kx2sPUIMozPC/g6E4w48h3FfH3k=",
 			"path": "github.com/cncd/pipeline/pipeline/frontend/yaml/matrix",
-			"revision": "087d10834b19bbb8d1665152696ca63883610021",
-			"revisionTime": "2017-04-06T15:46:03Z"
+			"revision": "94d637b94d0439ed4853e8089d8a1b1820b67c65",
+			"revisionTime": "2017-04-09T09:45:58Z"
 		},
 		{
 			"checksumSHA1": "2/3f3oNmxXy5kcrRLCFa24Oc9O4=",
 			"path": "github.com/cncd/pipeline/pipeline/interrupt",
-			"revision": "087d10834b19bbb8d1665152696ca63883610021",
-			"revisionTime": "2017-04-06T15:46:03Z"
+			"revision": "94d637b94d0439ed4853e8089d8a1b1820b67c65",
+			"revisionTime": "2017-04-09T09:45:58Z"
 		},
 		{
 			"checksumSHA1": "uOjTfke7Qxosrivgz/nVTHeIP5g=",
 			"path": "github.com/cncd/pipeline/pipeline/multipart",
-			"revision": "087d10834b19bbb8d1665152696ca63883610021",
-			"revisionTime": "2017-04-06T15:46:03Z"
+			"revision": "94d637b94d0439ed4853e8089d8a1b1820b67c65",
+			"revisionTime": "2017-04-09T09:45:58Z"
 		},
 		{
 			"checksumSHA1": "TP5lK1T8cOKv5QjZ2nqdlYczSTo=",
 			"path": "github.com/cncd/pipeline/pipeline/rpc",
-			"revision": "087d10834b19bbb8d1665152696ca63883610021",
-			"revisionTime": "2017-04-06T15:46:03Z"
+			"revision": "94d637b94d0439ed4853e8089d8a1b1820b67c65",
+			"revisionTime": "2017-04-09T09:45:58Z"
 		},
 		{
 			"checksumSHA1": "7Qj1DK0ceAXkYztW0l3+L6sn+V8=",