mirrors/woodpecker
1
0
Fork 0
mirror of https://github.com/woodpecker-ci/woodpecker.git synced 2024-12-23 00:46:30 +00:00
Code Issues Projects Releases Wiki Activity

use new .drone.sig signature file

Browse source
This commit is contained in:
Brad Rydzewski 2016-04-21 17:10:19 -07:00
parent 8684210241
commit faf7ff675d
15 changed files with 301 additions and 64 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

28
api/build.go
View file

@ -18,6 +18,7 @@ import (
"github.com/drone/drone/shared/httputil"
"github.com/drone/drone/store"
"github.com/gin-gonic/gin"
"github.com/square/go-jose"
"github.com/drone/drone/model"
"github.com/drone/drone/router/middleware/session"
@ -33,6 +34,9 @@ func init() {
droneYml = ".drone.yml"
}
droneSec = fmt.Sprintf("%s.sec", strings.TrimSuffix(droneYml, filepath.Ext(droneYml)))
if os.Getenv("CANARY") == "true" {
droneSec = fmt.Sprintf("%s.sig", strings.TrimSuffix(droneYml, filepath.Ext(droneYml)))
}
}
func GetBuilds(c *gin.Context) {
@ -296,25 +300,33 @@ func PostBuild(c *gin.Context) {
// enabled using with the environment variable CANARY=true
if os.Getenv("CANARY") == "true" {
var signed bool
var verified bool
signature, err := jose.ParseSigned(string(sec))
if err == nil && len(sec) != 0 {
signed = true
output, err := signature.Verify(repo.Hash)
if err == nil && string(output) == string(raw) {
verified = true
}
}
bus.Publish(c, bus.NewBuildEvent(bus.Enqueued, repo, build))
for _, job := range jobs {
queue.Publish(c, &queue.Work{
Signed: signed,
Verified: verified,
User: user,
Repo: repo,
Build: build,
BuildLast: last,
Job: job,
Keys: key,
Netrc: netrc,
Yaml: string(raw),
YamlEnc: string(sec),
Secrets: secs,
System: &model.System{
Link: httputil.GetURL(c.Request),
Plugins: strings.Split(os.Getenv("PLUGIN_FILTER"), " "),
Globals: strings.Split(os.Getenv("PLUGIN_PARAMS"), " "),
Escalates: strings.Split(os.Getenv("ESCALATE_FILTER"), " "),
},
System: &model.System{Link: httputil.GetURL(c.Request)},
})
}
return // EXIT NOT TO AVOID THE 0.4 ENGINE CODE BELOW

6
client/client.go
View file

@ -18,7 +18,7 @@ import (
)
const (
pathPull = "%s/api/queue/pull"
pathPull = "%s/api/queue/pull/%s/%s"
pathWait = "%s/api/queue/wait/%d"
pathStream = "%s/api/queue/stream/%d"
pathPush = "%s/api/queue/status/%d"
@ -43,9 +43,9 @@ func NewClientToken(uri, token string) Client {
}
// Pull pulls work from the server queue.
func (c *client) Pull() (*queue.Work, error) {
func (c *client) Pull(os, arch string) (*queue.Work, error) {
out := new(queue.Work)
uri := fmt.Sprintf(pathPull, c.base)
uri := fmt.Sprintf(pathPull, c.base, os, arch)
err := c.post(uri, nil, out)
return out, err
}

2
client/interface.go
View file

@ -9,7 +9,7 @@ import (
// Client is used to communicate with a Drone server.
type Client interface {
// Pull pulls work from the server queue.
Pull() (*queue.Work, error)
Pull(os, arch string) (*queue.Work, error)
// Push pushes an update to the server.
Push(*queue.Work) error

59
drone/agent/agent.go
View file

@ -40,6 +40,18 @@ var AgentCmd = cli.Command{
Usage: "limit number of running docker processes",
Value: 2,
},
cli.StringFlag{
EnvVar: "DOCKER_OS",
Name: "docker-os",
Usage: "docker operating system",
Value: "linux",
},
cli.StringFlag{
EnvVar: "DOCKER_ARCH",
Name: "docker-arch",
Usage: "docker architecture system",
Value: "amd64",
},
cli.StringFlag{
EnvVar: "DRONE_SERVER",
Name: "drone-server",
@ -68,16 +80,40 @@ var AgentCmd = cli.Command{
Usage: "start the agent with experimental features",
},
cli.StringSliceFlag{
EnvVar: "DRONE_NETRC_PLUGIN",
EnvVar: "DRONE_PLUGIN_NETRC",
Name: "netrc-plugin",
Usage: "plugins that receive the netrc file",
Value: &cli.StringSlice{"git", "hg"},
},
cli.StringSliceFlag{
EnvVar: "DRONE_PRIVILEGED_PLUGIN",
Name: "privileged-plugin",
EnvVar: "DRONE_PLUGIN_PRIVILEGED",
Name: "privileged",
Usage: "plugins that require privileged mode",
Value: &cli.StringSlice{"docker", "gcr", "ecr"},
Value: &cli.StringSlice{
"plugins/docker",
"plugins/docker:*",
"plguins/gcr",
"plguins/gcr:*",
"plugins/ecr",
"plugins/ecr:*",
},
},
cli.BoolFlag{
EnvVar: "DRONE_PLUGIN_PULL",
Name: "pull",
Usage: "always pull latest plugin images",
},
cli.StringFlag{
EnvVar: "DRONE_PLUGIN_NAMESPACE",
Name: "namespace",
Value: "plugins",
Usage: "default plugin image namespace",
},
cli.StringSliceFlag{
EnvVar: "DRONE_PLUGIN_WHITELIST",
Name: "whitelist",
Usage: "plugins that are permitted to run on the host",
Value: &cli.StringSlice{"plugins/*"},
},
},
}
@ -109,10 +145,21 @@ func start(c *cli.Context) {
for i := 0; i < c.Int("docker-max-procs"); i++ {
wg.Add(1)
go func() {
r := pipeline{
drone: client,
docker: docker,
config: config{
whitelist: c.StringSlice("whitelist"),
namespace: c.String("namespace"),
privileged: c.StringSlice("privileged"),
netrc: c.StringSlice("netrc-plugin"),
pull: c.Bool("pull"),
},
}
for {
if err := recoverExec(client, docker); err != nil {
if err := r.run(); err != nil {
dur := c.Duration("backoff")
logrus.Debugf("Attempting to reconnect in %v", dur)
logrus.Warnf("Attempting to reconnect in %v", dur)
time.Sleep(dur)
}
}

69
drone/agent/exec.go
View file

@ -14,7 +14,7 @@ import (
"github.com/drone/drone/engine/compiler"
"github.com/drone/drone/engine/compiler/builtin"
"github.com/drone/drone/engine/runner"
engine "github.com/drone/drone/engine/runner/docker"
"github.com/drone/drone/engine/runner/docker"
"github.com/drone/drone/model"
"github.com/drone/drone/queue"
"github.com/drone/drone/yaml/expander"
@ -23,15 +23,23 @@ import (
"golang.org/x/net/context"
)
func recoverExec(client client.Client, docker dockerclient.Client) error {
defer func() {
recover()
}()
return exec(client, docker)
type config struct {
platform string
namespace string
whitelist []string
privileged []string
netrc []string
pull bool
}
func exec(client client.Client, docker dockerclient.Client) error {
w, err := client.Pull()
type pipeline struct {
drone client.Client
docker dockerclient.Client
config config
}
func (r *pipeline) run() error {
w, err := r.drone.Pull("linux", "amd64")
if err != nil {
return err
}
@ -46,23 +54,34 @@ func exec(client client.Client, docker dockerclient.Client) error {
envs := toEnv(w)
w.Yaml = expander.ExpandString(w.Yaml, envs)
if w.Verified {
}
if w.Signed {
}
// inject the netrc file into the clone plugin if the repositroy is
// private and requires authentication.
var secrets []*model.Secret
if w.Verified {
secrets = append(secrets, w.Secrets...)
}
if w.Repo.IsPrivate {
w.Secrets = append(w.Secrets, &model.Secret{
secrets = append(secrets, &model.Secret{
Name: "DRONE_NETRC_USERNAME",
Value: w.Netrc.Login,
Images: []string{"git", "hg"}, // TODO(bradrydzewski) use the command line parameters here
Events: []string{model.EventDeploy, model.EventPull, model.EventPush, model.EventTag},
})
w.Secrets = append(w.Secrets, &model.Secret{
secrets = append(secrets, &model.Secret{
Name: "DRONE_NETRC_PASSWORD",
Value: w.Netrc.Password,
Images: []string{w.Repo.Kind},
Images: []string{"git", "hg"},
Events: []string{model.EventDeploy, model.EventPull, model.EventPush, model.EventTag},
})
w.Secrets = append(w.Secrets, &model.Secret{
secrets = append(secrets, &model.Secret{
Name: "DRONE_NETRC_MACHINE",
Value: w.Netrc.Machine,
Images: []string{"git", "hg"},
@ -71,25 +90,26 @@ func exec(client client.Client, docker dockerclient.Client) error {
}
trans := []compiler.Transform{
builtin.NewCloneOp("plugins/"+w.Repo.Kind+":latest", true),
builtin.NewCloneOp("plugins/git:latest", true),
builtin.NewCacheOp(
"plugins/cache:latest",
"/var/lib/drone/cache/"+w.Repo.FullName,
false,
),
builtin.NewSecretOp(w.Build.Event, w.Secrets),
builtin.NewNormalizeOp("plugins"),
builtin.NewWorkspaceOp("/drone", "drone/src/github.com/"+w.Repo.FullName),
builtin.NewSecretOp(w.Build.Event, secrets),
builtin.NewNormalizeOp(r.config.namespace),
builtin.NewWorkspaceOp("/drone", "/drone/src/github.com/"+w.Repo.FullName),
builtin.NewValidateOp(
w.Repo.IsTrusted,
[]string{"plugins/*"},
r.config.whitelist,
),
builtin.NewEnvOp(envs),
builtin.NewShellOp(builtin.Linux_adm64),
builtin.NewArgsOp(),
builtin.NewEscalateOp(r.config.privileged),
builtin.NewPodOp(prefix),
builtin.NewAliasOp(prefix),
builtin.NewPullOp(false),
builtin.NewPullOp(r.config.pull),
builtin.NewFilterOp(
model.StatusSuccess, // TODO(bradrydzewski) please add the last build status here
w.Build.Branch,
@ -109,14 +129,14 @@ func exec(client client.Client, docker dockerclient.Client) error {
return err
}
if err := client.Push(w); err != nil {
if err := r.drone.Push(w); err != nil {
logrus.Errorf("Error persisting update %s/%s#%d.%d. %s",
w.Repo.Owner, w.Repo.Name, w.Build.Number, w.Job.Number, err)
return err
}
conf := runner.Config{
Engine: engine.New(docker),
Engine: docker.New(r.docker),
}
ctx := context.TODO()
@ -126,7 +146,7 @@ func exec(client client.Client, docker dockerclient.Client) error {
run.Run()
defer cancel()
wait := client.Wait(w.Job.ID)
wait := r.drone.Wait(w.Job.ID)
if err != nil {
return err
}
@ -142,7 +162,7 @@ func exec(client client.Client, docker dockerclient.Client) error {
rc, wc := io.Pipe()
go func() {
err := client.Stream(w.Job.ID, rc)
err := r.drone.Stream(w.Job.ID, rc)
if err != nil && err != io.ErrClosedPipe {
logrus.Errorf("Error streaming build logs. %s", err)
}
@ -187,7 +207,7 @@ func exec(client client.Client, docker dockerclient.Client) error {
logrus.Infof("Finished build %s/%s#%d.%d",
w.Repo.Owner, w.Repo.Name, w.Build.Number, w.Job.Number)
return client.Push(w)
return r.drone.Push(w)
}
func toEnv(w *queue.Work) map[string]string {
@ -218,7 +238,8 @@ func toEnv(w *queue.Work) map[string]string {
"DRONE_BUILD_CREATED": fmt.Sprintf("%d", w.Build.Created),
"DRONE_BUILD_STARTED": fmt.Sprintf("%d", w.Build.Started),
"DRONE_BUILD_FINISHED": fmt.Sprintf("%d", w.Build.Finished),
"DRONE_BUILD_VERIFIED": fmt.Sprintf("%v", false),
"DRONE_YAML_VERIFIED": fmt.Sprintf("%v", w.Verified),
"DRONE_YAML_SIGNED": fmt.Sprintf("%v", w.Signed),
// SHORTER ALIASES
"DRONE_BRANCH": w.Build.Branch,

30
engine/compiler/builtin/escalate.go Normal file
View file

@ -0,0 +1,30 @@
package builtin
import (
"path/filepath"
"github.com/drone/drone/engine/compiler/parse"
)
type escalateOp struct {
visitor
plugins []string
}
// NewEscalateOp returns a transformer that configures plugins to automatically
// execute in privileged mode. This is intended for plugins running dind.
func NewEscalateOp(plugins []string) Visitor {
return &escalateOp{
plugins: plugins,
}
}
func (v *escalateOp) VisitContainer(node *parse.ContainerNode) error {
for _, pattern := range v.plugins {
ok, _ := filepath.Match(pattern, node.Container.Image)
if ok {
node.Container.Privileged = true
}
}
return nil
}

54
engine/compiler/builtin/escalate_test.go Normal file
View file

@ -0,0 +1,54 @@
package builtin
import (
"testing"
"github.com/drone/drone/engine/compiler/parse"
"github.com/drone/drone/engine/runner"
"github.com/franela/goblin"
)
func Test_escalate(t *testing.T) {
root := parse.NewRootNode()
g := goblin.Goblin(t)
g.Describe("privileged transform", func() {
g.It("should handle matches", func() {
c := root.NewPluginNode()
c.Container = runner.Container{Image: "plugins/docker"}
op := NewEscalateOp([]string{"plugins/docker"})
op.VisitContainer(c)
g.Assert(c.Container.Privileged).IsTrue()
})
g.It("should handle glob matches", func() {
c := root.NewPluginNode()
c.Container = runner.Container{Image: "plugins/docker"}
op := NewEscalateOp([]string{"plugins/*"})
op.VisitContainer(c)
g.Assert(c.Container.Privileged).IsTrue()
})
g.It("should handle non matches", func() {
c := root.NewPluginNode()
c.Container = runner.Container{Image: "plugins/git"}
op := NewEscalateOp([]string{"plugins/docker"})
op.VisitContainer(c)
g.Assert(c.Container.Privileged).IsFalse()
})
g.It("should handle non glob matches", func() {
c := root.NewPluginNode()
c.Container = runner.Container{Image: "plugins/docker:develop"}
op := NewEscalateOp([]string{"plugins/docker"})
op.VisitContainer(c)
g.Assert(c.Container.Privileged).IsFalse()
})
})
}

3
engine/compiler/builtin/normalize.go
View file

@ -43,6 +43,9 @@ func (v *normalizeOp) normalizePlugin(node *parse.ContainerNode) {
if strings.Contains(node.Container.Image, "/") {
return
}
if strings.Contains(node.Container.Image, "_") {
node.Container.Image = strings.Replace(node.Container.Image, "_", "-", -1)
}
node.Container.Image = filepath.Join(v.namespace, node.Container.Image)
}

9
engine/compiler/builtin/normalize_test.go
View file

@ -56,6 +56,15 @@ func Test_normalize(t *testing.T) {
g.Assert(c.Container.Image).Equal("index.docker.io/drone/git:latest")
})
g.It("should replace underscores with dashes", func() {
c := root.NewPluginNode()
c.Container = runner.Container{Image: "gh_pages"}
op := NewNormalizeOp("plugins")
op.VisitContainer(c)
g.Assert(c.Container.Image).Equal("plugins/gh-pages:latest")
})
g.It("should ignore shell or service types", func() {
c := root.NewShellNode()
c.Container = runner.Container{Image: "golang"}

2
queue/types.go
View file

@ -5,6 +5,8 @@ import "github.com/drone/drone/model"
// Work represents an item for work to be
// processed by a worker.
type Work struct {
Signed bool `json:"signed"`
Verified bool `json:"verified"`
Yaml string `json:"config"`
YamlEnc string `json:"secret"`
Repo *model.Repo `json:"repo"`

45
router/middleware/agent.go Normal file
View file

@ -0,0 +1,45 @@
package middleware
import (
"github.com/drone/drone/shared/token"
"github.com/Sirupsen/logrus"
"github.com/gin-gonic/gin"
"github.com/ianschenck/envflag"
)
var (
secret = envflag.String("AGENT_SECRET", "", "")
noauth = envflag.Bool("AGENT_NO_AUTH", false, "")
)
// Agent is a middleware function that initializes the authorization middleware
// for agents to connect to the queue.
func AgentMust() gin.HandlerFunc {
if *secret == "" {
logrus.Fatalf("please provide the agent secret to authenticate agent requests")
}
t := token.New(token.AgentToken, "")
s, err := t.Sign(*secret)
if err != nil {
logrus.Fatalf("invalid agent secret. %s", err)
}
logrus.Infof("using agent secret %s", *secret)
logrus.Warnf("agents can connect with token %s", s)
return func(c *gin.Context) {
parsed, err := token.ParseRequest(c.Request, func(t *token.Token) (string, error) {
return *secret, nil
})
if err != nil {
c.AbortWithError(403, err)
} else if parsed.Kind != token.AgentToken {
c.AbortWithStatus(403)
} else {
c.Next()
}
}
}

14
router/middleware/session/user.go
View file

@ -70,15 +70,14 @@ func MustAdmin() gin.HandlerFunc {
user := User(c)
switch {
case user == nil:
c.AbortWithStatus(http.StatusUnauthorized)
// c.HTML(http.StatusUnauthorized, "401.html", gin.H{})
c.String(401, "User not authorized")
c.Abort()
case user.Admin == false:
c.AbortWithStatus(http.StatusForbidden)
// c.HTML(http.StatusForbidden, "401.html", gin.H{})
c.String(413, "User not authorized")
c.Abort()
default:
c.Next()
}
}
}
@ -87,11 +86,10 @@ func MustUser() gin.HandlerFunc {
user := User(c)
switch {
case user == nil:
c.AbortWithStatus(http.StatusUnauthorized)
// c.HTML(http.StatusUnauthorized, "401.html", gin.H{})
c.String(401, "User not authorized")
c.Abort()
default:
c.Next()
}
}
}

7
router/router.go
View file

@ -8,6 +8,7 @@ import (
"github.com/gin-gonic/gin"
"github.com/drone/drone/api"
"github.com/drone/drone/router/middleware"
"github.com/drone/drone/router/middleware/header"
"github.com/drone/drone/router/middleware/session"
"github.com/drone/drone/router/middleware/token"
@ -16,7 +17,7 @@ import (
"github.com/drone/drone/web"
)
func Load(middleware ...gin.HandlerFunc) http.Handler {
func Load(middlewares ...gin.HandlerFunc) http.Handler {
e := gin.New()
e.Use(gin.Recovery())
@ -26,7 +27,7 @@ func Load(middleware ...gin.HandlerFunc) http.Handler {
e.Use(header.NoCache)
e.Use(header.Options)
e.Use(header.Secure)
e.Use(middleware...)
e.Use(middlewares...)
e.Use(session.SetUser())
e.Use(token.Refresh)
@ -163,7 +164,9 @@ func Load(middleware ...gin.HandlerFunc) http.Handler {
queue := e.Group("/api/queue")
{
queue.Use(middleware.AgentMust())
queue.POST("/pull", api.Pull)
queue.POST("/pull/:os/:arch", api.Pull)
queue.POST("/wait/:id", api.Wait)
queue.POST("/stream/:id", api.Stream)
queue.POST("/status/:id", api.Update)

9
shared/token/token.go
View file

@ -10,10 +10,11 @@ import (
type SecretFunc func(*Token) (string, error)
const (
UserToken = "user"
SessToken = "sess"
HookToken = "hook"
CsrfToken = "csrf"
UserToken = "user"
SessToken = "sess"
HookToken = "hook"
CsrfToken = "csrf"
AgentToken = "agent"
)
// Default algorithm used to sign JWT tokens.

28
web/hook.go
View file

@ -8,6 +8,7 @@ import (
"strings"
"github.com/gin-gonic/gin"
"github.com/square/go-jose"
log "github.com/Sirupsen/logrus"
"github.com/drone/drone/bus"
@ -31,6 +32,9 @@ func init() {
droneYml = ".drone.yml"
}
droneSec = fmt.Sprintf("%s.sec", strings.TrimSuffix(droneYml, filepath.Ext(droneYml)))
if os.Getenv("CANARY") == "true" {
droneSec = fmt.Sprintf("%s.sig", strings.TrimSuffix(droneYml, filepath.Ext(droneYml)))
}
}
var skipRe = regexp.MustCompile(`\[(?i:ci *skip|skip *ci)\]`)
@ -214,25 +218,33 @@ func PostHook(c *gin.Context) {
// enabled using with the environment variable CANARY=true
if os.Getenv("CANARY") == "true" {
var signed bool
var verified bool
signature, err := jose.ParseSigned(string(sec))
if err == nil && len(sec) != 0 {
signed = true
output, err := signature.Verify(repo.Hash)
if err == nil && string(output) == string(raw) {
verified = true
}
}
bus.Publish(c, bus.NewBuildEvent(bus.Enqueued, repo, build))
for _, job := range jobs {
queue.Publish(c, &queue.Work{
Signed: signed,
Verified: verified,
User: user,
Repo: repo,
Build: build,
BuildLast: last,
Job: job,
Keys: key,
Netrc: netrc,
Yaml: string(raw),
YamlEnc: string(sec),
Secrets: secs,
System: &model.System{
Link: httputil.GetURL(c.Request),
Plugins: strings.Split(os.Getenv("PLUGIN_FILTER"), " "),
Globals: strings.Split(os.Getenv("PLUGIN_PARAMS"), " "),
Escalates: strings.Split(os.Getenv("ESCALATE_FILTER"), " "),
},
System: &model.System{Link: httputil.GetURL(c.Request)},
})
}
return // EXIT NOT TO AVOID THE 0.4 ENGINE CODE BELOW