woodpecker/server/hook.go

545 lines
13 KiB
Go
Raw Normal View History

2016-05-02 19:21:25 +00:00
package server
2015-04-08 22:43:59 +00:00
import (
2017-03-16 10:14:02 +00:00
"context"
2017-05-05 16:59:37 +00:00
"crypto/sha256"
2017-03-16 10:14:02 +00:00
"encoding/json"
2015-09-30 01:21:17 +00:00
"fmt"
2017-05-12 13:02:24 +00:00
"math/rand"
2017-03-16 10:14:02 +00:00
"regexp"
"strconv"
2017-03-16 10:14:02 +00:00
"time"
"github.com/gin-gonic/gin"
2015-04-08 22:43:59 +00:00
2017-03-16 10:14:02 +00:00
"github.com/Sirupsen/logrus"
2015-09-30 01:21:17 +00:00
"github.com/drone/drone/model"
"github.com/drone/drone/remote"
2015-09-30 01:21:17 +00:00
"github.com/drone/drone/shared/httputil"
"github.com/drone/drone/shared/token"
"github.com/drone/drone/store"
2017-03-16 10:14:02 +00:00
"github.com/drone/envsubst"
"github.com/cncd/pipeline/pipeline/backend"
"github.com/cncd/pipeline/pipeline/frontend"
"github.com/cncd/pipeline/pipeline/frontend/yaml"
"github.com/cncd/pipeline/pipeline/frontend/yaml/compiler"
"github.com/cncd/pipeline/pipeline/frontend/yaml/linter"
"github.com/cncd/pipeline/pipeline/frontend/yaml/matrix"
"github.com/cncd/pipeline/pipeline/rpc"
"github.com/cncd/pubsub"
"github.com/cncd/queue"
2015-04-08 22:43:59 +00:00
)
2017-03-16 10:14:02 +00:00
//
// CANARY IMPLEMENTATION
//
// This file is a complete disaster because I'm trying to wedge in some
// experimental code. Please pardon our appearance during renovations.
//
var skipRe = regexp.MustCompile(`\[(?i:ci *skip|skip *ci)\]`)
2017-05-12 13:02:24 +00:00
func init() {
rand.Seed(time.Now().UnixNano())
}
2017-03-16 10:14:02 +00:00
func GetQueueInfo(c *gin.Context) {
c.IndentedJSON(200,
Config.Services.Queue.Info(c),
2017-03-16 10:14:02 +00:00
)
}
2015-04-08 22:43:59 +00:00
func PostHook(c *gin.Context) {
remote_ := remote.FromContext(c)
2015-04-08 22:43:59 +00:00
tmprepo, build, err := remote_.Hook(c.Request)
2015-04-08 22:43:59 +00:00
if err != nil {
2017-03-16 10:14:02 +00:00
logrus.Errorf("failure to parse hook. %s", err)
2015-09-30 01:21:17 +00:00
c.AbortWithError(400, err)
2015-04-08 22:43:59 +00:00
return
}
2015-09-30 01:21:17 +00:00
if build == nil {
2015-04-08 22:43:59 +00:00
c.Writer.WriteHeader(200)
return
}
2015-09-30 01:21:17 +00:00
if tmprepo == nil {
2017-03-16 10:14:02 +00:00
logrus.Errorf("failure to ascertain repo from hook.")
2015-04-08 22:43:59 +00:00
c.Writer.WriteHeader(400)
return
}
// skip the build if any case-insensitive combination of the words "skip" and "ci"
// wrapped in square brackets appear in the commit message
skipMatch := skipRe.FindString(build.Message)
if len(skipMatch) > 0 {
2017-03-16 10:14:02 +00:00
logrus.Infof("ignoring hook. %s found in %s", skipMatch, build.Commit)
2015-04-08 22:43:59 +00:00
c.Writer.WriteHeader(204)
return
}
repo, err := store.GetRepoOwnerName(c, tmprepo.Owner, tmprepo.Name)
2015-04-08 22:43:59 +00:00
if err != nil {
2017-03-16 10:14:02 +00:00
logrus.Errorf("failure to find repo %s/%s from hook. %s", tmprepo.Owner, tmprepo.Name, err)
2015-09-30 01:21:17 +00:00
c.AbortWithError(404, err)
2015-04-08 22:43:59 +00:00
return
}
// get the token and verify the hook is authorized
parsed, err := token.ParseRequest(c.Request, func(t *token.Token) (string, error) {
return repo.Hash, nil
})
if err != nil {
2017-03-16 10:14:02 +00:00
logrus.Errorf("failure to parse token from hook for %s. %s", repo.FullName, err)
2015-09-30 01:21:17 +00:00
c.AbortWithError(400, err)
return
}
if parsed.Text != repo.FullName {
2017-03-16 10:14:02 +00:00
logrus.Errorf("failure to verify token from hook. Expected %s, got %s", repo.FullName, parsed.Text)
c.AbortWithStatus(403)
return
}
2015-09-30 01:21:17 +00:00
if repo.UserID == 0 {
2017-03-16 10:14:02 +00:00
logrus.Warnf("ignoring hook. repo %s has no owner.", repo.FullName)
c.Writer.WriteHeader(204)
return
2015-09-30 01:21:17 +00:00
}
var skipped = true
if (build.Event == model.EventPush && repo.AllowPush) ||
(build.Event == model.EventPull && repo.AllowPull) ||
(build.Event == model.EventDeploy && repo.AllowDeploy) ||
(build.Event == model.EventTag && repo.AllowTag) {
skipped = false
}
if skipped {
2017-03-16 10:14:02 +00:00
logrus.Infof("ignoring hook. repo %s is disabled for %s events.", repo.FullName, build.Event)
2015-04-08 22:43:59 +00:00
c.Writer.WriteHeader(204)
return
}
user, err := store.GetUser(c, repo.UserID)
2015-04-08 22:43:59 +00:00
if err != nil {
2017-03-16 10:14:02 +00:00
logrus.Errorf("failure to find repo owner %s. %s", repo.FullName, err)
2015-09-30 01:21:17 +00:00
c.AbortWithError(500, err)
2015-04-08 22:43:59 +00:00
return
}
// if the remote has a refresh token, the current access token
// may be stale. Therefore, we should refresh prior to dispatching
2017-04-02 14:13:26 +00:00
// the build.
if refresher, ok := remote_.(remote.Refresher); ok {
ok, _ := refresher.Refresh(user)
if ok {
store.UpdateUser(c, user)
}
}
// fetch the build file from the database
2017-05-05 16:59:37 +00:00
confb, err := remote_.File(user, repo, build, repo.Config)
2015-04-08 22:43:59 +00:00
if err != nil {
2017-03-16 10:14:02 +00:00
logrus.Errorf("failure to get build config for %s. %s", repo.FullName, err)
2015-09-30 01:21:17 +00:00
c.AbortWithError(404, err)
2015-04-08 22:43:59 +00:00
return
}
2017-05-05 16:59:37 +00:00
sha := shasum(confb)
conf, err := Config.Storage.Config.ConfigFind(repo, sha)
if err != nil {
conf = &model.Config{
2017-05-05 18:05:42 +00:00
RepoID: repo.ID,
Data: string(confb),
Hash: sha,
2017-05-05 16:59:37 +00:00
}
2017-05-05 18:05:42 +00:00
err = Config.Storage.Config.ConfigCreate(conf)
2017-05-05 16:59:37 +00:00
if err != nil {
logrus.Errorf("failure to persist config for %s. %s", repo.FullName, err)
c.AbortWithError(500, err)
return
}
}
build.ConfigID = conf.ID
2015-04-08 22:43:59 +00:00
netrc, err := remote_.Netrc(user, repo)
2015-04-28 22:08:21 +00:00
if err != nil {
c.String(500, "Failed to generate netrc file. %s", err)
2015-04-28 22:08:21 +00:00
return
}
2015-04-08 22:43:59 +00:00
// verify the branches can be built vs skipped
2017-05-05 16:59:37 +00:00
branches, err := yaml.ParseString(conf.Data)
2017-03-16 11:00:56 +00:00
if err == nil {
if !branches.Branches.Match(build.Branch) && build.Event != model.EventTag && build.Event != model.EventDeploy {
c.String(200, "Branch does not match restrictions defined in yaml")
return
}
2015-04-30 17:39:16 +00:00
}
2015-04-08 22:43:59 +00:00
2015-09-30 01:21:17 +00:00
// update some build fields
build.RepoID = repo.ID
2017-03-18 11:25:53 +00:00
build.Verified = true
build.Status = model.StatusPending
2015-09-30 01:21:17 +00:00
2017-05-05 17:13:40 +00:00
if repo.IsGated {
allowed, _ := Config.Services.Senders.SenderAllowed(user, repo, build, conf)
if !allowed {
build.Status = model.StatusBlocked
}
}
2017-05-14 17:57:38 +00:00
build.Trim()
err = store.CreateBuild(c, build, build.Procs...)
if err != nil {
2017-03-16 10:14:02 +00:00
logrus.Errorf("failure to save commit for %s. %s", repo.FullName, err)
2015-09-30 01:21:17 +00:00
c.AbortWithError(500, err)
2015-04-24 21:25:03 +00:00
return
}
c.JSON(200, build)
if build.Status == model.StatusBlocked {
return
}
2017-03-18 11:25:53 +00:00
2017-05-19 21:36:08 +00:00
secs, err := Config.Services.Secrets.SecretListBuild(repo, build)
if err != nil {
logrus.Debugf("Error getting secrets for %s#%d. %s", repo.FullName, build.Number, err)
}
regs, err := Config.Services.Registries.RegistryList(repo)
if err != nil {
logrus.Debugf("Error getting registry credentials for %s#%d. %s", repo.FullName, build.Number, err)
}
// get the previous build so that we can send
// on status change notifications
last, _ := store.GetBuildLastBefore(c, repo, build.Branch, build.ID)
2017-03-16 10:14:02 +00:00
//
// BELOW: NEW
//
defer func() {
uri := fmt.Sprintf("%s/%s/%d", httputil.GetURL(c.Request), repo.FullName, build.Number)
err = remote_.Status(user, repo, build, uri)
if err != nil {
logrus.Errorf("error setting commit status for %s/%d", repo.FullName, build.Number)
}
}()
b := builder{
Repo: repo,
Curr: build,
Last: last,
Netrc: netrc,
Secs: secs,
2017-04-06 16:04:25 +00:00
Regs: regs,
2017-03-16 10:14:02 +00:00
Link: httputil.GetURL(c.Request),
2017-05-05 16:59:37 +00:00
Yaml: conf.Data,
2017-03-16 10:14:02 +00:00
}
items, err := b.Build()
if err != nil {
build.Status = model.StatusError
build.Started = time.Now().Unix()
build.Finished = build.Started
build.Error = err.Error()
2017-03-16 11:00:56 +00:00
store.UpdateBuild(c, build)
2017-03-16 10:14:02 +00:00
return
}
2017-04-02 14:13:26 +00:00
var pcounter = len(items)
2017-03-16 10:14:02 +00:00
for _, item := range items {
2017-04-02 14:13:26 +00:00
build.Procs = append(build.Procs, item.Proc)
item.Proc.BuildID = build.ID
for _, stage := range item.Config.Stages {
var gid int
for _, step := range stage.Steps {
pcounter++
if gid == 0 {
gid = pcounter
}
proc := &model.Proc{
BuildID: build.ID,
Name: step.Alias,
PID: pcounter,
PPID: item.Proc.PID,
PGID: gid,
State: model.StatusPending,
}
build.Procs = append(build.Procs, proc)
}
}
2017-03-16 10:14:02 +00:00
}
2017-04-04 09:30:06 +00:00
err = store.FromContext(c).ProcCreate(build.Procs)
if err != nil {
logrus.Errorf("error persisting procs %s/%d: %s", repo.FullName, build.Number, err)
}
2017-03-16 10:14:02 +00:00
//
// publish topic
//
message := pubsub.Message{
Labels: map[string]string{
"repo": repo.FullName,
"private": strconv.FormatBool(repo.IsPrivate),
},
}
2017-04-04 09:30:06 +00:00
buildCopy := *build
buildCopy.Procs = model.Tree(buildCopy.Procs)
2017-03-16 10:14:02 +00:00
message.Data, _ = json.Marshal(model.Event{
2016-09-28 01:30:28 +00:00
Type: model.Enqueued,
Repo: *repo,
2017-04-04 09:30:06 +00:00
Build: buildCopy,
2017-03-16 10:14:02 +00:00
})
// TODO remove global reference
Config.Services.Pubsub.Publish(c, "topic/events", message)
2017-03-16 10:14:02 +00:00
//
// end publish topic
//
2017-03-16 10:14:02 +00:00
for _, item := range items {
task := new(queue.Task)
2017-04-02 14:13:26 +00:00
task.ID = fmt.Sprint(item.Proc.ID)
2017-03-16 10:14:02 +00:00
task.Labels = map[string]string{}
task.Labels["platform"] = item.Platform
for k, v := range item.Labels {
task.Labels[k] = v
}
task.Data, _ = json.Marshal(rpc.Pipeline{
2017-04-02 14:13:26 +00:00
ID: fmt.Sprint(item.Proc.ID),
2017-03-16 10:14:02 +00:00
Config: item.Config,
Timeout: b.Repo.Timeout,
})
Config.Services.Logs.Open(context.Background(), task.ID)
Config.Services.Queue.Push(context.Background(), task)
}
2017-03-05 07:56:08 +00:00
}
2017-03-16 11:00:56 +00:00
// return the metadata from the cli context.
2017-04-02 14:13:26 +00:00
func metadataFromStruct(repo *model.Repo, build, last *model.Build, proc *model.Proc, link string) frontend.Metadata {
2017-03-16 11:00:56 +00:00
return frontend.Metadata{
Repo: frontend.Repo{
2017-04-21 15:31:13 +00:00
Name: repo.FullName,
2017-03-16 11:00:56 +00:00
Link: repo.Link,
Remote: repo.Clone,
Private: repo.IsPrivate,
},
Curr: frontend.Build{
Number: build.Number,
2017-05-12 19:31:08 +00:00
Parent: build.Parent,
2017-03-16 11:00:56 +00:00
Created: build.Created,
Started: build.Started,
Finished: build.Finished,
Status: build.Status,
Event: build.Event,
Link: build.Link,
Target: build.Deploy,
Commit: frontend.Commit{
Sha: build.Commit,
Ref: build.Ref,
Refspec: build.Refspec,
Branch: build.Branch,
Message: build.Message,
Author: frontend.Author{
Name: build.Author,
Email: build.Email,
Avatar: build.Avatar,
},
},
},
Prev: frontend.Build{
Number: last.Number,
Created: last.Created,
Started: last.Started,
Finished: last.Finished,
Status: last.Status,
Event: last.Event,
Link: last.Link,
Target: last.Deploy,
Commit: frontend.Commit{
Sha: last.Commit,
Ref: last.Ref,
Refspec: last.Refspec,
Branch: last.Branch,
Message: last.Message,
Author: frontend.Author{
Name: last.Author,
Email: last.Email,
Avatar: last.Avatar,
},
},
},
Job: frontend.Job{
2017-04-02 14:13:26 +00:00
Number: proc.PID,
Matrix: proc.Environ,
2017-03-16 11:00:56 +00:00
},
Sys: frontend.System{
Name: "drone",
Link: link,
Arch: "linux/amd64",
},
}
}
type builder struct {
Repo *model.Repo
Curr *model.Build
Last *model.Build
Netrc *model.Netrc
Secs []*model.Secret
2017-04-06 16:04:25 +00:00
Regs []*model.Registry
2017-03-16 11:00:56 +00:00
Link string
Yaml string
Envs map[string]string
2017-03-16 11:00:56 +00:00
}
type buildItem struct {
2017-04-02 14:13:26 +00:00
Proc *model.Proc
2017-03-16 11:00:56 +00:00
Platform string
Labels map[string]string
Config *backend.Config
}
func (b *builder) Build() ([]*buildItem, error) {
axes, err := matrix.ParseString(b.Yaml)
if err != nil {
return nil, err
}
if len(axes) == 0 {
axes = append(axes, matrix.Axis{})
}
var items []*buildItem
for i, axis := range axes {
2017-04-02 14:13:26 +00:00
proc := &model.Proc{
BuildID: b.Curr.ID,
PID: i + 1,
PGID: i + 1,
State: model.StatusPending,
Environ: axis,
2017-03-16 11:00:56 +00:00
}
2017-04-02 14:13:26 +00:00
metadata := metadataFromStruct(b.Repo, b.Curr, b.Last, proc, b.Link)
2017-03-16 11:00:56 +00:00
environ := metadata.Environ()
for k, v := range metadata.EnvironDrone() {
environ[k] = v
}
2017-03-21 14:30:01 +00:00
for k, v := range axis {
environ[k] = v
}
var secrets []compiler.Secret
2017-03-16 11:00:56 +00:00
for _, sec := range b.Secs {
if !sec.Match(b.Curr.Event) {
2017-03-16 11:00:56 +00:00
continue
}
2017-04-10 16:27:34 +00:00
secrets = append(secrets, compiler.Secret{
Name: sec.Name,
Value: sec.Value,
Match: sec.Images,
})
2017-03-16 11:00:56 +00:00
}
y := b.Yaml
s, err := envsubst.Eval(y, func(name string) string {
return environ[name]
})
2017-03-21 14:30:01 +00:00
if err != nil {
return nil, err
2017-03-16 11:00:56 +00:00
}
2017-03-21 14:30:01 +00:00
y = s
2017-03-16 11:00:56 +00:00
parsed, err := yaml.ParseString(y)
if err != nil {
return nil, err
}
metadata.Sys.Arch = parsed.Platform
if metadata.Sys.Arch == "" {
metadata.Sys.Arch = "linux/amd64"
}
lerr := linter.New(
linter.WithTrusted(b.Repo.IsTrusted),
).Lint(parsed)
if lerr != nil {
2017-05-05 16:59:37 +00:00
return nil, lerr
2017-03-16 11:00:56 +00:00
}
2017-04-06 16:04:25 +00:00
var registries []compiler.Registry
for _, reg := range b.Regs {
registries = append(registries, compiler.Registry{
2017-05-05 14:40:54 +00:00
Hostname: reg.Address,
2017-04-06 16:04:25 +00:00
Username: reg.Username,
Password: reg.Password,
Email: reg.Email,
})
}
2017-03-16 11:00:56 +00:00
ir := compiler.New(
compiler.WithEnviron(environ),
compiler.WithEnviron(b.Envs),
compiler.WithEscalated(Config.Pipeline.Privileged...),
compiler.WithVolumes(Config.Pipeline.Volumes...),
compiler.WithNetworks(Config.Pipeline.Networks...),
2017-03-16 11:00:56 +00:00
compiler.WithLocal(false),
2017-03-19 09:07:21 +00:00
compiler.WithOption(
compiler.WithNetrc(
b.Netrc.Login,
b.Netrc.Password,
b.Netrc.Machine,
),
b.Repo.IsPrivate,
),
2017-04-06 16:04:25 +00:00
compiler.WithRegistry(registries...),
compiler.WithSecret(secrets...),
2017-03-16 11:00:56 +00:00
compiler.WithPrefix(
fmt.Sprintf(
"%d_%d",
2017-04-02 14:13:26 +00:00
proc.ID,
2017-05-12 13:02:24 +00:00
rand.Int(),
2017-03-16 11:00:56 +00:00
),
),
2017-04-02 14:13:26 +00:00
compiler.WithEnviron(proc.Environ),
2017-03-16 11:00:56 +00:00
compiler.WithProxy(),
compiler.WithWorkspaceFromURL("/drone", b.Curr.Link),
2017-04-01 11:17:04 +00:00
compiler.WithMetadata(metadata),
2017-03-16 11:00:56 +00:00
).Compile(parsed)
// for _, sec := range b.Secs {
// if !sec.MatchEvent(b.Curr.Event) {
// continue
// }
// if b.Curr.Verified || sec.SkipVerify {
// ir.Secrets = append(ir.Secrets, &backend.Secret{
// Mask: sec.Conceal,
// Name: sec.Name,
// Value: sec.Value,
// })
// }
// }
2017-03-16 11:00:56 +00:00
item := &buildItem{
2017-04-02 14:13:26 +00:00
Proc: proc,
2017-03-16 11:00:56 +00:00
Config: ir,
Labels: parsed.Labels,
Platform: metadata.Sys.Arch,
}
if item.Labels == nil {
item.Labels = map[string]string{}
}
items = append(items, item)
}
return items, nil
}
2017-05-05 16:59:37 +00:00
func shasum(raw []byte) string {
sum := sha256.Sum256(raw)
return fmt.Sprintf("%x", sum)
}