woodpecker/server/hook.go

428 lines
11 KiB
Go
Raw Normal View History

2018-02-19 22:24:10 +00:00
// Copyright 2018 Drone.IO Inc.
2018-03-21 13:02:17 +00:00
//
2018-02-19 22:24:10 +00:00
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
2018-03-21 13:02:17 +00:00
//
2018-02-19 22:24:10 +00:00
// http://www.apache.org/licenses/LICENSE-2.0
2018-03-21 13:02:17 +00:00
//
2018-02-19 22:24:10 +00:00
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
2016-05-02 19:21:25 +00:00
package server
2015-04-08 22:43:59 +00:00
import (
2017-03-16 10:14:02 +00:00
"context"
2017-05-05 16:59:37 +00:00
"crypto/sha256"
2017-03-16 10:14:02 +00:00
"encoding/json"
2015-09-30 01:21:17 +00:00
"fmt"
2017-05-12 13:02:24 +00:00
"math/rand"
2019-06-28 06:29:57 +00:00
"net/http"
2017-03-16 10:14:02 +00:00
"regexp"
"strconv"
2017-03-16 10:14:02 +00:00
"time"
"github.com/gin-gonic/gin"
2015-04-08 22:43:59 +00:00
2017-03-16 10:14:02 +00:00
"github.com/Sirupsen/logrus"
2019-08-27 11:01:29 +00:00
"github.com/laszlocph/woodpecker/model"
"github.com/laszlocph/woodpecker/remote"
"github.com/laszlocph/woodpecker/shared/httputil"
"github.com/laszlocph/woodpecker/shared/token"
"github.com/laszlocph/woodpecker/store"
2017-03-16 10:14:02 +00:00
2019-08-27 11:01:29 +00:00
"github.com/laszlocph/woodpecker/cncd/pipeline/pipeline/frontend/yaml"
"github.com/laszlocph/woodpecker/cncd/pipeline/pipeline/rpc"
"github.com/laszlocph/woodpecker/cncd/pubsub"
"github.com/laszlocph/woodpecker/cncd/queue"
2015-04-08 22:43:59 +00:00
)
2017-03-16 10:14:02 +00:00
var skipRe = regexp.MustCompile(`\[(?i:ci *skip|skip *ci)\]`)
2017-05-12 13:02:24 +00:00
func init() {
rand.Seed(time.Now().UnixNano())
}
2017-03-16 10:14:02 +00:00
func GetQueueInfo(c *gin.Context) {
c.IndentedJSON(200,
Config.Services.Queue.Info(c),
2017-03-16 10:14:02 +00:00
)
}
2019-06-28 06:29:57 +00:00
func PauseQueue(c *gin.Context) {
Config.Services.Queue.Pause()
c.Status(http.StatusOK)
}
func ResumeQueue(c *gin.Context) {
Config.Services.Queue.Resume()
c.Status(http.StatusOK)
}
func BlockTilQueueHasRunningItem(c *gin.Context) {
for {
info := Config.Services.Queue.Info(c)
if info.Stats.Running == 0 {
break
}
}
c.Status(http.StatusOK)
}
2015-04-08 22:43:59 +00:00
func PostHook(c *gin.Context) {
remote_ := remote.FromContext(c)
2015-04-08 22:43:59 +00:00
tmprepo, build, err := remote_.Hook(c.Request)
2015-04-08 22:43:59 +00:00
if err != nil {
2017-03-16 10:14:02 +00:00
logrus.Errorf("failure to parse hook. %s", err)
2015-09-30 01:21:17 +00:00
c.AbortWithError(400, err)
2015-04-08 22:43:59 +00:00
return
}
2015-09-30 01:21:17 +00:00
if build == nil {
2015-04-08 22:43:59 +00:00
c.Writer.WriteHeader(200)
return
}
2015-09-30 01:21:17 +00:00
if tmprepo == nil {
2017-03-16 10:14:02 +00:00
logrus.Errorf("failure to ascertain repo from hook.")
2015-04-08 22:43:59 +00:00
c.Writer.WriteHeader(400)
return
}
// skip the build if any case-insensitive combination of the words "skip" and "ci"
// wrapped in square brackets appear in the commit message
skipMatch := skipRe.FindString(build.Message)
if len(skipMatch) > 0 {
2017-03-16 10:14:02 +00:00
logrus.Infof("ignoring hook. %s found in %s", skipMatch, build.Commit)
2015-04-08 22:43:59 +00:00
c.Writer.WriteHeader(204)
return
}
repo, err := store.GetRepoOwnerName(c, tmprepo.Owner, tmprepo.Name)
2015-04-08 22:43:59 +00:00
if err != nil {
2017-03-16 10:14:02 +00:00
logrus.Errorf("failure to find repo %s/%s from hook. %s", tmprepo.Owner, tmprepo.Name, err)
2015-09-30 01:21:17 +00:00
c.AbortWithError(404, err)
2015-04-08 22:43:59 +00:00
return
}
2017-07-14 19:58:38 +00:00
if !repo.IsActive {
logrus.Errorf("ignoring hook. %s/%s is inactive.", tmprepo.Owner, tmprepo.Name)
c.AbortWithError(204, err)
return
}
2015-04-08 22:43:59 +00:00
// get the token and verify the hook is authorized
parsed, err := token.ParseRequest(c.Request, func(t *token.Token) (string, error) {
return repo.Hash, nil
})
if err != nil {
2017-03-16 10:14:02 +00:00
logrus.Errorf("failure to parse token from hook for %s. %s", repo.FullName, err)
2015-09-30 01:21:17 +00:00
c.AbortWithError(400, err)
return
}
if parsed.Text != repo.FullName {
2017-03-16 10:14:02 +00:00
logrus.Errorf("failure to verify token from hook. Expected %s, got %s", repo.FullName, parsed.Text)
c.AbortWithStatus(403)
return
}
2015-09-30 01:21:17 +00:00
if repo.UserID == 0 {
2017-03-16 10:14:02 +00:00
logrus.Warnf("ignoring hook. repo %s has no owner.", repo.FullName)
c.Writer.WriteHeader(204)
return
2015-09-30 01:21:17 +00:00
}
var skipped = true
if (build.Event == model.EventPush && repo.AllowPush) ||
(build.Event == model.EventPull && repo.AllowPull) ||
(build.Event == model.EventDeploy && repo.AllowDeploy) ||
(build.Event == model.EventTag && repo.AllowTag) {
skipped = false
}
if skipped {
2017-03-16 10:14:02 +00:00
logrus.Infof("ignoring hook. repo %s is disabled for %s events.", repo.FullName, build.Event)
2015-04-08 22:43:59 +00:00
c.Writer.WriteHeader(204)
return
}
user, err := store.GetUser(c, repo.UserID)
2015-04-08 22:43:59 +00:00
if err != nil {
2017-03-16 10:14:02 +00:00
logrus.Errorf("failure to find repo owner %s. %s", repo.FullName, err)
2015-09-30 01:21:17 +00:00
c.AbortWithError(500, err)
2015-04-08 22:43:59 +00:00
return
}
// if the remote has a refresh token, the current access token
// may be stale. Therefore, we should refresh prior to dispatching
2017-04-02 14:13:26 +00:00
// the build.
if refresher, ok := remote_.(remote.Refresher); ok {
ok, _ := refresher.Refresh(user)
if ok {
store.UpdateUser(c, user)
}
}
2019-06-01 10:52:02 +00:00
// fetch the build file from the remote
configFetcher := &configFetcher{remote_: remote_, user: user, repo: repo, build: build}
remoteYamlConfigs, err := configFetcher.Fetch()
2015-04-08 22:43:59 +00:00
if err != nil {
2017-07-31 19:15:05 +00:00
logrus.Errorf("error: %s: cannot find %s in %s: %s", repo.FullName, repo.Config, build.Ref, err)
2015-09-30 01:21:17 +00:00
c.AbortWithError(404, err)
2015-04-08 22:43:59 +00:00
return
}
2019-09-09 07:53:54 +00:00
filtered, err := branchFiltered(build, remoteYamlConfigs)
if err != nil {
logrus.Errorf("failure to parse yaml from hook for %s. %s", repo.FullName, err)
c.AbortWithError(400, err)
}
if filtered {
2019-06-19 07:36:54 +00:00
c.String(200, "Branch does not match restrictions defined in yaml")
return
}
2019-07-22 11:43:18 +00:00
if zeroSteps(build, remoteYamlConfigs) {
c.String(200, "Step conditions yield zero runnable steps")
return
}
// update some build fields
build.RepoID = repo.ID
build.Verified = true
build.Status = model.StatusPending
2019-07-18 12:21:05 +00:00
if repo.IsGated && build.Sender != user.Login {
build.Status = model.StatusBlocked
}
err = store.CreateBuild(c, build, build.Procs...)
if err != nil {
2017-03-16 10:14:02 +00:00
logrus.Errorf("failure to save commit for %s. %s", repo.FullName, err)
2015-09-30 01:21:17 +00:00
c.AbortWithError(500, err)
2015-04-24 21:25:03 +00:00
return
}
// persist the build config for historical correctness, restarts, etc
for _, remoteYamlConfig := range remoteYamlConfigs {
2019-06-13 15:38:19 +00:00
_, err := findOrPersistPipelineConfig(build, remoteYamlConfig)
if err != nil {
logrus.Errorf("failure to find or persist build config for %s. %s", repo.FullName, err)
c.AbortWithError(500, err)
return
}
}
c.JSON(200, build)
if build.Status == model.StatusBlocked {
return
}
2017-03-18 11:25:53 +00:00
netrc, err := remote_.Netrc(user, repo)
if err != nil {
c.String(500, "Failed to generate netrc file. %s", err)
return
}
2017-06-26 19:27:53 +00:00
envs := map[string]string{}
if Config.Services.Environ != nil {
globals, _ := Config.Services.Environ.EnvironList(repo)
for _, global := range globals {
envs[global.Name] = global.Value
}
}
2017-05-19 21:36:08 +00:00
secs, err := Config.Services.Secrets.SecretListBuild(repo, build)
if err != nil {
logrus.Debugf("Error getting secrets for %s#%d. %s", repo.FullName, build.Number, err)
}
regs, err := Config.Services.Registries.RegistryList(repo)
if err != nil {
logrus.Debugf("Error getting registry credentials for %s#%d. %s", repo.FullName, build.Number, err)
}
// get the previous build so that we can send status change notifications
last, _ := store.GetBuildLastBefore(c, repo, build.Branch, build.ID)
2019-06-01 08:17:02 +00:00
b := procBuilder{
2017-03-16 10:14:02 +00:00
Repo: repo,
Curr: build,
Last: last,
Netrc: netrc,
Secs: secs,
2017-04-06 16:04:25 +00:00
Regs: regs,
2017-06-26 19:27:53 +00:00
Envs: envs,
2017-03-16 10:14:02 +00:00
Link: httputil.GetURL(c.Request),
2019-06-13 15:38:19 +00:00
Yamls: remoteYamlConfigs,
2017-03-16 10:14:02 +00:00
}
2019-06-01 08:08:41 +00:00
buildItems, err := b.Build()
2017-03-16 10:14:02 +00:00
if err != nil {
build.Status = model.StatusError
build.Started = time.Now().Unix()
build.Finished = build.Started
build.Error = err.Error()
2017-03-16 11:00:56 +00:00
store.UpdateBuild(c, build)
2017-03-16 10:14:02 +00:00
return
}
2019-07-22 11:43:18 +00:00
build = setBuildStepsOnBuild(b.Curr, buildItems)
2017-03-16 10:14:02 +00:00
2019-06-01 08:08:41 +00:00
err = store.FromContext(c).ProcCreate(build.Procs)
if err != nil {
logrus.Errorf("error persisting procs %s/%d: %s", repo.FullName, build.Number, err)
}
defer func() {
for _, item := range buildItems {
uri := fmt.Sprintf("%s/%s/%d", httputil.GetURL(c.Request), repo.FullName, build.Number)
if len(buildItems) > 1 {
err = remote_.Status(user, repo, build, uri, item.Proc)
} else {
err = remote_.Status(user, repo, build, uri, nil)
}
if err != nil {
logrus.Errorf("error setting commit status for %s/%d: %v", repo.FullName, build.Number, err)
}
}
}()
2019-06-01 08:08:41 +00:00
publishToTopic(c, build, repo)
queueBuild(build, repo, buildItems)
}
2019-09-09 07:53:54 +00:00
func branchFiltered(build *model.Build, remoteYamlConfigs []*remote.FileMeta) (bool, error) {
2019-06-19 07:36:54 +00:00
for _, remoteYamlConfig := range remoteYamlConfigs {
parsedPipelineConfig, err := yaml.ParseString(string(remoteYamlConfig.Data))
2019-09-09 07:53:54 +00:00
if err != nil {
return false, err
}
if !parsedPipelineConfig.Branches.Match(build.Branch) && build.Event != model.EventTag && build.Event != model.EventDeploy {
} else {
return false, nil
2019-06-19 07:36:54 +00:00
}
}
2019-09-09 07:53:54 +00:00
return true, nil
2019-06-19 07:36:54 +00:00
}
2019-07-22 11:43:18 +00:00
func zeroSteps(build *model.Build, remoteYamlConfigs []*remote.FileMeta) bool {
b := procBuilder{
Repo: &model.Repo{},
2019-07-22 11:43:18 +00:00
Curr: build,
Last: &model.Build{},
Netrc: &model.Netrc{},
Secs: []*model.Secret{},
Regs: []*model.Registry{},
Link: "",
Yamls: remoteYamlConfigs,
}
buildItems, err := b.Build()
if err != nil {
return false
}
if len(buildItems) == 0 {
return true
}
return false
}
2019-06-13 15:38:19 +00:00
func findOrPersistPipelineConfig(build *model.Build, remoteYamlConfig *remote.FileMeta) (*model.Config, error) {
sha := shasum(remoteYamlConfig.Data)
conf, err := Config.Storage.Config.ConfigFindIdentical(build.RepoID, sha)
2019-06-01 10:52:02 +00:00
if err != nil {
conf = &model.Config{
RepoID: build.RepoID,
2019-06-13 15:38:19 +00:00
Data: string(remoteYamlConfig.Data),
2019-06-01 10:52:02 +00:00
Hash: sha,
2019-06-13 15:38:19 +00:00
Name: sanitizePath(remoteYamlConfig.Name),
2019-06-01 10:52:02 +00:00
}
err = Config.Storage.Config.ConfigCreate(conf)
if err != nil {
// retry in case we receive two hooks at the same time
conf, err = Config.Storage.Config.ConfigFindIdentical(build.RepoID, sha)
2019-06-01 10:52:02 +00:00
if err != nil {
return nil, err
}
}
}
buildConfig := &model.BuildConfig{
ConfigID: conf.ID,
BuildID: build.ID,
}
err = Config.Storage.Config.BuildConfigCreate(buildConfig)
if err != nil {
return nil, err
}
2019-06-01 10:52:02 +00:00
return conf, nil
}
2019-06-13 15:38:19 +00:00
// publishes message to UI clients
2019-06-01 08:08:41 +00:00
func publishToTopic(c *gin.Context, build *model.Build, repo *model.Repo) {
2017-03-16 10:14:02 +00:00
message := pubsub.Message{
Labels: map[string]string{
"repo": repo.FullName,
"private": strconv.FormatBool(repo.IsPrivate),
},
}
2017-04-04 09:30:06 +00:00
buildCopy := *build
buildCopy.Procs = model.Tree(buildCopy.Procs)
2017-03-16 10:14:02 +00:00
message.Data, _ = json.Marshal(model.Event{
2016-09-28 01:30:28 +00:00
Type: model.Enqueued,
Repo: *repo,
2017-04-04 09:30:06 +00:00
Build: buildCopy,
2017-03-16 10:14:02 +00:00
})
Config.Services.Pubsub.Publish(c, "topic/events", message)
2019-06-01 08:08:41 +00:00
}
2019-06-01 08:08:41 +00:00
func queueBuild(build *model.Build, repo *model.Repo, buildItems []*buildItem) {
2019-06-13 15:38:19 +00:00
var tasks []*queue.Task
2019-06-01 08:08:41 +00:00
for _, item := range buildItems {
2019-06-19 07:36:54 +00:00
if item.Proc.State == model.StatusSkipped {
continue
}
2017-03-16 10:14:02 +00:00
task := new(queue.Task)
2017-04-02 14:13:26 +00:00
task.ID = fmt.Sprint(item.Proc.ID)
2017-03-16 10:14:02 +00:00
task.Labels = map[string]string{}
for k, v := range item.Labels {
task.Labels[k] = v
}
task.Labels["platform"] = item.Platform
2019-06-01 08:08:41 +00:00
task.Labels["repo"] = repo.FullName
2019-06-13 15:38:19 +00:00
task.Dependencies = taskIds(item.DependsOn, buildItems)
task.RunOn = item.RunsOn
task.DepStatus = make(map[string]string)
2017-03-16 10:14:02 +00:00
task.Data, _ = json.Marshal(rpc.Pipeline{
2017-04-02 14:13:26 +00:00
ID: fmt.Sprint(item.Proc.ID),
2017-03-16 10:14:02 +00:00
Config: item.Config,
2019-06-01 08:08:41 +00:00
Timeout: repo.Timeout,
2017-03-16 10:14:02 +00:00
})
Config.Services.Logs.Open(context.Background(), task.ID)
2019-06-13 15:38:19 +00:00
tasks = append(tasks, task)
}
Config.Services.Queue.PushAtOnce(context.Background(), tasks)
}
func taskIds(dependsOn []string, buildItems []*buildItem) []string {
taskIds := []string{}
for _, dep := range dependsOn {
for _, buildItem := range buildItems {
if buildItem.Proc.Name == dep {
taskIds = append(taskIds, fmt.Sprint(buildItem.Proc.ID))
}
}
}
2019-06-13 15:38:19 +00:00
return taskIds
2017-03-05 07:56:08 +00:00
}
2017-03-16 11:00:56 +00:00
2017-05-05 16:59:37 +00:00
func shasum(raw []byte) string {
sum := sha256.Sum256(raw)
return fmt.Sprintf("%x", sum)
}