mirrors/woodpecker
1
0
Fork 0
mirror of https://github.com/woodpecker-ci/woodpecker.git synced 2024-11-27 04:11:03 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #28 from laszlocph/dryness

Browse source 
Dryness
This commit is contained in:
Laszlo Fogas 2019-06-05 06:51:45 +02:00 committed by GitHub
commit bb00a8d7e3
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
13 changed files with 354 additions and 541 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
cmd/drone-server/server.go
View file

@ -671,7 +671,6 @@ func setupEvilGlobals(c *cli.Context, v store.Store, r remote.Remote) {
droneserver.Config.Services.Secrets = setupSecretService(c, v)
droneserver.Config.Services.Senders = sender.New(v, v)
droneserver.Config.Services.Environ = setupEnvironService(c, v)
droneserver.Config.Services.Limiter = setupLimiter(c, v)
if endpoint := c.String("gating-service"); endpoint != "" {
droneserver.Config.Services.Senders = sender.NewRemote(endpoint)

4
cmd/drone-server/setup.go
View file

@ -66,10 +66,6 @@ func setupEnvironService(c *cli.Context, s store.Store) model.EnvironService {
return nil
}
func setupLimiter(c *cli.Context, s store.Store) model.Limiter {
return new(model.NoLimit)
}
func setupPubsub(c *cli.Context) {}
func setupStream(c *cli.Context) {}
func setupGatingService(c *cli.Context) {}

41
model/limit.go
View file

@ -1,41 +0,0 @@
// Copyright 2018 Drone.IO Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package model
// Limiter defines an interface for limiting repository creation.
// This could be used, for example, to limit repository creation to
// a specific organization or a specific set of users.
type Limiter interface {
LimitUser(*User) error
LimitRepo(*User, *Repo) error
LimitRepos(*User, []*Repo) []*Repo
LimitBuild(*User, *Repo, *Build) error
}
// NoLimit implements the Limiter interface without enforcing any
// actual limits. All limiting functions are no-ops.
type NoLimit struct{}
// LimitUser is a no-op for limiting user creation.
func (NoLimit) LimitUser(*User) error { return nil }
// LimitRepo is a no-op for limiting repo creation.
func (NoLimit) LimitRepo(*User, *Repo) error { return nil }
// LimitRepos is a no-op for limiting repository listings.
func (NoLimit) LimitRepos(user *User, repos []*Repo) []*Repo { return repos }
// LimitBuild is a no-op for limiting build creation.
func (NoLimit) LimitBuild(*User, *Repo, *Build) error { return nil }

173
server/build.go
View file

@ -17,7 +17,6 @@ package server
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net/http"
@ -25,13 +24,11 @@ import (
"time"
"github.com/Sirupsen/logrus"
"github.com/laszlocph/drone-oss-08/cncd/pipeline/pipeline/rpc"
"github.com/laszlocph/drone-oss-08/cncd/pubsub"
"github.com/gin-gonic/gin"
"github.com/laszlocph/drone-oss-08/cncd/queue"
"github.com/laszlocph/drone-oss-08/remote"
"github.com/laszlocph/drone-oss-08/shared/httputil"
"github.com/laszlocph/drone-oss-08/store"
"github.com/gin-gonic/gin"
"github.com/laszlocph/drone-oss-08/model"
"github.com/laszlocph/drone-oss-08/router/middleware/session"
@ -270,14 +267,6 @@ func PostApproval(c *gin.Context) {
build.Reviewed = time.Now().Unix()
build.Reviewer = user.Login
//
//
// This code is copied pasted until I have a chance
// to refactor into a proper function. Lots of changes
// and technical debt. No judgement please!
//
//
// fetch the build file from the database
conf, err := Config.Storage.Config.ConfigLoad(build.ConfigID)
if err != nil {
@ -326,7 +315,7 @@ func PostApproval(c *gin.Context) {
}
}()
b := builder{
b := procBuilder{
Repo: repo,
Curr: build,
Last: last,
@ -337,7 +326,7 @@ func PostApproval(c *gin.Context) {
Yaml: conf.Data,
Envs: envs,
}
items, err := b.Build()
buildItems, err := b.Build()
if err != nil {
build.Status = model.StatusError
build.Started = time.Now().Unix()
@ -347,73 +336,14 @@ func PostApproval(c *gin.Context) {
return
}
var pcounter = len(items)
for _, item := range items {
build.Procs = append(build.Procs, item.Proc)
item.Proc.BuildID = build.ID
for _, stage := range item.Config.Stages {
var gid int
for _, step := range stage.Steps {
pcounter++
if gid == 0 {
gid = pcounter
}
proc := &model.Proc{
BuildID: build.ID,
Name: step.Alias,
PID: pcounter,
PPID: item.Proc.PID,
PGID: gid,
State: model.StatusPending,
}
build.Procs = append(build.Procs, proc)
}
}
setBuildProcs(build, buildItems)
err = store.FromContext(c).ProcCreate(build.Procs)
if err != nil {
logrus.Errorf("error persisting procs %s/%d: %s", repo.FullName, build.Number, err)
}
store.FromContext(c).ProcCreate(build.Procs)
//
// publish topic
//
buildCopy := *build
buildCopy.Procs = model.Tree(buildCopy.Procs)
message := pubsub.Message{
Labels: map[string]string{
"repo": repo.FullName,
"private": strconv.FormatBool(repo.IsPrivate),
},
}
message.Data, _ = json.Marshal(model.Event{
Type: model.Enqueued,
Repo: *repo,
Build: buildCopy,
})
// TODO remove global reference
Config.Services.Pubsub.Publish(c, "topic/events", message)
//
// end publish topic
//
for _, item := range items {
task := new(queue.Task)
task.ID = fmt.Sprint(item.Proc.ID)
task.Labels = map[string]string{}
task.Labels["platform"] = item.Platform
for k, v := range item.Labels {
task.Labels[k] = v
}
task.Data, _ = json.Marshal(rpc.Pipeline{
ID: fmt.Sprint(item.Proc.ID),
Config: item.Config,
Timeout: b.Repo.Timeout,
})
Config.Services.Logs.Open(context.Background(), task.ID)
Config.Services.Queue.Push(context.Background(), task)
}
publishToTopic(c, build, repo)
queueBuild(build, repo, buildItems)
}
func PostDecline(c *gin.Context) {
@ -463,15 +393,8 @@ func GetBuildQueue(c *gin.Context) {
c.JSON(200, out)
}
//
//
//
//
//
//
// PostBuild restarts a build
func PostBuild(c *gin.Context) {
remote_ := remote.FromContext(c)
repo := session.Repo(c)
@ -581,7 +504,7 @@ func PostBuild(c *gin.Context) {
}
}
b := builder{
b := procBuilder{
Repo: repo,
Curr: build,
Last: last,
@ -592,7 +515,7 @@ func PostBuild(c *gin.Context) {
Yaml: conf.Data,
Envs: buildParams,
}
items, err := b.Build()
buildItems, err := b.Build()
if err != nil {
build.Status = model.StatusError
build.Started = time.Now().Unix()
@ -602,30 +525,7 @@ func PostBuild(c *gin.Context) {
return
}
var pcounter = len(items)
for _, item := range items {
build.Procs = append(build.Procs, item.Proc)
item.Proc.BuildID = build.ID
for _, stage := range item.Config.Stages {
var gid int
for _, step := range stage.Steps {
pcounter++
if gid == 0 {
gid = pcounter
}
proc := &model.Proc{
BuildID: build.ID,
Name: step.Alias,
PID: pcounter,
PPID: item.Proc.PID,
PGID: gid,
State: model.StatusPending,
}
build.Procs = append(build.Procs, proc)
}
}
}
setBuildProcs(build, buildItems)
err = store.FromContext(c).ProcCreate(build.Procs)
if err != nil {
@ -637,55 +537,12 @@ func PostBuild(c *gin.Context) {
c.JSON(500, build)
return
}
c.JSON(202, build)
//
// publish topic
//
buildCopy := *build
buildCopy.Procs = model.Tree(buildCopy.Procs)
message := pubsub.Message{
Labels: map[string]string{
"repo": repo.FullName,
"private": strconv.FormatBool(repo.IsPrivate),
},
}
message.Data, _ = json.Marshal(model.Event{
Type: model.Enqueued,
Repo: *repo,
Build: buildCopy,
})
// TODO remove global reference
Config.Services.Pubsub.Publish(c, "topic/events", message)
//
// end publish topic
//
for _, item := range items {
task := new(queue.Task)
task.ID = fmt.Sprint(item.Proc.ID)
task.Labels = map[string]string{}
task.Labels["platform"] = item.Platform
for k, v := range item.Labels {
task.Labels[k] = v
}
task.Data, _ = json.Marshal(rpc.Pipeline{
ID: fmt.Sprint(item.Proc.ID),
Config: item.Config,
Timeout: b.Repo.Timeout,
})
Config.Services.Logs.Open(context.Background(), task.ID)
Config.Services.Queue.Push(context.Background(), task)
}
publishToTopic(c, build, repo)
queueBuild(build, repo, buildItems)
}
//
///
//
func DeleteBuildLogs(c *gin.Context) {
repo := session.Repo(c)
user := session.User(c)

372
server/hook.go
View file

@ -20,10 +20,8 @@ import (
"encoding/json"
"fmt"
"math/rand"
"net/url"
"regexp"
"strconv"
"strings"
"time"
"github.com/gin-gonic/gin"
@ -34,26 +32,13 @@ import (
"github.com/laszlocph/drone-oss-08/shared/httputil"
"github.com/laszlocph/drone-oss-08/shared/token"
"github.com/laszlocph/drone-oss-08/store"
"github.com/drone/envsubst"
"github.com/laszlocph/drone-oss-08/cncd/pipeline/pipeline/backend"
"github.com/laszlocph/drone-oss-08/cncd/pipeline/pipeline/frontend"
"github.com/laszlocph/drone-oss-08/cncd/pipeline/pipeline/frontend/yaml"
"github.com/laszlocph/drone-oss-08/cncd/pipeline/pipeline/frontend/yaml/compiler"
"github.com/laszlocph/drone-oss-08/cncd/pipeline/pipeline/frontend/yaml/linter"
"github.com/laszlocph/drone-oss-08/cncd/pipeline/pipeline/frontend/yaml/matrix"
"github.com/laszlocph/drone-oss-08/cncd/pipeline/pipeline/rpc"
"github.com/laszlocph/drone-oss-08/cncd/pubsub"
"github.com/laszlocph/drone-oss-08/cncd/queue"
)
//
// CANARY IMPLEMENTATION
//
// This file is a complete disaster because I'm trying to wedge in some
// experimental code. Please pardon our appearance during renovations.
//
var skipRe = regexp.MustCompile(`\[(?i:ci *skip|skip *ci)\]`)
func init() {
@ -157,54 +142,30 @@ func PostHook(c *gin.Context) {
}
}
// fetch the build file from the database
confb, err := remote.FileBackoff(remote_, user, repo, build, repo.Config)
// fetch the build file from the remote
remoteYamlConfig, err := remote.FileBackoff(remote_, user, repo, build, repo.Config)
if err != nil {
logrus.Errorf("error: %s: cannot find %s in %s: %s", repo.FullName, repo.Config, build.Ref, err)
c.AbortWithError(404, err)
return
}
sha := shasum(confb)
conf, err := Config.Storage.Config.ConfigFind(repo, sha)
conf, err := findOrPersistPipelineConfig(repo, remoteYamlConfig)
if err != nil {
conf = &model.Config{
RepoID: repo.ID,
Data: string(confb),
Hash: sha,
}
err = Config.Storage.Config.ConfigCreate(conf)
if err != nil {
// retry in case we receive two hooks at the same time
conf, err = Config.Storage.Config.ConfigFind(repo, sha)
if err != nil {
logrus.Errorf("failure to find or persist build config for %s. %s", repo.FullName, err)
c.AbortWithError(500, err)
return
}
}
logrus.Errorf("failure to find or persist build config for %s. %s", repo.FullName, err)
c.AbortWithError(500, err)
return
}
build.ConfigID = conf.ID
netrc, err := remote_.Netrc(user, repo)
if err != nil {
c.String(500, "Failed to generate netrc file. %s", err)
return
}
// verify the branches can be built vs skipped
branches, err := yaml.ParseString(conf.Data)
// verify that pipeline can be built at all
parsedPipelineConfig, err := yaml.ParseString(conf.Data)
if err == nil {
if !branches.Branches.Match(build.Branch) && build.Event != model.EventTag && build.Event != model.EventDeploy {
if !parsedPipelineConfig.Branches.Match(build.Branch) && build.Event != model.EventTag && build.Event != model.EventDeploy {
c.String(200, "Branch does not match restrictions defined in yaml")
return
}
}
// update some build fields
build.RepoID = repo.ID
build.Verified = true
build.Status = model.StatusPending
if repo.IsGated {
allowed, _ := Config.Services.Senders.SenderAllowed(user, repo, build, conf)
if !allowed {
@ -212,12 +173,11 @@ func PostHook(c *gin.Context) {
}
}
if err = Config.Services.Limiter.LimitBuild(user, repo, build); err != nil {
c.String(403, "Build blocked by limiter")
return
}
// update some build fields
build.RepoID = repo.ID
build.Verified = true
build.Status = model.StatusPending
build.Trim()
err = store.CreateBuild(c, build, build.Procs...)
if err != nil {
logrus.Errorf("failure to save commit for %s. %s", repo.FullName, err)
@ -231,6 +191,12 @@ func PostHook(c *gin.Context) {
return
}
netrc, err := remote_.Netrc(user, repo)
if err != nil {
c.String(500, "Failed to generate netrc file. %s", err)
return
}
envs := map[string]string{}
if Config.Services.Environ != nil {
globals, _ := Config.Services.Environ.EnvironList(repo)
@ -249,14 +215,9 @@ func PostHook(c *gin.Context) {
logrus.Debugf("Error getting registry credentials for %s#%d. %s", repo.FullName, build.Number, err)
}
// get the previous build so that we can send
// on status change notifications
// get the previous build so that we can send status change notifications
last, _ := store.GetBuildLastBefore(c, repo, build.Branch, build.ID)
//
// BELOW: NEW
//
defer func() {
uri := fmt.Sprintf("%s/%s/%d", httputil.GetURL(c.Request), repo.FullName, build.Number)
err = remote_.Status(user, repo, build, uri)
@ -265,7 +226,7 @@ func PostHook(c *gin.Context) {
}
}()
b := builder{
b := procBuilder{
Repo: repo,
Curr: build,
Last: last,
@ -276,7 +237,7 @@ func PostHook(c *gin.Context) {
Link: httputil.GetURL(c.Request),
Yaml: conf.Data,
}
items, err := b.Build()
buildItems, err := b.Build()
if err != nil {
build.Status = model.StatusError
build.Started = time.Now().Unix()
@ -286,9 +247,42 @@ func PostHook(c *gin.Context) {
return
}
var pcounter = len(items)
setBuildProcs(build, buildItems)
for _, item := range items {
err = store.FromContext(c).ProcCreate(build.Procs)
if err != nil {
logrus.Errorf("error persisting procs %s/%d: %s", repo.FullName, build.Number, err)
}
publishToTopic(c, build, repo)
queueBuild(build, repo, buildItems)
}
func findOrPersistPipelineConfig(repo *model.Repo, remoteYamlConfig []byte) (*model.Config, error) {
sha := shasum(remoteYamlConfig)
conf, err := Config.Storage.Config.ConfigFind(repo, sha)
if err != nil {
conf = &model.Config{
RepoID: repo.ID,
Data: string(remoteYamlConfig),
Hash: sha,
}
err = Config.Storage.Config.ConfigCreate(conf)
if err != nil {
// retry in case we receive two hooks at the same time
conf, err = Config.Storage.Config.ConfigFind(repo, sha)
if err != nil {
return nil, err
}
}
}
return conf, nil
}
func setBuildProcs(build *model.Build, buildItems []*buildItem) {
pcounter := len(buildItems)
for _, item := range buildItems {
build.Procs = append(build.Procs, item.Proc)
item.Proc.BuildID = build.ID
@ -311,14 +305,9 @@ func PostHook(c *gin.Context) {
}
}
}
err = store.FromContext(c).ProcCreate(build.Procs)
if err != nil {
logrus.Errorf("error persisting procs %s/%d: %s", repo.FullName, build.Number, err)
}
}
//
// publish topic
//
func publishToTopic(c *gin.Context, build *model.Build, repo *model.Repo) {
message := pubsub.Message{
Labels: map[string]string{
"repo": repo.FullName,
@ -332,13 +321,11 @@ func PostHook(c *gin.Context) {
Repo: *repo,
Build: buildCopy,
})
// TODO remove global reference
Config.Services.Pubsub.Publish(c, "topic/events", message)
//
// end publish topic
//
}
for _, item := range items {
func queueBuild(build *model.Build, repo *model.Repo, buildItems []*buildItem) {
for _, item := range buildItems {
task := new(queue.Task)
task.ID = fmt.Sprint(item.Proc.ID)
task.Labels = map[string]string{}
@ -346,12 +333,12 @@ func PostHook(c *gin.Context) {
task.Labels[k] = v
}
task.Labels["platform"] = item.Platform
task.Labels["repo"] = b.Repo.FullName
task.Labels["repo"] = repo.FullName
task.Data, _ = json.Marshal(rpc.Pipeline{
ID: fmt.Sprint(item.Proc.ID),
Config: item.Config,
Timeout: b.Repo.Timeout,
Timeout: repo.Timeout,
})
Config.Services.Logs.Open(context.Background(), task.ID)
@ -359,237 +346,6 @@ func PostHook(c *gin.Context) {
}
}
// return the metadata from the cli context.
func metadataFromStruct(repo *model.Repo, build, last *model.Build, proc *model.Proc, link string) frontend.Metadata {
host := link
uri, err := url.Parse(link)
if err == nil {
host = uri.Host
}
return frontend.Metadata{
Repo: frontend.Repo{
Name: repo.FullName,
Link: repo.Link,
Remote: repo.Clone,
Private: repo.IsPrivate,
Branch: repo.Branch,
},
Curr: frontend.Build{
Number: build.Number,
Parent: build.Parent,
Created: build.Created,
Started: build.Started,
Finished: build.Finished,
Status: build.Status,
Event: build.Event,
Link: build.Link,
Target: build.Deploy,
Commit: frontend.Commit{
Sha: build.Commit,
Ref: build.Ref,
Refspec: build.Refspec,
Branch: build.Branch,
Message: build.Message,
Author: frontend.Author{
Name: build.Author,
Email: build.Email,
Avatar: build.Avatar,
},
},
},
Prev: frontend.Build{
Number: last.Number,
Created: last.Created,
Started: last.Started,
Finished: last.Finished,
Status: last.Status,
Event: last.Event,
Link: last.Link,
Target: last.Deploy,
Commit: frontend.Commit{
Sha: last.Commit,
Ref: last.Ref,
Refspec: last.Refspec,
Branch: last.Branch,
Message: last.Message,
Author: frontend.Author{
Name: last.Author,
Email: last.Email,
Avatar: last.Avatar,
},
},
},
Job: frontend.Job{
Number: proc.PID,
Matrix: proc.Environ,
},
Sys: frontend.System{
Name: "drone",
Link: link,
Host: host,
Arch: "linux/amd64",
},
}
}
type builder struct {
Repo *model.Repo
Curr *model.Build
Last *model.Build
Netrc *model.Netrc
Secs []*model.Secret
Regs []*model.Registry
Link string
Yaml string
Envs map[string]string
}
type buildItem struct {
Proc *model.Proc
Platform string
Labels map[string]string
Config *backend.Config
}
func (b *builder) Build() ([]*buildItem, error) {
axes, err := matrix.ParseString(b.Yaml)
if err != nil {
return nil, err
}
if len(axes) == 0 {
axes = append(axes, matrix.Axis{})
}
var items []*buildItem
for i, axis := range axes {
proc := &model.Proc{
BuildID: b.Curr.ID,
PID: i + 1,
PGID: i + 1,
State: model.StatusPending,
Environ: axis,
}
metadata := metadataFromStruct(b.Repo, b.Curr, b.Last, proc, b.Link)
environ := metadata.Environ()
for k, v := range metadata.EnvironDrone() {
environ[k] = v
}
for k, v := range axis {
environ[k] = v
}
var secrets []compiler.Secret
for _, sec := range b.Secs {
if !sec.Match(b.Curr.Event) {
continue
}
secrets = append(secrets, compiler.Secret{
Name: sec.Name,
Value: sec.Value,
Match: sec.Images,
})
}
y := b.Yaml
s, err := envsubst.Eval(y, func(name string) string {
env := environ[name]
if strings.Contains(env, "\n") {
env = fmt.Sprintf("%q", env)
}
return env
})
if err != nil {
return nil, err
}
y = s
parsed, err := yaml.ParseString(y)
if err != nil {
return nil, err
}
metadata.Sys.Arch = parsed.Platform
if metadata.Sys.Arch == "" {
metadata.Sys.Arch = "linux/amd64"
}
lerr := linter.New(
linter.WithTrusted(b.Repo.IsTrusted),
).Lint(parsed)
if lerr != nil {
return nil, lerr
}
var registries []compiler.Registry
for _, reg := range b.Regs {
registries = append(registries, compiler.Registry{
Hostname: reg.Address,
Username: reg.Username,
Password: reg.Password,
Email: reg.Email,
})
}
ir := compiler.New(
compiler.WithEnviron(environ),
compiler.WithEnviron(b.Envs),
compiler.WithEscalated(Config.Pipeline.Privileged...),
compiler.WithResourceLimit(Config.Pipeline.Limits.MemSwapLimit, Config.Pipeline.Limits.MemLimit, Config.Pipeline.Limits.ShmSize, Config.Pipeline.Limits.CPUQuota, Config.Pipeline.Limits.CPUShares, Config.Pipeline.Limits.CPUSet),
compiler.WithVolumes(Config.Pipeline.Volumes...),
compiler.WithNetworks(Config.Pipeline.Networks...),
compiler.WithLocal(false),
compiler.WithOption(
compiler.WithNetrc(
b.Netrc.Login,
b.Netrc.Password,
b.Netrc.Machine,
),
b.Repo.IsPrivate,
),
compiler.WithRegistry(registries...),
compiler.WithSecret(secrets...),
compiler.WithPrefix(
fmt.Sprintf(
"%d_%d",
proc.ID,
rand.Int(),
),
),
compiler.WithEnviron(proc.Environ),
compiler.WithProxy(),
compiler.WithWorkspaceFromURL("/drone", b.Repo.Link),
compiler.WithMetadata(metadata),
).Compile(parsed)
// for _, sec := range b.Secs {
// if !sec.MatchEvent(b.Curr.Event) {
// continue
// }
// if b.Curr.Verified || sec.SkipVerify {
// ir.Secrets = append(ir.Secrets, &backend.Secret{
// Mask: sec.Conceal,
// Name: sec.Name,
// Value: sec.Value,
// })
// }
// }
item := &buildItem{
Proc: proc,
Config: ir,
Labels: parsed.Labels,
Platform: metadata.Sys.Arch,
}
if item.Labels == nil {
item.Labels = map[string]string{}
}
items = append(items, item)
}
return items, nil
}
func shasum(raw []byte) string {
sum := sha256.Sum256(raw)
return fmt.Sprintf("%x", sum)

2
server/hook_test.go
View file

@ -21,7 +21,7 @@ import (
)
func TestMultilineEnvsubst(t *testing.T) {
b := builder{
b := procBuilder{
Repo: &model.Repo{},
Curr: &model.Build{
Message: `aaa

7
server/login.go
View file

@ -19,12 +19,12 @@ import (
"net/http"
"time"
"github.com/gorilla/securecookie"
"github.com/laszlocph/drone-oss-08/model"
"github.com/laszlocph/drone-oss-08/remote"
"github.com/laszlocph/drone-oss-08/shared/httputil"
"github.com/laszlocph/drone-oss-08/shared/token"
"github.com/laszlocph/drone-oss-08/store"
"github.com/gorilla/securecookie"
"github.com/Sirupsen/logrus"
"github.com/gin-gonic/gin"
@ -95,11 +95,6 @@ func HandleAuth(c *gin.Context) {
),
}
if err = Config.Services.Limiter.LimitUser(u); err != nil {
c.String(403, "User activation blocked by limiter")
return
}
// insert the user into the database
if err := store.CreateUser(c, u); err != nil {
logrus.Errorf("cannot insert %s. %s", u.Login, err)

263
server/procBuilder.go Normal file
View file

@ -0,0 +1,263 @@
// Copyright 2018 Drone.IO Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package server
import (
"fmt"
"math/rand"
"net/url"
"strings"
"github.com/drone/envsubst"
"github.com/laszlocph/drone-oss-08/cncd/pipeline/pipeline/backend"
"github.com/laszlocph/drone-oss-08/cncd/pipeline/pipeline/frontend"
"github.com/laszlocph/drone-oss-08/cncd/pipeline/pipeline/frontend/yaml"
"github.com/laszlocph/drone-oss-08/cncd/pipeline/pipeline/frontend/yaml/compiler"
"github.com/laszlocph/drone-oss-08/cncd/pipeline/pipeline/frontend/yaml/linter"
"github.com/laszlocph/drone-oss-08/cncd/pipeline/pipeline/frontend/yaml/matrix"
"github.com/laszlocph/drone-oss-08/model"
)
// Takes the hook data and the yaml and returns in internal data model
type procBuilder struct {
Repo *model.Repo
Curr *model.Build
Last *model.Build
Netrc *model.Netrc
Secs []*model.Secret
Regs []*model.Registry
Link string
Yaml string
Envs map[string]string
}
type buildItem struct {
Proc *model.Proc
Platform string
Labels map[string]string
Config *backend.Config
}
func (b *procBuilder) Build() ([]*buildItem, error) {
axes, err := matrix.ParseString(b.Yaml)
if err != nil {
return nil, err
}
if len(axes) == 0 {
axes = append(axes, matrix.Axis{})
}
var items []*buildItem
for i, axis := range axes {
proc := &model.Proc{
BuildID: b.Curr.ID,
PID: i + 1,
PGID: i + 1,
State: model.StatusPending,
Environ: axis,
}
metadata := metadataFromStruct(b.Repo, b.Curr, b.Last, proc, b.Link)
environ := metadata.Environ()
for k, v := range metadata.EnvironDrone() {
environ[k] = v
}
for k, v := range axis {
environ[k] = v
}
var secrets []compiler.Secret
for _, sec := range b.Secs {
if !sec.Match(b.Curr.Event) {
continue
}
secrets = append(secrets, compiler.Secret{
Name: sec.Name,
Value: sec.Value,
Match: sec.Images,
})
}
y := b.Yaml
s, err := envsubst.Eval(y, func(name string) string {
env := environ[name]
if strings.Contains(env, "\n") {
env = fmt.Sprintf("%q", env)
}
return env
})
if err != nil {
return nil, err
}
y = s
parsed, err := yaml.ParseString(y)
if err != nil {
return nil, err
}
metadata.Sys.Arch = parsed.Platform
if metadata.Sys.Arch == "" {
metadata.Sys.Arch = "linux/amd64"
}
lerr := linter.New(
linter.WithTrusted(b.Repo.IsTrusted),
).Lint(parsed)
if lerr != nil {
return nil, lerr
}
var registries []compiler.Registry
for _, reg := range b.Regs {
registries = append(registries, compiler.Registry{
Hostname: reg.Address,
Username: reg.Username,
Password: reg.Password,
Email: reg.Email,
})
}
ir := compiler.New(
compiler.WithEnviron(environ),
compiler.WithEnviron(b.Envs),
compiler.WithEscalated(Config.Pipeline.Privileged...),
compiler.WithResourceLimit(Config.Pipeline.Limits.MemSwapLimit, Config.Pipeline.Limits.MemLimit, Config.Pipeline.Limits.ShmSize, Config.Pipeline.Limits.CPUQuota, Config.Pipeline.Limits.CPUShares, Config.Pipeline.Limits.CPUSet),
compiler.WithVolumes(Config.Pipeline.Volumes...),
compiler.WithNetworks(Config.Pipeline.Networks...),
compiler.WithLocal(false),
compiler.WithOption(
compiler.WithNetrc(
b.Netrc.Login,
b.Netrc.Password,
b.Netrc.Machine,
),
b.Repo.IsPrivate,
),
compiler.WithRegistry(registries...),
compiler.WithSecret(secrets...),
compiler.WithPrefix(
fmt.Sprintf(
"%d_%d",
proc.ID,
rand.Int(),
),
),
compiler.WithEnviron(proc.Environ),
compiler.WithProxy(),
compiler.WithWorkspaceFromURL("/drone", b.Repo.Link),
compiler.WithMetadata(metadata),
).Compile(parsed)
// for _, sec := range b.Secs {
// if !sec.MatchEvent(b.Curr.Event) {
// continue
// }
// if b.Curr.Verified || sec.SkipVerify {
// ir.Secrets = append(ir.Secrets, &backend.Secret{
// Mask: sec.Conceal,
// Name: sec.Name,
// Value: sec.Value,
// })
// }
// }
item := &buildItem{
Proc: proc,
Config: ir,
Labels: parsed.Labels,
Platform: metadata.Sys.Arch,
}
if item.Labels == nil {
item.Labels = map[string]string{}
}
items = append(items, item)
}
return items, nil
}
// return the metadata from the cli context.
func metadataFromStruct(repo *model.Repo, build, last *model.Build, proc *model.Proc, link string) frontend.Metadata {
host := link
uri, err := url.Parse(link)
if err == nil {
host = uri.Host
}
return frontend.Metadata{
Repo: frontend.Repo{
Name: repo.FullName,
Link: repo.Link,
Remote: repo.Clone,
Private: repo.IsPrivate,
Branch: repo.Branch,
},
Curr: frontend.Build{
Number: build.Number,
Parent: build.Parent,
Created: build.Created,
Started: build.Started,
Finished: build.Finished,
Status: build.Status,
Event: build.Event,
Link: build.Link,
Target: build.Deploy,
Commit: frontend.Commit{
Sha: build.Commit,
Ref: build.Ref,
Refspec: build.Refspec,
Branch: build.Branch,
Message: build.Message,
Author: frontend.Author{
Name: build.Author,
Email: build.Email,
Avatar: build.Avatar,
},
},
},
Prev: frontend.Build{
Number: last.Number,
Created: last.Created,
Started: last.Started,
Finished: last.Finished,
Status: last.Status,
Event: last.Event,
Link: last.Link,
Target: last.Deploy,
Commit: frontend.Commit{
Sha: last.Commit,
Ref: last.Ref,
Refspec: last.Refspec,
Branch: last.Branch,
Message: last.Message,
Author: frontend.Author{
Name: last.Author,
Email: last.Email,
Avatar: last.Avatar,
},
},
},
Job: frontend.Job{
Number: proc.PID,
Matrix: proc.Environ,
},
Sys: frontend.System{
Name: "drone",
Link: link,
Host: host,
Arch: "linux/amd64",
},
}
}

5
server/repo.go
View file

@ -41,11 +41,6 @@ func PostRepo(c *gin.Context) {
return
}
if err := Config.Services.Limiter.LimitRepo(user, repo); err != nil {
c.String(403, "Repository activation blocked by limiter")
return
}
repo.IsActive = true
repo.UserID = user.ID
if !repo.AllowPush && !repo.AllowPull && !repo.AllowDeploy && !repo.AllowTag {

1
server/rpc.go
View file

@ -55,7 +55,6 @@ var Config = struct {
Secrets model.SecretService
Registries model.RegistryService
Environ model.EnvironService
Limiter model.Limiter
}
Storage struct {
// Users model.UserStore

11
server/sync.go
View file

@ -28,10 +28,9 @@ type Syncer interface {
}
type syncer struct {
remote remote.Remote
store store.Store
perms model.PermStore
limiter model.Limiter
remote remote.Remote
store store.Store
perms model.PermStore
}
func (s *syncer) Sync(user *model.User) error {
@ -41,10 +40,6 @@ func (s *syncer) Sync(user *model.User) error {
return err
}
if s.limiter != nil {
repos = s.limiter.LimitRepos(user, repos)
}
var perms []*model.Perm
for _, repo := range repos {
perm := model.Perm{

14
server/user.go
View file

@ -46,10 +46,9 @@ func GetFeed(c *gin.Context) {
store.FromContext(c).UpdateUser(user)
sync := syncer{
remote: remote.FromContext(c),
store: store.FromContext(c),
perms: store.FromContext(c),
limiter: Config.Services.Limiter,
remote: remote.FromContext(c),
store: store.FromContext(c),
perms: store.FromContext(c),
}
if err := sync.Sync(user); err != nil {
logrus.Debugf("sync error: %s: %s", user.Login, err)
@ -89,10 +88,9 @@ func GetRepos(c *gin.Context) {
store.FromContext(c).UpdateUser(user)
sync := syncer{
remote: remote.FromContext(c),
store: store.FromContext(c),
perms: store.FromContext(c),
limiter: Config.Services.Limiter,
remote: remote.FromContext(c),
store: store.FromContext(c),
perms: store.FromContext(c),
}
if err := sync.Sync(user); err != nil {
logrus.Debugf("sync error: %s: %s", user.Login, err)

1
store/datastore/builds.go
View file

@ -72,6 +72,7 @@ func (db *datastore) GetBuildQueue() ([]*model.Feed, error) {
}
func (db *datastore) CreateBuild(build *model.Build, procs ...*model.Proc) error {
build.Trim()
id, err := db.incrementRepoRetry(build.RepoID)
if err != nil {
return err