diff --git a/cmd/drone-server/server.go b/cmd/drone-server/server.go index 903c92a4c..be82d6fb5 100644 --- a/cmd/drone-server/server.go +++ b/cmd/drone-server/server.go @@ -671,7 +671,6 @@ func setupEvilGlobals(c *cli.Context, v store.Store, r remote.Remote) { droneserver.Config.Services.Secrets = setupSecretService(c, v) droneserver.Config.Services.Senders = sender.New(v, v) droneserver.Config.Services.Environ = setupEnvironService(c, v) - droneserver.Config.Services.Limiter = setupLimiter(c, v) if endpoint := c.String("gating-service"); endpoint != "" { droneserver.Config.Services.Senders = sender.NewRemote(endpoint) diff --git a/cmd/drone-server/setup.go b/cmd/drone-server/setup.go index f16affa7e..16f5934db 100644 --- a/cmd/drone-server/setup.go +++ b/cmd/drone-server/setup.go @@ -66,10 +66,6 @@ func setupEnvironService(c *cli.Context, s store.Store) model.EnvironService { return nil } -func setupLimiter(c *cli.Context, s store.Store) model.Limiter { - return new(model.NoLimit) -} - func setupPubsub(c *cli.Context) {} func setupStream(c *cli.Context) {} func setupGatingService(c *cli.Context) {} diff --git a/model/limit.go b/model/limit.go deleted file mode 100644 index 2465700f4..000000000 --- a/model/limit.go +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright 2018 Drone.IO Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package model - -// Limiter defines an interface for limiting repository creation. -// This could be used, for example, to limit repository creation to -// a specific organization or a specific set of users. -type Limiter interface { - LimitUser(*User) error - LimitRepo(*User, *Repo) error - LimitRepos(*User, []*Repo) []*Repo - LimitBuild(*User, *Repo, *Build) error -} - -// NoLimit implements the Limiter interface without enforcing any -// actual limits. All limiting functions are no-ops. -type NoLimit struct{} - -// LimitUser is a no-op for limiting user creation. -func (NoLimit) LimitUser(*User) error { return nil } - -// LimitRepo is a no-op for limiting repo creation. -func (NoLimit) LimitRepo(*User, *Repo) error { return nil } - -// LimitRepos is a no-op for limiting repository listings. -func (NoLimit) LimitRepos(user *User, repos []*Repo) []*Repo { return repos } - -// LimitBuild is a no-op for limiting build creation. -func (NoLimit) LimitBuild(*User, *Repo, *Build) error { return nil } diff --git a/server/build.go b/server/build.go index 26ca4f4da..33161c3c6 100644 --- a/server/build.go +++ b/server/build.go @@ -17,7 +17,6 @@ package server import ( "bytes" "context" - "encoding/json" "fmt" "io" "net/http" @@ -25,13 +24,11 @@ import ( "time" "github.com/Sirupsen/logrus" - "github.com/laszlocph/drone-oss-08/cncd/pipeline/pipeline/rpc" - "github.com/laszlocph/drone-oss-08/cncd/pubsub" + "github.com/gin-gonic/gin" "github.com/laszlocph/drone-oss-08/cncd/queue" "github.com/laszlocph/drone-oss-08/remote" "github.com/laszlocph/drone-oss-08/shared/httputil" "github.com/laszlocph/drone-oss-08/store" - "github.com/gin-gonic/gin" "github.com/laszlocph/drone-oss-08/model" "github.com/laszlocph/drone-oss-08/router/middleware/session" @@ -270,14 +267,6 @@ func PostApproval(c *gin.Context) { build.Reviewed = time.Now().Unix() build.Reviewer = user.Login - // - // - // This code is copied pasted until I have a chance - // to refactor into a proper function. Lots of changes - // and technical debt. No judgement please! - // - // - // fetch the build file from the database conf, err := Config.Storage.Config.ConfigLoad(build.ConfigID) if err != nil { @@ -326,7 +315,7 @@ func PostApproval(c *gin.Context) { } }() - b := builder{ + b := procBuilder{ Repo: repo, Curr: build, Last: last, @@ -337,7 +326,7 @@ func PostApproval(c *gin.Context) { Yaml: conf.Data, Envs: envs, } - items, err := b.Build() + buildItems, err := b.Build() if err != nil { build.Status = model.StatusError build.Started = time.Now().Unix() @@ -347,73 +336,14 @@ func PostApproval(c *gin.Context) { return } - var pcounter = len(items) - for _, item := range items { - build.Procs = append(build.Procs, item.Proc) - item.Proc.BuildID = build.ID - - for _, stage := range item.Config.Stages { - var gid int - for _, step := range stage.Steps { - pcounter++ - if gid == 0 { - gid = pcounter - } - proc := &model.Proc{ - BuildID: build.ID, - Name: step.Alias, - PID: pcounter, - PPID: item.Proc.PID, - PGID: gid, - State: model.StatusPending, - } - build.Procs = append(build.Procs, proc) - } - } + setBuildProcs(build, buildItems) + err = store.FromContext(c).ProcCreate(build.Procs) + if err != nil { + logrus.Errorf("error persisting procs %s/%d: %s", repo.FullName, build.Number, err) } - store.FromContext(c).ProcCreate(build.Procs) - // - // publish topic - // - buildCopy := *build - buildCopy.Procs = model.Tree(buildCopy.Procs) - message := pubsub.Message{ - Labels: map[string]string{ - "repo": repo.FullName, - "private": strconv.FormatBool(repo.IsPrivate), - }, - } - message.Data, _ = json.Marshal(model.Event{ - Type: model.Enqueued, - Repo: *repo, - Build: buildCopy, - }) - // TODO remove global reference - Config.Services.Pubsub.Publish(c, "topic/events", message) - - // - // end publish topic - // - - for _, item := range items { - task := new(queue.Task) - task.ID = fmt.Sprint(item.Proc.ID) - task.Labels = map[string]string{} - task.Labels["platform"] = item.Platform - for k, v := range item.Labels { - task.Labels[k] = v - } - - task.Data, _ = json.Marshal(rpc.Pipeline{ - ID: fmt.Sprint(item.Proc.ID), - Config: item.Config, - Timeout: b.Repo.Timeout, - }) - - Config.Services.Logs.Open(context.Background(), task.ID) - Config.Services.Queue.Push(context.Background(), task) - } + publishToTopic(c, build, repo) + queueBuild(build, repo, buildItems) } func PostDecline(c *gin.Context) { @@ -463,15 +393,8 @@ func GetBuildQueue(c *gin.Context) { c.JSON(200, out) } -// -// -// -// -// -// - +// PostBuild restarts a build func PostBuild(c *gin.Context) { - remote_ := remote.FromContext(c) repo := session.Repo(c) @@ -581,7 +504,7 @@ func PostBuild(c *gin.Context) { } } - b := builder{ + b := procBuilder{ Repo: repo, Curr: build, Last: last, @@ -592,7 +515,7 @@ func PostBuild(c *gin.Context) { Yaml: conf.Data, Envs: buildParams, } - items, err := b.Build() + buildItems, err := b.Build() if err != nil { build.Status = model.StatusError build.Started = time.Now().Unix() @@ -602,30 +525,7 @@ func PostBuild(c *gin.Context) { return } - var pcounter = len(items) - for _, item := range items { - build.Procs = append(build.Procs, item.Proc) - item.Proc.BuildID = build.ID - - for _, stage := range item.Config.Stages { - var gid int - for _, step := range stage.Steps { - pcounter++ - if gid == 0 { - gid = pcounter - } - proc := &model.Proc{ - BuildID: build.ID, - Name: step.Alias, - PID: pcounter, - PPID: item.Proc.PID, - PGID: gid, - State: model.StatusPending, - } - build.Procs = append(build.Procs, proc) - } - } - } + setBuildProcs(build, buildItems) err = store.FromContext(c).ProcCreate(build.Procs) if err != nil { @@ -637,55 +537,12 @@ func PostBuild(c *gin.Context) { c.JSON(500, build) return } - c.JSON(202, build) - // - // publish topic - // - buildCopy := *build - buildCopy.Procs = model.Tree(buildCopy.Procs) - message := pubsub.Message{ - Labels: map[string]string{ - "repo": repo.FullName, - "private": strconv.FormatBool(repo.IsPrivate), - }, - } - message.Data, _ = json.Marshal(model.Event{ - Type: model.Enqueued, - Repo: *repo, - Build: buildCopy, - }) - // TODO remove global reference - Config.Services.Pubsub.Publish(c, "topic/events", message) - // - // end publish topic - // - - for _, item := range items { - task := new(queue.Task) - task.ID = fmt.Sprint(item.Proc.ID) - task.Labels = map[string]string{} - task.Labels["platform"] = item.Platform - for k, v := range item.Labels { - task.Labels[k] = v - } - - task.Data, _ = json.Marshal(rpc.Pipeline{ - ID: fmt.Sprint(item.Proc.ID), - Config: item.Config, - Timeout: b.Repo.Timeout, - }) - - Config.Services.Logs.Open(context.Background(), task.ID) - Config.Services.Queue.Push(context.Background(), task) - } + publishToTopic(c, build, repo) + queueBuild(build, repo, buildItems) } -// -/// -// - func DeleteBuildLogs(c *gin.Context) { repo := session.Repo(c) user := session.User(c) diff --git a/server/hook.go b/server/hook.go index 65093d0bd..7604f22d0 100644 --- a/server/hook.go +++ b/server/hook.go @@ -20,10 +20,8 @@ import ( "encoding/json" "fmt" "math/rand" - "net/url" "regexp" "strconv" - "strings" "time" "github.com/gin-gonic/gin" @@ -34,26 +32,13 @@ import ( "github.com/laszlocph/drone-oss-08/shared/httputil" "github.com/laszlocph/drone-oss-08/shared/token" "github.com/laszlocph/drone-oss-08/store" - "github.com/drone/envsubst" - "github.com/laszlocph/drone-oss-08/cncd/pipeline/pipeline/backend" - "github.com/laszlocph/drone-oss-08/cncd/pipeline/pipeline/frontend" "github.com/laszlocph/drone-oss-08/cncd/pipeline/pipeline/frontend/yaml" - "github.com/laszlocph/drone-oss-08/cncd/pipeline/pipeline/frontend/yaml/compiler" - "github.com/laszlocph/drone-oss-08/cncd/pipeline/pipeline/frontend/yaml/linter" - "github.com/laszlocph/drone-oss-08/cncd/pipeline/pipeline/frontend/yaml/matrix" "github.com/laszlocph/drone-oss-08/cncd/pipeline/pipeline/rpc" "github.com/laszlocph/drone-oss-08/cncd/pubsub" "github.com/laszlocph/drone-oss-08/cncd/queue" ) -// -// CANARY IMPLEMENTATION -// -// This file is a complete disaster because I'm trying to wedge in some -// experimental code. Please pardon our appearance during renovations. -// - var skipRe = regexp.MustCompile(`\[(?i:ci *skip|skip *ci)\]`) func init() { @@ -157,54 +142,30 @@ func PostHook(c *gin.Context) { } } - // fetch the build file from the database - confb, err := remote.FileBackoff(remote_, user, repo, build, repo.Config) + // fetch the build file from the remote + remoteYamlConfig, err := remote.FileBackoff(remote_, user, repo, build, repo.Config) if err != nil { logrus.Errorf("error: %s: cannot find %s in %s: %s", repo.FullName, repo.Config, build.Ref, err) c.AbortWithError(404, err) return } - sha := shasum(confb) - conf, err := Config.Storage.Config.ConfigFind(repo, sha) + conf, err := findOrPersistPipelineConfig(repo, remoteYamlConfig) if err != nil { - conf = &model.Config{ - RepoID: repo.ID, - Data: string(confb), - Hash: sha, - } - err = Config.Storage.Config.ConfigCreate(conf) - if err != nil { - // retry in case we receive two hooks at the same time - conf, err = Config.Storage.Config.ConfigFind(repo, sha) - if err != nil { - logrus.Errorf("failure to find or persist build config for %s. %s", repo.FullName, err) - c.AbortWithError(500, err) - return - } - } + logrus.Errorf("failure to find or persist build config for %s. %s", repo.FullName, err) + c.AbortWithError(500, err) + return } build.ConfigID = conf.ID - netrc, err := remote_.Netrc(user, repo) - if err != nil { - c.String(500, "Failed to generate netrc file. %s", err) - return - } - - // verify the branches can be built vs skipped - branches, err := yaml.ParseString(conf.Data) + // verify that pipeline can be built at all + parsedPipelineConfig, err := yaml.ParseString(conf.Data) if err == nil { - if !branches.Branches.Match(build.Branch) && build.Event != model.EventTag && build.Event != model.EventDeploy { + if !parsedPipelineConfig.Branches.Match(build.Branch) && build.Event != model.EventTag && build.Event != model.EventDeploy { c.String(200, "Branch does not match restrictions defined in yaml") return } } - // update some build fields - build.RepoID = repo.ID - build.Verified = true - build.Status = model.StatusPending - if repo.IsGated { allowed, _ := Config.Services.Senders.SenderAllowed(user, repo, build, conf) if !allowed { @@ -212,12 +173,11 @@ func PostHook(c *gin.Context) { } } - if err = Config.Services.Limiter.LimitBuild(user, repo, build); err != nil { - c.String(403, "Build blocked by limiter") - return - } + // update some build fields + build.RepoID = repo.ID + build.Verified = true + build.Status = model.StatusPending - build.Trim() err = store.CreateBuild(c, build, build.Procs...) if err != nil { logrus.Errorf("failure to save commit for %s. %s", repo.FullName, err) @@ -231,6 +191,12 @@ func PostHook(c *gin.Context) { return } + netrc, err := remote_.Netrc(user, repo) + if err != nil { + c.String(500, "Failed to generate netrc file. %s", err) + return + } + envs := map[string]string{} if Config.Services.Environ != nil { globals, _ := Config.Services.Environ.EnvironList(repo) @@ -249,14 +215,9 @@ func PostHook(c *gin.Context) { logrus.Debugf("Error getting registry credentials for %s#%d. %s", repo.FullName, build.Number, err) } - // get the previous build so that we can send - // on status change notifications + // get the previous build so that we can send status change notifications last, _ := store.GetBuildLastBefore(c, repo, build.Branch, build.ID) - // - // BELOW: NEW - // - defer func() { uri := fmt.Sprintf("%s/%s/%d", httputil.GetURL(c.Request), repo.FullName, build.Number) err = remote_.Status(user, repo, build, uri) @@ -265,7 +226,7 @@ func PostHook(c *gin.Context) { } }() - b := builder{ + b := procBuilder{ Repo: repo, Curr: build, Last: last, @@ -276,7 +237,7 @@ func PostHook(c *gin.Context) { Link: httputil.GetURL(c.Request), Yaml: conf.Data, } - items, err := b.Build() + buildItems, err := b.Build() if err != nil { build.Status = model.StatusError build.Started = time.Now().Unix() @@ -286,9 +247,42 @@ func PostHook(c *gin.Context) { return } - var pcounter = len(items) + setBuildProcs(build, buildItems) - for _, item := range items { + err = store.FromContext(c).ProcCreate(build.Procs) + if err != nil { + logrus.Errorf("error persisting procs %s/%d: %s", repo.FullName, build.Number, err) + } + + publishToTopic(c, build, repo) + queueBuild(build, repo, buildItems) +} + +func findOrPersistPipelineConfig(repo *model.Repo, remoteYamlConfig []byte) (*model.Config, error) { + sha := shasum(remoteYamlConfig) + conf, err := Config.Storage.Config.ConfigFind(repo, sha) + if err != nil { + conf = &model.Config{ + RepoID: repo.ID, + Data: string(remoteYamlConfig), + Hash: sha, + } + err = Config.Storage.Config.ConfigCreate(conf) + if err != nil { + // retry in case we receive two hooks at the same time + conf, err = Config.Storage.Config.ConfigFind(repo, sha) + if err != nil { + return nil, err + } + } + } + + return conf, nil +} + +func setBuildProcs(build *model.Build, buildItems []*buildItem) { + pcounter := len(buildItems) + for _, item := range buildItems { build.Procs = append(build.Procs, item.Proc) item.Proc.BuildID = build.ID @@ -311,14 +305,9 @@ func PostHook(c *gin.Context) { } } } - err = store.FromContext(c).ProcCreate(build.Procs) - if err != nil { - logrus.Errorf("error persisting procs %s/%d: %s", repo.FullName, build.Number, err) - } +} - // - // publish topic - // +func publishToTopic(c *gin.Context, build *model.Build, repo *model.Repo) { message := pubsub.Message{ Labels: map[string]string{ "repo": repo.FullName, @@ -332,13 +321,11 @@ func PostHook(c *gin.Context) { Repo: *repo, Build: buildCopy, }) - // TODO remove global reference Config.Services.Pubsub.Publish(c, "topic/events", message) - // - // end publish topic - // +} - for _, item := range items { +func queueBuild(build *model.Build, repo *model.Repo, buildItems []*buildItem) { + for _, item := range buildItems { task := new(queue.Task) task.ID = fmt.Sprint(item.Proc.ID) task.Labels = map[string]string{} @@ -346,12 +333,12 @@ func PostHook(c *gin.Context) { task.Labels[k] = v } task.Labels["platform"] = item.Platform - task.Labels["repo"] = b.Repo.FullName + task.Labels["repo"] = repo.FullName task.Data, _ = json.Marshal(rpc.Pipeline{ ID: fmt.Sprint(item.Proc.ID), Config: item.Config, - Timeout: b.Repo.Timeout, + Timeout: repo.Timeout, }) Config.Services.Logs.Open(context.Background(), task.ID) @@ -359,237 +346,6 @@ func PostHook(c *gin.Context) { } } -// return the metadata from the cli context. -func metadataFromStruct(repo *model.Repo, build, last *model.Build, proc *model.Proc, link string) frontend.Metadata { - host := link - uri, err := url.Parse(link) - if err == nil { - host = uri.Host - } - return frontend.Metadata{ - Repo: frontend.Repo{ - Name: repo.FullName, - Link: repo.Link, - Remote: repo.Clone, - Private: repo.IsPrivate, - Branch: repo.Branch, - }, - Curr: frontend.Build{ - Number: build.Number, - Parent: build.Parent, - Created: build.Created, - Started: build.Started, - Finished: build.Finished, - Status: build.Status, - Event: build.Event, - Link: build.Link, - Target: build.Deploy, - Commit: frontend.Commit{ - Sha: build.Commit, - Ref: build.Ref, - Refspec: build.Refspec, - Branch: build.Branch, - Message: build.Message, - Author: frontend.Author{ - Name: build.Author, - Email: build.Email, - Avatar: build.Avatar, - }, - }, - }, - Prev: frontend.Build{ - Number: last.Number, - Created: last.Created, - Started: last.Started, - Finished: last.Finished, - Status: last.Status, - Event: last.Event, - Link: last.Link, - Target: last.Deploy, - Commit: frontend.Commit{ - Sha: last.Commit, - Ref: last.Ref, - Refspec: last.Refspec, - Branch: last.Branch, - Message: last.Message, - Author: frontend.Author{ - Name: last.Author, - Email: last.Email, - Avatar: last.Avatar, - }, - }, - }, - Job: frontend.Job{ - Number: proc.PID, - Matrix: proc.Environ, - }, - Sys: frontend.System{ - Name: "drone", - Link: link, - Host: host, - Arch: "linux/amd64", - }, - } -} - -type builder struct { - Repo *model.Repo - Curr *model.Build - Last *model.Build - Netrc *model.Netrc - Secs []*model.Secret - Regs []*model.Registry - Link string - Yaml string - Envs map[string]string -} - -type buildItem struct { - Proc *model.Proc - Platform string - Labels map[string]string - Config *backend.Config -} - -func (b *builder) Build() ([]*buildItem, error) { - - axes, err := matrix.ParseString(b.Yaml) - if err != nil { - return nil, err - } - if len(axes) == 0 { - axes = append(axes, matrix.Axis{}) - } - - var items []*buildItem - for i, axis := range axes { - proc := &model.Proc{ - BuildID: b.Curr.ID, - PID: i + 1, - PGID: i + 1, - State: model.StatusPending, - Environ: axis, - } - - metadata := metadataFromStruct(b.Repo, b.Curr, b.Last, proc, b.Link) - environ := metadata.Environ() - for k, v := range metadata.EnvironDrone() { - environ[k] = v - } - for k, v := range axis { - environ[k] = v - } - - var secrets []compiler.Secret - for _, sec := range b.Secs { - if !sec.Match(b.Curr.Event) { - continue - } - secrets = append(secrets, compiler.Secret{ - Name: sec.Name, - Value: sec.Value, - Match: sec.Images, - }) - } - - y := b.Yaml - s, err := envsubst.Eval(y, func(name string) string { - env := environ[name] - if strings.Contains(env, "\n") { - env = fmt.Sprintf("%q", env) - } - return env - }) - if err != nil { - return nil, err - } - y = s - - parsed, err := yaml.ParseString(y) - if err != nil { - return nil, err - } - metadata.Sys.Arch = parsed.Platform - if metadata.Sys.Arch == "" { - metadata.Sys.Arch = "linux/amd64" - } - - lerr := linter.New( - linter.WithTrusted(b.Repo.IsTrusted), - ).Lint(parsed) - if lerr != nil { - return nil, lerr - } - - var registries []compiler.Registry - for _, reg := range b.Regs { - registries = append(registries, compiler.Registry{ - Hostname: reg.Address, - Username: reg.Username, - Password: reg.Password, - Email: reg.Email, - }) - } - - ir := compiler.New( - compiler.WithEnviron(environ), - compiler.WithEnviron(b.Envs), - compiler.WithEscalated(Config.Pipeline.Privileged...), - compiler.WithResourceLimit(Config.Pipeline.Limits.MemSwapLimit, Config.Pipeline.Limits.MemLimit, Config.Pipeline.Limits.ShmSize, Config.Pipeline.Limits.CPUQuota, Config.Pipeline.Limits.CPUShares, Config.Pipeline.Limits.CPUSet), - compiler.WithVolumes(Config.Pipeline.Volumes...), - compiler.WithNetworks(Config.Pipeline.Networks...), - compiler.WithLocal(false), - compiler.WithOption( - compiler.WithNetrc( - b.Netrc.Login, - b.Netrc.Password, - b.Netrc.Machine, - ), - b.Repo.IsPrivate, - ), - compiler.WithRegistry(registries...), - compiler.WithSecret(secrets...), - compiler.WithPrefix( - fmt.Sprintf( - "%d_%d", - proc.ID, - rand.Int(), - ), - ), - compiler.WithEnviron(proc.Environ), - compiler.WithProxy(), - compiler.WithWorkspaceFromURL("/drone", b.Repo.Link), - compiler.WithMetadata(metadata), - ).Compile(parsed) - - // for _, sec := range b.Secs { - // if !sec.MatchEvent(b.Curr.Event) { - // continue - // } - // if b.Curr.Verified || sec.SkipVerify { - // ir.Secrets = append(ir.Secrets, &backend.Secret{ - // Mask: sec.Conceal, - // Name: sec.Name, - // Value: sec.Value, - // }) - // } - // } - - item := &buildItem{ - Proc: proc, - Config: ir, - Labels: parsed.Labels, - Platform: metadata.Sys.Arch, - } - if item.Labels == nil { - item.Labels = map[string]string{} - } - items = append(items, item) - } - - return items, nil -} - func shasum(raw []byte) string { sum := sha256.Sum256(raw) return fmt.Sprintf("%x", sum) diff --git a/server/hook_test.go b/server/hook_test.go index 3bd10bfe2..c34b0070d 100644 --- a/server/hook_test.go +++ b/server/hook_test.go @@ -21,7 +21,7 @@ import ( ) func TestMultilineEnvsubst(t *testing.T) { - b := builder{ + b := procBuilder{ Repo: &model.Repo{}, Curr: &model.Build{ Message: `aaa diff --git a/server/login.go b/server/login.go index bb783aca7..c6354036f 100644 --- a/server/login.go +++ b/server/login.go @@ -19,12 +19,12 @@ import ( "net/http" "time" + "github.com/gorilla/securecookie" "github.com/laszlocph/drone-oss-08/model" "github.com/laszlocph/drone-oss-08/remote" "github.com/laszlocph/drone-oss-08/shared/httputil" "github.com/laszlocph/drone-oss-08/shared/token" "github.com/laszlocph/drone-oss-08/store" - "github.com/gorilla/securecookie" "github.com/Sirupsen/logrus" "github.com/gin-gonic/gin" @@ -95,11 +95,6 @@ func HandleAuth(c *gin.Context) { ), } - if err = Config.Services.Limiter.LimitUser(u); err != nil { - c.String(403, "User activation blocked by limiter") - return - } - // insert the user into the database if err := store.CreateUser(c, u); err != nil { logrus.Errorf("cannot insert %s. %s", u.Login, err) diff --git a/server/procBuilder.go b/server/procBuilder.go new file mode 100644 index 000000000..f5ddee813 --- /dev/null +++ b/server/procBuilder.go @@ -0,0 +1,263 @@ +// Copyright 2018 Drone.IO Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "fmt" + "math/rand" + "net/url" + "strings" + + "github.com/drone/envsubst" + "github.com/laszlocph/drone-oss-08/cncd/pipeline/pipeline/backend" + "github.com/laszlocph/drone-oss-08/cncd/pipeline/pipeline/frontend" + "github.com/laszlocph/drone-oss-08/cncd/pipeline/pipeline/frontend/yaml" + "github.com/laszlocph/drone-oss-08/cncd/pipeline/pipeline/frontend/yaml/compiler" + "github.com/laszlocph/drone-oss-08/cncd/pipeline/pipeline/frontend/yaml/linter" + "github.com/laszlocph/drone-oss-08/cncd/pipeline/pipeline/frontend/yaml/matrix" + "github.com/laszlocph/drone-oss-08/model" +) + +// Takes the hook data and the yaml and returns in internal data model +type procBuilder struct { + Repo *model.Repo + Curr *model.Build + Last *model.Build + Netrc *model.Netrc + Secs []*model.Secret + Regs []*model.Registry + Link string + Yaml string + Envs map[string]string +} + +type buildItem struct { + Proc *model.Proc + Platform string + Labels map[string]string + Config *backend.Config +} + +func (b *procBuilder) Build() ([]*buildItem, error) { + + axes, err := matrix.ParseString(b.Yaml) + if err != nil { + return nil, err + } + if len(axes) == 0 { + axes = append(axes, matrix.Axis{}) + } + + var items []*buildItem + for i, axis := range axes { + proc := &model.Proc{ + BuildID: b.Curr.ID, + PID: i + 1, + PGID: i + 1, + State: model.StatusPending, + Environ: axis, + } + + metadata := metadataFromStruct(b.Repo, b.Curr, b.Last, proc, b.Link) + environ := metadata.Environ() + for k, v := range metadata.EnvironDrone() { + environ[k] = v + } + for k, v := range axis { + environ[k] = v + } + + var secrets []compiler.Secret + for _, sec := range b.Secs { + if !sec.Match(b.Curr.Event) { + continue + } + secrets = append(secrets, compiler.Secret{ + Name: sec.Name, + Value: sec.Value, + Match: sec.Images, + }) + } + + y := b.Yaml + s, err := envsubst.Eval(y, func(name string) string { + env := environ[name] + if strings.Contains(env, "\n") { + env = fmt.Sprintf("%q", env) + } + return env + }) + if err != nil { + return nil, err + } + y = s + + parsed, err := yaml.ParseString(y) + if err != nil { + return nil, err + } + metadata.Sys.Arch = parsed.Platform + if metadata.Sys.Arch == "" { + metadata.Sys.Arch = "linux/amd64" + } + + lerr := linter.New( + linter.WithTrusted(b.Repo.IsTrusted), + ).Lint(parsed) + if lerr != nil { + return nil, lerr + } + + var registries []compiler.Registry + for _, reg := range b.Regs { + registries = append(registries, compiler.Registry{ + Hostname: reg.Address, + Username: reg.Username, + Password: reg.Password, + Email: reg.Email, + }) + } + + ir := compiler.New( + compiler.WithEnviron(environ), + compiler.WithEnviron(b.Envs), + compiler.WithEscalated(Config.Pipeline.Privileged...), + compiler.WithResourceLimit(Config.Pipeline.Limits.MemSwapLimit, Config.Pipeline.Limits.MemLimit, Config.Pipeline.Limits.ShmSize, Config.Pipeline.Limits.CPUQuota, Config.Pipeline.Limits.CPUShares, Config.Pipeline.Limits.CPUSet), + compiler.WithVolumes(Config.Pipeline.Volumes...), + compiler.WithNetworks(Config.Pipeline.Networks...), + compiler.WithLocal(false), + compiler.WithOption( + compiler.WithNetrc( + b.Netrc.Login, + b.Netrc.Password, + b.Netrc.Machine, + ), + b.Repo.IsPrivate, + ), + compiler.WithRegistry(registries...), + compiler.WithSecret(secrets...), + compiler.WithPrefix( + fmt.Sprintf( + "%d_%d", + proc.ID, + rand.Int(), + ), + ), + compiler.WithEnviron(proc.Environ), + compiler.WithProxy(), + compiler.WithWorkspaceFromURL("/drone", b.Repo.Link), + compiler.WithMetadata(metadata), + ).Compile(parsed) + + // for _, sec := range b.Secs { + // if !sec.MatchEvent(b.Curr.Event) { + // continue + // } + // if b.Curr.Verified || sec.SkipVerify { + // ir.Secrets = append(ir.Secrets, &backend.Secret{ + // Mask: sec.Conceal, + // Name: sec.Name, + // Value: sec.Value, + // }) + // } + // } + + item := &buildItem{ + Proc: proc, + Config: ir, + Labels: parsed.Labels, + Platform: metadata.Sys.Arch, + } + if item.Labels == nil { + item.Labels = map[string]string{} + } + items = append(items, item) + } + + return items, nil +} + +// return the metadata from the cli context. +func metadataFromStruct(repo *model.Repo, build, last *model.Build, proc *model.Proc, link string) frontend.Metadata { + host := link + uri, err := url.Parse(link) + if err == nil { + host = uri.Host + } + return frontend.Metadata{ + Repo: frontend.Repo{ + Name: repo.FullName, + Link: repo.Link, + Remote: repo.Clone, + Private: repo.IsPrivate, + Branch: repo.Branch, + }, + Curr: frontend.Build{ + Number: build.Number, + Parent: build.Parent, + Created: build.Created, + Started: build.Started, + Finished: build.Finished, + Status: build.Status, + Event: build.Event, + Link: build.Link, + Target: build.Deploy, + Commit: frontend.Commit{ + Sha: build.Commit, + Ref: build.Ref, + Refspec: build.Refspec, + Branch: build.Branch, + Message: build.Message, + Author: frontend.Author{ + Name: build.Author, + Email: build.Email, + Avatar: build.Avatar, + }, + }, + }, + Prev: frontend.Build{ + Number: last.Number, + Created: last.Created, + Started: last.Started, + Finished: last.Finished, + Status: last.Status, + Event: last.Event, + Link: last.Link, + Target: last.Deploy, + Commit: frontend.Commit{ + Sha: last.Commit, + Ref: last.Ref, + Refspec: last.Refspec, + Branch: last.Branch, + Message: last.Message, + Author: frontend.Author{ + Name: last.Author, + Email: last.Email, + Avatar: last.Avatar, + }, + }, + }, + Job: frontend.Job{ + Number: proc.PID, + Matrix: proc.Environ, + }, + Sys: frontend.System{ + Name: "drone", + Link: link, + Host: host, + Arch: "linux/amd64", + }, + } +} diff --git a/server/repo.go b/server/repo.go index 7f51db3b6..70c68269e 100644 --- a/server/repo.go +++ b/server/repo.go @@ -41,11 +41,6 @@ func PostRepo(c *gin.Context) { return } - if err := Config.Services.Limiter.LimitRepo(user, repo); err != nil { - c.String(403, "Repository activation blocked by limiter") - return - } - repo.IsActive = true repo.UserID = user.ID if !repo.AllowPush && !repo.AllowPull && !repo.AllowDeploy && !repo.AllowTag { diff --git a/server/rpc.go b/server/rpc.go index 383e61639..61ff8648a 100644 --- a/server/rpc.go +++ b/server/rpc.go @@ -55,7 +55,6 @@ var Config = struct { Secrets model.SecretService Registries model.RegistryService Environ model.EnvironService - Limiter model.Limiter } Storage struct { // Users model.UserStore diff --git a/server/sync.go b/server/sync.go index b448ee338..59da137cc 100644 --- a/server/sync.go +++ b/server/sync.go @@ -28,10 +28,9 @@ type Syncer interface { } type syncer struct { - remote remote.Remote - store store.Store - perms model.PermStore - limiter model.Limiter + remote remote.Remote + store store.Store + perms model.PermStore } func (s *syncer) Sync(user *model.User) error { @@ -41,10 +40,6 @@ func (s *syncer) Sync(user *model.User) error { return err } - if s.limiter != nil { - repos = s.limiter.LimitRepos(user, repos) - } - var perms []*model.Perm for _, repo := range repos { perm := model.Perm{ diff --git a/server/user.go b/server/user.go index 24b068277..be4fc663d 100644 --- a/server/user.go +++ b/server/user.go @@ -46,10 +46,9 @@ func GetFeed(c *gin.Context) { store.FromContext(c).UpdateUser(user) sync := syncer{ - remote: remote.FromContext(c), - store: store.FromContext(c), - perms: store.FromContext(c), - limiter: Config.Services.Limiter, + remote: remote.FromContext(c), + store: store.FromContext(c), + perms: store.FromContext(c), } if err := sync.Sync(user); err != nil { logrus.Debugf("sync error: %s: %s", user.Login, err) @@ -89,10 +88,9 @@ func GetRepos(c *gin.Context) { store.FromContext(c).UpdateUser(user) sync := syncer{ - remote: remote.FromContext(c), - store: store.FromContext(c), - perms: store.FromContext(c), - limiter: Config.Services.Limiter, + remote: remote.FromContext(c), + store: store.FromContext(c), + perms: store.FromContext(c), } if err := sync.Sync(user); err != nil { logrus.Debugf("sync error: %s: %s", user.Login, err) diff --git a/store/datastore/builds.go b/store/datastore/builds.go index 5a347fc4f..ed7977152 100644 --- a/store/datastore/builds.go +++ b/store/datastore/builds.go @@ -72,6 +72,7 @@ func (db *datastore) GetBuildQueue() ([]*model.Feed, error) { } func (db *datastore) CreateBuild(build *model.Build, procs ...*model.Proc) error { + build.Trim() id, err := db.incrementRepoRetry(build.RepoID) if err != nil { return err