From 105a0708fe94c889f9840f45dd90e447163ea21c Mon Sep 17 00:00:00 2001 From: Laszlo Fogas Date: Wed, 5 Jun 2019 15:58:27 +0200 Subject: [PATCH] Refactor --- cncd/pipeline/pipeline/frontend/metadata.go | 7 + docker-compose.yml | 2 +- server/build.go | 3 - server/procBuilder.go | 161 +++++++++++--------- server/procBuilder_test.go | 3 - 5 files changed, 94 insertions(+), 82 deletions(-) diff --git a/cncd/pipeline/pipeline/frontend/metadata.go b/cncd/pipeline/pipeline/frontend/metadata.go index edd4b0b62..c46791add 100644 --- a/cncd/pipeline/pipeline/frontend/metadata.go +++ b/cncd/pipeline/pipeline/frontend/metadata.go @@ -222,3 +222,10 @@ func (m *Metadata) EnvironDrone() map[string]string { } var pullRegexp = regexp.MustCompile("\\d+") + +func (m *Metadata) SetPlatform(platform string) { + if platform == "" { + platform = "linux/amd64" + } + m.Sys.Arch = platform +} diff --git a/docker-compose.yml b/docker-compose.yml index 4a2594fb6..e90443644 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -28,4 +28,4 @@ services: environment: - DRONE_SERVER=drone-server:9000 - DRONE_SECRET=${DRONE_SECRET} - - DRONE_MAX_PROCS=1 \ No newline at end of file + - DRONE_MAX_PROCS=2 \ No newline at end of file diff --git a/server/build.go b/server/build.go index 280dd6287..ed024938d 100644 --- a/server/build.go +++ b/server/build.go @@ -336,7 +336,6 @@ func PostApproval(c *gin.Context) { return } - setBuildProcs(build, buildItems) err = store.FromContext(c).ProcCreate(build.Procs) if err != nil { logrus.Errorf("error persisting procs %s/%d: %s", repo.FullName, build.Number, err) @@ -525,8 +524,6 @@ func PostBuild(c *gin.Context) { return } - setBuildProcs(build, buildItems) - err = store.FromContext(c).ProcCreate(build.Procs) if err != nil { logrus.Errorf("cannot restart %s#%d: %s", repo.FullName, build.Number, err) diff --git a/server/procBuilder.go b/server/procBuilder.go index e12c05df8..7cd70e13a 100644 --- a/server/procBuilder.go +++ b/server/procBuilder.go @@ -54,6 +54,7 @@ func (b *procBuilder) Build() ([]*buildItem, error) { var items []*buildItem for j, y := range b.Yamls { + // matrix axes axes, err := matrix.ParseString(y) if err != nil { return nil, err @@ -70,49 +71,24 @@ func (b *procBuilder) Build() ([]*buildItem, error) { State: model.StatusPending, Environ: axis, } + b.Curr.Procs = append(b.Curr.Procs, proc) metadata := metadataFromStruct(b.Repo, b.Curr, b.Last, proc, b.Link) - environ := metadata.Environ() - for k, v := range metadata.EnvironDrone() { - environ[k] = v - } - for k, v := range axis { - environ[k] = v - } + environ := b.environmentVariables(metadata, axis) - var secrets []compiler.Secret - for _, sec := range b.Secs { - if !sec.Match(b.Curr.Event) { - continue - } - secrets = append(secrets, compiler.Secret{ - Name: sec.Name, - Value: sec.Value, - Match: sec.Images, - }) - } - - s, err := envsubst.Eval(y, func(name string) string { - env := environ[name] - if strings.Contains(env, "\n") { - env = fmt.Sprintf("%q", env) - } - return env - }) + // substitute vars + y, err := b.envsubst_(y, environ) if err != nil { return nil, err } - y = s + // parse yaml pipeline parsed, err := yaml.ParseString(y) if err != nil { return nil, err } - metadata.Sys.Arch = parsed.Platform - if metadata.Sys.Arch == "" { - metadata.Sys.Arch = "linux/amd64" - } + // lint pipeline lerr := linter.New( linter.WithTrusted(b.Repo.IsTrusted), ).Lint(parsed) @@ -120,46 +96,9 @@ func (b *procBuilder) Build() ([]*buildItem, error) { return nil, lerr } - var registries []compiler.Registry - for _, reg := range b.Regs { - registries = append(registries, compiler.Registry{ - Hostname: reg.Address, - Username: reg.Username, - Password: reg.Password, - Email: reg.Email, - }) - } + metadata.SetPlatform(parsed.Platform) - ir := compiler.New( - compiler.WithEnviron(environ), - compiler.WithEnviron(b.Envs), - compiler.WithEscalated(Config.Pipeline.Privileged...), - compiler.WithResourceLimit(Config.Pipeline.Limits.MemSwapLimit, Config.Pipeline.Limits.MemLimit, Config.Pipeline.Limits.ShmSize, Config.Pipeline.Limits.CPUQuota, Config.Pipeline.Limits.CPUShares, Config.Pipeline.Limits.CPUSet), - compiler.WithVolumes(Config.Pipeline.Volumes...), - compiler.WithNetworks(Config.Pipeline.Networks...), - compiler.WithLocal(false), - compiler.WithOption( - compiler.WithNetrc( - b.Netrc.Login, - b.Netrc.Password, - b.Netrc.Machine, - ), - b.Repo.IsPrivate, - ), - compiler.WithRegistry(registries...), - compiler.WithSecret(secrets...), - compiler.WithPrefix( - fmt.Sprintf( - "%d_%d", - proc.ID, - rand.Int(), - ), - ), - compiler.WithEnviron(proc.Environ), - compiler.WithProxy(), - compiler.WithWorkspaceFromURL("/drone", b.Repo.Link), - compiler.WithMetadata(metadata), - ).Compile(parsed) + ir := b.toInternalRepresentation(parsed, environ, metadata, proc.ID) item := &buildItem{ Proc: proc, @@ -174,17 +113,89 @@ func (b *procBuilder) Build() ([]*buildItem, error) { } } - setBuildProcs(b.Curr, items) + setBuildSteps(b.Curr, items) return items, nil } -func setBuildProcs(build *model.Build, buildItems []*buildItem) { +func (b *procBuilder) envsubst_(y string, environ map[string]string) (string, error) { + return envsubst.Eval(y, func(name string) string { + env := environ[name] + if strings.Contains(env, "\n") { + env = fmt.Sprintf("%q", env) + } + return env + }) +} + +func (b *procBuilder) environmentVariables(metadata frontend.Metadata, axis matrix.Axis) map[string]string { + environ := metadata.Environ() + for k, v := range metadata.EnvironDrone() { + environ[k] = v + } + for k, v := range axis { + environ[k] = v + } + return environ +} + +func (b *procBuilder) toInternalRepresentation(parsed *yaml.Config, environ map[string]string, metadata frontend.Metadata, procID int64) *backend.Config { + var secrets []compiler.Secret + for _, sec := range b.Secs { + if !sec.Match(b.Curr.Event) { + continue + } + secrets = append(secrets, compiler.Secret{ + Name: sec.Name, + Value: sec.Value, + Match: sec.Images, + }) + } + + var registries []compiler.Registry + for _, reg := range b.Regs { + registries = append(registries, compiler.Registry{ + Hostname: reg.Address, + Username: reg.Username, + Password: reg.Password, + Email: reg.Email, + }) + } + + return compiler.New( + compiler.WithEnviron(environ), + compiler.WithEnviron(b.Envs), + compiler.WithEscalated(Config.Pipeline.Privileged...), + compiler.WithResourceLimit(Config.Pipeline.Limits.MemSwapLimit, Config.Pipeline.Limits.MemLimit, Config.Pipeline.Limits.ShmSize, Config.Pipeline.Limits.CPUQuota, Config.Pipeline.Limits.CPUShares, Config.Pipeline.Limits.CPUSet), + compiler.WithVolumes(Config.Pipeline.Volumes...), + compiler.WithNetworks(Config.Pipeline.Networks...), + compiler.WithLocal(false), + compiler.WithOption( + compiler.WithNetrc( + b.Netrc.Login, + b.Netrc.Password, + b.Netrc.Machine, + ), + b.Repo.IsPrivate, + ), + compiler.WithRegistry(registries...), + compiler.WithSecret(secrets...), + compiler.WithPrefix( + fmt.Sprintf( + "%d_%d", + procID, + rand.Int(), + ), + ), + compiler.WithProxy(), + compiler.WithWorkspaceFromURL("/drone", b.Repo.Link), + compiler.WithMetadata(metadata), + ).Compile(parsed) +} + +func setBuildSteps(build *model.Build, buildItems []*buildItem) { pcounter := len(buildItems) for _, item := range buildItems { - build.Procs = append(build.Procs, item.Proc) - item.Proc.BuildID = build.ID - for _, stage := range item.Config.Stages { var gid int for _, step := range stage.Steps { diff --git a/server/procBuilder_test.go b/server/procBuilder_test.go index 9ba858ed9..aaf002821 100644 --- a/server/procBuilder_test.go +++ b/server/procBuilder_test.go @@ -49,9 +49,6 @@ bbb`, t.Fatal(err) } else { fmt.Println(buildItems) - build := &model.Build{} - setBuildProcs(build, buildItems) - fmt.Println(build) } }