Refactor

cncd/pipeline/pipeline/frontend/metadata.go

@@ -222,3 +222,10 @@ func (m *Metadata) EnvironDrone() map[string]string {
 }
 
 var pullRegexp = regexp.MustCompile("\\d+")
+
+func (m *Metadata) SetPlatform(platform string) {
+	if platform == "" {
+		platform = "linux/amd64"
+	}
+	m.Sys.Arch = platform
+}

docker-compose.yml

@@ -28,4 +28,4 @@ services:
     environment:
       - DRONE_SERVER=drone-server:9000
       - DRONE_SECRET=${DRONE_SECRET}
-      - DRONE_MAX_PROCS=1
+      - DRONE_MAX_PROCS=2

server/build.go

@@ -336,7 +336,6 @@ func PostApproval(c *gin.Context) {
 		return
 	}
 
-	setBuildProcs(build, buildItems)
 	err = store.FromContext(c).ProcCreate(build.Procs)
 	if err != nil {
 		logrus.Errorf("error persisting procs %s/%d: %s", repo.FullName, build.Number, err)
@@ -525,8 +524,6 @@ func PostBuild(c *gin.Context) {
 		return
 	}
 
-	setBuildProcs(build, buildItems)
-
 	err = store.FromContext(c).ProcCreate(build.Procs)
 	if err != nil {
 		logrus.Errorf("cannot restart %s#%d: %s", repo.FullName, build.Number, err)

server/procBuilder.go

@@ -54,6 +54,7 @@ func (b *procBuilder) Build() ([]*buildItem, error) {
 	var items []*buildItem
 
 	for j, y := range b.Yamls {
+		// matrix axes
 		axes, err := matrix.ParseString(y)
 		if err != nil {
 			return nil, err
@@ -70,8 +71,64 @@ func (b *procBuilder) Build() ([]*buildItem, error) {
 				State:   model.StatusPending,
 				Environ: axis,
 			}
+			b.Curr.Procs = append(b.Curr.Procs, proc)
 
 			metadata := metadataFromStruct(b.Repo, b.Curr, b.Last, proc, b.Link)
+			environ := b.environmentVariables(metadata, axis)
+
+			// substitute vars
+			y, err := b.envsubst_(y, environ)
+			if err != nil {
+				return nil, err
+			}
+
+			// parse yaml pipeline
+			parsed, err := yaml.ParseString(y)
+			if err != nil {
+				return nil, err
+			}
+
+			// lint pipeline
+			lerr := linter.New(
+				linter.WithTrusted(b.Repo.IsTrusted),
+			).Lint(parsed)
+			if lerr != nil {
+				return nil, lerr
+			}
+
+			metadata.SetPlatform(parsed.Platform)
+
+			ir := b.toInternalRepresentation(parsed, environ, metadata, proc.ID)
+
+			item := &buildItem{
+				Proc:     proc,
+				Config:   ir,
+				Labels:   parsed.Labels,
+				Platform: metadata.Sys.Arch,
+			}
+			if item.Labels == nil {
+				item.Labels = map[string]string{}
+			}
+			items = append(items, item)
+		}
+	}
+
+	setBuildSteps(b.Curr, items)
+
+	return items, nil
+}
+
+func (b *procBuilder) envsubst_(y string, environ map[string]string) (string, error) {
+	return envsubst.Eval(y, func(name string) string {
+		env := environ[name]
+		if strings.Contains(env, "\n") {
+			env = fmt.Sprintf("%q", env)
+		}
+		return env
+	})
+}
+
+func (b *procBuilder) environmentVariables(metadata frontend.Metadata, axis matrix.Axis) map[string]string {
 	environ := metadata.Environ()
 	for k, v := range metadata.EnvironDrone() {
 		environ[k] = v
@@ -79,7 +136,10 @@ func (b *procBuilder) Build() ([]*buildItem, error) {
 	for k, v := range axis {
 		environ[k] = v
 	}
+	return environ
+}
 
+func (b *procBuilder) toInternalRepresentation(parsed *yaml.Config, environ map[string]string, metadata frontend.Metadata, procID int64) *backend.Config {
 	var secrets []compiler.Secret
 	for _, sec := range b.Secs {
 		if !sec.Match(b.Curr.Event) {
@@ -92,34 +152,6 @@ func (b *procBuilder) Build() ([]*buildItem, error) {
 		})
 	}
 
-			s, err := envsubst.Eval(y, func(name string) string {
-				env := environ[name]
-				if strings.Contains(env, "\n") {
-					env = fmt.Sprintf("%q", env)
-				}
-				return env
-			})
-			if err != nil {
-				return nil, err
-			}
-			y = s
-
-			parsed, err := yaml.ParseString(y)
-			if err != nil {
-				return nil, err
-			}
-			metadata.Sys.Arch = parsed.Platform
-			if metadata.Sys.Arch == "" {
-				metadata.Sys.Arch = "linux/amd64"
-			}
-
-			lerr := linter.New(
-				linter.WithTrusted(b.Repo.IsTrusted),
-			).Lint(parsed)
-			if lerr != nil {
-				return nil, lerr
-			}
-
 	var registries []compiler.Registry
 	for _, reg := range b.Regs {
 		registries = append(registries, compiler.Registry{
@@ -130,7 +162,7 @@ func (b *procBuilder) Build() ([]*buildItem, error) {
 		})
 	}
 
-			ir := compiler.New(
+	return compiler.New(
 		compiler.WithEnviron(environ),
 		compiler.WithEnviron(b.Envs),
 		compiler.WithEscalated(Config.Pipeline.Privileged...),
@@ -151,40 +183,19 @@ func (b *procBuilder) Build() ([]*buildItem, error) {
 		compiler.WithPrefix(
 			fmt.Sprintf(
 				"%d_%d",
-						proc.ID,
+				procID,
 				rand.Int(),
 			),
 		),
-				compiler.WithEnviron(proc.Environ),
 		compiler.WithProxy(),
 		compiler.WithWorkspaceFromURL("/drone", b.Repo.Link),
 		compiler.WithMetadata(metadata),
 	).Compile(parsed)
-
-			item := &buildItem{
-				Proc:     proc,
-				Config:   ir,
-				Labels:   parsed.Labels,
-				Platform: metadata.Sys.Arch,
-			}
-			if item.Labels == nil {
-				item.Labels = map[string]string{}
-			}
-			items = append(items, item)
-		}
-	}
-
-	setBuildProcs(b.Curr, items)
-
-	return items, nil
 }
 
-func setBuildProcs(build *model.Build, buildItems []*buildItem) {
+func setBuildSteps(build *model.Build, buildItems []*buildItem) {
 	pcounter := len(buildItems)
 	for _, item := range buildItems {
-		build.Procs = append(build.Procs, item.Proc)
-		item.Proc.BuildID = build.ID
-
 		for _, stage := range item.Config.Stages {
 			var gid int
 			for _, step := range stage.Steps {

server/procBuilder_test.go

@@ -49,9 +49,6 @@ bbb`,
 		t.Fatal(err)
 	} else {
 		fmt.Println(buildItems)
-		build := &model.Build{}
-		setBuildProcs(build, buildItems)
-		fmt.Println(build)
 	}
 }