woodpecker/pipeline/stepBuilder.go
qwerty287 abb2f280eb
Unique status for matrix (#2695)
implement this fix but with an additional field on workflows to not
change the workflow name

closes #1840 
closes #713

---------

Co-authored-by: 6543 <6543@obermui.de>
2023-11-01 17:28:02 +01:00

296 lines
8.1 KiB
Go

// Copyright 2022 Woodpecker Authors
// Copyright 2018 Drone.IO Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pipeline
import (
"fmt"
"path/filepath"
"strings"
"github.com/oklog/ulid/v2"
"github.com/rs/zerolog/log"
backend_types "github.com/woodpecker-ci/woodpecker/pipeline/backend/types"
yaml_types "github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml/types"
forge_types "github.com/woodpecker-ci/woodpecker/server/forge/types"
"github.com/woodpecker-ci/woodpecker/pipeline/frontend"
"github.com/woodpecker-ci/woodpecker/pipeline/frontend/metadata"
"github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml"
"github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml/compiler"
"github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml/linter"
"github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml/matrix"
"github.com/woodpecker-ci/woodpecker/server"
"github.com/woodpecker-ci/woodpecker/server/model"
)
// StepBuilder Takes the hook data and the yaml and returns in internal data model
type StepBuilder struct {
Repo *model.Repo
Curr *model.Pipeline
Last *model.Pipeline
Netrc *model.Netrc
Secs []*model.Secret
Regs []*model.Registry
Link string
Yamls []*forge_types.FileMeta
Envs map[string]string
Forge metadata.ServerForge
ProxyOpts compiler.ProxyOptions
}
type Item struct {
Workflow *model.Workflow
Labels map[string]string
DependsOn []string
RunsOn []string
Config *backend_types.Config
}
func (b *StepBuilder) Build() ([]*Item, error) {
var items []*Item
b.Yamls = forge_types.SortByName(b.Yamls)
pidSequence := 1
for _, y := range b.Yamls {
// matrix axes
axes, err := matrix.ParseString(string(y.Data))
if err != nil {
return nil, err
}
if len(axes) == 0 {
axes = append(axes, matrix.Axis{})
}
for i, axis := range axes {
workflow := &model.Workflow{
PID: pidSequence,
State: model.StatusPending,
Environ: axis,
Name: SanitizePath(y.Name),
AxisID: i + 1,
}
item, err := b.genItemForWorkflow(workflow, axis, string(y.Data))
if err != nil {
return nil, err
}
if item == nil {
continue
}
items = append(items, item)
pidSequence++
}
// TODO: add summary workflow that send status back based on workflows generated by matrix function
// depend on https://github.com/woodpecker-ci/woodpecker/issues/778
}
items = filterItemsWithMissingDependencies(items)
// check if at least one step can start if slice is not empty
if len(items) > 0 && !stepListContainsItemsToRun(items) {
return nil, fmt.Errorf("pipeline has no startpoint")
}
return items, nil
}
func (b *StepBuilder) genItemForWorkflow(workflow *model.Workflow, axis matrix.Axis, data string) (*Item, error) {
workflowMetadata := frontend.MetadataFromStruct(b.Forge, b.Repo, b.Curr, b.Last, workflow, b.Link)
environ := b.environmentVariables(workflowMetadata, axis)
// add global environment variables for substituting
for k, v := range b.Envs {
if _, exists := environ[k]; exists {
// don't override existing values
continue
}
environ[k] = v
}
// substitute vars
substituted, err := frontend.EnvVarSubst(data, environ)
if err != nil {
return nil, err
}
// parse yaml pipeline
parsed, err := yaml.ParseString(substituted)
if err != nil {
return nil, &yaml.PipelineParseError{Err: err}
}
// lint pipeline
if err := linter.New(
linter.WithTrusted(b.Repo.IsTrusted),
).Lint(parsed); err != nil {
return nil, &yaml.PipelineParseError{Err: err}
}
// checking if filtered.
if match, err := parsed.When.Match(workflowMetadata, true, environ); !match && err == nil {
log.Debug().Str("pipeline", workflow.Name).Msg(
"Marked as skipped, does not match metadata",
)
return nil, nil
} else if err != nil {
log.Debug().Str("pipeline", workflow.Name).Msg(
"Pipeline config could not be parsed",
)
return nil, err
}
ir, err := b.toInternalRepresentation(parsed, environ, workflowMetadata, workflow.ID)
if err != nil {
return nil, err
}
if len(ir.Stages) == 0 {
return nil, nil
}
item := &Item{
Workflow: workflow,
Config: ir,
Labels: parsed.Labels,
DependsOn: parsed.DependsOn,
RunsOn: parsed.RunsOn,
}
if item.Labels == nil {
item.Labels = map[string]string{}
}
return item, nil
}
func stepListContainsItemsToRun(items []*Item) bool {
for i := range items {
if items[i].Workflow.State == model.StatusPending {
return true
}
}
return false
}
func filterItemsWithMissingDependencies(items []*Item) []*Item {
itemsToRemove := make([]*Item, 0)
for _, item := range items {
for _, dep := range item.DependsOn {
if !containsItemWithName(dep, items) {
itemsToRemove = append(itemsToRemove, item)
}
}
}
if len(itemsToRemove) > 0 {
filtered := make([]*Item, 0)
for _, item := range items {
if !containsItemWithName(item.Workflow.Name, itemsToRemove) {
filtered = append(filtered, item)
}
}
// Recursive to handle transitive deps
return filterItemsWithMissingDependencies(filtered)
}
return items
}
func containsItemWithName(name string, items []*Item) bool {
for _, item := range items {
if name == item.Workflow.Name {
return true
}
}
return false
}
func (b *StepBuilder) environmentVariables(metadata metadata.Metadata, axis matrix.Axis) map[string]string {
environ := metadata.Environ()
for k, v := range axis {
environ[k] = v
}
return environ
}
func (b *StepBuilder) toInternalRepresentation(parsed *yaml_types.Workflow, environ map[string]string, metadata metadata.Metadata, stepID int64) (*backend_types.Config, error) {
var secrets []compiler.Secret
for _, sec := range b.Secs {
if !sec.Match(b.Curr.Event) {
continue
}
secrets = append(secrets, compiler.Secret{
Name: sec.Name,
Value: sec.Value,
AllowedPlugins: sec.Images,
})
}
var registries []compiler.Registry
for _, reg := range b.Regs {
registries = append(registries, compiler.Registry{
Hostname: reg.Address,
Username: reg.Username,
Password: reg.Password,
Email: reg.Email,
})
}
return compiler.New(
compiler.WithEnviron(environ),
compiler.WithEnviron(b.Envs),
// TODO: server deps should be moved into StepBuilder fields and set on StepBuilder creation
compiler.WithEscalated(server.Config.Pipeline.Privileged...),
compiler.WithResourceLimit(server.Config.Pipeline.Limits.MemSwapLimit, server.Config.Pipeline.Limits.MemLimit, server.Config.Pipeline.Limits.ShmSize, server.Config.Pipeline.Limits.CPUQuota, server.Config.Pipeline.Limits.CPUShares, server.Config.Pipeline.Limits.CPUSet),
compiler.WithVolumes(server.Config.Pipeline.Volumes...),
compiler.WithNetworks(server.Config.Pipeline.Networks...),
compiler.WithLocal(false),
compiler.WithOption(
compiler.WithNetrc(
b.Netrc.Login,
b.Netrc.Password,
b.Netrc.Machine,
),
b.Repo.IsSCMPrivate || server.Config.Pipeline.AuthenticatePublicRepos,
),
compiler.WithDefaultCloneImage(server.Config.Pipeline.DefaultCloneImage),
compiler.WithRegistry(registries...),
compiler.WithSecret(secrets...),
compiler.WithPrefix(
fmt.Sprintf(
"wp_%s_%d",
strings.ToLower(ulid.Make().String()),
stepID,
),
),
compiler.WithProxy(b.ProxyOpts),
compiler.WithWorkspaceFromURL("/woodpecker", b.Repo.Link),
compiler.WithMetadata(metadata),
compiler.WithTrusted(b.Repo.IsTrusted),
compiler.WithNetrcOnlyTrusted(b.Repo.NetrcOnlyTrusted),
).Compile(parsed)
}
func SanitizePath(path string) string {
path = filepath.Base(path)
path = strings.TrimSuffix(path, ".yml")
path = strings.TrimSuffix(path, ".yaml")
path = strings.TrimPrefix(path, ".")
return path
}