mirrors/woodpecker
1
0
Fork 0
mirror of https://github.com/woodpecker-ci/woodpecker.git synced 2024-06-26 09:10:48 +00:00
Code Issues Projects Releases Wiki Activity

Rename pipeline frontend types (#1829)

Browse source 
this adjust the packages that parse the yaml-config-file to match
[Terminology](https://woodpecker-ci.org/docs/next/usage/terminology)
This commit is contained in:
6543 2023-06-06 09:14:21 +02:00 committed by GitHub
parent b82ed13586
commit 971cb52032
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
28 changed files with 535 additions and 517 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
docs/docs/20-usage/25-workflows.md
View file

@ -1,7 +1,7 @@
# Workflows
:::info
This Feature is only available for GitHub, Gitea & GitLab repositories. Follow [this](https://github.com/woodpecker-ci/woodpecker/issues/131) issue to support further development.
This Feature is only available for GitHub, Gitea & GitLab repositories. Follow [this](https://github.com/woodpecker-ci/woodpecker/issues/1138) issue to support further development.
:::
A pipeline has at least one workflow. A workflow is a set of steps that are executed in sequence using the same workspace which is a shared folder containing the repository and all the generated data from previous steps.

31
pipeline/frontend/yaml/compiler/cacher.go
View file

@ -4,23 +4,22 @@ import (
"path"
"strings"
"github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml"
"github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml/types"
yaml_types "github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml/types"
)
// Cacher defines a compiler transform that can be used
// to implement default caching for a repository.
type Cacher interface {
Restore(repo, branch string, mounts []string) *yaml.Container
Rebuild(repo, branch string, mounts []string) *yaml.Container
Restore(repo, branch string, mounts []string) *yaml_types.Container
Rebuild(repo, branch string, mounts []string) *yaml_types.Container
}
type volumeCacher struct {
base string
}
func (c *volumeCacher) Restore(repo, branch string, mounts []string) *yaml.Container {
return &yaml.Container{
func (c *volumeCacher) Restore(repo, branch string, mounts []string) *yaml_types.Container {
return &yaml_types.Container{
Name: "rebuild_cache",
Image: "plugins/volume-cache:1.0.0",
Settings: map[string]interface{}{
@ -30,8 +29,8 @@ func (c *volumeCacher) Restore(repo, branch string, mounts []string) *yaml.Conta
"file": strings.Replace(branch, "/", "_", -1) + ".tar",
"fallback_to": "master.tar",
},
Volumes: types.Volumes{
Volumes: []*types.Volume{
Volumes: yaml_types.Volumes{
Volumes: []*yaml_types.Volume{
{
Source: path.Join(c.base, repo),
Destination: "/cache",
@ -42,8 +41,8 @@ func (c *volumeCacher) Restore(repo, branch string, mounts []string) *yaml.Conta
}
}
func (c *volumeCacher) Rebuild(repo, branch string, mounts []string) *yaml.Container {
return &yaml.Container{
func (c *volumeCacher) Rebuild(repo, branch string, mounts []string) *yaml_types.Container {
return &yaml_types.Container{
Name: "rebuild_cache",
Image: "plugins/volume-cache:1.0.0",
Settings: map[string]interface{}{
@ -53,8 +52,8 @@ func (c *volumeCacher) Rebuild(repo, branch string, mounts []string) *yaml.Conta
"flush": true,
"file": strings.Replace(branch, "/", "_", -1) + ".tar",
},
Volumes: types.Volumes{
Volumes: []*types.Volume{
Volumes: yaml_types.Volumes{
Volumes: []*yaml_types.Volume{
{
Source: path.Join(c.base, repo),
Destination: "/cache",
@ -72,8 +71,8 @@ type s3Cacher struct {
region string
}
func (c *s3Cacher) Restore(_, _ string, mounts []string) *yaml.Container {
return &yaml.Container{
func (c *s3Cacher) Restore(_, _ string, mounts []string) *yaml_types.Container {
return &yaml_types.Container{
Name: "rebuild_cache",
Image: "plugins/s3-cache:latest",
Settings: map[string]interface{}{
@ -87,8 +86,8 @@ func (c *s3Cacher) Restore(_, _ string, mounts []string) *yaml.Container {
}
}
func (c *s3Cacher) Rebuild(_, _ string, mounts []string) *yaml.Container {
return &yaml.Container{
func (c *s3Cacher) Rebuild(_, _ string, mounts []string) *yaml_types.Container {
return &yaml_types.Container{
Name: "rebuild_cache",
Image: "plugins/s3-cache:latest",
Settings: map[string]interface{}{

48
pipeline/frontend/yaml/compiler/compiler.go
View file

@ -5,9 +5,9 @@ import (
"path"
"strings"
backend "github.com/woodpecker-ci/woodpecker/pipeline/backend/types"
backend_types "github.com/woodpecker-ci/woodpecker/pipeline/backend/types"
"github.com/woodpecker-ci/woodpecker/pipeline/frontend/metadata"
"github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml"
yaml_types "github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml/types"
"github.com/woodpecker-ci/woodpecker/shared/constant"
)
@ -42,7 +42,7 @@ type Secret struct {
PluginOnly bool
}
func (s *Secret) Available(container *yaml.Container) bool {
func (s *Secret) Available(container *yaml_types.Container) bool {
return (len(s.Match) == 0 || matchImage(container.Image, s.Match...)) && (!s.PluginOnly || container.IsPlugin())
}
@ -101,8 +101,8 @@ func New(opts ...Option) *Compiler {
// Compile compiles the YAML configuration to the pipeline intermediate
// representation configuration format.
func (c *Compiler) Compile(conf *yaml.Config) (*backend.Config, error) {
config := new(backend.Config)
func (c *Compiler) Compile(conf *yaml_types.Workflow) (*backend_types.Config, error) {
config := new(backend_types.Config)
if match, err := conf.When.Match(c.metadata, true); !match && err == nil {
// This pipeline does not match the configured filter so return an empty config and stop further compilation.
@ -113,19 +113,19 @@ func (c *Compiler) Compile(conf *yaml.Config) (*backend.Config, error) {
}
// create a default volume
config.Volumes = append(config.Volumes, &backend.Volume{
config.Volumes = append(config.Volumes, &backend_types.Volume{
Name: fmt.Sprintf("%s_default", c.prefix),
Driver: "local",
})
// create a default network
if strings.HasPrefix(c.metadata.Sys.Platform, windowsPrefix) {
config.Networks = append(config.Networks, &backend.Network{
config.Networks = append(config.Networks, &backend_types.Network{
Name: fmt.Sprintf("%s_default", c.prefix),
Driver: networkDriverNAT,
})
} else {
config.Networks = append(config.Networks, &backend.Network{
config.Networks = append(config.Networks, &backend_types.Network{
Name: fmt.Sprintf("%s_default", c.prefix),
Driver: networkDriverBridge,
})
@ -133,7 +133,7 @@ func (c *Compiler) Compile(conf *yaml.Config) (*backend.Config, error) {
// create secrets for mask
for _, sec := range c.secrets {
config.Secrets = append(config.Secrets, &backend.Secret{
config.Secrets = append(config.Secrets, &backend_types.Secret{
Name: sec.Name,
Value: sec.Value,
Mask: true,
@ -155,12 +155,12 @@ func (c *Compiler) Compile(conf *yaml.Config) (*backend.Config, error) {
}
// add default clone step
if !c.local && len(conf.Clone.Containers) == 0 && !conf.SkipClone {
if !c.local && len(conf.Clone.ContainerList) == 0 && !conf.SkipClone {
cloneSettings := map[string]interface{}{"depth": "0"}
if c.metadata.Curr.Event == metadata.EventTag {
cloneSettings["tags"] = "true"
}
container := &yaml.Container{
container := &yaml_types.Container{
Name: defaultCloneName,
Image: cloneImage,
Settings: cloneSettings,
@ -169,21 +169,21 @@ func (c *Compiler) Compile(conf *yaml.Config) (*backend.Config, error) {
name := fmt.Sprintf("%s_clone", c.prefix)
step := c.createProcess(name, container, defaultCloneName)
stage := new(backend.Stage)
stage := new(backend_types.Stage)
stage.Name = name
stage.Alias = defaultCloneName
stage.Steps = append(stage.Steps, step)
config.Stages = append(config.Stages, stage)
} else if !c.local && !conf.SkipClone {
for i, container := range conf.Clone.Containers {
for i, container := range conf.Clone.ContainerList {
if match, err := container.When.Match(c.metadata, false); !match && err == nil {
continue
} else if err != nil {
return nil, err
}
stage := new(backend.Stage)
stage := new(backend_types.Stage)
stage.Name = fmt.Sprintf("%s_clone_%v", c.prefix, i)
stage.Alias = container.Name
@ -206,12 +206,12 @@ func (c *Compiler) Compile(conf *yaml.Config) (*backend.Config, error) {
c.setupCache(conf, config)
// add services steps
if len(conf.Services.Containers) != 0 {
stage := new(backend.Stage)
if len(conf.Services.ContainerList) != 0 {
stage := new(backend_types.Stage)
stage.Name = fmt.Sprintf("%s_%s", c.prefix, nameServices)
stage.Alias = nameServices
for i, container := range conf.Services.Containers {
for i, container := range conf.Services.ContainerList {
if match, err := container.When.Match(c.metadata, false); !match && err == nil {
continue
} else if err != nil {
@ -226,9 +226,9 @@ func (c *Compiler) Compile(conf *yaml.Config) (*backend.Config, error) {
}
// add pipeline steps. 1 pipeline step per stage, at the moment
var stage *backend.Stage
var stage *backend_types.Stage
var group string
for i, container := range conf.Pipeline.Containers {
for i, container := range conf.Steps.ContainerList {
// Skip if local and should not run local
if c.local && !container.When.IsLocal() {
continue
@ -243,7 +243,7 @@ func (c *Compiler) Compile(conf *yaml.Config) (*backend.Config, error) {
if stage == nil || group != container.Group || container.Group == "" {
group = container.Group
stage = new(backend.Stage)
stage = new(backend_types.Stage)
stage.Name = fmt.Sprintf("%s_stage_%v", c.prefix, i)
stage.Alias = container.Name
config.Stages = append(config.Stages, stage)
@ -259,7 +259,7 @@ func (c *Compiler) Compile(conf *yaml.Config) (*backend.Config, error) {
return config, nil
}
func (c *Compiler) setupCache(conf *yaml.Config, ir *backend.Config) {
func (c *Compiler) setupCache(conf *yaml_types.Workflow, ir *backend_types.Config) {
if c.local || len(conf.Cache) == 0 || c.cacher == nil {
return
}
@ -268,7 +268,7 @@ func (c *Compiler) setupCache(conf *yaml.Config, ir *backend.Config) {
name := fmt.Sprintf("%s_restore_cache", c.prefix)
step := c.createProcess(name, container, "cache")
stage := new(backend.Stage)
stage := new(backend_types.Stage)
stage.Name = name
stage.Alias = "restore_cache"
stage.Steps = append(stage.Steps, step)
@ -276,7 +276,7 @@ func (c *Compiler) setupCache(conf *yaml.Config, ir *backend.Config) {
ir.Stages = append(ir.Stages, stage)
}
func (c *Compiler) setupCacheRebuild(conf *yaml.Config, ir *backend.Config) {
func (c *Compiler) setupCacheRebuild(conf *yaml_types.Workflow, ir *backend_types.Config) {
if c.local || len(conf.Cache) == 0 || c.metadata.Curr.Event != metadata.EventPush || c.cacher == nil {
return
}
@ -285,7 +285,7 @@ func (c *Compiler) setupCacheRebuild(conf *yaml.Config, ir *backend.Config) {
name := fmt.Sprintf("%s_rebuild_cache", c.prefix)
step := c.createProcess(name, container, "cache")
stage := new(backend.Stage)
stage := new(backend_types.Stage)
stage.Name = name
stage.Alias = "rebuild_cache"
stage.Steps = append(stage.Steps, step)

26
pipeline/frontend/yaml/compiler/compiler_test.go
View file

@ -3,10 +3,10 @@ package compiler
import (
"testing"
"github.com/docker/docker/api/types/strslice"
"github.com/stretchr/testify/assert"
"github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml"
"github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml/types"
yaml_types "github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml/types"
yaml_base_types "github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml/types/base"
)
func TestSecretAvailable(t *testing.T) {
@ -14,29 +14,29 @@ func TestSecretAvailable(t *testing.T) {
Match: []string{"golang"},
PluginOnly: false,
}
assert.True(t, secret.Available(&yaml.Container{
assert.True(t, secret.Available(&yaml_types.Container{
Image: "golang",
Commands: types.StringOrSlice(strslice.StrSlice{"echo 'this is not a plugin'"}),
Commands: yaml_base_types.StringOrSlice{"echo 'this is not a plugin'"},
}))
assert.False(t, secret.Available(&yaml.Container{
assert.False(t, secret.Available(&yaml_types.Container{
Image: "not-golang",
Commands: types.StringOrSlice(strslice.StrSlice{"echo 'this is not a plugin'"}),
Commands: yaml_base_types.StringOrSlice{"echo 'this is not a plugin'"},
}))
// secret only available for "golang" plugin
secret = Secret{
Match: []string{"golang"},
PluginOnly: true,
}
assert.True(t, secret.Available(&yaml.Container{
assert.True(t, secret.Available(&yaml_types.Container{
Image: "golang",
Commands: types.StringOrSlice(strslice.StrSlice{}),
Commands: yaml_base_types.StringOrSlice{},
}))
assert.False(t, secret.Available(&yaml.Container{
assert.False(t, secret.Available(&yaml_types.Container{
Image: "not-golang",
Commands: types.StringOrSlice(strslice.StrSlice{}),
Commands: yaml_base_types.StringOrSlice{},
}))
assert.False(t, secret.Available(&yaml.Container{
assert.False(t, secret.Available(&yaml_types.Container{
Image: "not-golang",
Commands: types.StringOrSlice(strslice.StrSlice{"echo 'this is not a plugin'"}),
Commands: yaml_base_types.StringOrSlice{"echo 'this is not a plugin'"},
}))
}

26
pipeline/frontend/yaml/compiler/convert.go
View file

@ -8,13 +8,13 @@ import (
"github.com/rs/zerolog/log"
backend "github.com/woodpecker-ci/woodpecker/pipeline/backend/types"
backend_types "github.com/woodpecker-ci/woodpecker/pipeline/backend/types"
"github.com/woodpecker-ci/woodpecker/pipeline/frontend/metadata"
"github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml"
"github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml/compiler/settings"
yaml_types "github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml/types"
)
func (c *Compiler) createProcess(name string, container *yaml.Container, section string) *backend.Step {
func (c *Compiler) createProcess(name string, container *yaml_types.Container, section string) *backend_types.Step {
var (
detached bool
workingdir string
@ -26,14 +26,14 @@ func (c *Compiler) createProcess(name string, container *yaml.Container, section
// network = container.Network
)
networks := []backend.Conn{
networks := []backend_types.Conn{
{
Name: fmt.Sprintf("%s_default", c.prefix),
Aliases: []string{container.Name},
},
}
for _, network := range c.networks {
networks = append(networks, backend.Conn{
networks = append(networks, backend_types.Conn{
Name: network,
})
}
@ -89,11 +89,7 @@ func (c *Compiler) createProcess(name string, container *yaml.Container, section
privileged = true
}
authConfig := backend.Auth{
Username: container.AuthConfig.Username,
Password: container.AuthConfig.Password,
Email: container.AuthConfig.Email,
}
authConfig := backend_types.Auth{}
for _, registry := range c.registries {
if matchHostname(container.Image, registry.Hostname) {
authConfig.Username = registry.Username
@ -111,9 +107,9 @@ func (c *Compiler) createProcess(name string, container *yaml.Container, section
}
// Kubernetes advanced settings
backendOptions := backend.BackendOptions{
Kubernetes: backend.KubernetesBackendOptions{
Resources: backend.Resources{
backendOptions := backend_types.BackendOptions{
Kubernetes: backend_types.KubernetesBackendOptions{
Resources: backend_types.Resources{
Limits: container.BackendOptions.Kubernetes.Resources.Limits,
Requests: container.BackendOptions.Kubernetes.Resources.Requests,
},
@ -155,7 +151,7 @@ func (c *Compiler) createProcess(name string, container *yaml.Container, section
failure = metadata.FailureFail
}
return &backend.Step{
return &backend_types.Step{
Name: name,
Alias: container.Name,
Image: container.Image,
@ -189,7 +185,7 @@ func (c *Compiler) createProcess(name string, container *yaml.Container, section
}
}
func (c *Compiler) stepWorkdir(container *yaml.Container) string {
func (c *Compiler) stepWorkdir(container *yaml_types.Container) string {
if filepath.IsAbs(container.Directory) {
return container.Directory
}

18
pipeline/frontend/yaml/constraint/constraint.go
View file

@ -11,7 +11,7 @@ import (
"gopkg.in/yaml.v3"
"github.com/woodpecker-ci/woodpecker/pipeline/frontend/metadata"
"github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml/types"
yaml_base_types "github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml/types/base"
)
type (
@ -32,7 +32,7 @@ type (
Cron List
Status List
Matrix Map
Local types.BoolTrue
Local yaml_base_types.BoolTrue
Path Path
Evaluate string `yaml:"evaluate,omitempty"`
}
@ -240,11 +240,11 @@ func (c *List) Excludes(v string) bool {
// UnmarshalYAML unmarshals the constraint.
func (c *List) UnmarshalYAML(value *yaml.Node) error {
out1 := struct {
Include types.StringOrSlice
Exclude types.StringOrSlice
Include yaml_base_types.StringOrSlice
Exclude yaml_base_types.StringOrSlice
}{}
var out2 types.StringOrSlice
var out2 yaml_base_types.StringOrSlice
err1 := value.Decode(&out1)
err2 := value.Decode(&out2)
@ -319,12 +319,12 @@ func (c *Map) UnmarshalYAML(unmarshal func(interface{}) error) error {
// UnmarshalYAML unmarshal the constraint.
func (c *Path) UnmarshalYAML(value *yaml.Node) error {
out1 := struct {
Include types.StringOrSlice `yaml:"include,omitempty"`
Exclude types.StringOrSlice `yaml:"exclude,omitempty"`
IgnoreMessage string `yaml:"ignore_message,omitempty"`
Include yaml_base_types.StringOrSlice `yaml:"include,omitempty"`
Exclude yaml_base_types.StringOrSlice `yaml:"exclude,omitempty"`
IgnoreMessage string `yaml:"ignore_message,omitempty"`
}{}
var out2 types.StringOrSlice
var out2 yaml_base_types.StringOrSlice
err1 := value.Decode(&out1)
err2 := value.Decode(&out2)

136
pipeline/frontend/yaml/container.go
View file

@ -1,136 +0,0 @@
package yaml
import (
"fmt"
"golang.org/x/exp/slices"
"gopkg.in/yaml.v3"
"github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml/constraint"
"github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml/types"
"github.com/woodpecker-ci/woodpecker/shared/constant"
)
type (
// AuthConfig defines registry authentication credentials.
AuthConfig struct {
Username string
Password string
Email string
}
// Advanced backend options
BackendOptions struct {
Kubernetes KubernetesBackendOptions `yaml:"kubernetes,omitempty"`
}
KubernetesBackendOptions struct {
Resources Resources `yaml:"resources,omitempty"`
}
Resources struct {
Requests map[string]string `yaml:"requests,omitempty"`
Limits map[string]string `yaml:"limits,omitempty"`
}
// Containers denotes an ordered collection of containers.
Containers struct {
Containers []*Container
}
// Container defines a container.
Container struct {
AuthConfig AuthConfig `yaml:"auth_config,omitempty"`
CapAdd []string `yaml:"cap_add,omitempty"`
CapDrop []string `yaml:"cap_drop,omitempty"`
Commands types.StringOrSlice `yaml:"commands,omitempty"`
CPUQuota types.StringorInt `yaml:"cpu_quota,omitempty"`
CPUSet string `yaml:"cpuset,omitempty"`
CPUShares types.StringorInt `yaml:"cpu_shares,omitempty"`
Detached bool `yaml:"detach,omitempty"`
Devices []string `yaml:"devices,omitempty"`
Tmpfs []string `yaml:"tmpfs,omitempty"`
DNS types.StringOrSlice `yaml:"dns,omitempty"`
DNSSearch types.StringOrSlice `yaml:"dns_search,omitempty"`
Directory string `yaml:"directory,omitempty"`
Environment types.SliceorMap `yaml:"environment,omitempty"`
ExtraHosts []string `yaml:"extra_hosts,omitempty"`
Group string `yaml:"group,omitempty"`
Image string `yaml:"image,omitempty"`
Failure string `yaml:"failure,omitempty"`
Isolation string `yaml:"isolation,omitempty"`
MemLimit types.MemStringorInt `yaml:"mem_limit,omitempty"`
MemSwapLimit types.MemStringorInt `yaml:"memswap_limit,omitempty"`
MemSwappiness types.MemStringorInt `yaml:"mem_swappiness,omitempty"`
Name string `yaml:"name,omitempty"`
NetworkMode string `yaml:"network_mode,omitempty"`
IpcMode string `yaml:"ipc_mode,omitempty"`
Networks types.Networks `yaml:"networks,omitempty"`
Privileged bool `yaml:"privileged,omitempty"`
Pull bool `yaml:"pull,omitempty"`
ShmSize types.MemStringorInt `yaml:"shm_size,omitempty"`
Ulimits types.Ulimits `yaml:"ulimits,omitempty"`
Volumes types.Volumes `yaml:"volumes,omitempty"`
Secrets Secrets `yaml:"secrets,omitempty"`
Sysctls types.SliceorMap `yaml:"sysctls,omitempty"`
When constraint.When `yaml:"when,omitempty"`
Settings map[string]interface{} `yaml:"settings"`
BackendOptions BackendOptions `yaml:"backend_options,omitempty"`
}
)
// UnmarshalYAML implements the Unmarshaler interface.
func (c *Containers) UnmarshalYAML(value *yaml.Node) error {
switch value.Kind {
// We support maps ...
case yaml.MappingNode:
c.Containers = make([]*Container, 0, len(value.Content)/2+1)
// We cannot use decode on specific values
// since if we try to load from a map, the order
// will not be kept. Therefore use value.Content
// and take the map values i%2=1
for i, n := range value.Content {
if i%2 == 1 {
container := &Container{}
if err := n.Decode(container); err != nil {
return err
}
if container.Name == "" {
container.Name = fmt.Sprintf("%v", value.Content[i-1].Value)
}
c.Containers = append(c.Containers, container)
}
}
// ... and lists
case yaml.SequenceNode:
c.Containers = make([]*Container, 0, len(value.Content))
for i, n := range value.Content {
container := &Container{}
if err := n.Decode(container); err != nil {
return err
}
if container.Name == "" {
container.Name = fmt.Sprintf("step-%d", i)
}
c.Containers = append(c.Containers, container)
}
default:
return fmt.Errorf("yaml node type[%d]: '%s' not supported", value.Kind, value.Tag)
}
return nil
}
func (c *Container) IsPlugin() bool {
return len(c.Commands) == 0
}
func (c *Container) IsTrustedCloneImage() bool {
return c.IsPlugin() && slices.Contains(constant.TrustedCloneImages, c.Image)
}

20
pipeline/frontend/yaml/linter/linter.go
View file

@ -3,7 +3,7 @@ package linter
import (
"fmt"
"github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml"
"github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml/types"
)
const (
@ -27,20 +27,20 @@ func New(opts ...Option) *Linter {
}
// Lint lints the configuration.
func (l *Linter) Lint(c *yaml.Config) error {
if len(c.Pipeline.Containers) == 0 {
func (l *Linter) Lint(c *types.Workflow) error {
if len(c.Steps.ContainerList) == 0 {
return fmt.Errorf("Invalid or missing pipeline section")
}
if err := l.lint(c.Clone.Containers, blockClone); err != nil {
if err := l.lint(c.Clone.ContainerList, blockClone); err != nil {
return err
}
if err := l.lint(c.Pipeline.Containers, blockPipeline); err != nil {
if err := l.lint(c.Steps.ContainerList, blockPipeline); err != nil {
return err
}
return l.lint(c.Services.Containers, blockServices)
return l.lint(c.Services.ContainerList, blockServices)
}
func (l *Linter) lint(containers []*yaml.Container, _ uint8) error {
func (l *Linter) lint(containers []*types.Container, _ uint8) error {
for _, container := range containers {
if err := l.lintImage(container); err != nil {
return err
@ -57,14 +57,14 @@ func (l *Linter) lint(containers []*yaml.Container, _ uint8) error {
return nil
}
func (l *Linter) lintImage(c *yaml.Container) error {
func (l *Linter) lintImage(c *types.Container) error {
if len(c.Image) == 0 {
return fmt.Errorf("Invalid or missing image")
}
return nil
}
func (l *Linter) lintCommands(c *yaml.Container) error {
func (l *Linter) lintCommands(c *types.Container) error {
if len(c.Commands) == 0 {
return nil
}
@ -78,7 +78,7 @@ func (l *Linter) lintCommands(c *yaml.Container) error {
return nil
}
func (l *Linter) lintTrusted(c *yaml.Container) error {
func (l *Linter) lintTrusted(c *types.Container) error {
if c.Privileged {
return fmt.Errorf("Insufficient privileges to use privileged mode")
}

33
pipeline/frontend/yaml/config.go → pipeline/frontend/yaml/parse.go
View file

@ -9,36 +9,9 @@ import (
"github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml/types"
)
type (
// Config defines a pipeline configuration.
Config struct {
When constraint.When `yaml:"when,omitempty"`
Cache types.StringOrSlice
Platform string
Workspace Workspace
Clone Containers
Pipeline Containers
Services Containers
Networks Networks
Volumes Volumes
Labels types.SliceorMap
DependsOn []string `yaml:"depends_on,omitempty"`
RunsOn []string `yaml:"runs_on,omitempty"`
SkipClone bool `yaml:"skip_clone"`
// Deprecated use When.Branch
BranchesDontUseIt *constraint.List `yaml:"branches,omitempty"`
}
// Workspace defines a pipeline workspace.
Workspace struct {
Base string
Path string
}
)
// ParseBytes parses the configuration from bytes b.
func ParseBytes(b []byte) (*Config, error) {
out := new(Config)
func ParseBytes(b []byte) (*types.Workflow, error) {
out := new(types.Workflow)
err := xyaml.Unmarshal(b, out)
if err != nil {
return nil, err
@ -60,7 +33,7 @@ func ParseBytes(b []byte) (*Config, error) {
}
// ParseString parses the configuration from string s.
func ParseString(s string) (*Config, error) {
func ParseString(s string) (*types.Workflow, error) {
return ParseBytes(
[]byte(s),
)

52
pipeline/frontend/yaml/config_test.go → pipeline/frontend/yaml/parse_test.go
View file

@ -6,7 +6,7 @@ import (
"github.com/franela/goblin"
"github.com/woodpecker-ci/woodpecker/pipeline/frontend/metadata"
"github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml/types"
yaml_base_types "github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml/types/base"
)
func TestParse(t *testing.T) {
@ -24,21 +24,21 @@ func TestParse(t *testing.T) {
g.Assert(out.Workspace.Base).Equal("/go")
g.Assert(out.Workspace.Path).Equal("src/github.com/octocat/hello-world")
g.Assert(out.Volumes.Volumes[0].Name).Equal("custom")
g.Assert(out.Volumes.Volumes[0].Driver).Equal("blockbridge")
g.Assert(out.Networks.Networks[0].Name).Equal("custom")
g.Assert(out.Networks.Networks[0].Driver).Equal("overlay")
g.Assert(out.Services.Containers[0].Name).Equal("database")
g.Assert(out.Services.Containers[0].Image).Equal("mysql")
g.Assert(out.Pipeline.Containers[0].Name).Equal("test")
g.Assert(out.Pipeline.Containers[0].Image).Equal("golang")
g.Assert(out.Pipeline.Containers[0].Commands).Equal(types.StringOrSlice{"go install", "go test"})
g.Assert(out.Pipeline.Containers[1].Name).Equal("build")
g.Assert(out.Pipeline.Containers[1].Image).Equal("golang")
g.Assert(out.Pipeline.Containers[1].Commands).Equal(types.StringOrSlice{"go build"})
g.Assert(out.Pipeline.Containers[2].Name).Equal("notify")
g.Assert(out.Pipeline.Containers[2].Image).Equal("slack")
// g.Assert(out.Pipeline.Containers[2].NetworkMode).Equal("container:name")
g.Assert(out.Volumes.WorkflowVolumes[0].Name).Equal("custom")
g.Assert(out.Volumes.WorkflowVolumes[0].Driver).Equal("blockbridge")
g.Assert(out.Networks.WorkflowNetworks[0].Name).Equal("custom")
g.Assert(out.Networks.WorkflowNetworks[0].Driver).Equal("overlay")
g.Assert(out.Services.ContainerList[0].Name).Equal("database")
g.Assert(out.Services.ContainerList[0].Image).Equal("mysql")
g.Assert(out.Steps.ContainerList[0].Name).Equal("test")
g.Assert(out.Steps.ContainerList[0].Image).Equal("golang")
g.Assert(out.Steps.ContainerList[0].Commands).Equal(yaml_base_types.StringOrSlice{"go install", "go test"})
g.Assert(out.Steps.ContainerList[1].Name).Equal("build")
g.Assert(out.Steps.ContainerList[1].Image).Equal("golang")
g.Assert(out.Steps.ContainerList[1].Commands).Equal(yaml_base_types.StringOrSlice{"go build"})
g.Assert(out.Steps.ContainerList[2].Name).Equal("notify")
g.Assert(out.Steps.ContainerList[2].Image).Equal("slack")
// g.Assert(out.Steps.ContainerList[2].NetworkMode).Equal("container:name")
g.Assert(out.Labels["com.example.team"]).Equal("frontend")
g.Assert(out.Labels["com.example.type"]).Equal("build")
g.Assert(out.DependsOn[0]).Equal("lint")
@ -53,8 +53,8 @@ func TestParse(t *testing.T) {
if err != nil {
g.Fail(err)
}
g.Assert(out.Pipeline.Containers[0].Name).Equal("notify_success")
g.Assert(out.Pipeline.Containers[0].Image).Equal("plugins/slack")
g.Assert(out.Steps.ContainerList[0].Name).Equal("notify_success")
g.Assert(out.Steps.ContainerList[0].Image).Equal("plugins/slack")
})
g.It("Should unmarshal variables", func() {
@ -62,15 +62,15 @@ func TestParse(t *testing.T) {
if err != nil {
g.Fail(err)
}
g.Assert(out.Pipeline.Containers[0].Name).Equal("notify_fail")
g.Assert(out.Pipeline.Containers[0].Image).Equal("plugins/slack")
g.Assert(out.Pipeline.Containers[1].Name).Equal("notify_success")
g.Assert(out.Pipeline.Containers[1].Image).Equal("plugins/slack")
g.Assert(out.Steps.ContainerList[0].Name).Equal("notify_fail")
g.Assert(out.Steps.ContainerList[0].Image).Equal("plugins/slack")
g.Assert(out.Steps.ContainerList[1].Name).Equal("notify_success")
g.Assert(out.Steps.ContainerList[1].Image).Equal("plugins/slack")
g.Assert(len(out.Pipeline.Containers[0].When.Constraints)).Equal(0)
g.Assert(out.Pipeline.Containers[1].Name).Equal("notify_success")
g.Assert(out.Pipeline.Containers[1].Image).Equal("plugins/slack")
g.Assert(out.Pipeline.Containers[1].When.Constraints[0].Event.Include).Equal([]string{"success"})
g.Assert(len(out.Steps.ContainerList[0].When.Constraints)).Equal(0)
g.Assert(out.Steps.ContainerList[1].Name).Equal("notify_success")
g.Assert(out.Steps.ContainerList[1].Image).Equal("plugins/slack")
g.Assert(out.Steps.ContainerList[1].When.Constraints[0].Event.Include).Equal([]string{"success"})
})
matchConfig, err := ParseString(sampleYaml)

29
pipeline/frontend/yaml/types/backend_options.go Normal file
View file

@ -0,0 +1,29 @@
// Copyright 2023 Woodpecker Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package types
// BackendOptions are advanced options for specific backends
type BackendOptions struct {
Kubernetes KubernetesBackendOptions `yaml:"kubernetes,omitempty"`
}
type KubernetesBackendOptions struct {
Resources Resources `yaml:"resources,omitempty"`
}
type Resources struct {
Requests map[string]string `yaml:"requests,omitempty"`
Limits map[string]string `yaml:"limits,omitempty"`
}

16
pipeline/frontend/yaml/types/types_yaml_test.go → pipeline/frontend/yaml/types/base/base_types_test.go
View file

@ -1,4 +1,4 @@
package types
package base
import (
"fmt"
@ -10,7 +10,7 @@ import (
)
type StructStringorInt struct {
Foo StringorInt
Foo StringOrInt
}
func TestStringorIntYaml(t *testing.T) {
@ -18,7 +18,7 @@ func TestStringorIntYaml(t *testing.T) {
s := StructStringorInt{}
assert.NoError(t, yaml.Unmarshal([]byte(str), &s))
assert.Equal(t, StringorInt(10), s.Foo)
assert.Equal(t, StringOrInt(10), s.Foo)
d, err := yaml.Marshal(&s)
assert.Nil(t, err)
@ -26,7 +26,7 @@ func TestStringorIntYaml(t *testing.T) {
s2 := StructStringorInt{}
assert.NoError(t, yaml.Unmarshal(d, &s2))
assert.Equal(t, StringorInt(10), s2.Foo)
assert.Equal(t, StringOrInt(10), s2.Foo)
}
}
@ -52,7 +52,7 @@ func TestStringOrSliceYaml(t *testing.T) {
}
type StructSliceorMap struct {
Foos SliceorMap `yaml:"foos,omitempty"`
Foos SliceOrMap `yaml:"foos,omitempty"`
Bars []string `yaml:"bars"`
}
@ -62,7 +62,7 @@ func TestSliceOrMapYaml(t *testing.T) {
s := StructSliceorMap{}
assert.NoError(t, yaml.Unmarshal([]byte(str), &s))
assert.Equal(t, SliceorMap{"bar": "baz", "far": "faz"}, s.Foos)
assert.Equal(t, SliceOrMap{"bar": "baz", "far": "faz"}, s.Foos)
d, err := yaml.Marshal(&s)
assert.Nil(t, err)
@ -70,7 +70,7 @@ func TestSliceOrMapYaml(t *testing.T) {
s2 := StructSliceorMap{}
assert.NoError(t, yaml.Unmarshal(d, &s2))
assert.Equal(t, SliceorMap{"bar": "baz", "far": "faz"}, s2.Foos)
assert.Equal(t, SliceOrMap{"bar": "baz", "far": "faz"}, s2.Foos)
}
var sampleStructSliceorMap = `
@ -88,7 +88,7 @@ func TestUnmarshalSliceOrMap(t *testing.T) {
func TestStr2SliceOrMapPtrMap(t *testing.T) {
s := map[string]*StructSliceorMap{"udav": {
Foos: SliceorMap{"io.rancher.os.bar": "baz", "io.rancher.os.far": "true"},
Foos: SliceOrMap{"io.rancher.os.bar": "baz", "io.rancher.os.far": "true"},
Bars: []string{},
}}
d, err := yaml.Marshal(&s)

2
pipeline/frontend/yaml/types/bool.go → pipeline/frontend/yaml/types/base/bool.go
View file

@ -1,4 +1,4 @@
package types
package base
import (
"strconv"

2
pipeline/frontend/yaml/types/bool_test.go → pipeline/frontend/yaml/types/base/bool_test.go
View file

@ -1,4 +1,4 @@
package types
package base
import (
"testing"

57
pipeline/frontend/yaml/types/base/int.go Normal file
View file

@ -0,0 +1,57 @@
package base
import (
"errors"
"strconv"
"github.com/docker/go-units"
)
// StringOrInt represents a string or an integer.
type StringOrInt int64
// UnmarshalYAML implements the Unmarshaler interface.
func (s *StringOrInt) UnmarshalYAML(unmarshal func(interface{}) error) error {
var intType int64
if err := unmarshal(&intType); err == nil {
*s = StringOrInt(intType)
return nil
}
var stringType string
if err := unmarshal(&stringType); err == nil {
intType, err := strconv.ParseInt(stringType, 10, 64)
if err != nil {
return err
}
*s = StringOrInt(intType)
return nil
}
return errors.New("Failed to unmarshal StringOrInt")
}
// MemStringOrInt represents a string or an integer
// the String supports notations like 10m for then Megabyte of memory
type MemStringOrInt int64
// UnmarshalYAML implements the Unmarshaler interface.
func (s *MemStringOrInt) UnmarshalYAML(unmarshal func(interface{}) error) error {
var intType int64
if err := unmarshal(&intType); err == nil {
*s = MemStringOrInt(intType)
return nil
}
var stringType string
if err := unmarshal(&stringType); err == nil {
intType, err := units.RAMInBytes(stringType)
if err != nil {
return err
}
*s = MemStringOrInt(intType)
return nil
}
return errors.New("Failed to unmarshal MemStringOrInt")
}

55
pipeline/frontend/yaml/types/base/map.go Normal file
View file

@ -0,0 +1,55 @@
package base
import (
"errors"
"fmt"
"strings"
)
// SliceOrMap represents a slice or a map of strings.
type SliceOrMap map[string]string
// UnmarshalYAML implements the Unmarshaler interface.
func (s *SliceOrMap) UnmarshalYAML(unmarshal func(interface{}) error) error {
var sliceType []interface{}
if err := unmarshal(&sliceType); err == nil {
parts := map[string]string{}
for _, s := range sliceType {
if str, ok := s.(string); ok {
str := strings.TrimSpace(str)
keyValueSlice := strings.SplitN(str, "=", 2)
key := keyValueSlice[0]
val := ""
if len(keyValueSlice) == 2 {
val = keyValueSlice[1]
}
parts[key] = val
} else {
return fmt.Errorf("Cannot unmarshal '%v' of type %T into a string value", s, s)
}
}
*s = parts
return nil
}
var mapType map[interface{}]interface{}
if err := unmarshal(&mapType); err == nil {
parts := map[string]string{}
for k, v := range mapType {
if sk, ok := k.(string); ok {
if sv, ok := v.(string); ok {
parts[sk] = sv
} else {
return fmt.Errorf("Cannot unmarshal '%v' of type %T into a string value", v, v)
}
} else {
return fmt.Errorf("Cannot unmarshal '%v' of type %T into a string value", k, k)
}
}
*s = parts
return nil
}
return errors.New("Failed to unmarshal SliceOrMap")
}

46
pipeline/frontend/yaml/types/base/slice.go Normal file
View file

@ -0,0 +1,46 @@
package base
import (
"errors"
"fmt"
)
// StringOrSlice represents a string or an array of strings.
// We need to override the yaml decoder to accept both options.
type StringOrSlice []string
// UnmarshalYAML implements the Unmarshaler interface.
func (s *StringOrSlice) UnmarshalYAML(unmarshal func(interface{}) error) error {
var stringType string
if err := unmarshal(&stringType); err == nil {
*s = []string{stringType}
return nil
}
var sliceType []interface{}
if err := unmarshal(&sliceType); err == nil {
parts, err := toStrings(sliceType)
if err != nil {
return err
}
*s = parts
return nil
}
return errors.New("Failed to unmarshal StringOrSlice")
}
func toStrings(s []interface{}) ([]string, error) {
if len(s) == 0 {
return nil, nil
}
r := make([]string, len(s))
for k, v := range s {
if sv, ok := v.(string); ok {
r[k] = sv
} else {
return nil, fmt.Errorf("Cannot unmarshal '%v' of type %T into a string value", v, v)
}
}
return r, nil
}

118
pipeline/frontend/yaml/types/container.go Normal file
View file

@ -0,0 +1,118 @@
package types
import (
"fmt"
"golang.org/x/exp/slices"
"gopkg.in/yaml.v3"
"github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml/constraint"
"github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml/types/base"
"github.com/woodpecker-ci/woodpecker/shared/constant"
)
type (
// ContainerList denotes an ordered collection of containers.
ContainerList struct {
ContainerList []*Container
}
// Container defines a container.
Container struct {
BackendOptions BackendOptions `yaml:"backend_options,omitempty"`
Commands base.StringOrSlice `yaml:"commands,omitempty"`
Detached bool `yaml:"detach,omitempty"`
Directory string `yaml:"directory,omitempty"`
Environment base.SliceOrMap `yaml:"environment,omitempty"`
Failure string `yaml:"failure,omitempty"`
Group string `yaml:"group,omitempty"`
Image string `yaml:"image,omitempty"`
Name string `yaml:"name,omitempty"`
Pull bool `yaml:"pull,omitempty"`
Secrets Secrets `yaml:"secrets,omitempty"`
Settings map[string]interface{} `yaml:"settings"`
Volumes Volumes `yaml:"volumes,omitempty"`
When constraint.When `yaml:"when,omitempty"`
// Docker Specific
Privileged bool `yaml:"privileged,omitempty"`
// Undocumented
CapAdd []string `yaml:"cap_add,omitempty"`
CapDrop []string `yaml:"cap_drop,omitempty"`
CPUQuota base.StringOrInt `yaml:"cpu_quota,omitempty"`
CPUSet string `yaml:"cpuset,omitempty"`
CPUShares base.StringOrInt `yaml:"cpu_shares,omitempty"`
Devices []string `yaml:"devices,omitempty"`
DNSSearch base.StringOrSlice `yaml:"dns_search,omitempty"`
DNS base.StringOrSlice `yaml:"dns,omitempty"`
ExtraHosts []string `yaml:"extra_hosts,omitempty"`
IpcMode string `yaml:"ipc_mode,omitempty"`
Isolation string `yaml:"isolation,omitempty"`
MemLimit base.MemStringOrInt `yaml:"mem_limit,omitempty"`
MemSwapLimit base.MemStringOrInt `yaml:"memswap_limit,omitempty"`
MemSwappiness base.MemStringOrInt `yaml:"mem_swappiness,omitempty"`
NetworkMode string `yaml:"network_mode,omitempty"`
Networks Networks `yaml:"networks,omitempty"`
ShmSize base.MemStringOrInt `yaml:"shm_size,omitempty"`
Sysctls base.SliceOrMap `yaml:"sysctls,omitempty"`
Tmpfs []string `yaml:"tmpfs,omitempty"`
Ulimits Ulimits `yaml:"ulimits,omitempty"`
}
)
// UnmarshalYAML implements the Unmarshaler interface.
func (c *ContainerList) UnmarshalYAML(value *yaml.Node) error {
switch value.Kind {
// We support maps ...
case yaml.MappingNode:
c.ContainerList = make([]*Container, 0, len(value.Content)/2+1)
// We cannot use decode on specific values
// since if we try to load from a map, the order
// will not be kept. Therefor use value.Content
// and take the map values i%2=1
for i, n := range value.Content {
if i%2 == 1 {
container := &Container{}
if err := n.Decode(container); err != nil {
return err
}
if container.Name == "" {
container.Name = fmt.Sprintf("%v", value.Content[i-1].Value)
}
c.ContainerList = append(c.ContainerList, container)
}
}
// ... and lists
case yaml.SequenceNode:
c.ContainerList = make([]*Container, 0, len(value.Content))
for i, n := range value.Content {
container := &Container{}
if err := n.Decode(container); err != nil {
return err
}
if container.Name == "" {
container.Name = fmt.Sprintf("step-%d", i)
}
c.ContainerList = append(c.ContainerList, container)
}
default:
return fmt.Errorf("yaml node type[%d]: '%s' not supported", value.Kind, value.Tag)
}
return nil
}
func (c *Container) IsPlugin() bool {
return len(c.Commands) == 0
}
func (c *Container) IsTrustedCloneImage() bool {
return c.IsPlugin() && slices.Contains(constant.TrustedCloneImages, c.Image)
}

51
pipeline/frontend/yaml/container_test.go → pipeline/frontend/yaml/types/container_test.go
View file

@ -1,4 +1,4 @@
package yaml
package types
import (
"testing"
@ -8,14 +8,11 @@ import (
"gopkg.in/yaml.v3"
"github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml/constraint"
"github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml/types"
"github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml/types/base"
)
var containerYaml = []byte(`
image: golang:latest
auth_config:
username: janedoe
password: password
cap_add: [ ALL ]
cap_drop: [ NET_ADMIN, SYS_ADMIN ]
commands:
@ -65,31 +62,27 @@ settings:
func TestUnmarshalContainer(t *testing.T) {
want := Container{
AuthConfig: AuthConfig{
Username: "janedoe",
Password: "password",
},
CapAdd: []string{"ALL"},
CapDrop: []string{"NET_ADMIN", "SYS_ADMIN"},
Commands: types.StringOrSlice{"go build", "go test"},
CPUQuota: types.StringorInt(11),
Commands: base.StringOrSlice{"go build", "go test"},
CPUQuota: base.StringOrInt(11),
CPUSet: "1,2",
CPUShares: types.StringorInt(99),
CPUShares: base.StringOrInt(99),
Detached: true,
Devices: []string{"/dev/ttyUSB0:/dev/ttyUSB0"},
Directory: "example/",
DNS: types.StringOrSlice{"8.8.8.8"},
DNSSearch: types.StringOrSlice{"example.com"},
Environment: types.SliceorMap{"RACK_ENV": "development", "SHOW": "true"},
DNS: base.StringOrSlice{"8.8.8.8"},
DNSSearch: base.StringOrSlice{"example.com"},
Environment: base.SliceOrMap{"RACK_ENV": "development", "SHOW": "true"},
ExtraHosts: []string{"somehost:162.242.195.82", "otherhost:50.31.209.229"},
Image: "golang:latest",
Isolation: "hyperv",
MemLimit: types.MemStringorInt(1024),
MemSwapLimit: types.MemStringorInt(1024),
MemSwappiness: types.MemStringorInt(1024),
MemLimit: base.MemStringOrInt(1024),
MemSwapLimit: base.MemStringOrInt(1024),
MemSwappiness: base.MemStringOrInt(1024),
Name: "my-build-container",
Networks: types.Networks{
Networks: []*types.Network{
Networks: Networks{
Networks: []*Network{
{Name: "some-network"},
{Name: "other-network"},
},
@ -97,10 +90,10 @@ func TestUnmarshalContainer(t *testing.T) {
NetworkMode: "bridge",
Pull: true,
Privileged: true,
ShmSize: types.MemStringorInt(1024),
Tmpfs: types.StringOrSlice{"/var/lib/test"},
Volumes: types.Volumes{
Volumes: []*types.Volume{
ShmSize: base.MemStringOrInt(1024),
Tmpfs: base.StringOrSlice{"/var/lib/test"},
Volumes: Volumes{
Volumes: []*Volume{
{Source: "", Destination: "/var/lib/mysql"},
{Source: "/opt/data", Destination: "/var/lib/mysql"},
{Source: "/etc/configs", Destination: "/etc/configs/", AccessMode: "ro"},
@ -265,10 +258,10 @@ func TestUnmarshalContainers(t *testing.T) {
}
for _, test := range testdata {
in := []byte(test.from)
got := Containers{}
got := ContainerList{}
err := yaml.Unmarshal(in, &got)
assert.NoError(t, err)
assert.EqualValues(t, test.want, got.Containers, "problem parsing containers %q", test.from)
assert.EqualValues(t, test.want, got.ContainerList, "problem parsing containers %q", test.from)
}
}
@ -281,7 +274,7 @@ func TestUnmarshalContainersErr(t *testing.T) {
}
for _, test := range testdata {
in := []byte(test)
containers := new(Containers)
containers := new(ContainerList)
err := yaml.Unmarshal(in, &containers)
assert.Error(t, err, "wanted error for containers %q", test)
}
@ -298,9 +291,9 @@ func stringsToInterface(val ...string) []interface{} {
func TestIsPlugin(t *testing.T) {
assert.True(t, (&Container{}).IsPlugin())
assert.True(t, (&Container{
Commands: types.StringOrSlice(strslice.StrSlice{}),
Commands: base.StringOrSlice(strslice.StrSlice{}),
}).IsPlugin())
assert.False(t, (&Container{
Commands: types.StringOrSlice(strslice.StrSlice{"echo 'this is not a plugin'"}),
Commands: base.StringOrSlice(strslice.StrSlice{"echo 'this is not a plugin'"}),
}).IsPlugin())
}

2
pipeline/frontend/yaml/secret.go → pipeline/frontend/yaml/types/secret.go
View file

@ -1,4 +1,4 @@
package yaml
package types
import "gopkg.in/yaml.v3"

2
pipeline/frontend/yaml/secret_test.go → pipeline/frontend/yaml/types/secret_test.go
View file

@ -1,4 +1,4 @@
package yaml
package types
import (
"testing"

148
pipeline/frontend/yaml/types/types_yaml.go
View file

@ -1,148 +0,0 @@
package types
import (
"errors"
"fmt"
"strconv"
"strings"
"github.com/docker/docker/api/types/strslice"
"github.com/docker/go-units"
)
// StringorInt represents a string or an integer.
type StringorInt int64
// UnmarshalYAML implements the Unmarshaler interface.
func (s *StringorInt) UnmarshalYAML(unmarshal func(interface{}) error) error {
var intType int64
if err := unmarshal(&intType); err == nil {
*s = StringorInt(intType)
return nil
}
var stringType string
if err := unmarshal(&stringType); err == nil {
intType, err := strconv.ParseInt(stringType, 10, 64)
if err != nil {
return err
}
*s = StringorInt(intType)
return nil
}
return errors.New("Failed to unmarshal StringorInt")
}
// MemStringorInt represents a string or an integer
// the String supports notations like 10m for then Megabyte of memory
type MemStringorInt int64
// UnmarshalYAML implements the Unmarshaler interface.
func (s *MemStringorInt) UnmarshalYAML(unmarshal func(interface{}) error) error {
var intType int64
if err := unmarshal(&intType); err == nil {
*s = MemStringorInt(intType)
return nil
}
var stringType string
if err := unmarshal(&stringType); err == nil {
intType, err := units.RAMInBytes(stringType)
if err != nil {
return err
}
*s = MemStringorInt(intType)
return nil
}
return errors.New("Failed to unmarshal MemStringorInt")
}
// StringOrSlice represents
// Using engine-api Strslice and augment it with YAML marshaling stuff. a string or an array of strings.
type StringOrSlice strslice.StrSlice
// UnmarshalYAML implements the Unmarshaler interface.
func (s *StringOrSlice) UnmarshalYAML(unmarshal func(interface{}) error) error {
var stringType string
if err := unmarshal(&stringType); err == nil {
*s = []string{stringType}
return nil
}
var sliceType []interface{}
if err := unmarshal(&sliceType); err == nil {
parts, err := toStrings(sliceType)
if err != nil {
return err
}
*s = parts
return nil
}
return errors.New("Failed to unmarshal StringOrSlice")
}
// SliceorMap represents a slice or a map of strings.
type SliceorMap map[string]string
// UnmarshalYAML implements the Unmarshaler interface.
func (s *SliceorMap) UnmarshalYAML(unmarshal func(interface{}) error) error {
var sliceType []interface{}
if err := unmarshal(&sliceType); err == nil {
parts := map[string]string{}
for _, s := range sliceType {
if str, ok := s.(string); ok {
str := strings.TrimSpace(str)
keyValueSlice := strings.SplitN(str, "=", 2)
key := keyValueSlice[0]
val := ""
if len(keyValueSlice) == 2 {
val = keyValueSlice[1]
}
parts[key] = val
} else {
return fmt.Errorf("Cannot unmarshal '%v' of type %T into a string value", s, s)
}
}
*s = parts
return nil
}
var mapType map[interface{}]interface{}
if err := unmarshal(&mapType); err == nil {
parts := map[string]string{}
for k, v := range mapType {
if sk, ok := k.(string); ok {
if sv, ok := v.(string); ok {
parts[sk] = sv
} else {
return fmt.Errorf("Cannot unmarshal '%v' of type %T into a string value", v, v)
}
} else {
return fmt.Errorf("Cannot unmarshal '%v' of type %T into a string value", k, k)
}
}
*s = parts
return nil
}
return errors.New("Failed to unmarshal SliceorMap")
}
func toStrings(s []interface{}) ([]string, error) {
if len(s) == 0 {
return nil, nil
}
r := make([]string, len(s))
for k, v := range s {
if sv, ok := v.(string); ok {
r[k] = sv
} else {
return nil, fmt.Errorf("Cannot unmarshal '%v' of type %T into a string value", v, v)
}
}
return r, nil
}

34
pipeline/frontend/yaml/types/workflow.go Normal file
View file

@ -0,0 +1,34 @@
package types
import (
"github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml/constraint"
"github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml/types/base"
)
type (
// Workflow defines a workflow configuration.
Workflow struct {
When constraint.When `yaml:"when,omitempty"`
Platform string `yaml:"platform,omitempty"`
Workspace Workspace `yaml:"workspace,omitempty"`
Clone ContainerList `yaml:"clone,omitempty"`
Steps ContainerList `yaml:"pipeline"` // TODO: discussed if we should rename it to "steps"
Services ContainerList `yaml:"services,omitempty"`
Labels base.SliceOrMap `yaml:"labels,omitempty"`
DependsOn []string `yaml:"depends_on,omitempty"`
RunsOn []string `yaml:"runs_on,omitempty"`
SkipClone bool `yaml:"skip_clone"`
// Undocumented
Cache base.StringOrSlice `yaml:"cache,omitempty"`
Networks WorkflowNetworks `yaml:"networks,omitempty"`
Volumes WorkflowVolumes `yaml:"volumes,omitempty"`
// Deprecated
BranchesDontUseIt *constraint.List `yaml:"branches,omitempty"`
}
// Workspace defines a pipeline workspace.
Workspace struct {
Base string
Path string
}
)

18
pipeline/frontend/yaml/network.go → pipeline/frontend/yaml/types/workflow_network.go
View file

@ -1,4 +1,4 @@
package yaml
package types
import (
"fmt"
@ -7,13 +7,13 @@ import (
)
type (
// Networks defines a collection of networks.
Networks struct {
Networks []*Network
// WorkflowNetworks defines a collection of networks.
WorkflowNetworks struct {
WorkflowNetworks []*WorkflowNetwork
}
// Network defines a container network.
Network struct {
// WorkflowNetwork defines a container network.
WorkflowNetwork struct {
Name string `yaml:"name,omitempty"`
Driver string `yaml:"driver,omitempty"`
DriverOpts map[string]string `yaml:"driver_opts,omitempty"`
@ -21,8 +21,8 @@ type (
)
// UnmarshalYAML implements the Unmarshaler interface.
func (n *Networks) UnmarshalYAML(value *yaml.Node) error {
networks := map[string]Network{}
func (n *WorkflowNetworks) UnmarshalYAML(value *yaml.Node) error {
networks := map[string]WorkflowNetwork{}
err := value.Decode(&networks)
for key, nn := range networks {
@ -32,7 +32,7 @@ func (n *Networks) UnmarshalYAML(value *yaml.Node) error {
if nn.Driver == "" {
nn.Driver = "bridge"
}
n.Networks = append(n.Networks, &nn)
n.WorkflowNetworks = append(n.WorkflowNetworks, &nn)
}
return err
}

26
pipeline/frontend/yaml/network_test.go → pipeline/frontend/yaml/types/workflow_network_test.go
View file

@ -1,4 +1,4 @@
package yaml
package types
import (
"testing"
@ -10,18 +10,18 @@ import (
func TestUnmarshalNetwork(t *testing.T) {
testdata := []struct {
from string
want Network
want WorkflowNetwork
}{
{
from: "{ name: foo, driver: bar }",
want: Network{
want: WorkflowNetwork{
Name: "foo",
Driver: "bar",
},
},
{
from: "{ name: foo, driver: bar, driver_opts: { baz: qux } }",
want: Network{
want: WorkflowNetwork{
Name: "foo",
Driver: "bar",
DriverOpts: map[string]string{
@ -33,21 +33,21 @@ func TestUnmarshalNetwork(t *testing.T) {
for _, test := range testdata {
in := []byte(test.from)
got := Network{}
got := WorkflowNetwork{}
err := yaml.Unmarshal(in, &got)
assert.NoError(t, err)
assert.EqualValues(t, test.want, got, "problem parsing network %q", test.from)
}
}
func TestUnmarshalNetworks(t *testing.T) {
func TestUnmarshalWorkflowNetworks(t *testing.T) {
testdata := []struct {
from string
want []*Network
want []*WorkflowNetwork
}{
{
from: "foo: { driver: bar }",
want: []*Network{
want: []*WorkflowNetwork{
{
Name: "foo",
Driver: "bar",
@ -56,7 +56,7 @@ func TestUnmarshalNetworks(t *testing.T) {
},
{
from: "foo: { name: baz }",
want: []*Network{
want: []*WorkflowNetwork{
{
Name: "baz",
Driver: "bridge",
@ -65,7 +65,7 @@ func TestUnmarshalNetworks(t *testing.T) {
},
{
from: "foo: { name: baz, driver: bar }",
want: []*Network{
want: []*WorkflowNetwork{
{
Name: "baz",
Driver: "bar",
@ -76,10 +76,10 @@ func TestUnmarshalNetworks(t *testing.T) {
for _, test := range testdata {
in := []byte(test.from)
got := Networks{}
got := WorkflowNetworks{}
err := yaml.Unmarshal(in, &got)
assert.NoError(t, err)
assert.EqualValues(t, test.want, got.Networks, "problem parsing network %q", test.from)
assert.EqualValues(t, test.want, got.WorkflowNetworks, "problem parsing network %q", test.from)
}
}
@ -91,7 +91,7 @@ func TestUnmarshalNetworkErr(t *testing.T) {
for _, test := range testdata {
in := []byte(test)
err := yaml.Unmarshal(in, new(Networks))
err := yaml.Unmarshal(in, new(WorkflowNetworks))
assert.Error(t, err, "wanted error for networks %q", test)
}
}

18
pipeline/frontend/yaml/volume.go → pipeline/frontend/yaml/types/workflow_volume.go
View file

@ -1,4 +1,4 @@
package yaml
package types
import (
"fmt"
@ -7,13 +7,13 @@ import (
)
type (
// Volumes defines a collection of volumes.
Volumes struct {
Volumes []*Volume
// WorkflowVolumes defines a collection of volumes.
WorkflowVolumes struct {
WorkflowVolumes []*WorkflowVolume
}
// Volume defines a container volume.
Volume struct {
// WorkflowVolume defines a container volume.
WorkflowVolume struct {
Name string `yaml:"name,omitempty"`
Driver string `yaml:"driver,omitempty"`
DriverOpts map[string]string `yaml:"driver_opts,omitempty"`
@ -21,10 +21,10 @@ type (
)
// UnmarshalYAML implements the Unmarshaler interface.
func (v *Volumes) UnmarshalYAML(value *yaml.Node) error {
func (v *WorkflowVolumes) UnmarshalYAML(value *yaml.Node) error {
y, _ := yaml.Marshal(value)
volumes := map[string]Volume{}
volumes := map[string]WorkflowVolume{}
err := yaml.Unmarshal(y, &volumes)
if err != nil {
return err
@ -37,7 +37,7 @@ func (v *Volumes) UnmarshalYAML(value *yaml.Node) error {
if vv.Driver == "" {
vv.Driver = "local"
}
v.Volumes = append(v.Volumes, &vv)
v.WorkflowVolumes = append(v.WorkflowVolumes, &vv)
}
return err
}

26
pipeline/frontend/yaml/volume_test.go → pipeline/frontend/yaml/types/workflow_volume_test.go
View file

@ -1,4 +1,4 @@
package yaml
package types
import (
"testing"
@ -10,18 +10,18 @@ import (
func TestUnmarshalVolume(t *testing.T) {
testdata := []struct {
from string
want Volume
want WorkflowVolume
}{
{
from: "{ name: foo, driver: bar }",
want: Volume{
want: WorkflowVolume{
Name: "foo",
Driver: "bar",
},
},
{
from: "{ name: foo, driver: bar, driver_opts: { baz: qux } }",
want: Volume{
want: WorkflowVolume{
Name: "foo",
Driver: "bar",
DriverOpts: map[string]string{
@ -33,21 +33,21 @@ func TestUnmarshalVolume(t *testing.T) {
for _, test := range testdata {
in := []byte(test.from)
got := Volume{}
got := WorkflowVolume{}
err := yaml.Unmarshal(in, &got)
assert.NoError(t, err)
assert.EqualValues(t, test.want, got, "problem parsing volume %q", test.from)
}
}
func TestUnmarshalVolumes(t *testing.T) {
func TestUnmarshalWorkflowVolumes(t *testing.T) {
testdata := []struct {
from string
want []*Volume
want []*WorkflowVolume
}{
{
from: "foo: { driver: bar }",
want: []*Volume{
want: []*WorkflowVolume{
{
Name: "foo",
Driver: "bar",
@ -56,7 +56,7 @@ func TestUnmarshalVolumes(t *testing.T) {
},
{
from: "foo: { name: baz }",
want: []*Volume{
want: []*WorkflowVolume{
{
Name: "baz",
Driver: "local",
@ -65,7 +65,7 @@ func TestUnmarshalVolumes(t *testing.T) {
},
{
from: "foo: { name: baz, driver: bar }",
want: []*Volume{
want: []*WorkflowVolume{
{
Name: "baz",
Driver: "bar",
@ -76,10 +76,10 @@ func TestUnmarshalVolumes(t *testing.T) {
for _, test := range testdata {
in := []byte(test.from)
got := Volumes{}
got := WorkflowVolumes{}
err := yaml.Unmarshal(in, &got)
assert.NoError(t, err)
assert.EqualValues(t, test.want, got.Volumes, "problem parsing volumes %q", test.from)
assert.EqualValues(t, test.want, got.WorkflowVolumes, "problem parsing volumes %q", test.from)
}
}
@ -91,7 +91,7 @@ func TestUnmarshalVolumesErr(t *testing.T) {
for _, test := range testdata {
in := []byte(test)
err := yaml.Unmarshal(in, new(Volumes))
err := yaml.Unmarshal(in, new(WorkflowVolumes))
assert.Error(t, err, "wanted error for volumes %q", test)
}
}

10
pipeline/stepBuilder.go
View file

@ -23,7 +23,10 @@ import (
"github.com/oklog/ulid/v2"
"github.com/rs/zerolog/log"
backend "github.com/woodpecker-ci/woodpecker/pipeline/backend/types"
backend_types "github.com/woodpecker-ci/woodpecker/pipeline/backend/types"
yaml_types "github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml/types"
forge_types "github.com/woodpecker-ci/woodpecker/server/forge/types"
"github.com/woodpecker-ci/woodpecker/pipeline/frontend"
"github.com/woodpecker-ci/woodpecker/pipeline/frontend/metadata"
"github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml"
@ -31,7 +34,6 @@ import (
"github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml/linter"
"github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml/matrix"
"github.com/woodpecker-ci/woodpecker/server"
forge_types "github.com/woodpecker-ci/woodpecker/server/forge/types"
"github.com/woodpecker-ci/woodpecker/server/model"
)
@ -55,7 +57,7 @@ type Item struct {
Labels map[string]string
DependsOn []string
RunsOn []string
Config *backend.Config
Config *backend_types.Config
}
func (b *StepBuilder) Build() ([]*Item, error) {
@ -216,7 +218,7 @@ func (b *StepBuilder) environmentVariables(metadata metadata.Metadata, axis matr
return environ
}
func (b *StepBuilder) toInternalRepresentation(parsed *yaml.Config, environ map[string]string, metadata metadata.Metadata, stepID int64) (*backend.Config, error) {
func (b *StepBuilder) toInternalRepresentation(parsed *yaml_types.Workflow, environ map[string]string, metadata metadata.Metadata, stepID int64) (*backend_types.Config, error) {
var secrets []compiler.Secret
for _, sec := range b.Secs {
if !sec.Match(b.Curr.Event) {