woodpecker/pipeline/frontend/yaml/compiler/compiler.go

234 lines
6 KiB
Go
Raw Normal View History

2017-03-05 07:56:08 +00:00
package compiler
import (
"fmt"
backend "github.com/woodpecker-ci/woodpecker/pipeline/backend/types"
"github.com/woodpecker-ci/woodpecker/pipeline/frontend"
"github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml"
2017-03-05 07:56:08 +00:00
)
// TODO(bradrydzewski) compiler should handle user-defined volumes from YAML
// TODO(bradrydzewski) compiler should handle user-defined networks from YAML
2017-04-06 16:04:25 +00:00
type Registry struct {
Hostname string
Username string
Password string
Email string
Token string
}
2017-04-10 10:39:50 +00:00
type Secret struct {
Name string
Value string
Match []string
}
2017-06-22 19:06:28 +00:00
type ResourceLimit struct {
MemSwapLimit int64
MemLimit int64
ShmSize int64
CPUQuota int64
CPUShares int64
CPUSet string
}
2017-03-05 07:56:08 +00:00
// Compiler compiles the yaml
type Compiler struct {
2017-04-06 16:04:25 +00:00
local bool
escalated []string
prefix string
volumes []string
2017-04-10 16:27:34 +00:00
networks []string
2017-04-06 16:04:25 +00:00
env map[string]string
cloneEnv map[string]string
2017-04-06 16:04:25 +00:00
base string
path string
metadata frontend.Metadata
registries []Registry
2017-04-10 10:39:50 +00:00
secrets map[string]Secret
2017-06-22 19:06:28 +00:00
cacher Cacher
reslimit ResourceLimit
2017-03-05 07:56:08 +00:00
}
// New creates a new Compiler with options.
func New(opts ...Option) *Compiler {
2017-04-10 10:39:50 +00:00
compiler := &Compiler{
env: map[string]string{},
cloneEnv: map[string]string{},
secrets: map[string]Secret{},
2017-04-10 10:39:50 +00:00
}
2017-03-05 07:56:08 +00:00
for _, opt := range opts {
opt(compiler)
}
return compiler
}
// Compile compiles the YAML configuration to the pipeline intermediate
// representation configuration format.
func (c *Compiler) Compile(conf *yaml.Config) *backend.Config {
config := new(backend.Config)
// create a default volume
config.Volumes = append(config.Volumes, &backend.Volume{
Name: fmt.Sprintf("%s_default", c.prefix),
Driver: "local",
})
// create a default network
2019-04-06 13:44:04 +00:00
if c.metadata.Sys.Arch == "windows/amd64" {
config.Networks = append(config.Networks, &backend.Network{
Name: fmt.Sprintf("%s_default", c.prefix),
Driver: "nat",
})
} else {
config.Networks = append(config.Networks, &backend.Network{
Name: fmt.Sprintf("%s_default", c.prefix),
Driver: "bridge",
})
}
2017-03-05 07:56:08 +00:00
2020-11-19 07:42:18 +00:00
// create secrets for mask
for _, sec := range c.secrets {
config.Secrets = append(config.Secrets, &backend.Secret{
Name: sec.Name,
Value: sec.Value,
Mask: true,
})
}
2017-03-05 07:56:08 +00:00
// overrides the default workspace paths when specified
// in the YAML file.
if len(conf.Workspace.Base) != 0 {
c.base = conf.Workspace.Base
}
if len(conf.Workspace.Path) != 0 {
c.path = conf.Workspace.Path
}
// add default clone step
if !c.local && len(conf.Clone.Containers) == 0 && !conf.SkipClone {
2017-03-05 07:56:08 +00:00
container := &yaml.Container{
Name: "clone",
// TODO: switch to `:latest` once v1.1.0 got released
// https://github.com/woodpecker-ci/plugin-git/issues/3
Image: "woodpeckerci/plugin-git:next",
Settings: map[string]interface{}{"depth": "0"},
Environment: c.cloneEnv,
}
2017-03-05 07:56:08 +00:00
name := fmt.Sprintf("%s_clone", c.prefix)
2017-07-21 21:52:52 +00:00
step := c.createProcess(name, container, "clone")
2017-03-05 07:56:08 +00:00
stage := new(backend.Stage)
stage.Name = name
stage.Alias = "clone"
stage.Steps = append(stage.Steps, step)
config.Stages = append(config.Stages, stage)
} else if !c.local && !conf.SkipClone {
2017-03-05 07:56:08 +00:00
for i, container := range conf.Clone.Containers {
if !container.Constraints.Match(c.metadata) {
continue
}
stage := new(backend.Stage)
stage.Name = fmt.Sprintf("%s_clone_%v", c.prefix, i)
stage.Alias = container.Name
name := fmt.Sprintf("%s_clone_%d", c.prefix, i)
2017-07-21 21:52:52 +00:00
step := c.createProcess(name, container, "clone")
for k, v := range c.cloneEnv {
step.Environment[k] = v
}
2017-03-05 07:56:08 +00:00
stage.Steps = append(stage.Steps, step)
config.Stages = append(config.Stages, stage)
}
}
2017-06-22 19:06:28 +00:00
c.setupCache(conf, config)
2017-03-05 07:56:08 +00:00
// add services steps
if len(conf.Services.Containers) != 0 {
stage := new(backend.Stage)
stage.Name = fmt.Sprintf("%s_services", c.prefix)
stage.Alias = "services"
for i, container := range conf.Services.Containers {
2017-05-14 17:28:17 +00:00
if !container.Constraints.Match(c.metadata) {
continue
}
2017-03-05 07:56:08 +00:00
name := fmt.Sprintf("%s_services_%d", c.prefix, i)
2017-07-21 21:52:52 +00:00
step := c.createProcess(name, container, "services")
2017-03-05 07:56:08 +00:00
stage.Steps = append(stage.Steps, step)
}
config.Stages = append(config.Stages, stage)
}
// add pipeline steps. 1 pipeline step per stage, at the moment
var stage *backend.Stage
var group string
for i, container := range conf.Pipeline.Containers {
2017-05-06 01:15:47 +00:00
//Skip if local and should not run local
if c.local && !container.Constraints.Local.Bool() {
continue
}
2017-03-05 07:56:08 +00:00
if !container.Constraints.Match(c.metadata) {
continue
}
if stage == nil || group != container.Group || container.Group == "" {
group = container.Group
stage = new(backend.Stage)
stage.Name = fmt.Sprintf("%s_stage_%v", c.prefix, i)
stage.Alias = container.Name
config.Stages = append(config.Stages, stage)
}
name := fmt.Sprintf("%s_step_%d", c.prefix, i)
2017-07-21 21:52:52 +00:00
step := c.createProcess(name, container, "pipeline")
2017-03-05 07:56:08 +00:00
stage.Steps = append(stage.Steps, step)
}
2017-06-22 19:06:28 +00:00
c.setupCacheRebuild(conf, config)
2017-03-05 07:56:08 +00:00
return config
}
2017-06-22 19:06:28 +00:00
func (c *Compiler) setupCache(conf *yaml.Config, ir *backend.Config) {
if c.local || len(conf.Cache) == 0 || c.cacher == nil {
return
}
container := c.cacher.Restore(c.metadata.Repo.Name, c.metadata.Curr.Commit.Branch, conf.Cache)
name := fmt.Sprintf("%s_restore_cache", c.prefix)
2017-07-21 21:52:52 +00:00
step := c.createProcess(name, container, "cache")
2017-06-22 19:06:28 +00:00
stage := new(backend.Stage)
stage.Name = name
stage.Alias = "restore_cache"
stage.Steps = append(stage.Steps, step)
ir.Stages = append(ir.Stages, stage)
}
func (c *Compiler) setupCacheRebuild(conf *yaml.Config, ir *backend.Config) {
if c.local || len(conf.Cache) == 0 || c.metadata.Curr.Event != "push" || c.cacher == nil {
return
}
container := c.cacher.Rebuild(c.metadata.Repo.Name, c.metadata.Curr.Commit.Branch, conf.Cache)
name := fmt.Sprintf("%s_rebuild_cache", c.prefix)
2017-07-21 21:52:52 +00:00
step := c.createProcess(name, container, "cache")
2017-06-22 19:06:28 +00:00
stage := new(backend.Stage)
stage.Name = name
stage.Alias = "rebuild_cache"
stage.Steps = append(stage.Steps, step)
ir.Stages = append(ir.Stages, stage)
}