From 204d05f44709e82726831d3ca17a5f673fcfb302 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Sat, 29 Apr 2023 14:49:41 +0200 Subject: [PATCH] Implement YAML Map Merge, Overrides, and Sequence Merge Support (#1720) close #1192 --- docs/docs/20-usage/35-advanced-yaml-syntax.md | 59 +++++++++++++------ go.mod | 3 +- go.sum | 6 +- pipeline/frontend/yaml/config.go | 5 +- pipeline/frontend/yaml/linter/linter_test.go | 17 +++++- pipeline/frontend/yaml/matrix/matrix.go | 6 +- .../test-merge-map-and-sequence.yml | 18 ++++++ pipeline/schema/schema.go | 22 +++++-- pipeline/schema/schema_test.go | 4 ++ 9 files changed, 109 insertions(+), 31 deletions(-) create mode 100644 pipeline/schema/.woodpecker/test-merge-map-and-sequence.yml diff --git a/docs/docs/20-usage/35-advanced-yaml-syntax.md b/docs/docs/20-usage/35-advanced-yaml-syntax.md index 204d7d0ed..83f9e4f19 100644 --- a/docs/docs/20-usage/35-advanced-yaml-syntax.md +++ b/docs/docs/20-usage/35-advanced-yaml-syntax.md @@ -32,34 +32,59 @@ Just add a new section called **variables** like this: commands: build ``` - + +## Sequence merges + +```yaml +variables: + pre_cmds: &pre_cmds + - echo start + - whoami + post_cmds: &post_cmds + - echo stop + hello_cmd: &hello_cmd + - echo hello + +pipeline: + step1: + image: debian + commands: + - <<: *pre_cmds # prepend a sequence + - echo exec step now do dedicated things + - <<: *post_cmds # append a sequence + step2: + image: debian + commands: + - <<: [*pre_cmds, *hello_cmd] # prepend two sequences + - echo echo from second step + - <<: *post_cmds +``` diff --git a/go.mod b/go.mod index e5eef5cb1..3f596c90a 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,8 @@ go 1.20 require ( code.gitea.io/sdk/gitea v0.15.1-0.20221016183512-2d9ee57af1e0 - codeberg.org/6543/go-yaml2json v0.3.0 + codeberg.org/6543/go-yaml2json v1.0.0 + codeberg.org/6543/xyaml v1.1.0 github.com/alessio/shellescape v1.4.1 github.com/antonmedv/expr v1.12.3 github.com/bmatcuk/doublestar/v4 v4.6.0 diff --git a/go.sum b/go.sum index 23d717954..35ffe978f 100644 --- a/go.sum +++ b/go.sum @@ -32,8 +32,10 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= code.gitea.io/sdk/gitea v0.15.1-0.20221016183512-2d9ee57af1e0 h1:AKpsCoOtVrWWBtANM9319pwCB5ihx1Sdvr1HSbAwr54= code.gitea.io/sdk/gitea v0.15.1-0.20221016183512-2d9ee57af1e0/go.mod h1:ndkDk99BnfiUCCYEUhpNzi0lpmApXlwRFqClBlOlEBg= -codeberg.org/6543/go-yaml2json v0.3.0 h1:BlvjmY0Gous8P+rr8aBdgPYnIfUAqFepF8q7Tp0R5t8= -codeberg.org/6543/go-yaml2json v0.3.0/go.mod h1:mz61q14LWF4ZABrgMEDMmk3t9dPi6zgR1uBh2VKV2RQ= +codeberg.org/6543/go-yaml2json v1.0.0 h1:heGqo9VEi7gY2yNqjj7X4ADs5nzlFIbGsJtgYDLrnig= +codeberg.org/6543/go-yaml2json v1.0.0/go.mod h1:mz61q14LWF4ZABrgMEDMmk3t9dPi6zgR1uBh2VKV2RQ= +codeberg.org/6543/xyaml v1.1.0 h1:0PWTy8OUqshshjrrnAXFWXSPUEa8R49DIh2ah07SxFc= +codeberg.org/6543/xyaml v1.1.0/go.mod h1:jI7afXLZUxeL4rNNsG1SlHh78L+gma9lK1bIebyFZwA= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= gitea.com/xorm/sqlfiddle v0.0.0-20180821085327-62ce714f951a h1:lSA0F4e9A2NcQSqGqTOXqu2aRi/XEQxDCBwM8yJtE6s= gitea.com/xorm/sqlfiddle v0.0.0-20180821085327-62ce714f951a/go.mod h1:EXuID2Zs0pAQhH8yz+DNjUbjppKQzKFAn28TMYPB6IU= diff --git a/pipeline/frontend/yaml/config.go b/pipeline/frontend/yaml/config.go index c5c73b9fb..25e30ed0d 100644 --- a/pipeline/frontend/yaml/config.go +++ b/pipeline/frontend/yaml/config.go @@ -1,8 +1,7 @@ package yaml import ( - "gopkg.in/yaml.v3" - + "codeberg.org/6543/xyaml" "github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml/constraint" "github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml/types" ) @@ -37,7 +36,7 @@ type ( // ParseBytes parses the configuration from bytes b. func ParseBytes(b []byte) (*Config, error) { out := new(Config) - err := yaml.Unmarshal(b, out) + err := xyaml.Unmarshal(b, out) if err != nil { return nil, err } diff --git a/pipeline/frontend/yaml/linter/linter_test.go b/pipeline/frontend/yaml/linter/linter_test.go index 8bbf03e35..1be131826 100644 --- a/pipeline/frontend/yaml/linter/linter_test.go +++ b/pipeline/frontend/yaml/linter/linter_test.go @@ -42,7 +42,22 @@ pipeline: repo: foo/bar settings: foo: bar -`}} +`}, { + Title: "merge maps", Data: ` +variables: + step_template: &base-step + image: golang:1.19 + commands: + - go version + +pipeline: + test base step: + <<: *base-step + test base step with latest image: + <<: *base-step + image: golang:latest +`, + }} for _, testd := range testdatas { t.Run(testd.Title, func(t *testing.T) { diff --git a/pipeline/frontend/yaml/matrix/matrix.go b/pipeline/frontend/yaml/matrix/matrix.go index da5e9f247..d9b5454d2 100644 --- a/pipeline/frontend/yaml/matrix/matrix.go +++ b/pipeline/frontend/yaml/matrix/matrix.go @@ -19,7 +19,7 @@ import ( pipeline "github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml" - "gopkg.in/yaml.v3" + "codeberg.org/6543/xyaml" ) const ( @@ -115,7 +115,7 @@ func parse(raw []byte) (Matrix, error) { data := struct { Matrix map[string][]string }{} - if err := yaml.Unmarshal(raw, &data); err != nil { + if err := xyaml.Unmarshal(raw, &data); err != nil { return nil, &pipeline.PipelineParseError{Err: err} } return data.Matrix, nil @@ -128,7 +128,7 @@ func parseList(raw []byte) ([]Axis, error) { } }{} - if err := yaml.Unmarshal(raw, &data); err != nil { + if err := xyaml.Unmarshal(raw, &data); err != nil { return nil, &pipeline.PipelineParseError{Err: err} } return data.Matrix.Include, nil diff --git a/pipeline/schema/.woodpecker/test-merge-map-and-sequence.yml b/pipeline/schema/.woodpecker/test-merge-map-and-sequence.yml new file mode 100644 index 000000000..94e0558ab --- /dev/null +++ b/pipeline/schema/.woodpecker/test-merge-map-and-sequence.yml @@ -0,0 +1,18 @@ +variables: + step_template: &base-step + image: golang:1.19 + commands: &base-cmds + - go version + - whoami + +pipeline: + test-base-step: + <<: *base-step + test base step with latest image: + <<: *base-step + image: golang:latest + test list overwrite: + <<: *base-step + commands: + - <<: *base-cmds + - hostname diff --git a/pipeline/schema/schema.go b/pipeline/schema/schema.go index f2cd40eb6..f4e0fa8f5 100644 --- a/pipeline/schema/schema.go +++ b/pipeline/schema/schema.go @@ -1,13 +1,14 @@ package schema import ( - "bytes" _ "embed" "fmt" "io" "codeberg.org/6543/go-yaml2json" + "codeberg.org/6543/xyaml" "github.com/xeipuuv/gojsonschema" + "gopkg.in/yaml.v3" ) //go:embed schema.json @@ -16,13 +17,26 @@ var schemaDefinition []byte // Lint lints an io.Reader against the Woodpecker schema.json func Lint(r io.Reader) ([]gojsonschema.ResultError, error) { schemaLoader := gojsonschema.NewBytesLoader(schemaDefinition) - buff := new(bytes.Buffer) - err := yaml2json.StreamConvert(r, buff) + + // read yaml config + rBytes, err := io.ReadAll(r) if err != nil { return nil, fmt.Errorf("Failed to load yml file %w", err) } - documentLoader := gojsonschema.NewBytesLoader(buff.Bytes()) + // resolve sequence merges + yamlDoc := new(yaml.Node) + if err := xyaml.Unmarshal(rBytes, yamlDoc); err != nil { + return nil, fmt.Errorf("Failed to parse yml file %w", err) + } + + // convert to json + jsonDoc, err := yaml2json.ConvertNode(yamlDoc) + if err != nil { + return nil, fmt.Errorf("Failed to convert yaml %w", err) + } + + documentLoader := gojsonschema.NewBytesLoader(jsonDoc) result, err := gojsonschema.Validate(schemaLoader, documentLoader) if err != nil { return nil, fmt.Errorf("Validation failed %w", err) diff --git a/pipeline/schema/schema_test.go b/pipeline/schema/schema_test.go index 3cf956849..5ed943b12 100644 --- a/pipeline/schema/schema_test.go +++ b/pipeline/schema/schema_test.go @@ -78,6 +78,10 @@ func TestSchema(t *testing.T) { name: "Labels", testFile: ".woodpecker/test-labels.yml", }, + { + name: "Map and Sequence Merge", // https://woodpecker-ci.org/docs/next/usage/advanced-yaml-syntax + testFile: ".woodpecker/test-merge-map-and-sequence.yml", + }, { name: "Broken Config", testFile: ".woodpecker/test-broken.yml",