Implement YAML Map Merge, Overrides, and Sequence Merge Support (#1720)

close  #1192

docs/docs/20-usage/35-advanced-yaml-syntax.md

@@ -32,34 +32,59 @@ Just add a new section called **variables** like this:
      commands: build
 ```
 
-<!--
-TODO(1192): Support YAML override and extension
+## Map merges and overwrites
 
-## Example of YAML override and extension
-
-```yml
+```yaml
 variables:
-  &some-plugin-settings
+  - &base-plugin-settings
     target: dist
     recursive: false
     try: true
+  - &special-setting
+    special: true
+  - &some-plugin codeberg.org/6543/docker-images/print_env
 
-pipelines:
+pipeline:
   develop:
-    name: Build and test
-    image: some-plugin
-    settings: *some-plugin-settings
+    image: *some-plugin
+    settings:
+      <<: [*base-plugin-settings, *special-setting] # merge two maps into an empty map
     when:
       branch: develop
 
   main:
-    name: Build and test
-    image: some-plugin
+    image: *some-plugin
     settings:
-      <<: *some-plugin-settings
-      try: false # replacing original value from `some-plugin-settings`
-      ongoing: false # adding a new value to `some-plugin-settings`
+      <<: *base-plugin-settings # merge one map and ...
+      try: false # ... overwrite original value
+      ongoing: false # ... adding a new value
     when:
       branch: main
 ```
--->
+
+## Sequence merges
+
+```yaml
+variables:
+  pre_cmds: &pre_cmds
+   - echo start
+   - whoami
+  post_cmds: &post_cmds
+   - echo stop
+  hello_cmd: &hello_cmd
+   - echo hello
+
+pipeline:
+  step1:
+    image: debian
+    commands:
+     - <<: *pre_cmds # prepend a sequence
+     - echo exec step now do dedicated things
+     - <<: *post_cmds # append a sequence
+  step2:
+    image: debian
+    commands:
+     - <<: [*pre_cmds, *hello_cmd] # prepend two sequences
+     - echo echo from second step
+     - <<: *post_cmds
+```

go.mod

@@ -4,7 +4,8 @@ go 1.20
 
 require (
 	code.gitea.io/sdk/gitea v0.15.1-0.20221016183512-2d9ee57af1e0
-	codeberg.org/6543/go-yaml2json v0.3.0
+	codeberg.org/6543/go-yaml2json v1.0.0
+	codeberg.org/6543/xyaml v1.1.0
 	github.com/alessio/shellescape v1.4.1
 	github.com/antonmedv/expr v1.12.3
 	github.com/bmatcuk/doublestar/v4 v4.6.0

go.sum

@@ -32,8 +32,10 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX
 cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
 code.gitea.io/sdk/gitea v0.15.1-0.20221016183512-2d9ee57af1e0 h1:AKpsCoOtVrWWBtANM9319pwCB5ihx1Sdvr1HSbAwr54=
 code.gitea.io/sdk/gitea v0.15.1-0.20221016183512-2d9ee57af1e0/go.mod h1:ndkDk99BnfiUCCYEUhpNzi0lpmApXlwRFqClBlOlEBg=
-codeberg.org/6543/go-yaml2json v0.3.0 h1:BlvjmY0Gous8P+rr8aBdgPYnIfUAqFepF8q7Tp0R5t8=
-codeberg.org/6543/go-yaml2json v0.3.0/go.mod h1:mz61q14LWF4ZABrgMEDMmk3t9dPi6zgR1uBh2VKV2RQ=
+codeberg.org/6543/go-yaml2json v1.0.0 h1:heGqo9VEi7gY2yNqjj7X4ADs5nzlFIbGsJtgYDLrnig=
+codeberg.org/6543/go-yaml2json v1.0.0/go.mod h1:mz61q14LWF4ZABrgMEDMmk3t9dPi6zgR1uBh2VKV2RQ=
+codeberg.org/6543/xyaml v1.1.0 h1:0PWTy8OUqshshjrrnAXFWXSPUEa8R49DIh2ah07SxFc=
+codeberg.org/6543/xyaml v1.1.0/go.mod h1:jI7afXLZUxeL4rNNsG1SlHh78L+gma9lK1bIebyFZwA=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 gitea.com/xorm/sqlfiddle v0.0.0-20180821085327-62ce714f951a h1:lSA0F4e9A2NcQSqGqTOXqu2aRi/XEQxDCBwM8yJtE6s=
 gitea.com/xorm/sqlfiddle v0.0.0-20180821085327-62ce714f951a/go.mod h1:EXuID2Zs0pAQhH8yz+DNjUbjppKQzKFAn28TMYPB6IU=

pipeline/frontend/yaml/config.go

@@ -1,8 +1,7 @@
 package yaml
 
 import (
-	"gopkg.in/yaml.v3"
-
+	"codeberg.org/6543/xyaml"
 	"github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml/constraint"
 	"github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml/types"
 )
@@ -37,7 +36,7 @@ type (
 // ParseBytes parses the configuration from bytes b.
 func ParseBytes(b []byte) (*Config, error) {
 	out := new(Config)
-	err := yaml.Unmarshal(b, out)
+	err := xyaml.Unmarshal(b, out)
 	if err != nil {
 		return nil, err
 	}

pipeline/frontend/yaml/linter/linter_test.go

@@ -42,7 +42,22 @@ pipeline:
     repo: foo/bar
     settings:
       foo: bar
-`}}
+`}, {
+		Title: "merge maps", Data: `
+variables:
+  step_template: &base-step
+    image: golang:1.19
+    commands:
+      - go version
+
+pipeline:
+  test base step:
+    <<: *base-step
+  test base step with latest image:
+    <<: *base-step
+    image: golang:latest
+`,
+	}}
 
 	for _, testd := range testdatas {
 		t.Run(testd.Title, func(t *testing.T) {

pipeline/frontend/yaml/matrix/matrix.go

@@ -19,7 +19,7 @@ import (
 
 	pipeline "github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml"
 
-	"gopkg.in/yaml.v3"
+	"codeberg.org/6543/xyaml"
 )
 
 const (
@@ -115,7 +115,7 @@ func parse(raw []byte) (Matrix, error) {
 	data := struct {
 		Matrix map[string][]string
 	}{}
-	if err := yaml.Unmarshal(raw, &data); err != nil {
+	if err := xyaml.Unmarshal(raw, &data); err != nil {
 		return nil, &pipeline.PipelineParseError{Err: err}
 	}
 	return data.Matrix, nil
@@ -128,7 +128,7 @@ func parseList(raw []byte) ([]Axis, error) {
 		}
 	}{}
 
-	if err := yaml.Unmarshal(raw, &data); err != nil {
+	if err := xyaml.Unmarshal(raw, &data); err != nil {
 		return nil, &pipeline.PipelineParseError{Err: err}
 	}
 	return data.Matrix.Include, nil

pipeline/schema/.woodpecker/test-merge-map-and-sequence.yml

@@ -0,0 +1,18 @@
+variables:
+  step_template: &base-step
+    image: golang:1.19
+    commands: &base-cmds
+      - go version
+      - whoami
+
+pipeline:
+  test-base-step:
+    <<: *base-step
+  test base step with latest image:
+    <<: *base-step
+    image: golang:latest
+  test list overwrite:
+    <<: *base-step
+    commands:
+      - <<: *base-cmds
+      - hostname

pipeline/schema/schema.go

@@ -1,13 +1,14 @@
 package schema
 
 import (
-	"bytes"
 	_ "embed"
 	"fmt"
 	"io"
 
 	"codeberg.org/6543/go-yaml2json"
+	"codeberg.org/6543/xyaml"
 	"github.com/xeipuuv/gojsonschema"
+	"gopkg.in/yaml.v3"
 )
 
 //go:embed schema.json
@@ -16,13 +17,26 @@ var schemaDefinition []byte
 // Lint lints an io.Reader against the Woodpecker schema.json
 func Lint(r io.Reader) ([]gojsonschema.ResultError, error) {
 	schemaLoader := gojsonschema.NewBytesLoader(schemaDefinition)
-	buff := new(bytes.Buffer)
-	err := yaml2json.StreamConvert(r, buff)
+
+	// read yaml config
+	rBytes, err := io.ReadAll(r)
 	if err != nil {
 		return nil, fmt.Errorf("Failed to load yml file %w", err)
 	}
 
-	documentLoader := gojsonschema.NewBytesLoader(buff.Bytes())
+	// resolve sequence merges
+	yamlDoc := new(yaml.Node)
+	if err := xyaml.Unmarshal(rBytes, yamlDoc); err != nil {
+		return nil, fmt.Errorf("Failed to parse yml file %w", err)
+	}
+
+	// convert to json
+	jsonDoc, err := yaml2json.ConvertNode(yamlDoc)
+	if err != nil {
+		return nil, fmt.Errorf("Failed to convert yaml %w", err)
+	}
+
+	documentLoader := gojsonschema.NewBytesLoader(jsonDoc)
 	result, err := gojsonschema.Validate(schemaLoader, documentLoader)
 	if err != nil {
 		return nil, fmt.Errorf("Validation failed %w", err)

pipeline/schema/schema_test.go

@@ -78,6 +78,10 @@ func TestSchema(t *testing.T) {
 			name:     "Labels",
 			testFile: ".woodpecker/test-labels.yml",
 		},
+		{
+			name:     "Map and Sequence Merge", // https://woodpecker-ci.org/docs/next/usage/advanced-yaml-syntax
+			testFile: ".woodpecker/test-merge-map-and-sequence.yml",
+		},
 		{
 			name:     "Broken Config",
 			testFile: ".woodpecker/test-broken.yml",