From 8da0ee47f76b5016c0f2c4978dd2bc457d47cfe5 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Sun, 17 Jul 2022 18:25:56 +0200 Subject: [PATCH] Use variables in pipeline (#1026) use yaml aliases (https://yaml.org/spec/1.2.2/#3222-anchors-and-aliases) to have pipeline `variables` Co-authored-by: qwerty287 <80460567+qwerty287@users.noreply.github.com> Co-authored-by: Anbraten --- .woodpecker/binaries.yml | 121 +++++------------- .woodpecker/docker.yml | 13 +- .woodpecker/test.yml | 110 +++++----------- .woodpecker/web.yml | 26 ++-- cli/lint/lint.go | 16 ++- docs/docs/20-usage/20-pipeline-syntax.md | 6 + docs/docs/20-usage/35-advanced-yaml-syntax.md | 62 +++++++++ pipeline/schema/schema.json | 3 + 8 files changed, 173 insertions(+), 184 deletions(-) create mode 100644 docs/docs/20-usage/35-advanced-yaml-syntax.md diff --git a/.woodpecker/binaries.yml b/.woodpecker/binaries.yml index af6fdaf15..f451e6612 100644 --- a/.woodpecker/binaries.yml +++ b/.woodpecker/binaries.yml @@ -2,153 +2,94 @@ depends_on: - test - web +variables: + - &golang_image 'golang:1.18' + - &node_image 'node:16-alpine' + - &when_path + # related config files + - ".woodpecker/binaries.yml" + - "nfpm/*.yml" + # go source code + - "**/*.go" + - "go.*" + # web source code + - "web/**" + pipeline: build-web: group: prepare - image: node:16-alpine + image: *node_image commands: - cd web/ - yarn install --frozen-lockfile - yarn build when: - path: - # related config files - - ".woodpecker/binaries.yml" - - "nfpm/*.yml" - # go source code - - "**/*.go" - - "go.*" - # web source code - - "web/**" + path: *when_path vendor: group: prepare - image: golang:1.18 + image: *golang_image commands: - go mod vendor when: - path: - # related config files - - ".woodpecker/binaries.yml" - - "nfpm/*.yml" - # go source code - - "**/*.go" - - "go.*" - # web source code - - "web/**" + path: *when_path build-server: group: build - image: golang:1.18 + image: *golang_image commands: - make release-server when: - path: - # related config files - - ".woodpecker/binaries.yml" - - "nfpm/*.yml" - # go source code - - "**/*.go" - - "go.*" - # web source code - - "web/**" + path: *when_path build-agent: group: build - image: golang:1.18 + image: *golang_image commands: - make release-agent when: - path: - # related config files - - ".woodpecker/binaries.yml" - - "nfpm/*.yml" - # go source code - - "**/*.go" - - "go.*" - # web source code - - "web/**" + path: *when_path build-cli: group: build - image: golang:1.18 + image: *golang_image commands: - make release-cli when: - path: - # related config files - - ".woodpecker/binaries.yml" - - "nfpm/*.yml" - # go source code - - "**/*.go" - - "go.*" - # web source code - - "web/**" + path: *when_path build-deb-rpm: group: bundle - image: golang:1.18 + image: *golang_image commands: - make bundle when: - path: - # related config files - - ".woodpecker/binaries.yml" - - "nfpm/*.yml" - # go source code - - "**/*.go" - - "go.*" - # web source code - - "web/**" + path: *when_path build-tarball: group: bundle - image: golang:1.18 + image: *golang_image commands: - make release-tarball when: - path: - # related config files - - ".woodpecker/binaries.yml" - - "nfpm/*.yml" - # go source code - - "**/*.go" - - "go.*" - # web source code - - "web/**" + path: *when_path checksums: - image: golang:1.18 + image: *golang_image commands: - make release-checksums when: - path: - # related config files - - ".woodpecker/binaries.yml" - - "nfpm/*.yml" - # go source code - - "**/*.go" - - "go.*" - # web source code - - "web/**" + path: *when_path # TODO: upload build artifacts for pushes to ${CI_REPO_DEFAULT_BRANCH} release-dryrun: - image: golang:1.18 + image: *golang_image commands: - ls -la dist/*.* - cat dist/checksums.txt when: - path: - # related config files - - ".woodpecker/binaries.yml" - - "nfpm/*.yml" - # go source code - - "**/*.go" - - "go.*" - # web source code - - "web/**" + path: *when_path release: image: plugins/github-release diff --git a/.woodpecker/docker.yml b/.woodpecker/docker.yml index 88ef0f62a..8eaa722b2 100644 --- a/.woodpecker/docker.yml +++ b/.woodpecker/docker.yml @@ -2,9 +2,15 @@ depends_on: - test - web +variables: + - &golang_image 'golang:1.18' + - &node_image 'node:16-alpine' + - &xgo_image 'techknowlogick/xgo:go-1.18.x' + - &xgo_version 'go-1.18.x' + pipeline: vendor: - image: golang:1.18 + image: *golang_image commands: - go mod vendor @@ -12,14 +18,14 @@ pipeline: # S e r v e r # ############### build-web: - image: node:16-alpine + image: *node_image commands: - cd web/ - yarn install --frozen-lockfile - yarn build cross-compile-server: - image: techknowlogick/xgo:go-1.18.x + image: *xgo_image commands: - apt update - apt install -y tree @@ -27,6 +33,7 @@ pipeline: environment: PLATFORMS: linux|arm/v7;linux|arm64/v8;linux|amd64;linux|ppc64le TAGS: bindata sqlite sqlite_unlock_notify + XGO_VERSION: *xgo_version publish-server-dryrun: image: woodpeckerci/plugin-docker-buildx diff --git a/.woodpecker/test.yml b/.woodpecker/test.yml index 3a699ed19..5f56a9bb9 100644 --- a/.woodpecker/test.yml +++ b/.woodpecker/test.yml @@ -1,63 +1,56 @@ +variables: + - &golang_image 'golang:1.18' + - &node_image 'node:16-alpine' + - &when_path + # related config files + - ".woodpecker/test.yml" + - ".golangci.yml" + # go source code + - "**/*.go" + - "go.*" + # schema changes + - "pipeline/schema/**" + pipeline: lint-pipeline: - image: woodpeckerci/woodpecker-cli:next-alpine + image: *golang_image commands: - - woodpecker-cli lint + - go run github.com/woodpecker-ci/woodpecker/cmd/cli lint when: path: - ".woodpecker/**" vendor: - image: golang:1.18 + image: *golang_image group: prepare commands: - go mod vendor when: - path: - # related config files - - ".woodpecker/test.yml" - # go source code - - "**/*.go" - - "go.*" - # schema changes - - "pipeline/schema/**" + path: *when_path dummy-web: - image: golang:1.18 + image: *golang_image group: prepare commands: - mkdir -p web/dist/ - echo "test" > web/dist/index.html when: - path: - # related config files - - ".woodpecker/test.yml" - # go source code - - "**/*.go" - - "go.*" - # schema changes - - "pipeline/schema/**" + path: *when_path lint: - image: golang:1.18 + image: *golang_image group: test commands: - make lint when: - path: - # related config files - - ".woodpecker/test.yml" - - ".golangci.yml" - # go source code - - "**/*.go" - - "go.*" + path: *when_path lint-editorconfig: image: mstruebing/editorconfig-checker group: test test: - image: golang:1.18 + image: *golang_image group: test commands: - make test-agent @@ -65,32 +58,20 @@ pipeline: - make test-cli - make test-lib when: - path: - # related config files - - ".woodpecker/test.yml" - # go source code - - "**/*.go" - - "go.*" - # schema changes - - "pipeline/schema/**" + path: *when_path sqlite: - image: golang:1.18 + image: *golang_image group: test environment: - WOODPECKER_DATABASE_DRIVER=sqlite3 commands: - make test-server-datastore-coverage when: - path: - # related config files - - ".woodpecker/test.yml" - # go source code - - "**/*.go" - - "go.*" + path: *when_path postgres: - image: golang:1.18 + image: *golang_image group: test environment: - WOODPECKER_DATABASE_DRIVER=postgres @@ -98,15 +79,10 @@ pipeline: commands: - make test-server-datastore when: - path: - # related config files - - ".woodpecker/test.yml" - # go source code - - "**/*.go" - - "go.*" + path: *when_path mysql: - image: golang:1.18 + image: *golang_image group: test environment: - WOODPECKER_DATABASE_DRIVER=mysql @@ -114,12 +90,7 @@ pipeline: commands: - make test-server-datastore when: - path: - # related config files - - ".woodpecker/test.yml" - # go source code - - "**/*.go" - - "go.*" + path: *when_path codecov: pull: true @@ -133,14 +104,7 @@ pipeline: token: from_secret: codecov_token when: - path: - # related config files - - ".woodpecker/test.yml" - # go source code - - "**/*.go" - - "go.*" - # schema changes - - "pipeline/schema/**" + path: *when_path services: service-postgres: @@ -150,12 +114,7 @@ services: - POSTGRES_USER=postgres - POSTGRES_HOST_AUTH_METHOD=trust when: - path: - # related config files - - ".woodpecker/test.yml" - # go source code - - "**/*.go" - - "go.*" + path: *when_path service-mysql: image: mysql:5.6.27 @@ -164,9 +123,4 @@ services: - MYSQL_DATABASE=test - MYSQL_ALLOW_EMPTY_PASSWORD=yes when: - path: - # related config files - - ".woodpecker/test.yml" - # go source code - - "**/*.go" - - "go.*" + path: *when_path diff --git a/.woodpecker/web.yml b/.woodpecker/web.yml index 3f29479a2..2a56c5364 100644 --- a/.woodpecker/web.yml +++ b/.woodpecker/web.yml @@ -1,45 +1,51 @@ +variables: + - &node_image 'node:16-alpine' + - &when_path + # web source code + - "web/**" + pipeline: deps: - image: node:16-alpine + image: *node_image commands: - cd web/ - yarn install --frozen-lockfile when: - path: "web/**" + path: *when_path lint: group: test - image: node:16-alpine + image: *node_image commands: - cd web/ - yarn lint when: - path: "web/**" + path: *when_path formatcheck: group: test - image: node:16-alpine + image: *node_image commands: - cd web/ - yarn formatcheck when: - path: "web/**" + path: *when_path typecheck: group: test - image: node:16-alpine + image: *node_image commands: - cd web/ - yarn typecheck when: - path: "web/**" + path: *when_path test: group: test - image: node:16-alpine + image: *node_image commands: - cd web/ - yarn test when: - path: "web/**" + path: *when_path diff --git a/cli/lint/lint.go b/cli/lint/lint.go index 052d1ee07..a4bf7f64e 100644 --- a/cli/lint/lint.go +++ b/cli/lint/lint.go @@ -26,7 +26,8 @@ func lint(c *cli.Context) error { } func lintDir(c *cli.Context, dir string) error { - return filepath.Walk(dir, func(path string, info os.FileInfo, e error) error { + var errorStrings []string + if err := filepath.Walk(dir, func(path string, info os.FileInfo, e error) error { if e != nil { return e } @@ -34,13 +35,22 @@ func lintDir(c *cli.Context, dir string) error { // check if it is a regular file (not dir) if info.Mode().IsRegular() && strings.HasSuffix(info.Name(), ".yml") { fmt.Println("#", info.Name()) - _ = lintFile(c, path) // TODO: should we drop errors or store them and report back? + if err := lintFile(c, path); err != nil { + errorStrings = append(errorStrings, err.Error()) + } fmt.Println("") return nil } return nil - }) + }); err != nil { + return err + } + + if len(errorStrings) != 0 { + return fmt.Errorf("ERRORS: %s", strings.Join(errorStrings, "; ")) + } + return nil } func lintFile(_ *cli.Context, file string) error { diff --git a/docs/docs/20-usage/20-pipeline-syntax.md b/docs/docs/20-usage/20-pipeline-syntax.md index 41e2277c7..44c388f80 100644 --- a/docs/docs/20-usage/20-pipeline-syntax.md +++ b/docs/docs/20-usage/20-pipeline-syntax.md @@ -664,6 +664,12 @@ Example configuration to clone Mercurial repository: + path: bitbucket.org/foo/bar ``` +## `variables` + +Woodpecker supports [YAML anchors & aliases](https://yaml.org/spec/1.2.2/#3222-anchors-and-aliases) in the pipeline configuration. These can be used as variables to not repeat yourself. + +For more details and examples check the [Advanced YAML syntax docs](/docs/usage/advanced-yaml-syntax) + ### Git Submodules To use the credentials that cloned the repository to clone it's submodules, update `.gitmodules` to use `https` instead of `git`: diff --git a/docs/docs/20-usage/35-advanced-yaml-syntax.md b/docs/docs/20-usage/35-advanced-yaml-syntax.md new file mode 100644 index 000000000..748e08b34 --- /dev/null +++ b/docs/docs/20-usage/35-advanced-yaml-syntax.md @@ -0,0 +1,62 @@ +# Advanced YAML syntax + +## Anchors & aliases + +You can use [YAML anchors & aliases](https://yaml.org/spec/1.2.2/#3222-anchors-and-aliases) as variables in your pipeline config. + +To convert this: +```yml +pipeline: + test: + image: golang:1.18 + command: go test ./... + build: + image: golang:1.18 + command: build +``` + +Just add a new section called **variables** like this: + +```diff ++variables: ++ - &golang_image 'golang:1.18' + + pipeline: + test: +- image: golang:1.18 ++ image: *golang_image + command: go test ./... + build: +- image: golang:1.18 ++ image: *golang_image + command: build +``` + +## Example of YAML override and extension + +```yml +variables: + &some-plugin-settings + settings: + target: dist + recursive: false + try: true + +pipelines: + develop: + name: Build and test + image: some-plugin + settings: *some-plugin-settings + when: + branch: develop + + main + name: Build and test + image: some-plugin + settings: + <<: *some-plugin-settings + try: false # replacing original value from `some-plugin-settings` + ongoing: false # adding a new value to `some-plugin-settings` + when: + branch: main +``` diff --git a/pipeline/schema/schema.json b/pipeline/schema/schema.json index 78f421358..5c4393fea 100644 --- a/pipeline/schema/schema.json +++ b/pipeline/schema/schema.json @@ -11,6 +11,9 @@ "type": "string", "format": "uri" }, + "variables": { + "description": "Use yaml aliases to define variables. Read more: https://woodpecker-ci.org/docs/usage/advanced-yaml-syntax" + }, "clone": { "$ref": "#/definitions/clone" }, "branches": { "$ref": "#/definitions/branches" }, "pipeline": { "$ref": "#/definitions/pipeline" },