From 929f1a16c4b33e8981a7cbac433c880f07a9cfd5 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Mon, 4 Nov 2024 22:32:06 +0100 Subject: [PATCH 1/3] Add changelog for v2.7.2 (#4292) --- CHANGELOG.md | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c0053be19..1acf8854a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,46 @@ # Changelog +## [2.7.2](https://github.com/woodpecker-ci/woodpecker/releases/tag/v2.7.2) - 2024-11-03 + +### Important + +To secure your instance, set `WOODPECKER_PLUGINS_PRIVILEGED` to only allow specific versions of the `woodpeckerci/plugin-docker-buildx` plugin, use version 5.0.0 or above. This prevents older, potentially unstable versions from being privileged. + +For example, to allow only version 5.0.0, use: + +```bash +WOODPECKER_PLUGINS_PRIVILEGED=woodpeckerci/plugin-docker-buildx:5.0.0 +``` + +To allow multiple versions, you can separate them with commas: + +```bash +WOODPECKER_PLUGINS_PRIVILEGED=woodpeckerci/plugin-docker-buildx:5.0.0,woodpeckerci/plugin-docker-buildx:5.1.0 +``` + +This setup ensures only specified, stable plugin versions are given privileged access. + +Read more about it in [#4213](https://github.com/woodpecker-ci/woodpecker/pull/4213) + +### ❤️ Thanks to all contributors! ❤️ + +@6543, @anbraten, @j04n-f, @pat-s, @qwerty287 + +### 🔒 Security + +- Chore(deps): update dependency vite to v5.4.6 [security] ([#4163](https://github.com/woodpecker-ci/woodpecker/pull/4163)) [[#4187](https://github.com/woodpecker-ci/woodpecker/pull/4187)] + +### 🐛 Bug Fixes + +- Don't parse forge config files multiple times if no error occured ([#4272](https://github.com/woodpecker-ci/woodpecker/pull/4272)) [[#4273](https://github.com/woodpecker-ci/woodpecker/pull/4273)] +- Fix repo/owner parsing for gitlab ([#4255](https://github.com/woodpecker-ci/woodpecker/pull/4255)) [[#4261](https://github.com/woodpecker-ci/woodpecker/pull/4261)] +- Run queue.process() in background [[#4115](https://github.com/woodpecker-ci/woodpecker/pull/4115)] +- Only update agent.LastWork if not done recently ([#4031](https://github.com/woodpecker-ci/woodpecker/pull/4031)) [[#4100](https://github.com/woodpecker-ci/woodpecker/pull/4100)] + +### Misc + +- Backport JS dependency updates [[#4189](https://github.com/woodpecker-ci/woodpecker/pull/4189)] + ## [2.7.1](https://github.com/woodpecker-ci/woodpecker/releases/tag/v2.7.1) - 2024-09-07 ### ❤️ Thanks to all contributors! ❤️ @@ -58,6 +99,7 @@ - Remove `unplugin-icons` [[#3809](https://github.com/woodpecker-ci/woodpecker/pull/3809)] - Release windows binaries as zip file [[#3906](https://github.com/woodpecker-ci/woodpecker/pull/3906)] - Convert to openapi 3.0 [[#3897](https://github.com/woodpecker-ci/woodpecker/pull/3897)] +- Enhance pipeline list [[#3898](https://github.com/woodpecker-ci/woodpecker/pull/3898)] - Add user registries UI [[#3888](https://github.com/woodpecker-ci/woodpecker/pull/3888)] - Sort users by login [[#3891](https://github.com/woodpecker-ci/woodpecker/pull/3891)] - Exclude dummy backend in production [[#3877](https://github.com/woodpecker-ci/woodpecker/pull/3877)] From 2fd0c3803274d0ee43f472cf2bbf24198863ad4d Mon Sep 17 00:00:00 2001 From: Anbraten <6918444+anbraten@users.noreply.github.com> Date: Tue, 5 Nov 2024 03:32:45 +0100 Subject: [PATCH 2/3] Improve migration notes (#4291) --- .cspell.json | 1 - docs/blog/2023-11-09-release-v2.0.0/index.md | 2 +- .../2023-12-12-podman-image-builds/index.md | 2 + .../2023-12-13-debug-pipeline-steps/index.md | 2 + docs/blog/2023-12-15-podman-sigstore/index.md | 2 + docs/blog/2024-05-27-release-v2.5.0/index.md | 2 +- .../05-deployment-methods/40-nixos.md | 2 +- docs/docusaurus.config.ts | 8 +- docs/src/pages/about.md | 18 +++++ .../92-awesome.md => src/pages/awesome.md} | 0 .../pages/migrations.md} | 75 +++++++++++++------ docs/src/pages/versions.md | 8 +- 12 files changed, 89 insertions(+), 33 deletions(-) create mode 100644 docs/src/pages/about.md rename docs/{docs/92-awesome.md => src/pages/awesome.md} (100%) rename docs/{docs/91-migrations.md => src/pages/migrations.md} (84%) diff --git a/.cspell.json b/.cspell.json index 34dc4ce65..f495a2e5e 100644 --- a/.cspell.json +++ b/.cspell.json @@ -245,7 +245,6 @@ "**/testdata/**", "docs/versioned_docs/", "package.json", - "91-migrations.md", // generated "go.sum", "flake.lock", diff --git a/docs/blog/2023-11-09-release-v2.0.0/index.md b/docs/blog/2023-11-09-release-v2.0.0/index.md index 1fc3db91e..1aa7c94b2 100644 --- a/docs/blog/2023-11-09-release-v2.0.0/index.md +++ b/docs/blog/2023-11-09-release-v2.0.0/index.md @@ -53,7 +53,7 @@ Security is pretty important to us and we want to make sure that no one can stea ## Migration notes -There have been a few more breaking changes. [Read more about what you need to do when upgrading!](../docs/migrations#200) +There have been a few more breaking changes. [Read more about what you need to do when upgrading!](/migrations#200) ## New features diff --git a/docs/blog/2023-12-12-podman-image-builds/index.md b/docs/blog/2023-12-12-podman-image-builds/index.md index 3e7fe9008..e0501de4a 100644 --- a/docs/blog/2023-12-12-podman-image-builds/index.md +++ b/docs/blog/2023-12-12-podman-image-builds/index.md @@ -14,6 +14,8 @@ tags: [community, image, podman] I run Woodpecker CI with podman backend instead of docker and just figured out how to build images with buildah. Since I couldn't find this anywhere documented, I thought I might as well just share it here. + + It's actually pretty straight forward. Here's what my repository structure looks like: ```bash diff --git a/docs/blog/2023-12-13-debug-pipeline-steps/index.md b/docs/blog/2023-12-13-debug-pipeline-steps/index.md index 5f5d116c9..7682d50bc 100644 --- a/docs/blog/2023-12-13-debug-pipeline-steps/index.md +++ b/docs/blog/2023-12-13-debug-pipeline-steps/index.md @@ -15,6 +15,8 @@ tags: [community, debug] Sometimes you want to debug a pipeline. Therefore I recently discovered: + + A simple step like should allow you to debug: ```yaml diff --git a/docs/blog/2023-12-15-podman-sigstore/index.md b/docs/blog/2023-12-15-podman-sigstore/index.md index db3d5b964..e2600946e 100644 --- a/docs/blog/2023-12-15-podman-sigstore/index.md +++ b/docs/blog/2023-12-15-podman-sigstore/index.md @@ -14,6 +14,8 @@ tags: [community, image, podman, sigstore, signature] This example shows how to build a container image with podman while verifying the base image and signing the resulting image. + + The image being pulled uses a keyless signature, while the image being built will be signed by a pre-generated private key. ## Prerequisites diff --git a/docs/blog/2024-05-27-release-v2.5.0/index.md b/docs/blog/2024-05-27-release-v2.5.0/index.md index 3f9bd3d6c..3de36b5ee 100644 --- a/docs/blog/2024-05-27-release-v2.5.0/index.md +++ b/docs/blog/2024-05-27-release-v2.5.0/index.md @@ -87,4 +87,4 @@ be removed in the next major release: - Use `WOODPECKER_EXPERT_FORGE_OAUTH_HOST` instead of `WOODPECKER_DEV_GITEA_OAUTH_URL` or `WOODPECKER_DEV_OAUTH_HOST` - Deprecated `WOODPECKER_WEBHOOK_HOST` in favor of `WOODPECKER_EXPERT_WEBHOOK_HOST` -For a full list of deprecations that will be dropped in the `next` major release `3.0.0` (no eta yet), please check the [migrations](/docs/migrations#next) section. +For a full list of deprecations that will be dropped in the `next` major release `3.0.0` (no eta yet), please check the [migrations](/migrations#next) section. diff --git a/docs/docs/30-administration/05-deployment-methods/40-nixos.md b/docs/docs/30-administration/05-deployment-methods/40-nixos.md index b599b7021..063016985 100644 --- a/docs/docs/30-administration/05-deployment-methods/40-nixos.md +++ b/docs/docs/30-administration/05-deployment-methods/40-nixos.md @@ -87,4 +87,4 @@ All configuration options can be found via [NixOS Search](https://search.nixos.o ## Tips and tricks -There are some resources on how to utilize Woodpecker more effectively with NixOS on the [Awesome Woodpecker](../../92-awesome.md) page, like using the runners nix-store in the pipeline. +There are some resources on how to utilize Woodpecker more effectively with NixOS on the [Awesome Woodpecker](/awesome) page, like using the runners nix-store in the pipeline. diff --git a/docs/docusaurus.config.ts b/docs/docusaurus.config.ts index e4cfe7fd6..8ace2d984 100644 --- a/docs/docusaurus.config.ts +++ b/docs/docusaurus.config.ts @@ -50,13 +50,13 @@ const config: Config = { position: 'left', items: [ { - to: '/docs/next/migrations', // Always point to newest migration guide - activeBaseRegex: 'docs/(next/)?migrations', + to: '/migrations', // Always point to newest migration guide + activeBaseRegex: 'migrations', label: 'Migrations', }, { - to: '/docs/next/awesome', // Always point to newest awesome list - activeBaseRegex: 'docs/(next/)?awesome', + to: '/awesome', // Always point to newest awesome list + activeBaseRegex: 'awesome', label: 'Awesome', }, { diff --git a/docs/src/pages/about.md b/docs/src/pages/about.md new file mode 100644 index 000000000..bec3304a1 --- /dev/null +++ b/docs/src/pages/about.md @@ -0,0 +1,18 @@ +# About + +Woodpecker has been originally forked from Drone 0.8 as the Drone CI license was changed after the 0.8 release from Apache 2.0 to a proprietary license. Woodpecker is based on this latest freely available version. + +## History + +Woodpecker was originally forked by [@laszlocph](https://github.com/laszlocph) in 2019. + +A few important time points: + +- [`2fbaa56`](https://github.com/woodpecker-ci/woodpecker/commit/2fbaa56eee0f4be7a3ca4be03dbd00c1bf5d1274) is the first commit of the fork, made on Apr 3, 2019. +- The first release [v0.8.91](https://github.com/woodpecker-ci/woodpecker/releases/tag/v0.8.91) was published on Apr 6, 2019. +- On Aug 27, 2019, the project was renamed to "Woodpecker" ([`630c383`](https://github.com/woodpecker-ci/woodpecker/commit/630c383181b10c4ec375e500c812c4b76b3c52b8)). +- The first release under the name "Woodpecker" was published on Sep 9, 2019 ([v0.8.104](https://github.com/woodpecker-ci/woodpecker/releases/tag/v0.8.104)). + +## Differences to Drone + +Woodpecker is a community-focused software that still stay free and open source forever, while Drone is managed by [Harness](https://harness.io/) and published under [Polyform Small Business](https://polyformproject.org/licenses/small-business/1.0.0/) license. diff --git a/docs/docs/92-awesome.md b/docs/src/pages/awesome.md similarity index 100% rename from docs/docs/92-awesome.md rename to docs/src/pages/awesome.md diff --git a/docs/docs/91-migrations.md b/docs/src/pages/migrations.md similarity index 84% rename from docs/docs/91-migrations.md rename to docs/src/pages/migrations.md index f1d9f4b6e..2eec56a80 100644 --- a/docs/docs/91-migrations.md +++ b/docs/src/pages/migrations.md @@ -1,50 +1,79 @@ # Migrations -Some versions need some changes to the server configuration or the pipeline configuration files. +Some versions need some changes to the server configuration or the pipeline configuration files. If you are an user check the `User migrations` section of an version. As an admin of a Woodpecker server or agent check the `Admin migrations` section. ## `next` -- Deprecate `WOODPECKER_FILTER_LABELS` use `WOODPECKER_AGENT_LABELS` +:::info +This will be the next version of Woodpecker. +::: + +## User migrations + - Removed built-in environment variables: - `CI_COMMIT_URL` use `CI_PIPELINE_FORGE_URL` - `CI_STEP_FINISHED` as empty during execution - `CI_PIPELINE_FINISHED` as empty during execution - `CI_PIPELINE_STATUS` was always `success` - `CI_STEP_STATUS` was always `success` -- Set `/woodpecker` as defautl workdir for the **woodpecker-cli** container -- Move docker resource limit settings from server into agent configuration -- Rename server environment variable `WOODPECKER_ESCALATE` to `WOODPECKER_PLUGINS_PRIVILEGED` -- All default privileged plugins (like `woodpeckerci/plugin-docker-buildx`) were removed. Please carefully [re-add those plugins](./30-administration/10-server-config.md#woodpecker_plugins_privileged) you trust and rely on. -- `WOODPECKER_DEFAULT_CLONE_IMAGE` got depricated use `WOODPECKER_DEFAULT_CLONE_PLUGIN` -- Check trusted-clone- and privileged-plugins by image name and tag (if tag is set) +- Set `/woodpecker` as default workdir for the **woodpecker-cli** container - Secret filters for plugins now check against tag if specified -- Removed `WOODPECKER_DEV_OAUTH_HOST` and `WOODPECKER_DEV_GITEA_OAUTH_URL` use `WOODPECKER_EXPERT_FORGE_OAUTH_HOST` - Compatibility mode of deprecated `pipeline:`, `platform:` and `branches:` pipeline config options are now removed and pipeline will now fail if still in use. -- Removed `steps.[name].group` in favor of `steps.[name].depends_on` (see [workflow syntax](./20-usage/20-workflow-syntax.md#depends_on) to learn how to set dependencies) -- Removed `WOODPECKER_ROOT_PATH` and `WOODPECKER_ROOT_URL` config variables. Use `WOODPECKER_HOST` with a path instead +- Removed `steps.[name].group` in favor of `steps.[name].depends_on` (see [workflow syntax](/docs/usage/workflow-syntax#depends_on) to learn how to set dependencies) - Pipelines without a config file will now be skipped instead of failing -- Removed implicitly defined `regcred` image pull secret name. Set it explicitly via `WOODPECKER_BACKEND_K8S_PULL_SECRET_NAMES` - Removed `includes` and `excludes` support from **event** filter -- Removed uppercasing all secret env vars, instead, the value of the `secrets` property is used. [Read more](./20-usage/40-secrets.md#usage) +- Removed upper-casing all secret env vars, instead, the value of the `secrets` property is used. [Read more](/docs/usage/secrets#usage) - Removed alternative names for secrets, use `environment` with `from_secret` -- Removed slice definition for env vars - Removed `environment` filter, use `when.evaluate` - Removed `WOODPECKER_WEBHOOK_HOST` in favor of `WOODPECKER_EXPERT_WEBHOOK_HOST` -- Migrated to rfc9421 for webhook signatures - Renamed `start_time`, `end_time`, `created_at`, `started_at`, `finished_at` and `reviewed_at` JSON fields to `started`, `finished`, `created`, `started`, `finished`, `reviewed` - JSON field `trusted` on repo model was changed from boolean to object - Update all webhooks by pressing the "Repair all" button in the admin settings as the webhook token claims have changed - Crons now use standard Linux syntax without seconds -- Replaced `configs` object by `netrc` in external configuration APIs - Removed old API routes: `registry/` -> `registries`, `/authorize/token` - Replaced `registry` command with `repo registry` in cli -- Disallow upgrades from 1.x, upgrade to 2.x first - Deprecated `secrets`, use `environment` with `from_secret` +## Admin migrations + +- Deprecate `WOODPECKER_FILTER_LABELS` use `WOODPECKER_AGENT_LABELS` +- Move docker resource limit settings from server into agent configuration +- Rename server environment variable `WOODPECKER_ESCALATE` to `WOODPECKER_PLUGINS_PRIVILEGED` +- All default privileged plugins (like `woodpeckerci/plugin-docker-buildx`) were removed. Please carefully [re-add those plugins](/docs/next/administration/server-config#woodpecker_plugins_privileged) you trust and rely on. +- `WOODPECKER_DEFAULT_CLONE_IMAGE` got deprecated use `WOODPECKER_DEFAULT_CLONE_PLUGIN` +- Check trusted-clone- and privileged-plugins by image name and tag (if tag is set) +- Removed `WOODPECKER_DEV_OAUTH_HOST` and `WOODPECKER_DEV_GITEA_OAUTH_URL` use `WOODPECKER_EXPERT_FORGE_OAUTH_HOST` +- Removed `WOODPECKER_ROOT_PATH` and `WOODPECKER_ROOT_URL` config variables. Use `WOODPECKER_HOST` with a path instead +- Removed implicitly defined `regcred` image pull secret name. Set it explicitly via `WOODPECKER_BACKEND_K8S_PULL_SECRET_NAMES` +- Removed slice definition for env vars +- Migrated to rfc9421 for webhook signatures +- Replaced `configs` object by `netrc` in external configuration APIs +- Disallow upgrades from 1.x, upgrade to 2.x first + +## 2.7.2 + +To secure your instance, set `WOODPECKER_PLUGINS_PRIVILEGED` to only allow specific versions of the `woodpeckerci/plugin-docker-buildx` plugin, use version 5.0.0 or above. This prevents older, potentially unstable versions from being privileged. + +For example, to allow only version 5.0.0, use: + +```bash +WOODPECKER_PLUGINS_PRIVILEGED=woodpeckerci/plugin-docker-buildx:5.0.0 +``` + +To allow multiple versions, you can separate them with commas: + +```bash +WOODPECKER_PLUGINS_PRIVILEGED=woodpeckerci/plugin-docker-buildx:5.0.0,woodpeckerci/plugin-docker-buildx:5.1.0 +``` + +This setup ensures only specified, stable plugin versions are given privileged access. + +Read more about it in [#4213](https://github.com/woodpecker-ci/woodpecker/pull/4213) + ## 2.0.0 - Dropped deprecated `CI_BUILD_*`, `CI_PREV_BUILD_*`, `CI_JOB_*`, `*_LINK`, `CI_SYSTEM_ARCH`, `CI_REPO_REMOTE` built-in environment variables -- Deprecated `platform:` filter in favor of `labels:`, [read more](./20-usage/20-workflow-syntax.md#filter-by-platform) +- Deprecated `platform:` filter in favor of `labels:`, [read more](/docs/usage/workflow-syntax#filter-by-platform) - Secrets `event` property was renamed to `events` and `image` to `images` as both are lists. The new property `events` / `images` has to be used in the api. The old properties `event` and `image` were removed. - The secrets `plugin_only` option was removed. Secrets with images are now always only available for plugins using listed by the `images` property. Existing secrets with a list of `images` will now only be available to the listed images if they are used as a plugin. - Removed `build` alias for `pipeline` command in CLI @@ -56,8 +85,8 @@ Some versions need some changes to the server configuration or the pipeline conf ## 1.0.0 -- The signature used to verify extension calls (like those used for the [config-extension](./30-administration/40-advanced/100-external-configuration-api.md)) done by the Woodpecker server switched from using a shared-secret HMac to an ed25519 key-pair. Read more about it at the [config-extensions](./30-administration/40-advanced/100-external-configuration-api.md) documentation. -- Refactored support for old agent filter labels and expressions. Learn how to use the new [filter](./20-usage/20-workflow-syntax.md#labels) +- The signature used to verify extension calls (like those used for the [config-extension](/docs/administration/advanced/external-configuration-api)) done by the Woodpecker server switched from using a shared-secret HMac to an ed25519 key-pair. Read more about it at the [config-extensions](/docs/administration/advanced/external-configuration-api) documentation. +- Refactored support for old agent filter labels and expressions. Learn how to use the new [filter](/docs/usage/workflow-syntax#labels) - Renamed step environment variable `CI_SYSTEM_ARCH` to `CI_SYSTEM_PLATFORM`. Same applies for the cli exec variable. - Renamed environment variables `CI_BUILD_*` and `CI_PREV_BUILD_*` to `CI_PIPELINE_*` and `CI_PREV_PIPELINE_*`, old ones are still available but deprecated - Renamed environment variables `CI_JOB_*` to `CI_STEP_*`, old ones are still available but deprecated @@ -66,7 +95,7 @@ Some versions need some changes to the server configuration or the pipeline conf - Renamed API endpoints for pipelines (`//builds/` -> `//pipelines/`), old ones are still available but deprecated - Updated Prometheus gauge `build_*` to `pipeline_*` - Updated Prometheus gauge `*_job_*` to `*_step_*` -- Renamed config env `WOODPECKER_MAX_PROCS` to `WOODPECKER_MAX_WORKFLOWS` (still available as fallback) +- Renamed config env `WOODPECKER_MAX_PROCS` to `WOODPECKER_MAX_WORKFLOWS` (still available as fallback) - The pipelines are now also read from `.yaml` files, the new default order is `.woodpecker/*.yml` and `.woodpecker/*.yaml` (without any prioritization) -> `.woodpecker.yml` -> `.woodpecker.yaml` - Dropped support for [Coding](https://coding.net/), [Gogs](https://gogs.io) and Bitbucket Server (Stash). - `/api/queue/resume` & `/api/queue/pause` endpoint methods were changed from `GET` to `POST` @@ -95,7 +124,7 @@ Some versions need some changes to the server configuration or the pipeline conf Only projects created after updating will have an empty value by default. Existing projects will stick to the current pipeline path which is `.drone.yml` in most cases. - Read more about it at the [Project Settings](./20-usage/75-project-settings.md#pipeline-path) + Read more about it at the [Project Settings](/docs/usage/project-settings#pipeline-path) - From version `0.15.0` ongoing there will be three types of docker images: `latest`, `next` and `x.x.x` with an alpine variant for each type like `latest-alpine`. If you used `latest` before to try pre-release features you should switch to `next` after this release. @@ -130,7 +159,7 @@ Some versions need some changes to the server configuration or the pipeline conf - CI_SOURCE_BRANCH => use CI_COMMIT_SOURCE_BRANCH - CI_TARGET_BRANCH => use CI_COMMIT_TARGET_BRANCH - For all available variables and their descriptions have a look at [built-in-environment-variables](./20-usage/50-environment.md#built-in-environment-variables). + For all available variables and their descriptions have a look at [built-in-environment-variables](/docs/usage/environment#built-in-environment-variables). - Prometheus metrics have been changed from `drone_*` to `woodpecker_*` diff --git a/docs/src/pages/versions.md b/docs/src/pages/versions.md index 4efe1bcf3..c74291edc 100644 --- a/docs/src/pages/versions.md +++ b/docs/src/pages/versions.md @@ -19,7 +19,7 @@ the actual release will be about a week later. ### Deprecations & migrations -All deprecations and migrations for Woodpecker users and instance admins are documented in the [migration guide](/docs/next/migrations). +All deprecations and migrations for Woodpecker users and instance admins are documented in the [migration guide](/migrations). ## Next version (current state of the `main` branch) @@ -33,7 +33,11 @@ Here you can find documentation for previous versions of Woodpecker. | | | | | ------- | ---------- | ------------------------------------------------------------------------------------- | -| 2.6.0 | 2024-07-18 | [Documentation](https://github.com/woodpecker-ci/woodpecker/tree/v2.6.0/docs/docs/) | +| 2.7.2 | 2024-11-03 | [Documentation](https://github.com/woodpecker-ci/woodpecker/tree/v2.7.2/docs/docs/) | +| 2.7.1 | 2024-09-07 | [Documentation](https://github.com/woodpecker-ci/woodpecker/tree/v2.7.1/docs/docs/) | +| 2.7.0 | 2024-07-18 | [Documentation](https://github.com/woodpecker-ci/woodpecker/tree/v2.7.0/docs/docs/) | +| 2.6.1 | 2024-07-19 | [Documentation](https://github.com/woodpecker-ci/woodpecker/tree/v2.6.1/docs/docs/) | +| 2.6.0 | 2024-06-13 | [Documentation](https://github.com/woodpecker-ci/woodpecker/tree/v2.6.0/docs/docs/) | | 2.5.0 | 2024-06-01 | [Documentation](https://github.com/woodpecker-ci/woodpecker/tree/v2.5.0/docs/docs/) | | 2.4.1 | 2024-03-20 | [Documentation](https://github.com/woodpecker-ci/woodpecker/tree/v2.4.1/docs/docs/) | | 2.4.0 | 2024-03-19 | [Documentation](https://github.com/woodpecker-ci/woodpecker/tree/v2.4.0/docs/docs/) | From 3ab579c03f30a6ad9babcf6a32ad5f0b594f5850 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Tue, 5 Nov 2024 04:03:40 +0100 Subject: [PATCH 3/3] Move Queue creation behind new func that evaluates queue type (#4252) --- cmd/server/setup.go | 18 +++++++++++------- server/queue/fifo.go | 4 ++-- server/queue/fifo_test.go | 28 ++++++++++++++-------------- server/queue/queue.go | 32 ++++++++++++++++++++++++++++++++ 4 files changed, 59 insertions(+), 23 deletions(-) diff --git a/cmd/server/setup.go b/cmd/server/setup.go index b46f1bd5c..c408e4023 100644 --- a/cmd/server/setup.go +++ b/cmd/server/setup.go @@ -103,8 +103,11 @@ func checkSqliteFileExist(path string) error { return err } -func setupQueue(ctx context.Context, s store.Store) queue.Queue { - return queue.WithTaskStore(ctx, queue.New(ctx), s) +func setupQueue(ctx context.Context, s store.Store) (queue.Queue, error) { + return queue.New(ctx, queue.Config{ + Backend: queue.TypeMemory, + Store: s, + }) } func setupMembershipService(_ context.Context, _store store.Store) cache.MembershipService { @@ -143,18 +146,19 @@ func setupJWTSecret(_store store.Store) (string, error) { return jwtSecret, nil } -func setupEvilGlobals(ctx context.Context, c *cli.Command, s store.Store) error { +func setupEvilGlobals(ctx context.Context, c *cli.Command, s store.Store) (err error) { // services - server.Config.Services.Queue = setupQueue(ctx, s) server.Config.Services.Logs = logging.New() server.Config.Services.Pubsub = pubsub.New() server.Config.Services.Membership = setupMembershipService(ctx, s) - serviceManager, err := services.NewManager(c, s, setup.Forge) + server.Config.Services.Queue, err = setupQueue(ctx, s) + if err != nil { + return fmt.Errorf("could not setup queue: %w", err) + } + server.Config.Services.Manager, err = services.NewManager(c, s, setup.Forge) if err != nil { return fmt.Errorf("could not setup service manager: %w", err) } - server.Config.Services.Manager = serviceManager - server.Config.Services.LogStore, err = setupLogStore(c, s) if err != nil { return fmt.Errorf("could not setup log store: %w", err) diff --git a/server/queue/fifo.go b/server/queue/fifo.go index f924055ad..d9acc66a4 100644 --- a/server/queue/fifo.go +++ b/server/queue/fifo.go @@ -59,8 +59,8 @@ const processTimeInterval = 100 * time.Millisecond var ErrWorkerKicked = fmt.Errorf("worker was kicked") -// New returns a new fifo queue. -func New(ctx context.Context) Queue { +// NewMemoryQueue returns a new fifo queue. +func NewMemoryQueue(ctx context.Context) Queue { q := &fifo{ ctx: ctx, workers: map[*worker]struct{}{}, diff --git a/server/queue/fifo_test.go b/server/queue/fifo_test.go index 01a790607..6c6d4252d 100644 --- a/server/queue/fifo_test.go +++ b/server/queue/fifo_test.go @@ -32,7 +32,7 @@ func TestFifo(t *testing.T) { want := &model.Task{ID: "1"} ctx := context.Background() - q := New(ctx) + q := NewMemoryQueue(ctx) assert.NoError(t, q.Push(ctx, want)) info := q.Info(ctx) assert.Len(t, info.Pending, 1, "expect task in pending queue") @@ -55,7 +55,7 @@ func TestFifoExpire(t *testing.T) { want := &model.Task{ID: "1"} ctx, cancel := context.WithCancelCause(context.Background()) - q, _ := New(ctx).(*fifo) + q, _ := NewMemoryQueue(ctx).(*fifo) q.extension = 0 assert.NoError(t, q.Push(ctx, want)) info := q.Info(ctx) @@ -78,7 +78,7 @@ func TestFifoWait(t *testing.T) { want := &model.Task{ID: "1"} ctx := context.Background() - q, _ := New(ctx).(*fifo) + q, _ := NewMemoryQueue(ctx).(*fifo) assert.NoError(t, q.Push(ctx, want)) got, err := q.Poll(ctx, 1, filterFnTrue) @@ -101,7 +101,7 @@ func TestFifoEvict(t *testing.T) { t1 := &model.Task{ID: "1"} ctx := context.Background() - q := New(ctx) + q := NewMemoryQueue(ctx) assert.NoError(t, q.Push(ctx, t1)) info := q.Info(ctx) assert.Len(t, info.Pending, 1, "expect task in pending queue") @@ -125,7 +125,7 @@ func TestFifoDependencies(t *testing.T) { DepStatus: make(map[string]model.StatusValue), } - q, _ := New(ctx).(*fifo) + q, _ := NewMemoryQueue(ctx).(*fifo) assert.NoError(t, q.PushAtOnce(ctx, []*model.Task{task2, task1})) got, err := q.Poll(ctx, 1, filterFnTrue) @@ -158,7 +158,7 @@ func TestFifoErrors(t *testing.T) { RunOn: []string{"success", "failure"}, } - q, _ := New(ctx).(*fifo) + q, _ := NewMemoryQueue(ctx).(*fifo) assert.NoError(t, q.PushAtOnce(ctx, []*model.Task{task2, task3, task1})) got, err := q.Poll(ctx, 1, filterFnTrue) @@ -194,7 +194,7 @@ func TestFifoErrors2(t *testing.T) { DepStatus: make(map[string]model.StatusValue), } - q, _ := New(ctx).(*fifo) + q, _ := NewMemoryQueue(ctx).(*fifo) assert.NoError(t, q.PushAtOnce(ctx, []*model.Task{task2, task3, task1})) for i := 0; i < 2; i++ { @@ -234,7 +234,7 @@ func TestFifoErrorsMultiThread(t *testing.T) { DepStatus: make(map[string]model.StatusValue), } - q, _ := New(ctx).(*fifo) + q, _ := NewMemoryQueue(ctx).(*fifo) assert.NoError(t, q.PushAtOnce(ctx, []*model.Task{task2, task3, task1})) obtainedWorkCh := make(chan *model.Task) @@ -314,7 +314,7 @@ func TestFifoTransitiveErrors(t *testing.T) { DepStatus: make(map[string]model.StatusValue), } - q, _ := New(ctx).(*fifo) + q, _ := NewMemoryQueue(ctx).(*fifo) assert.NoError(t, q.PushAtOnce(ctx, []*model.Task{task2, task3, task1})) got, err := q.Poll(ctx, 1, filterFnTrue) @@ -353,7 +353,7 @@ func TestFifoCancel(t *testing.T) { RunOn: []string{"success", "failure"}, } - q, _ := New(ctx).(*fifo) + q, _ := NewMemoryQueue(ctx).(*fifo) assert.NoError(t, q.PushAtOnce(ctx, []*model.Task{task2, task3, task1})) _, _ = q.Poll(ctx, 1, filterFnTrue) @@ -371,7 +371,7 @@ func TestFifoPause(t *testing.T) { ID: "1", } - q, _ := New(ctx).(*fifo) + q, _ := NewMemoryQueue(ctx).(*fifo) var wg sync.WaitGroup wg.Add(1) go func() { @@ -402,7 +402,7 @@ func TestFifoPauseResume(t *testing.T) { ID: "1", } - q, _ := New(ctx).(*fifo) + q, _ := NewMemoryQueue(ctx).(*fifo) q.Pause() assert.NoError(t, q.Push(ctx, task1)) q.Resume() @@ -429,7 +429,7 @@ func TestWaitingVsPending(t *testing.T) { RunOn: []string{"success", "failure"}, } - q, _ := New(ctx).(*fifo) + q, _ := NewMemoryQueue(ctx).(*fifo) assert.NoError(t, q.PushAtOnce(ctx, []*model.Task{task2, task3, task1})) got, _ := q.Poll(ctx, 1, filterFnTrue) @@ -519,7 +519,7 @@ func TestShouldRun(t *testing.T) { func TestFifoWithScoring(t *testing.T) { ctx := context.Background() - q := New(ctx) + q := NewMemoryQueue(ctx) // Create tasks with different labels tasks := []*model.Task{ diff --git a/server/queue/queue.go b/server/queue/queue.go index 682d0e0ac..3301f7e95 100644 --- a/server/queue/queue.go +++ b/server/queue/queue.go @@ -17,9 +17,11 @@ package queue import ( "context" "errors" + "fmt" "strings" "go.woodpecker-ci.org/woodpecker/v2/server/model" + "go.woodpecker-ci.org/woodpecker/v2/server/store" ) var ( @@ -115,3 +117,33 @@ type Queue interface { // KickAgentWorkers kicks all workers for a given agent. KickAgentWorkers(agentID int64) } + +// Config holds the configuration for the queue. +type Config struct { + Backend Type + Store store.Store +} + +// Queue type. +type Type string + +const ( + TypeMemory Type = "memory" +) + +// New creates a new queue based on the provided configuration. +func New(ctx context.Context, config Config) (Queue, error) { + var q Queue + + switch config.Backend { + case TypeMemory: + q = NewMemoryQueue(ctx) + if config.Store != nil { + q = WithTaskStore(ctx, q, config.Store) + } + default: + return nil, fmt.Errorf("unsupported queue backend: %s", config.Backend) + } + + return q, nil +}