From d7495357d53082baa1c72c17cb62eca6476b5b53 Mon Sep 17 00:00:00 2001 From: scottshotgg Date: Sun, 10 Aug 2025 03:12:42 -0500 Subject: [PATCH] Add Agent-level Tolerations setting (#5266) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .../11-backends/20-kubernetes.md | 18 ++ pipeline/backend/kubernetes/flags.go | 12 ++ pipeline/backend/kubernetes/kubernetes.go | 10 ++ pipeline/backend/kubernetes/pod.go | 8 + pipeline/backend/kubernetes/pod_test.go | 155 ++++++++++++++++++ .../frontend/yaml/linter/schema/schema.json | 6 + 6 files changed, 209 insertions(+) diff --git a/docs/docs/30-administration/10-configuration/11-backends/20-kubernetes.md b/docs/docs/30-administration/10-configuration/11-backends/20-kubernetes.md index cc7d3f430..7e904effe 100644 --- a/docs/docs/30-administration/10-configuration/11-backends/20-kubernetes.md +++ b/docs/docs/30-administration/10-configuration/11-backends/20-kubernetes.md @@ -380,6 +380,24 @@ Determines if Pod annotations can be defined from a step's backend options. --- +### BACKEND_K8S_POD_TOLERATIONS + +- Name: `WOODPECKER_BACKEND_K8S_POD_TOLERATIONS` +- Default: none + +Additional tolerations to apply to worker Pods. Must be a YAML object, e.g. `[{"effect":"NoSchedule","key":"jobs","operator":"Exists"}]`. + +--- + +### BACKEND_K8S_POD_TOLERATIONS_ALLOW_FROM_STEP + +- Name: `WOODPECKER_BACKEND_K8S_POD_TOLERATIONS_ALLOW_FROM_STEP` +- Default: `true` + +Determines if Pod tolerations can be defined from a step's backend options. + +--- + ### BACKEND_K8S_POD_NODE_SELECTOR - Name: `WOODPECKER_BACKEND_K8S_POD_NODE_SELECTOR` diff --git a/pipeline/backend/kubernetes/flags.go b/pipeline/backend/kubernetes/flags.go index f71f8c368..f91d77161 100644 --- a/pipeline/backend/kubernetes/flags.go +++ b/pipeline/backend/kubernetes/flags.go @@ -73,12 +73,24 @@ var Flags = []cli.Flag{ Usage: "backend k8s Agent-wide worker pod node selector", Value: "", }, + &cli.StringFlag{ + Sources: cli.EnvVars("WOODPECKER_BACKEND_K8S_POD_TOLERATIONS"), + Name: "backend-k8s-pod-tolerations", + Usage: "backend k8s Agent-wide worker pod tolerations", + Value: "", + }, &cli.BoolFlag{ Sources: cli.EnvVars("WOODPECKER_BACKEND_K8S_POD_ANNOTATIONS_ALLOW_FROM_STEP"), Name: "backend-k8s-pod-annotations-allow-from-step", Usage: "whether to allow using annotations from step's backend options", Value: false, }, + &cli.BoolFlag{ + Sources: cli.EnvVars("WOODPECKER_BACKEND_K8S_POD_TOLERATIONS_ALLOW_FROM_STEP"), + Name: "backend-k8s-pod-tolerations-allow-from-step", + Usage: "whether to allow using tolerations from step's backend options", + Value: true, + }, &cli.BoolFlag{ Sources: cli.EnvVars("WOODPECKER_BACKEND_K8S_SECCTX_NONROOT"), // cspell:words secctx nonroot Name: "backend-k8s-secctx-nonroot", diff --git a/pipeline/backend/kubernetes/kubernetes.go b/pipeline/backend/kubernetes/kubernetes.go index 90318175c..85e8804f0 100644 --- a/pipeline/backend/kubernetes/kubernetes.go +++ b/pipeline/backend/kubernetes/kubernetes.go @@ -67,6 +67,8 @@ type config struct { PodAnnotations map[string]string PodAnnotationsAllowFromStep bool PodNodeSelector map[string]string + PodTolerationsAllowFromStep bool + PodTolerations []Toleration ImagePullSecretNames []string SecurityContext SecurityContextConfig NativeSecretsAllowFromStep bool @@ -109,6 +111,7 @@ func configFromCliContext(ctx context.Context) (*config, error) { PodLabelsAllowFromStep: c.Bool("backend-k8s-pod-labels-allow-from-step"), PodAnnotations: make(map[string]string), // just init empty map to prevent nil panic PodAnnotationsAllowFromStep: c.Bool("backend-k8s-pod-annotations-allow-from-step"), + PodTolerationsAllowFromStep: c.Bool("backend-k8s-pod-tolerations-allow-from-step"), PodNodeSelector: make(map[string]string), // just init empty map to prevent nil panic ImagePullSecretNames: c.StringSlice("backend-k8s-pod-image-pull-secret-names"), SecurityContext: SecurityContextConfig{ @@ -136,6 +139,13 @@ func configFromCliContext(ctx context.Context) (*config, error) { return nil, err } } + if podTolerations := c.String("backend-k8s-pod-tolerations"); podTolerations != "" { + if err := yaml.Unmarshal([]byte(podTolerations), &config.PodTolerations); err != nil { + log.Error().Err(err).Msgf("could not unmarshal pod tolerations '%s'", podTolerations) + return nil, err + } + } + return &config, nil } } diff --git a/pipeline/backend/kubernetes/pod.go b/pipeline/backend/kubernetes/pod.go index f591d50d9..912967547 100644 --- a/pipeline/backend/kubernetes/pod.go +++ b/pipeline/backend/kubernetes/pod.go @@ -179,6 +179,14 @@ func podSpec(step *types.Step, config *config, options BackendOptions, nsp nativ Tolerations: tolerations(options.Tolerations), SecurityContext: podSecurityContext(options.SecurityContext, config.SecurityContext, step.Privileged), } + + // If there are tolerations and they are allowed + if config.PodTolerationsAllowFromStep && len(options.Tolerations) != 0 { + spec.Tolerations = tolerations(options.Tolerations) + } else { + spec.Tolerations = tolerations(config.PodTolerations) + } + spec.Volumes, err = pvcVolumes(step.Volumes) if err != nil { return spec, err diff --git a/pipeline/backend/kubernetes/pod_test.go b/pipeline/backend/kubernetes/pod_test.go index 927512517..4b97c1e76 100644 --- a/pipeline/backend/kubernetes/pod_test.go +++ b/pipeline/backend/kubernetes/pod_test.go @@ -387,6 +387,7 @@ func TestFullPod(t *testing.T) { PodLabelsAllowFromStep: true, PodAnnotations: map[string]string{"apps.kubernetes.io/pod-index": "0"}, PodAnnotationsAllowFromStep: true, + PodTolerationsAllowFromStep: true, PodNodeSelector: map[string]string{"topology.kubernetes.io/region": "eu-central-1"}, SecurityContext: SecurityContextConfig{RunAsNonRoot: false}, }, "wp-01he8bebctabr3kgk0qj36d2me-0", "linux/amd64", BackendOptions{ @@ -662,6 +663,160 @@ func TestSecrets(t *testing.T) { ja.Assertf(string(podJSON), expected) } +func TestPodTolerations(t *testing.T) { + const expected = ` + { + "metadata": { + "name": "wp-01he8bebctabr3kgk0qj36d2me-0", + "namespace": "woodpecker", + "creationTimestamp": null, + "labels": { + "step": "toleration-test", + "woodpecker-ci.org/step": "toleration-test" + } + }, + "spec": { + "containers": [ + { + "name": "wp-01he8bebctabr3kgk0qj36d2me-0", + "image": "alpine", + "resources": {} + } + ], + "restartPolicy": "Never", + "tolerations": [ + { + "key": "foo", + "value": "bar", + "effect": "NoSchedule" + }, + { + "key": "baz", + "value": "qux", + "effect": "NoExecute" + } + ] + }, + "status": {} + }` + + globalTolerations := []Toleration{ + {Key: "foo", Value: "bar", Effect: TaintEffectNoSchedule}, + {Key: "baz", Value: "qux", Effect: TaintEffectNoExecute}, + } + + pod, err := mkPod(&types.Step{ + Name: "toleration-test", + Image: "alpine", + UUID: "01he8bebctabr3kgk0qj36d2me-0", + }, &config{ + Namespace: "woodpecker", + PodTolerations: globalTolerations, + PodTolerationsAllowFromStep: false, + }, "wp-01he8bebctabr3kgk0qj36d2me-0", "linux/amd64", BackendOptions{}) + assert.NoError(t, err) + + podJSON, err := json.Marshal(pod) + assert.NoError(t, err) + + ja := jsonassert.New(t) + ja.Assertf(string(podJSON), expected) +} + +func TestPodTolerationsAllowFromStep(t *testing.T) { + const expectedDisallow = ` + { + "metadata": { + "name": "wp-01he8bebctabr3kgk0qj36d2me-0", + "namespace": "woodpecker", + "creationTimestamp": null, + "labels": { + "step": "toleration-test", + "woodpecker-ci.org/step": "toleration-test" + } + }, + "spec": { + "containers": [ + { + "name": "wp-01he8bebctabr3kgk0qj36d2me-0", + "image": "alpine", + "resources": {} + } + ], + "restartPolicy": "Never" + }, + "status": {} + }` + const expectedAllow = ` + { + "metadata": { + "name": "wp-01he8bebctabr3kgk0qj36d2me-0", + "namespace": "woodpecker", + "creationTimestamp": null, + "labels": { + "step": "toleration-test", + "woodpecker-ci.org/step": "toleration-test" + } + }, + "spec": { + "containers": [ + { + "name": "wp-01he8bebctabr3kgk0qj36d2me-0", + "image": "alpine", + "resources": {} + } + ], + "restartPolicy": "Never", + "tolerations": [ + { + "key": "custom", + "value": "value", + "effect": "NoSchedule" + } + ] + }, + "status": {} + }` + + stepTolerations := []Toleration{ + {Key: "custom", Value: "value", Effect: TaintEffectNoSchedule}, + } + + step := &types.Step{ + Name: "toleration-test", + Image: "alpine", + UUID: "01he8bebctabr3kgk0qj36d2me-0", + } + + pod, err := mkPod(step, &config{ + Namespace: "woodpecker", + PodTolerationsAllowFromStep: false, + }, "wp-01he8bebctabr3kgk0qj36d2me-0", "linux/amd64", BackendOptions{ + Tolerations: stepTolerations, + }) + assert.NoError(t, err) + + podJSON, err := json.Marshal(pod) + assert.NoError(t, err) + + ja := jsonassert.New(t) + ja.Assertf(string(podJSON), expectedDisallow) + + pod, err = mkPod(step, &config{ + Namespace: "woodpecker", + PodTolerationsAllowFromStep: true, + }, "wp-01he8bebctabr3kgk0qj36d2me-0", "linux/amd64", BackendOptions{ + Tolerations: stepTolerations, + }) + assert.NoError(t, err) + + podJSON, err = json.Marshal(pod) + assert.NoError(t, err) + + ja = jsonassert.New(t) + ja.Assertf(string(podJSON), expectedAllow) +} + func TestStepSecret(t *testing.T) { const expected = `{ "metadata": { diff --git a/pipeline/frontend/yaml/linter/schema/schema.json b/pipeline/frontend/yaml/linter/schema/schema.json index e1fd411f6..22ea52246 100644 --- a/pipeline/frontend/yaml/linter/schema/schema.json +++ b/pipeline/frontend/yaml/linter/schema/schema.json @@ -646,6 +646,12 @@ "type": ["boolean", "string", "number"] } }, + "tolerations": { + "type": "object", + "additionalProperties": { + "type": ["boolean", "string", "number"] + } + }, "securityContext": { "$ref": "#/definitions/step_backend_kubernetes_security_context" },