Add Agent-level Tolerations setting (#5266)

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

docs/docs/30-administration/10-configuration/11-backends/20-kubernetes.md

@@ -380,6 +380,24 @@ Determines if Pod annotations can be defined from a step's backend options.
 
 ---
 
+### BACKEND_K8S_POD_TOLERATIONS
+
+- Name: `WOODPECKER_BACKEND_K8S_POD_TOLERATIONS`
+- Default: none
+
+Additional tolerations to apply to worker Pods. Must be a YAML object, e.g. `[{"effect":"NoSchedule","key":"jobs","operator":"Exists"}]`.
+
+---
+
+### BACKEND_K8S_POD_TOLERATIONS_ALLOW_FROM_STEP
+
+- Name: `WOODPECKER_BACKEND_K8S_POD_TOLERATIONS_ALLOW_FROM_STEP`
+- Default: `true`
+
+Determines if Pod tolerations can be defined from a step's backend options.
+
+---
+
 ### BACKEND_K8S_POD_NODE_SELECTOR
 
 - Name: `WOODPECKER_BACKEND_K8S_POD_NODE_SELECTOR`

pipeline/backend/kubernetes/flags.go

@@ -73,12 +73,24 @@ var Flags = []cli.Flag{
 		Usage:   "backend k8s Agent-wide worker pod node selector",
 		Value:   "",
 	},
+	&cli.StringFlag{
+		Sources: cli.EnvVars("WOODPECKER_BACKEND_K8S_POD_TOLERATIONS"),
+		Name:    "backend-k8s-pod-tolerations",
+		Usage:   "backend k8s Agent-wide worker pod tolerations",
+		Value:   "",
+	},
 	&cli.BoolFlag{
 		Sources: cli.EnvVars("WOODPECKER_BACKEND_K8S_POD_ANNOTATIONS_ALLOW_FROM_STEP"),
 		Name:    "backend-k8s-pod-annotations-allow-from-step",
 		Usage:   "whether to allow using annotations from step's backend options",
 		Value:   false,
 	},
+	&cli.BoolFlag{
+		Sources: cli.EnvVars("WOODPECKER_BACKEND_K8S_POD_TOLERATIONS_ALLOW_FROM_STEP"),
+		Name:    "backend-k8s-pod-tolerations-allow-from-step",
+		Usage:   "whether to allow using tolerations from step's backend options",
+		Value:   true,
+	},
 	&cli.BoolFlag{
 		Sources: cli.EnvVars("WOODPECKER_BACKEND_K8S_SECCTX_NONROOT"), // cspell:words secctx nonroot
 		Name:    "backend-k8s-secctx-nonroot",

pipeline/backend/kubernetes/kubernetes.go

@@ -67,6 +67,8 @@ type config struct {
 	PodAnnotations              map[string]string
 	PodAnnotationsAllowFromStep bool
 	PodNodeSelector             map[string]string
+	PodTolerationsAllowFromStep bool
+	PodTolerations              []Toleration
 	ImagePullSecretNames        []string
 	SecurityContext             SecurityContextConfig
 	NativeSecretsAllowFromStep  bool
@@ -109,6 +111,7 @@ func configFromCliContext(ctx context.Context) (*config, error) {
 				PodLabelsAllowFromStep:      c.Bool("backend-k8s-pod-labels-allow-from-step"),
 				PodAnnotations:              make(map[string]string), // just init empty map to prevent nil panic
 				PodAnnotationsAllowFromStep: c.Bool("backend-k8s-pod-annotations-allow-from-step"),
+				PodTolerationsAllowFromStep: c.Bool("backend-k8s-pod-tolerations-allow-from-step"),
 				PodNodeSelector:             make(map[string]string), // just init empty map to prevent nil panic
 				ImagePullSecretNames:        c.StringSlice("backend-k8s-pod-image-pull-secret-names"),
 				SecurityContext: SecurityContextConfig{
@@ -136,6 +139,13 @@ func configFromCliContext(ctx context.Context) (*config, error) {
 					return nil, err
 				}
 			}
+			if podTolerations := c.String("backend-k8s-pod-tolerations"); podTolerations != "" {
+				if err := yaml.Unmarshal([]byte(podTolerations), &config.PodTolerations); err != nil {
+					log.Error().Err(err).Msgf("could not unmarshal pod tolerations '%s'", podTolerations)
+					return nil, err
+				}
+			}
+
 			return &config, nil
 		}
 	}

pipeline/backend/kubernetes/pod.go

@@ -179,6 +179,14 @@ func podSpec(step *types.Step, config *config, options BackendOptions, nsp nativ
 		Tolerations:        tolerations(options.Tolerations),
 		SecurityContext:    podSecurityContext(options.SecurityContext, config.SecurityContext, step.Privileged),
 	}
+
+	// If there are tolerations and they are allowed
+	if config.PodTolerationsAllowFromStep && len(options.Tolerations) != 0 {
+		spec.Tolerations = tolerations(options.Tolerations)
+	} else {
+		spec.Tolerations = tolerations(config.PodTolerations)
+	}
+
 	spec.Volumes, err = pvcVolumes(step.Volumes)
 	if err != nil {
 		return spec, err

pipeline/backend/kubernetes/pod_test.go

@@ -387,6 +387,7 @@ func TestFullPod(t *testing.T) {
 		PodLabelsAllowFromStep:      true,
 		PodAnnotations:              map[string]string{"apps.kubernetes.io/pod-index": "0"},
 		PodAnnotationsAllowFromStep: true,
+		PodTolerationsAllowFromStep: true,
 		PodNodeSelector:             map[string]string{"topology.kubernetes.io/region": "eu-central-1"},
 		SecurityContext:             SecurityContextConfig{RunAsNonRoot: false},
 	}, "wp-01he8bebctabr3kgk0qj36d2me-0", "linux/amd64", BackendOptions{
@@ -662,6 +663,160 @@ func TestSecrets(t *testing.T) {
 	ja.Assertf(string(podJSON), expected)
 }
 
+func TestPodTolerations(t *testing.T) {
+	const expected = `
+	{
+		"metadata": {
+			"name": "wp-01he8bebctabr3kgk0qj36d2me-0",
+			"namespace": "woodpecker",
+			"creationTimestamp": null,
+			"labels": {
+				"step": "toleration-test",
+				"woodpecker-ci.org/step": "toleration-test"
+			}
+		},
+		"spec": {
+			"containers": [
+				{
+					"name": "wp-01he8bebctabr3kgk0qj36d2me-0",
+					"image": "alpine",
+					"resources": {}
+				}
+			],
+			"restartPolicy": "Never",
+			"tolerations": [
+				{
+					"key": "foo",
+					"value": "bar",
+					"effect": "NoSchedule"
+				},
+				{
+					"key": "baz",
+					"value": "qux",
+					"effect": "NoExecute"
+				}
+			]
+		},
+		"status": {}
+	}`
+
+	globalTolerations := []Toleration{
+		{Key: "foo", Value: "bar", Effect: TaintEffectNoSchedule},
+		{Key: "baz", Value: "qux", Effect: TaintEffectNoExecute},
+	}
+
+	pod, err := mkPod(&types.Step{
+		Name:  "toleration-test",
+		Image: "alpine",
+		UUID:  "01he8bebctabr3kgk0qj36d2me-0",
+	}, &config{
+		Namespace:                   "woodpecker",
+		PodTolerations:              globalTolerations,
+		PodTolerationsAllowFromStep: false,
+	}, "wp-01he8bebctabr3kgk0qj36d2me-0", "linux/amd64", BackendOptions{})
+	assert.NoError(t, err)
+
+	podJSON, err := json.Marshal(pod)
+	assert.NoError(t, err)
+
+	ja := jsonassert.New(t)
+	ja.Assertf(string(podJSON), expected)
+}
+
+func TestPodTolerationsAllowFromStep(t *testing.T) {
+	const expectedDisallow = `
+	{
+		"metadata": {
+			"name": "wp-01he8bebctabr3kgk0qj36d2me-0",
+			"namespace": "woodpecker",
+			"creationTimestamp": null,
+			"labels": {
+				"step": "toleration-test",
+				"woodpecker-ci.org/step": "toleration-test"
+			}
+		},
+		"spec": {
+			"containers": [
+				{
+					"name": "wp-01he8bebctabr3kgk0qj36d2me-0",
+					"image": "alpine",
+					"resources": {}
+				}
+			],
+			"restartPolicy": "Never"
+		},
+		"status": {}
+	}`
+	const expectedAllow = `
+	{
+		"metadata": {
+			"name": "wp-01he8bebctabr3kgk0qj36d2me-0",
+			"namespace": "woodpecker",
+			"creationTimestamp": null,
+			"labels": {
+				"step": "toleration-test",
+				"woodpecker-ci.org/step": "toleration-test"
+			}
+		},
+		"spec": {
+			"containers": [
+				{
+					"name": "wp-01he8bebctabr3kgk0qj36d2me-0",
+					"image": "alpine",
+					"resources": {}
+				}
+			],
+			"restartPolicy": "Never",
+			"tolerations": [
+				{
+					"key": "custom",
+					"value": "value",
+					"effect": "NoSchedule"
+				}
+			]
+		},
+		"status": {}
+	}`
+
+	stepTolerations := []Toleration{
+		{Key: "custom", Value: "value", Effect: TaintEffectNoSchedule},
+	}
+
+	step := &types.Step{
+		Name:  "toleration-test",
+		Image: "alpine",
+		UUID:  "01he8bebctabr3kgk0qj36d2me-0",
+	}
+
+	pod, err := mkPod(step, &config{
+		Namespace:                   "woodpecker",
+		PodTolerationsAllowFromStep: false,
+	}, "wp-01he8bebctabr3kgk0qj36d2me-0", "linux/amd64", BackendOptions{
+		Tolerations: stepTolerations,
+	})
+	assert.NoError(t, err)
+
+	podJSON, err := json.Marshal(pod)
+	assert.NoError(t, err)
+
+	ja := jsonassert.New(t)
+	ja.Assertf(string(podJSON), expectedDisallow)
+
+	pod, err = mkPod(step, &config{
+		Namespace:                   "woodpecker",
+		PodTolerationsAllowFromStep: true,
+	}, "wp-01he8bebctabr3kgk0qj36d2me-0", "linux/amd64", BackendOptions{
+		Tolerations: stepTolerations,
+	})
+	assert.NoError(t, err)
+
+	podJSON, err = json.Marshal(pod)
+	assert.NoError(t, err)
+
+	ja = jsonassert.New(t)
+	ja.Assertf(string(podJSON), expectedAllow)
+}
+
 func TestStepSecret(t *testing.T) {
 	const expected = `{
 		"metadata": {

pipeline/frontend/yaml/linter/schema/schema.json

@@ -646,6 +646,12 @@
             "type": ["boolean", "string", "number"]
           }
         },
+        "tolerations": {
+          "type": "object",
+          "additionalProperties": {
+            "type": ["boolean", "string", "number"]
+          }
+        },
         "securityContext": {
           "$ref": "#/definitions/step_backend_kubernetes_security_context"
         },