Agent-wide node selector (#3608)

Co-authored-by: qwerty287 <80460567+qwerty287@users.noreply.github.com>

docs/docs/30-administration/22-backends/40-kubernetes.md

@@ -79,7 +79,8 @@ And then overwrite the `nodeSelector` in the `backend_options` section of the st
           kubernetes.io/arch: "${ARCH}"
 ```
 
-You can use [PodNodeSelector](https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers/#podnodeselector) admission controller if you want to set the node selector by per-namespace basis.
+You can use [WOODPECKER_BACKEND_K8S_POD_NODE_SELECTOR](#woodpecker_backend_k8s_pod_node_selector) if you want to set the node selector per Agent
+or [PodNodeSelector](https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers/#podnodeselector) admission controller if you want to set the node selector by per-namespace basis.
 
 ### Tolerations
 
@@ -279,6 +280,12 @@ Additional annotations to apply to worker Pods. Must be a YAML object, e.g. `{"e
 
 Determines if Pod annotations can be defined from a step's backend options.
 
+### `WOODPECKER_BACKEND_K8S_POD_NODE_SELECTOR`
+
+> Default: empty
+
+Additional node selector to apply to worker pods. Must be a YAML object, e.g. `{"topology.kubernetes.io/region":"eu-central-1"}`.
+
 ### `WOODPECKER_BACKEND_K8S_SECCTX_NONROOT`
 
 > Default: `false`

pipeline/backend/kubernetes/flags.go

@@ -61,6 +61,12 @@ var Flags = []cli.Flag{
 		Usage:   "backend k8s additional Agent-wide worker pod annotations",
 		Value:   "",
 	},
+	&cli.StringFlag{
+		EnvVars: []string{"WOODPECKER_BACKEND_K8S_POD_NODE_SELECTOR"},
+		Name:    "backend-k8s-pod-node-selector",
+		Usage:   "backend k8s Agent-wide worker pod node selector",
+		Value:   "",
+	},
 	&cli.BoolFlag{
 		EnvVars: []string{"WOODPECKER_BACKEND_K8S_POD_ANNOTATIONS_ALLOW_FROM_STEP"},
 		Name:    "backend-k8s-pod-annotations-allow-from-step",

pipeline/backend/kubernetes/kubernetes.go

@@ -62,6 +62,7 @@ type config struct {
 	PodLabelsAllowFromStep      bool
 	PodAnnotations              map[string]string
 	PodAnnotationsAllowFromStep bool
+	PodNodeSelector             map[string]string
 	ImagePullSecretNames        []string
 	SecurityContext             SecurityContextConfig
 }
@@ -91,6 +92,7 @@ func configFromCliContext(ctx context.Context) (*config, error) {
 				PodLabelsAllowFromStep:      c.Bool("backend-k8s-pod-labels-allow-from-step"),
 				PodAnnotations:              make(map[string]string), // just init empty map to prevent nil panic
 				PodAnnotationsAllowFromStep: c.Bool("backend-k8s-pod-annotations-allow-from-step"),
+				PodNodeSelector:             make(map[string]string), // just init empty map to prevent nil panic
 				ImagePullSecretNames:        c.StringSlice("backend-k8s-pod-image-pull-secret-names"),
 				SecurityContext: SecurityContextConfig{
 					RunAsNonRoot: c.Bool("backend-k8s-secctx-nonroot"), // cspell:words secctx nonroot
@@ -113,6 +115,12 @@ func configFromCliContext(ctx context.Context) (*config, error) {
 					return nil, err
 				}
 			}
+			if nodeSelector := c.String("backend-k8s-pod-node-selector"); nodeSelector != "" {
+				if err := yaml.Unmarshal([]byte(nodeSelector), &config.PodNodeSelector); err != nil {
+					log.Error().Err(err).Msgf("could not unmarshal pod node selector '%s'", nodeSelector)
+					return nil, err
+				}
+			}
 			return &config, nil
 		}
 	}
@@ -173,6 +181,7 @@ func (e *kube) getConfig() *config {
 	c := *e.config
 	c.PodLabels = maps.Clone(e.config.PodLabels)
 	c.PodAnnotations = maps.Clone(e.config.PodAnnotations)
+	c.PodNodeSelector = maps.Clone(e.config.PodNodeSelector)
 	c.ImagePullSecretNames = slices.Clone(e.config.ImagePullSecretNames)
 	return &c
 }

pipeline/backend/kubernetes/pod.go

@@ -154,7 +154,7 @@ func podSpec(step *types.Step, config *config, options BackendOptions) (v1.PodSp
 		ServiceAccountName: options.ServiceAccountName,
 		ImagePullSecrets:   imagePullSecretsReferences(config.ImagePullSecretNames),
 		HostAliases:        hostAliases(step.ExtraHosts),
-		NodeSelector:       nodeSelector(options.NodeSelector, step.Environment["CI_SYSTEM_PLATFORM"]),
+		NodeSelector:       nodeSelector(options.NodeSelector, config.PodNodeSelector, step.Environment["CI_SYSTEM_PLATFORM"]),
 		Tolerations:        tolerations(options.Tolerations),
 		SecurityContext:    podSecurityContext(options.SecurityContext, config.SecurityContext, step.Privileged),
 	}
@@ -331,7 +331,7 @@ func resourceList(resources map[string]string) (v1.ResourceList, error) {
 	return requestResources, nil
 }
 
-func nodeSelector(backendNodeSelector map[string]string, platform string) map[string]string {
+func nodeSelector(backendNodeSelector, configNodeSelector map[string]string, platform string) map[string]string {
 	nodeSelector := make(map[string]string)
 
 	if platform != "" {
@@ -340,6 +340,11 @@ func nodeSelector(backendNodeSelector map[string]string, platform string) map[st
 		log.Trace().Msgf("using the node selector from the Agent's platform: %v", nodeSelector)
 	}
 
+	if len(configNodeSelector) > 0 {
+		log.Trace().Msgf("appending labels to the node selector from the configuration: %v", configNodeSelector)
+		maps.Copy(nodeSelector, configNodeSelector)
+	}
+
 	if len(backendNodeSelector) > 0 {
 		log.Trace().Msgf("appending labels to the node selector from the backend options: %v", backendNodeSelector)
 		maps.Copy(nodeSelector, backendNodeSelector)

pipeline/backend/kubernetes/pod_test.go

@@ -240,7 +240,8 @@ func TestFullPod(t *testing.T) {
 			],
 			"restartPolicy": "Never",
 			"nodeSelector": {
-				"storage": "ssd"
+				"storage": "ssd",
+				"topology.kubernetes.io/region": "eu-central-1"
 			},
 			"runtimeClassName": "runc",
 			"serviceAccountName": "wp-svc-acc",
@@ -331,6 +332,7 @@ func TestFullPod(t *testing.T) {
 		PodLabelsAllowFromStep:      true,
 		PodAnnotations:              map[string]string{"apps.kubernetes.io/pod-index": "0"},
 		PodAnnotationsAllowFromStep: true,
+		PodNodeSelector:             map[string]string{"topology.kubernetes.io/region": "eu-central-1"},
 		SecurityContext:             SecurityContextConfig{RunAsNonRoot: false},
 	}, "wp-01he8bebctabr3kgk0qj36d2me-0", "linux/amd64", BackendOptions{
 		Labels:             map[string]string{"part-of": "woodpecker-ci"},