From 065eebd3069702dfd6d76f361a30af16d5ce4a85 Mon Sep 17 00:00:00 2001 From: Thomas Anderson <127358482+zc-devs@users.noreply.github.com> Date: Mon, 3 Jun 2024 18:25:28 +0300 Subject: [PATCH] Agent-wide node selector (#3608) Co-authored-by: qwerty287 <80460567+qwerty287@users.noreply.github.com> --- docs/docs/30-administration/22-backends/40-kubernetes.md | 9 ++++++++- pipeline/backend/kubernetes/flags.go | 6 ++++++ pipeline/backend/kubernetes/kubernetes.go | 9 +++++++++ pipeline/backend/kubernetes/pod.go | 9 +++++++-- pipeline/backend/kubernetes/pod_test.go | 4 +++- 5 files changed, 33 insertions(+), 4 deletions(-) diff --git a/docs/docs/30-administration/22-backends/40-kubernetes.md b/docs/docs/30-administration/22-backends/40-kubernetes.md index e9a71e0a5..ca19fe451 100644 --- a/docs/docs/30-administration/22-backends/40-kubernetes.md +++ b/docs/docs/30-administration/22-backends/40-kubernetes.md @@ -79,7 +79,8 @@ And then overwrite the `nodeSelector` in the `backend_options` section of the st kubernetes.io/arch: "${ARCH}" ``` -You can use [PodNodeSelector](https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers/#podnodeselector) admission controller if you want to set the node selector by per-namespace basis. +You can use [WOODPECKER_BACKEND_K8S_POD_NODE_SELECTOR](#woodpecker_backend_k8s_pod_node_selector) if you want to set the node selector per Agent +or [PodNodeSelector](https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers/#podnodeselector) admission controller if you want to set the node selector by per-namespace basis. ### Tolerations @@ -279,6 +280,12 @@ Additional annotations to apply to worker Pods. Must be a YAML object, e.g. `{"e Determines if Pod annotations can be defined from a step's backend options. +### `WOODPECKER_BACKEND_K8S_POD_NODE_SELECTOR` + +> Default: empty + +Additional node selector to apply to worker pods. Must be a YAML object, e.g. `{"topology.kubernetes.io/region":"eu-central-1"}`. + ### `WOODPECKER_BACKEND_K8S_SECCTX_NONROOT` > Default: `false` diff --git a/pipeline/backend/kubernetes/flags.go b/pipeline/backend/kubernetes/flags.go index 10ecd4017..219595f09 100644 --- a/pipeline/backend/kubernetes/flags.go +++ b/pipeline/backend/kubernetes/flags.go @@ -61,6 +61,12 @@ var Flags = []cli.Flag{ Usage: "backend k8s additional Agent-wide worker pod annotations", Value: "", }, + &cli.StringFlag{ + EnvVars: []string{"WOODPECKER_BACKEND_K8S_POD_NODE_SELECTOR"}, + Name: "backend-k8s-pod-node-selector", + Usage: "backend k8s Agent-wide worker pod node selector", + Value: "", + }, &cli.BoolFlag{ EnvVars: []string{"WOODPECKER_BACKEND_K8S_POD_ANNOTATIONS_ALLOW_FROM_STEP"}, Name: "backend-k8s-pod-annotations-allow-from-step", diff --git a/pipeline/backend/kubernetes/kubernetes.go b/pipeline/backend/kubernetes/kubernetes.go index a12f00b71..dc04403ff 100644 --- a/pipeline/backend/kubernetes/kubernetes.go +++ b/pipeline/backend/kubernetes/kubernetes.go @@ -62,6 +62,7 @@ type config struct { PodLabelsAllowFromStep bool PodAnnotations map[string]string PodAnnotationsAllowFromStep bool + PodNodeSelector map[string]string ImagePullSecretNames []string SecurityContext SecurityContextConfig } @@ -91,6 +92,7 @@ func configFromCliContext(ctx context.Context) (*config, error) { PodLabelsAllowFromStep: c.Bool("backend-k8s-pod-labels-allow-from-step"), PodAnnotations: make(map[string]string), // just init empty map to prevent nil panic PodAnnotationsAllowFromStep: c.Bool("backend-k8s-pod-annotations-allow-from-step"), + PodNodeSelector: make(map[string]string), // just init empty map to prevent nil panic ImagePullSecretNames: c.StringSlice("backend-k8s-pod-image-pull-secret-names"), SecurityContext: SecurityContextConfig{ RunAsNonRoot: c.Bool("backend-k8s-secctx-nonroot"), // cspell:words secctx nonroot @@ -113,6 +115,12 @@ func configFromCliContext(ctx context.Context) (*config, error) { return nil, err } } + if nodeSelector := c.String("backend-k8s-pod-node-selector"); nodeSelector != "" { + if err := yaml.Unmarshal([]byte(nodeSelector), &config.PodNodeSelector); err != nil { + log.Error().Err(err).Msgf("could not unmarshal pod node selector '%s'", nodeSelector) + return nil, err + } + } return &config, nil } } @@ -173,6 +181,7 @@ func (e *kube) getConfig() *config { c := *e.config c.PodLabels = maps.Clone(e.config.PodLabels) c.PodAnnotations = maps.Clone(e.config.PodAnnotations) + c.PodNodeSelector = maps.Clone(e.config.PodNodeSelector) c.ImagePullSecretNames = slices.Clone(e.config.ImagePullSecretNames) return &c } diff --git a/pipeline/backend/kubernetes/pod.go b/pipeline/backend/kubernetes/pod.go index 4b3c77503..b240d794a 100644 --- a/pipeline/backend/kubernetes/pod.go +++ b/pipeline/backend/kubernetes/pod.go @@ -154,7 +154,7 @@ func podSpec(step *types.Step, config *config, options BackendOptions) (v1.PodSp ServiceAccountName: options.ServiceAccountName, ImagePullSecrets: imagePullSecretsReferences(config.ImagePullSecretNames), HostAliases: hostAliases(step.ExtraHosts), - NodeSelector: nodeSelector(options.NodeSelector, step.Environment["CI_SYSTEM_PLATFORM"]), + NodeSelector: nodeSelector(options.NodeSelector, config.PodNodeSelector, step.Environment["CI_SYSTEM_PLATFORM"]), Tolerations: tolerations(options.Tolerations), SecurityContext: podSecurityContext(options.SecurityContext, config.SecurityContext, step.Privileged), } @@ -331,7 +331,7 @@ func resourceList(resources map[string]string) (v1.ResourceList, error) { return requestResources, nil } -func nodeSelector(backendNodeSelector map[string]string, platform string) map[string]string { +func nodeSelector(backendNodeSelector, configNodeSelector map[string]string, platform string) map[string]string { nodeSelector := make(map[string]string) if platform != "" { @@ -340,6 +340,11 @@ func nodeSelector(backendNodeSelector map[string]string, platform string) map[st log.Trace().Msgf("using the node selector from the Agent's platform: %v", nodeSelector) } + if len(configNodeSelector) > 0 { + log.Trace().Msgf("appending labels to the node selector from the configuration: %v", configNodeSelector) + maps.Copy(nodeSelector, configNodeSelector) + } + if len(backendNodeSelector) > 0 { log.Trace().Msgf("appending labels to the node selector from the backend options: %v", backendNodeSelector) maps.Copy(nodeSelector, backendNodeSelector) diff --git a/pipeline/backend/kubernetes/pod_test.go b/pipeline/backend/kubernetes/pod_test.go index b18e77e4c..db0da7488 100644 --- a/pipeline/backend/kubernetes/pod_test.go +++ b/pipeline/backend/kubernetes/pod_test.go @@ -240,7 +240,8 @@ func TestFullPod(t *testing.T) { ], "restartPolicy": "Never", "nodeSelector": { - "storage": "ssd" + "storage": "ssd", + "topology.kubernetes.io/region": "eu-central-1" }, "runtimeClassName": "runc", "serviceAccountName": "wp-svc-acc", @@ -331,6 +332,7 @@ func TestFullPod(t *testing.T) { PodLabelsAllowFromStep: true, PodAnnotations: map[string]string{"apps.kubernetes.io/pod-index": "0"}, PodAnnotationsAllowFromStep: true, + PodNodeSelector: map[string]string{"topology.kubernetes.io/region": "eu-central-1"}, SecurityContext: SecurityContextConfig{RunAsNonRoot: false}, }, "wp-01he8bebctabr3kgk0qj36d2me-0", "linux/amd64", BackendOptions{ Labels: map[string]string{"part-of": "woodpecker-ci"},