From 609ba481b5e912f59aaae8ca7bc22b44523c5e37 Mon Sep 17 00:00:00 2001 From: Ovidiu Calbajos Date: Mon, 12 Jun 2023 17:00:59 +0300 Subject: [PATCH] Support for k8s serviceAccount and nodeSelector (#1842) Add the possiblity to specify the Kubernetes serviceAccount and/or nodeSelector to be used on individual steps for Kubernetes executor --- .../22-backends/40-kubernetes.md | 21 ++++++++++++++++--- pipeline/backend/kubernetes/pod.go | 16 +++++++++++--- pipeline/backend/types/backend_kubernetes.go | 4 +++- pipeline/frontend/yaml/compiler/convert.go | 2 ++ .../frontend/yaml/types/backend_options.go | 4 +++- pipeline/schema/schema.json | 19 +++++++++++++++++ 6 files changed, 58 insertions(+), 8 deletions(-) diff --git a/docs/docs/30-administration/22-backends/40-kubernetes.md b/docs/docs/30-administration/22-backends/40-kubernetes.md index 69b9e04cf..51c1958f7 100644 --- a/docs/docs/30-administration/22-backends/40-kubernetes.md +++ b/docs/docs/30-administration/22-backends/40-kubernetes.md @@ -40,9 +40,23 @@ Additional labels to apply to worker pods. Must be a YAML object, e.g. `{"exampl Additional annotations to apply to worker pods. Must be a YAML object, e.g. `{"example.com/test-annotation":"test-value"}`. -## Resources +## Job specific configuration + +### Resources The kubernetes backend also allows for specifying requests and limits on a per-step basic, most commonly for CPU and memory. +See the [kubernetes documentation](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/) for more information on using resources. + +### serviceAccountName + +Specify the name of the ServiceAccount which the build pod will mount. This serviceAccount must be created externally. +See the [kubernetes documentation](https://kubernetes.io/docs/concepts/security/service-accounts/) for more information on using serviceAccounts. + +### nodeSelector + +Specify the label which is used to select the node where the job should be executed. Labels defined here will be appended to a list already containing "kubernetes.io/arch". +By default the pod will use "kubernetes.io/arch" inferred from top-level "platform" setting which is deducted from the agents' environment variable CI_SYSTEM_ARCH. To overwrite this, you need to specify this label in the nodeSelector section. +See the [kubernetes documentation](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector) for more information on using nodeSelector. Example pipeline configuration: ```yaml @@ -55,12 +69,13 @@ steps: - go test backend_options: kubernetes: + serviceAccountName: 'my-service-account' resources: requests: memory: 128Mi cpu: 1000m limits: memory: 256Mi + nodeSelector: + beta.kubernetes.io/instance-type: p3.8xlarge ``` - -See the [kubernetes documentation](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/) for more information on using resources. diff --git a/pipeline/backend/kubernetes/pod.go b/pipeline/backend/kubernetes/pod.go index d4e860530..fbc03c853 100644 --- a/pipeline/backend/kubernetes/pod.go +++ b/pipeline/backend/kubernetes/pod.go @@ -80,6 +80,11 @@ func Pod(namespace string, step *types.Step, labels, annotations map[string]stri } } + var ServiceAccountName string + if step.BackendOptions.Kubernetes.ServiceAccountName != "" { + ServiceAccountName = step.BackendOptions.Kubernetes.ServiceAccountName + } + podName, err := dnsName(step.Name) if err != nil { return nil, err @@ -97,6 +102,10 @@ func Pod(namespace string, step *types.Step, labels, annotations map[string]stri NodeSelector := map[string]string{"kubernetes.io/arch": strings.Split(platform, "/")[1]} + for key, val := range step.BackendOptions.Kubernetes.NodeSelector { + NodeSelector[key] = val + } + pod := &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: podName, @@ -105,9 +114,10 @@ func Pod(namespace string, step *types.Step, labels, annotations map[string]stri Annotations: annotations, }, Spec: v1.PodSpec{ - RestartPolicy: v1.RestartPolicyNever, - HostAliases: hostAliases, - NodeSelector: NodeSelector, + RestartPolicy: v1.RestartPolicyNever, + HostAliases: hostAliases, + NodeSelector: NodeSelector, + ServiceAccountName: ServiceAccountName, Containers: []v1.Container{{ Name: podName, Image: step.Image, diff --git a/pipeline/backend/types/backend_kubernetes.go b/pipeline/backend/types/backend_kubernetes.go index 96b71f17e..942978731 100644 --- a/pipeline/backend/types/backend_kubernetes.go +++ b/pipeline/backend/types/backend_kubernetes.go @@ -2,7 +2,9 @@ package types // KubernetesBackendOptions defines all the advanced options for the kubernetes backend type KubernetesBackendOptions struct { - Resources Resources `json:"resouces,omitempty"` + Resources Resources `json:"resouces,omitempty"` + ServiceAccountName string `json:"serviceAccountName,omitempty"` + NodeSelector map[string]string `json:"nodeSelector,omitempty"` } // Resources defines two maps for kubernetes resource definitions diff --git a/pipeline/frontend/yaml/compiler/convert.go b/pipeline/frontend/yaml/compiler/convert.go index e91e6acde..c3f45fa3f 100644 --- a/pipeline/frontend/yaml/compiler/convert.go +++ b/pipeline/frontend/yaml/compiler/convert.go @@ -116,6 +116,8 @@ func (c *Compiler) createProcess(name string, container *yaml_types.Container, s Limits: container.BackendOptions.Kubernetes.Resources.Limits, Requests: container.BackendOptions.Kubernetes.Resources.Requests, }, + ServiceAccountName: container.BackendOptions.Kubernetes.ServiceAccountName, + NodeSelector: container.BackendOptions.Kubernetes.NodeSelector, }, } diff --git a/pipeline/frontend/yaml/types/backend_options.go b/pipeline/frontend/yaml/types/backend_options.go index 1989c8825..b99999798 100644 --- a/pipeline/frontend/yaml/types/backend_options.go +++ b/pipeline/frontend/yaml/types/backend_options.go @@ -20,7 +20,9 @@ type BackendOptions struct { } type KubernetesBackendOptions struct { - Resources Resources `yaml:"resources,omitempty"` + Resources Resources `yaml:"resources,omitempty"` + ServiceAccountName string `yaml:"serviceAccountName,omitempty"` + NodeSelector map[string]string `yaml:"nodeSelector,omitempty"` } type Resources struct { diff --git a/pipeline/schema/schema.json b/pipeline/schema/schema.json index 199ee831a..8ffa52032 100644 --- a/pipeline/schema/schema.json +++ b/pipeline/schema/schema.json @@ -521,6 +521,25 @@ "type": "string" } }, + "step_backend_kubernetes_service_account": { + "description": "serviceAccountName to be use by job. Read more: https://woodpecker-ci.org/docs/administration/backends/kubernetes", + "type": "object", + "properties": { + "requests": { + "$ref": "#/definitions/step_kubernetes_service_account_object" + }, + "limits": { + "$ref": "#/definitions/step_kubernetes_service_account_object" + } + } + }, + "step_kubernetes_service_account_object": { + "description": "A list of kubernetes resource mappings", + "type": "object", + "additionalProperties": { + "type": "string" + } + }, "services": { "description": "Read more: https://woodpecker-ci.org/docs/usage/services", "type": "object",