diff --git a/pipeline/backend/kubernetes/backend_options.go b/pipeline/backend/kubernetes/backend_options.go index 3b1109a2c..a1d82067e 100644 --- a/pipeline/backend/kubernetes/backend_options.go +++ b/pipeline/backend/kubernetes/backend_options.go @@ -11,6 +11,8 @@ type BackendOptions struct { Resources Resources `mapstructure:"resources"` RuntimeClassName *string `mapstructure:"runtimeClassName"` ServiceAccountName string `mapstructure:"serviceAccountName"` + Labels map[string]string `mapstructure:"labels"` + Annotations map[string]string `mapstructure:"annotations"` NodeSelector map[string]string `mapstructure:"nodeSelector"` Tolerations []Toleration `mapstructure:"tolerations"` SecurityContext *SecurityContext `mapstructure:"securityContext"` diff --git a/pipeline/backend/kubernetes/backend_options_test.go b/pipeline/backend/kubernetes/backend_options_test.go index f9c21a5fe..3d5a73f32 100644 --- a/pipeline/backend/kubernetes/backend_options_test.go +++ b/pipeline/backend/kubernetes/backend_options_test.go @@ -20,6 +20,8 @@ func Test_parseBackendOptions(t *testing.T) { "kubernetes": map[string]any{ "nodeSelector": map[string]string{"storage": "ssd"}, "serviceAccountName": "wp-svc-acc", + "labels": map[string]string{"app": "test"}, + "annotations": map[string]string{"apps.kubernetes.io/pod-index": "0"}, "tolerations": []map[string]any{ {"key": "net-port", "value": "100Mbit", "effect": TaintEffectNoSchedule}, }, @@ -49,6 +51,8 @@ func Test_parseBackendOptions(t *testing.T) { assert.Equal(t, BackendOptions{ NodeSelector: map[string]string{"storage": "ssd"}, ServiceAccountName: "wp-svc-acc", + Labels: map[string]string{"app": "test"}, + Annotations: map[string]string{"apps.kubernetes.io/pod-index": "0"}, Tolerations: []Toleration{{Key: "net-port", Value: "100Mbit", Effect: TaintEffectNoSchedule}}, Resources: Resources{ Requests: map[string]string{"memory": "128Mi", "cpu": "1000m"}, diff --git a/pipeline/backend/kubernetes/flags.go b/pipeline/backend/kubernetes/flags.go index 6d5fd9c92..347d0bd64 100644 --- a/pipeline/backend/kubernetes/flags.go +++ b/pipeline/backend/kubernetes/flags.go @@ -46,15 +46,27 @@ var Flags = []cli.Flag{ &cli.StringFlag{ EnvVars: []string{"WOODPECKER_BACKEND_K8S_POD_LABELS"}, Name: "backend-k8s-pod-labels", - Usage: "backend k8s additional worker pod labels", + Usage: "backend k8s additional Agent-wide worker pod labels", Value: "", }, + &cli.BoolFlag{ + EnvVars: []string{"WOODPECKER_BACKEND_K8S_POD_LABELS_ALLOW_FROM_STEP"}, + Name: "backend-k8s-pod-labels-allow-from-step", + Usage: "whether to allow using labels from step's backend options", + Value: false, + }, &cli.StringFlag{ EnvVars: []string{"WOODPECKER_BACKEND_K8S_POD_ANNOTATIONS"}, Name: "backend-k8s-pod-annotations", - Usage: "backend k8s additional worker pod annotations", + Usage: "backend k8s additional Agent-wide worker pod annotations", Value: "", }, + &cli.BoolFlag{ + EnvVars: []string{"WOODPECKER_BACKEND_K8S_POD_ANNOTATIONS_ALLOW_FROM_STEP"}, + Name: "backend-k8s-pod-annotations-allow-from-step", + Usage: "whether to allow using annotations from step's backend options", + Value: false, + }, &cli.BoolFlag{ EnvVars: []string{"WOODPECKER_BACKEND_K8S_SECCTX_NONROOT"}, Name: "backend-k8s-secctx-nonroot", diff --git a/pipeline/backend/kubernetes/kubernetes.go b/pipeline/backend/kubernetes/kubernetes.go index 3019bba7a..00f5c1b7e 100644 --- a/pipeline/backend/kubernetes/kubernetes.go +++ b/pipeline/backend/kubernetes/kubernetes.go @@ -55,14 +55,16 @@ type kube struct { } type config struct { - Namespace string - StorageClass string - VolumeSize string - StorageRwx bool - PodLabels map[string]string - PodAnnotations map[string]string - ImagePullSecretNames []string - SecurityContext SecurityContextConfig + Namespace string + StorageClass string + VolumeSize string + StorageRwx bool + PodLabels map[string]string + PodLabelsAllowFromStep bool + PodAnnotations map[string]string + PodAnnotationsAllowFromStep bool + ImagePullSecretNames []string + SecurityContext SecurityContextConfig } type SecurityContextConfig struct { RunAsNonRoot bool @@ -82,13 +84,15 @@ func configFromCliContext(ctx context.Context) (*config, error) { if ctx != nil { if c, ok := ctx.Value(types.CliContext).(*cli.Context); ok { config := config{ - Namespace: c.String("backend-k8s-namespace"), - StorageClass: c.String("backend-k8s-storage-class"), - VolumeSize: c.String("backend-k8s-volume-size"), - StorageRwx: c.Bool("backend-k8s-storage-rwx"), - PodLabels: make(map[string]string), // just init empty map to prevent nil panic - PodAnnotations: make(map[string]string), // just init empty map to prevent nil panic - ImagePullSecretNames: c.StringSlice("backend-k8s-pod-image-pull-secret-names"), + Namespace: c.String("backend-k8s-namespace"), + StorageClass: c.String("backend-k8s-storage-class"), + VolumeSize: c.String("backend-k8s-volume-size"), + StorageRwx: c.Bool("backend-k8s-storage-rwx"), + PodLabels: make(map[string]string), // just init empty map to prevent nil panic + PodLabelsAllowFromStep: c.Bool("backend-k8s-pod-labels-allow-from-step"), + PodAnnotations: make(map[string]string), // just init empty map to prevent nil panic + PodAnnotationsAllowFromStep: c.Bool("backend-k8s-pod-annotations-allow-from-step"), + ImagePullSecretNames: c.StringSlice("backend-k8s-pod-image-pull-secret-names"), SecurityContext: SecurityContextConfig{ RunAsNonRoot: c.Bool("backend-k8s-secctx-nonroot"), }, diff --git a/pipeline/backend/kubernetes/pod.go b/pipeline/backend/kubernetes/pod.go index 20255ca88..2481e90c7 100644 --- a/pipeline/backend/kubernetes/pod.go +++ b/pipeline/backend/kubernetes/pod.go @@ -76,43 +76,76 @@ func podName(step *types.Step) (string, error) { func podMeta(step *types.Step, config *config, options BackendOptions, podName string) (metav1.ObjectMeta, error) { var err error meta := metav1.ObjectMeta{ - Name: podName, - Namespace: config.Namespace, + Name: podName, + Namespace: config.Namespace, + Annotations: podAnnotations(config, options, podName), } - meta.Labels = config.PodLabels - if meta.Labels == nil { - meta.Labels = make(map[string]string, 1) - } - meta.Labels[StepLabel], err = stepLabel(step) + meta.Labels, err = podLabels(step, config, options) if err != nil { return meta, err } - if step.Type == types.StepTypeService { - meta.Labels[ServiceLabel], _ = serviceName(step) - } - - meta.Annotations = config.PodAnnotations - if meta.Annotations == nil { - meta.Annotations = make(map[string]string) - } - - securityContext := options.SecurityContext - if securityContext != nil { - key, value := apparmorAnnotation(podName, securityContext.ApparmorProfile) - if key != nil && value != nil { - meta.Annotations[*key] = *value - } - } - return meta, nil } +func podLabels(step *types.Step, config *config, options BackendOptions) (map[string]string, error) { + var err error + labels := make(map[string]string) + + if len(options.Labels) > 0 { + if config.PodLabelsAllowFromStep { + log.Trace().Msgf("using labels from the backend options: %v", options.Labels) + maps.Copy(labels, options.Labels) + } else { + log.Debug().Msg("Pod labels were defined in backend options, but its using disallowed by instance configuration") + } + } + if len(config.PodLabels) > 0 { + log.Trace().Msgf("using labels from the configuration: %v", config.PodLabels) + maps.Copy(labels, config.PodLabels) + } + if step.Type == types.StepTypeService { + labels[ServiceLabel], _ = serviceName(step) + } + labels[StepLabel], err = stepLabel(step) + if err != nil { + return labels, err + } + + return labels, nil +} + func stepLabel(step *types.Step) (string, error) { return toDNSName(step.Name) } +func podAnnotations(config *config, options BackendOptions, podName string) map[string]string { + annotations := make(map[string]string) + + if len(options.Annotations) > 0 { + if config.PodAnnotationsAllowFromStep { + log.Trace().Msgf("using annotations from the backend options: %v", options.Annotations) + maps.Copy(annotations, options.Annotations) + } else { + log.Debug().Msg("Pod annotations were defined in backend options, but its using disallowed by instance configuration ") + } + } + if len(config.PodAnnotations) > 0 { + log.Trace().Msgf("using annotations from the configuration: %v", config.PodAnnotations) + maps.Copy(annotations, config.PodAnnotations) + } + securityContext := options.SecurityContext + if securityContext != nil { + key, value := apparmorAnnotation(podName, securityContext.ApparmorProfile) + if key != nil && value != nil { + annotations[*key] = *value + } + } + + return annotations +} + func podSpec(step *types.Step, config *config, options BackendOptions) (v1.PodSpec, error) { var err error spec := v1.PodSpec{ diff --git a/pipeline/backend/kubernetes/pod_test.go b/pipeline/backend/kubernetes/pod_test.go index 1e7fbedc5..33473d49b 100644 --- a/pipeline/backend/kubernetes/pod_test.go +++ b/pipeline/backend/kubernetes/pod_test.go @@ -157,11 +157,13 @@ func TestFullPod(t *testing.T) { "creationTimestamp": null, "labels": { "app": "test", + "part-of": "woodpecker-ci", "step": "go-test" }, "annotations": { "apps.kubernetes.io/pod-index": "0", - "container.apparmor.security.beta.kubernetes.io/wp-01he8bebctabr3kgk0qj36d2me-0": "localhost/k8s-apparmor-example-deny-write" + "container.apparmor.security.beta.kubernetes.io/wp-01he8bebctabr3kgk0qj36d2me-0": "localhost/k8s-apparmor-example-deny-write", + "kubernetes.io/limit-ranger": "LimitRanger plugin set: cpu, memory request and limit for container" } }, "spec": { @@ -323,12 +325,16 @@ func TestFullPod(t *testing.T) { ExtraHosts: hostAliases, Ports: ports, }, &config{ - Namespace: "woodpecker", - ImagePullSecretNames: []string{"regcred", "another-pull-secret"}, - PodLabels: map[string]string{"app": "test"}, - PodAnnotations: map[string]string{"apps.kubernetes.io/pod-index": "0"}, - SecurityContext: SecurityContextConfig{RunAsNonRoot: false}, + Namespace: "woodpecker", + ImagePullSecretNames: []string{"regcred", "another-pull-secret"}, + PodLabels: map[string]string{"app": "test"}, + PodLabelsAllowFromStep: true, + PodAnnotations: map[string]string{"apps.kubernetes.io/pod-index": "0"}, + PodAnnotationsAllowFromStep: true, + SecurityContext: SecurityContextConfig{RunAsNonRoot: false}, }, "wp-01he8bebctabr3kgk0qj36d2me-0", "linux/amd64", BackendOptions{ + Labels: map[string]string{"part-of": "woodpecker-ci"}, + Annotations: map[string]string{"kubernetes.io/limit-ranger": "LimitRanger plugin set: cpu, memory request and limit for container"}, NodeSelector: map[string]string{"storage": "ssd"}, RuntimeClassName: &runtimeClass, ServiceAccountName: "wp-svc-acc", diff --git a/pipeline/frontend/yaml/linter/schema/schema.json b/pipeline/frontend/yaml/linter/schema/schema.json index 289b524e2..f5869c15d 100644 --- a/pipeline/frontend/yaml/linter/schema/schema.json +++ b/pipeline/frontend/yaml/linter/schema/schema.json @@ -706,8 +706,17 @@ "description": "Advanced options for the kubernetes agent backends", "type": "object", "properties": { - "resources": { - "$ref": "#/definitions/step_backend_kubernetes_resources" + "labels": { + "type": "object", + "additionalProperties": { + "type": ["boolean", "string", "number"] + } + }, + "annotations": { + "type": "object", + "additionalProperties": { + "type": ["boolean", "string", "number"] + } }, "securityContext": { "$ref": "#/definitions/step_backend_kubernetes_security_context"