Ability to set pod annotations and labels from step (#3609)

Co-authored-by: qwerty287 <80460567+qwerty287@users.noreply.github.com>

pipeline/backend/kubernetes/backend_options.go

@@ -11,6 +11,8 @@ type BackendOptions struct {
 	Resources          Resources         `mapstructure:"resources"`
 	RuntimeClassName   *string           `mapstructure:"runtimeClassName"`
 	ServiceAccountName string            `mapstructure:"serviceAccountName"`
+	Labels             map[string]string `mapstructure:"labels"`
+	Annotations        map[string]string `mapstructure:"annotations"`
 	NodeSelector       map[string]string `mapstructure:"nodeSelector"`
 	Tolerations        []Toleration      `mapstructure:"tolerations"`
 	SecurityContext    *SecurityContext  `mapstructure:"securityContext"`

pipeline/backend/kubernetes/backend_options_test.go

@@ -20,6 +20,8 @@ func Test_parseBackendOptions(t *testing.T) {
 			"kubernetes": map[string]any{
 				"nodeSelector":       map[string]string{"storage": "ssd"},
 				"serviceAccountName": "wp-svc-acc",
+				"labels":             map[string]string{"app": "test"},
+				"annotations":        map[string]string{"apps.kubernetes.io/pod-index": "0"},
 				"tolerations": []map[string]any{
 					{"key": "net-port", "value": "100Mbit", "effect": TaintEffectNoSchedule},
 				},
@@ -49,6 +51,8 @@ func Test_parseBackendOptions(t *testing.T) {
 	assert.Equal(t, BackendOptions{
 		NodeSelector:       map[string]string{"storage": "ssd"},
 		ServiceAccountName: "wp-svc-acc",
+		Labels:             map[string]string{"app": "test"},
+		Annotations:        map[string]string{"apps.kubernetes.io/pod-index": "0"},
 		Tolerations:        []Toleration{{Key: "net-port", Value: "100Mbit", Effect: TaintEffectNoSchedule}},
 		Resources: Resources{
 			Requests: map[string]string{"memory": "128Mi", "cpu": "1000m"},

pipeline/backend/kubernetes/flags.go

@@ -46,15 +46,27 @@ var Flags = []cli.Flag{
 	&cli.StringFlag{
 		EnvVars: []string{"WOODPECKER_BACKEND_K8S_POD_LABELS"},
 		Name:    "backend-k8s-pod-labels",
-		Usage:   "backend k8s additional worker pod labels",
+		Usage:   "backend k8s additional Agent-wide worker pod labels",
 		Value:   "",
 	},
+	&cli.BoolFlag{
+		EnvVars: []string{"WOODPECKER_BACKEND_K8S_POD_LABELS_ALLOW_FROM_STEP"},
+		Name:    "backend-k8s-pod-labels-allow-from-step",
+		Usage:   "whether to allow using labels from step's backend options",
+		Value:   false,
+	},
 	&cli.StringFlag{
 		EnvVars: []string{"WOODPECKER_BACKEND_K8S_POD_ANNOTATIONS"},
 		Name:    "backend-k8s-pod-annotations",
-		Usage:   "backend k8s additional worker pod annotations",
+		Usage:   "backend k8s additional Agent-wide worker pod annotations",
 		Value:   "",
 	},
+	&cli.BoolFlag{
+		EnvVars: []string{"WOODPECKER_BACKEND_K8S_POD_ANNOTATIONS_ALLOW_FROM_STEP"},
+		Name:    "backend-k8s-pod-annotations-allow-from-step",
+		Usage:   "whether to allow using annotations from step's backend options",
+		Value:   false,
+	},
 	&cli.BoolFlag{
 		EnvVars: []string{"WOODPECKER_BACKEND_K8S_SECCTX_NONROOT"},
 		Name:    "backend-k8s-secctx-nonroot",

pipeline/backend/kubernetes/kubernetes.go

@@ -55,14 +55,16 @@ type kube struct {
 }
 
 type config struct {
-	Namespace            string
+	Namespace                   string
-	StorageClass         string
+	StorageClass                string
-	VolumeSize           string
+	VolumeSize                  string
-	StorageRwx           bool
+	StorageRwx                  bool
-	PodLabels            map[string]string
+	PodLabels                   map[string]string
-	PodAnnotations       map[string]string
+	PodLabelsAllowFromStep      bool
-	ImagePullSecretNames []string
+	PodAnnotations              map[string]string
-	SecurityContext      SecurityContextConfig
+	PodAnnotationsAllowFromStep bool
+	ImagePullSecretNames        []string
+	SecurityContext             SecurityContextConfig
 }
 type SecurityContextConfig struct {
 	RunAsNonRoot bool
@@ -82,13 +84,15 @@ func configFromCliContext(ctx context.Context) (*config, error) {
 	if ctx != nil {
 		if c, ok := ctx.Value(types.CliContext).(*cli.Context); ok {
 			config := config{
-				Namespace:            c.String("backend-k8s-namespace"),
+				Namespace:                   c.String("backend-k8s-namespace"),
-				StorageClass:         c.String("backend-k8s-storage-class"),
+				StorageClass:                c.String("backend-k8s-storage-class"),
-				VolumeSize:           c.String("backend-k8s-volume-size"),
+				VolumeSize:                  c.String("backend-k8s-volume-size"),
-				StorageRwx:           c.Bool("backend-k8s-storage-rwx"),
+				StorageRwx:                  c.Bool("backend-k8s-storage-rwx"),
-				PodLabels:            make(map[string]string), // just init empty map to prevent nil panic
+				PodLabels:                   make(map[string]string), // just init empty map to prevent nil panic
-				PodAnnotations:       make(map[string]string), // just init empty map to prevent nil panic
+				PodLabelsAllowFromStep:      c.Bool("backend-k8s-pod-labels-allow-from-step"),
-				ImagePullSecretNames: c.StringSlice("backend-k8s-pod-image-pull-secret-names"),
+				PodAnnotations:              make(map[string]string), // just init empty map to prevent nil panic
+				PodAnnotationsAllowFromStep: c.Bool("backend-k8s-pod-annotations-allow-from-step"),
+				ImagePullSecretNames:        c.StringSlice("backend-k8s-pod-image-pull-secret-names"),
 				SecurityContext: SecurityContextConfig{
 					RunAsNonRoot: c.Bool("backend-k8s-secctx-nonroot"),
 				},

pipeline/backend/kubernetes/pod.go

@@ -76,43 +76,76 @@ func podName(step *types.Step) (string, error) {
 func podMeta(step *types.Step, config *config, options BackendOptions, podName string) (metav1.ObjectMeta, error) {
 	var err error
 	meta := metav1.ObjectMeta{
-		Name:      podName,
+		Name:        podName,
-		Namespace: config.Namespace,
+		Namespace:   config.Namespace,
+		Annotations: podAnnotations(config, options, podName),
 	}
 
-	meta.Labels = config.PodLabels
+	meta.Labels, err = podLabels(step, config, options)
-	if meta.Labels == nil {
-		meta.Labels = make(map[string]string, 1)
-	}
-	meta.Labels[StepLabel], err = stepLabel(step)
 	if err != nil {
 		return meta, err
 	}
 
-	if step.Type == types.StepTypeService {
-		meta.Labels[ServiceLabel], _ = serviceName(step)
-	}
-
-	meta.Annotations = config.PodAnnotations
-	if meta.Annotations == nil {
-		meta.Annotations = make(map[string]string)
-	}
-
-	securityContext := options.SecurityContext
-	if securityContext != nil {
-		key, value := apparmorAnnotation(podName, securityContext.ApparmorProfile)
-		if key != nil && value != nil {
-			meta.Annotations[*key] = *value
-		}
-	}
-
 	return meta, nil
 }
 
+func podLabels(step *types.Step, config *config, options BackendOptions) (map[string]string, error) {
+	var err error
+	labels := make(map[string]string)
+
+	if len(options.Labels) > 0 {
+		if config.PodLabelsAllowFromStep {
+			log.Trace().Msgf("using labels from the backend options: %v", options.Labels)
+			maps.Copy(labels, options.Labels)
+		} else {
+			log.Debug().Msg("Pod labels were defined in backend options, but its using disallowed by instance configuration")
+		}
+	}
+	if len(config.PodLabels) > 0 {
+		log.Trace().Msgf("using labels from the configuration: %v", config.PodLabels)
+		maps.Copy(labels, config.PodLabels)
+	}
+	if step.Type == types.StepTypeService {
+		labels[ServiceLabel], _ = serviceName(step)
+	}
+	labels[StepLabel], err = stepLabel(step)
+	if err != nil {
+		return labels, err
+	}
+
+	return labels, nil
+}
+
 func stepLabel(step *types.Step) (string, error) {
 	return toDNSName(step.Name)
 }
 
+func podAnnotations(config *config, options BackendOptions, podName string) map[string]string {
+	annotations := make(map[string]string)
+
+	if len(options.Annotations) > 0 {
+		if config.PodAnnotationsAllowFromStep {
+			log.Trace().Msgf("using annotations from the backend options: %v", options.Annotations)
+			maps.Copy(annotations, options.Annotations)
+		} else {
+			log.Debug().Msg("Pod annotations were defined in backend options, but its using disallowed by instance configuration ")
+		}
+	}
+	if len(config.PodAnnotations) > 0 {
+		log.Trace().Msgf("using annotations from the configuration: %v", config.PodAnnotations)
+		maps.Copy(annotations, config.PodAnnotations)
+	}
+	securityContext := options.SecurityContext
+	if securityContext != nil {
+		key, value := apparmorAnnotation(podName, securityContext.ApparmorProfile)
+		if key != nil && value != nil {
+			annotations[*key] = *value
+		}
+	}
+
+	return annotations
+}
+
 func podSpec(step *types.Step, config *config, options BackendOptions) (v1.PodSpec, error) {
 	var err error
 	spec := v1.PodSpec{

pipeline/backend/kubernetes/pod_test.go

@@ -157,11 +157,13 @@ func TestFullPod(t *testing.T) {
 			"creationTimestamp": null,
 			"labels": {
 				"app": "test",
+				"part-of": "woodpecker-ci",
 				"step": "go-test"
 			},
 			"annotations": {
 				"apps.kubernetes.io/pod-index": "0",
-				"container.apparmor.security.beta.kubernetes.io/wp-01he8bebctabr3kgk0qj36d2me-0": "localhost/k8s-apparmor-example-deny-write"
+				"container.apparmor.security.beta.kubernetes.io/wp-01he8bebctabr3kgk0qj36d2me-0": "localhost/k8s-apparmor-example-deny-write",
+				"kubernetes.io/limit-ranger": "LimitRanger plugin set: cpu, memory request and limit for container"
 			}
 		},
 		"spec": {
@@ -323,12 +325,16 @@ func TestFullPod(t *testing.T) {
 		ExtraHosts:  hostAliases,
 		Ports:       ports,
 	}, &config{
-		Namespace:            "woodpecker",
+		Namespace:                   "woodpecker",
-		ImagePullSecretNames: []string{"regcred", "another-pull-secret"},
+		ImagePullSecretNames:        []string{"regcred", "another-pull-secret"},
-		PodLabels:            map[string]string{"app": "test"},
+		PodLabels:                   map[string]string{"app": "test"},
-		PodAnnotations:       map[string]string{"apps.kubernetes.io/pod-index": "0"},
+		PodLabelsAllowFromStep:      true,
-		SecurityContext:      SecurityContextConfig{RunAsNonRoot: false},
+		PodAnnotations:              map[string]string{"apps.kubernetes.io/pod-index": "0"},
+		PodAnnotationsAllowFromStep: true,
+		SecurityContext:             SecurityContextConfig{RunAsNonRoot: false},
 	}, "wp-01he8bebctabr3kgk0qj36d2me-0", "linux/amd64", BackendOptions{
+		Labels:             map[string]string{"part-of": "woodpecker-ci"},
+		Annotations:        map[string]string{"kubernetes.io/limit-ranger": "LimitRanger plugin set: cpu, memory request and limit for container"},
 		NodeSelector:       map[string]string{"storage": "ssd"},
 		RuntimeClassName:   &runtimeClass,
 		ServiceAccountName: "wp-svc-acc",

pipeline/frontend/yaml/linter/schema/schema.json

@@ -706,8 +706,17 @@
       "description": "Advanced options for the kubernetes agent backends",
       "type": "object",
       "properties": {
-        "resources": {
+        "labels": {
-          "$ref": "#/definitions/step_backend_kubernetes_resources"
+          "type": "object",
+          "additionalProperties": {
+            "type": ["boolean", "string", "number"]
+          }
+        },
+        "annotations": {
+          "type": "object",
+          "additionalProperties": {
+            "type": ["boolean", "string", "number"]
+          }
         },
         "securityContext": {
           "$ref": "#/definitions/step_backend_kubernetes_security_context"