Implement registries for Kubernetes backend (#4092)

According to [the documentation](https://woodpecker-ci.org/docs/administration/backends/kubernetes#images-from-private-registries), per-organization and per-pipeline registries are currently unsupported for the Kubernetes backend.

This patch implements this missing functionality by creating and deleting a matching secret for each pod with a matched registry, using the same name, labels, and annotations as the pod, and appending it to its `imagePullSecrets` list.

This patch adds tests for the new functionality, and has been manually end-to-end-tested in KinD by using a private image hosted in the matching gitea instance.

This will require updating the matching helm charts to add the create/delete permissions to the agent role, which **is already done**.

close  #2987

docs/docs/20-usage/41-registries.md

@@ -35,10 +35,6 @@ Example registry hostname matching logic:
 - Hostname `docker.io` matches `bradrydzewski/golang`
 - Hostname `docker.io` matches `bradrydzewski/golang:latest`
 
-:::note
-The flow above doesn't work in Kubernetes. There is [workaround](../30-administration/22-backends/40-kubernetes.md#images-from-private-registries).
-:::
-
 ## Global registry support
 
 To make a private registry globally available, check the [server configuration docs](../30-administration/10-server-config.md#global-registry-setting).

docs/docs/30-administration/22-backends/40-kubernetes.md

@@ -8,9 +8,9 @@ The Kubernetes backend executes steps inside standalone Pods. A temporary PVC is
 
 ## Images from private registries
 
-In order to pull private container images defined in your pipeline YAML you must provide [registry credentials in Kubernetes Secret](https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/).
-As the Secret is Agent-wide, it has to be placed in namespace defined by `WOODPECKER_BACKEND_K8S_NAMESPACE`.
-Besides, you need to provide the Secret name to Agent via `WOODPECKER_BACKEND_K8S_PULL_SECRET_NAMES`.
+In addition to [registries specified in the UI](../../20-usage/41-registries.md), you may provide [registry credentials in Kubernetes Secrets](https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/) to pull private container images defined in your pipeline YAML.
+
+Place these Secrets in namespace defined by `WOODPECKER_BACKEND_K8S_NAMESPACE` and provide the Secret names to Agents via `WOODPECKER_BACKEND_K8S_PULL_SECRET_NAMES`.
 
 ## Job specific configuration
 

pipeline/backend/kubernetes/kubernetes.go

@@ -16,6 +16,7 @@ package kubernetes
 
 import (
 	"context"
+	std_errs "errors"
 	"fmt"
 	"io"
 	"maps"
@@ -225,6 +226,13 @@ func (e *kube) StartStep(ctx context.Context, step *types.Step, taskUUID string)
 		log.Error().Err(err).Msg("could not parse backend options")
 	}
 
+	if needsRegistrySecret(step) {
+		err = startRegistrySecret(ctx, e, step)
+		if err != nil {
+			return err
+		}
+	}
+
 	log.Trace().Str("taskUUID", taskUUID).Msgf("starting step: %s", step.Name)
 	_, err = startPod(ctx, e, step, options)
 	return err
@@ -382,9 +390,20 @@ func (e *kube) TailStep(ctx context.Context, step *types.Step, taskUUID string)
 }
 
 func (e *kube) DestroyStep(ctx context.Context, step *types.Step, taskUUID string) error {
+	var errs []error
 	log.Trace().Str("taskUUID", taskUUID).Msgf("Stopping step: %s", step.Name)
+	if needsRegistrySecret(step) {
+		err := stopRegistrySecret(ctx, e, step, defaultDeleteOptions)
+		if err != nil {
+			errs = append(errs, err)
+		}
+	}
+
 	err := stopPod(ctx, e, step, defaultDeleteOptions)
-	return err
+	if err != nil {
+		errs = append(errs, err)
+	}
+	return std_errs.Join(errs...)
 }
 
 // DestroyWorkflow destroys the pipeline environment.

pipeline/backend/kubernetes/pod.go

@@ -163,6 +163,14 @@ func podSpec(step *types.Step, config *config, options BackendOptions, nsp nativ
 
 	log.Trace().Msgf("using the image pull secrets: %v", config.ImagePullSecretNames)
 	spec.ImagePullSecrets = secretsReferences(config.ImagePullSecretNames)
+	if needsRegistrySecret(step) {
+		log.Trace().Msgf("using an image pull secret from registries")
+		name, err := registrySecretName(step)
+		if err != nil {
+			return spec, err
+		}
+		spec.ImagePullSecrets = append(spec.ImagePullSecrets, secretReference(name))
+	}
 
 	spec.Volumes = append(spec.Volumes, nsp.volumes...)
 
@@ -514,6 +522,7 @@ func stopPod(ctx context.Context, engine *kube, step *types.Step, deleteOpts met
 	if err != nil {
 		return err
 	}
+
 	log.Trace().Str("name", podName).Msg("deleting pod")
 
 	err = engine.client.CoreV1().Pods(engine.config.Namespace).Delete(ctx, podName, deleteOpts)

pipeline/backend/kubernetes/pod_test.go

@@ -264,6 +264,9 @@ func TestFullPod(t *testing.T) {
 				},
 				{
 					"name": "another-pull-secret"
+				},
+				{
+					"name": "wp-01he8bebctabr3kgk0qj36d2me-0"
 				}
 			],
 			"tolerations": [
@@ -317,6 +320,7 @@ func TestFullPod(t *testing.T) {
 		},
 	}
 	pod, err := mkPod(&types.Step{
+		UUID:        "01he8bebctabr3kgk0qj36d2me-0",
 		Name:        "go-test",
 		Image:       "meltwater/drone-cache",
 		WorkingDir:  "/woodpecker/src",
@@ -328,6 +332,10 @@ func TestFullPod(t *testing.T) {
 		Environment: map[string]string{"CGO": "0"},
 		ExtraHosts:  hostAliases,
 		Ports:       ports,
+		AuthConfig: types.Auth{
+			Username: "foo",
+			Password: "bar",
+		},
 	}, &config{
 		Namespace:                   "woodpecker",
 		ImagePullSecretNames:        []string{"regcred", "another-pull-secret"},

pipeline/backend/kubernetes/secrets.go

@@ -15,11 +15,21 @@
 package kubernetes
 
 import (
+	"context"
+	"encoding/json"
 	"fmt"
 	"strings"
 
+	"github.com/distribution/reference"
+	config_file "github.com/docker/cli/cli/config/configfile"
+	config_file_types "github.com/docker/cli/cli/config/types"
 	"github.com/rs/zerolog/log"
 	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
+	meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"go.woodpecker-ci.org/woodpecker/v2/pipeline/backend/types"
+	"go.woodpecker-ci.org/woodpecker/v2/pipeline/frontend/yaml/utils"
 )
 
 type nativeSecretsProcessor struct {
@@ -189,3 +199,96 @@ func secretReference(name string) v1.LocalObjectReference {
 		Name: name,
 	}
 }
+
+func needsRegistrySecret(step *types.Step) bool {
+	return step.AuthConfig.Username != "" && step.AuthConfig.Password != ""
+}
+
+func mkRegistrySecret(step *types.Step, config *config) (*v1.Secret, error) {
+	name, err := registrySecretName(step)
+	if err != nil {
+		return nil, err
+	}
+
+	labels, err := registrySecretLabels(step)
+	if err != nil {
+		return nil, err
+	}
+
+	named, err := utils.ParseNamed(step.Image)
+	if err != nil {
+		return nil, err
+	}
+
+	authConfig := config_file.ConfigFile{
+		AuthConfigs: map[string]config_file_types.AuthConfig{
+			reference.Domain(named): {
+				Username: step.AuthConfig.Username,
+				Password: step.AuthConfig.Password,
+			},
+		},
+	}
+
+	configFileJSON, err := json.Marshal(authConfig)
+	if err != nil {
+		return nil, err
+	}
+
+	return &v1.Secret{
+		ObjectMeta: meta_v1.ObjectMeta{
+			Namespace: config.Namespace,
+			Name:      name,
+			Labels:    labels,
+		},
+		Type: v1.SecretTypeDockerConfigJson,
+		Data: map[string][]byte{
+			v1.DockerConfigJsonKey: configFileJSON,
+		},
+	}, nil
+}
+
+func registrySecretName(step *types.Step) (string, error) {
+	return podName(step)
+}
+
+func registrySecretLabels(step *types.Step) (map[string]string, error) {
+	var err error
+	labels := make(map[string]string)
+
+	if step.Type == types.StepTypeService {
+		labels[ServiceLabel], _ = serviceName(step)
+	}
+	labels[StepLabel], err = stepLabel(step)
+	if err != nil {
+		return labels, err
+	}
+
+	return labels, nil
+}
+
+func startRegistrySecret(ctx context.Context, engine *kube, step *types.Step) error {
+	secret, err := mkRegistrySecret(step, engine.config)
+	if err != nil {
+		return err
+	}
+	log.Trace().Msgf("creating secret: %s", secret.Name)
+	_, err = engine.client.CoreV1().Secrets(engine.config.Namespace).Create(ctx, secret, meta_v1.CreateOptions{})
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func stopRegistrySecret(ctx context.Context, engine *kube, step *types.Step, deleteOpts meta_v1.DeleteOptions) error {
+	name, err := registrySecretName(step)
+	if err != nil {
+		return err
+	}
+	log.Trace().Str("name", name).Msg("deleting secret")
+
+	err = engine.client.CoreV1().Secrets(engine.config.Namespace).Delete(ctx, name, deleteOpts)
+	if errors.IsNotFound(err) {
+		return nil
+	}
+	return err
+}

pipeline/backend/kubernetes/secrets_test.go

@@ -15,10 +15,14 @@
 package kubernetes
 
 import (
+	"encoding/json"
 	"testing"
 
+	"github.com/kinbiko/jsonassert"
 	"github.com/stretchr/testify/assert"
 	v1 "k8s.io/api/core/v1"
+
+	"go.woodpecker-ci.org/woodpecker/v2/pipeline/backend/types"
 )
 
 func TestNativeSecretsEnabled(t *testing.T) {
@@ -178,3 +182,61 @@ func TestFileSecret(t *testing.T) {
 		},
 	}, nsp.mounts)
 }
+
+func TestNoAuthNoSecret(t *testing.T) {
+	assert.False(t, needsRegistrySecret(&types.Step{}))
+}
+
+func TestNoPasswordNoSecret(t *testing.T) {
+	assert.False(t, needsRegistrySecret(&types.Step{
+		AuthConfig: types.Auth{Username: "foo"},
+	}))
+}
+
+func TestNoUsernameNoSecret(t *testing.T) {
+	assert.False(t, needsRegistrySecret(&types.Step{
+		AuthConfig: types.Auth{Password: "foo"},
+	}))
+}
+
+func TestUsernameAndPasswordNeedsSecret(t *testing.T) {
+	assert.True(t, needsRegistrySecret(&types.Step{
+		AuthConfig: types.Auth{Username: "foo", Password: "bar"},
+	}))
+}
+
+func TestRegistrySecret(t *testing.T) {
+	const expected = `{
+		"metadata": {
+			"name": "wp-01he8bebctabr3kgk0qj36d2me-0",
+			"namespace": "woodpecker",
+			"creationTimestamp": null,
+			"labels": {
+				"step": "go-test"
+			}
+		},
+		"type": "kubernetes.io/dockerconfigjson",
+		"data": {
+			".dockerconfigjson": "eyJhdXRocyI6eyJkb2NrZXIuaW8iOnsidXNlcm5hbWUiOiJmb28iLCJwYXNzd29yZCI6ImJhciJ9fX0="
+		}
+	}`
+
+	secret, err := mkRegistrySecret(&types.Step{
+		UUID:  "01he8bebctabr3kgk0qj36d2me-0",
+		Name:  "go-test",
+		Image: "meltwater/drone-cache",
+		AuthConfig: types.Auth{
+			Username: "foo",
+			Password: "bar",
+		},
+	}, &config{
+		Namespace: "woodpecker",
+	})
+	assert.NoError(t, err)
+
+	secretJSON, err := json.Marshal(secret)
+	assert.NoError(t, err)
+
+	ja := jsonassert.New(t)
+	ja.Assertf(string(secretJSON), expected)
+}

pipeline/frontend/yaml/utils/image.go

@@ -84,14 +84,19 @@ func imageHasTag(name string) bool {
 	return strings.Contains(name, ":")
 }
 
+// ParseNamed parses an image as a reference to validate it then parses it as a named reference.
+func ParseNamed(image string) (reference.Named, error) {
+	ref, err := reference.ParseAnyReference(image)
+	if err != nil {
+		return nil, err
+	}
+	return reference.ParseNamed(ref.String())
+}
+
 // MatchHostname returns true if the image hostname
 // matches the specified hostname.
 func MatchHostname(image, hostname string) bool {
-	ref, err := reference.ParseAnyReference(image)
-	if err != nil {
-		return false
-	}
-	named, err := reference.ParseNamed(ref.String())
+	named, err := ParseNamed(image)
 	if err != nil {
 		return false
 	}