From 7bc38a1d8b45553168230be5811a3dac590cf936 Mon Sep 17 00:00:00 2001 From: Thomas Anderson <127358482+zc-devs@users.noreply.github.com> Date: Sun, 23 Jun 2024 19:20:21 +0300 Subject: [PATCH] K8s secrets reference from step (#3655) --- .../backend/kubernetes/backend_options.go | 14 ++ .../kubernetes/backend_options_test.go | 28 +++ pipeline/backend/kubernetes/flags.go | 10 +- pipeline/backend/kubernetes/kubernetes.go | 2 + pipeline/backend/kubernetes/pod.go | 48 +++-- pipeline/backend/kubernetes/pod_test.go | 120 ++++++++++- pipeline/backend/kubernetes/secrets.go | 191 ++++++++++++++++++ pipeline/backend/kubernetes/secrets_test.go | 180 +++++++++++++++++ .../frontend/yaml/linter/schema/schema.json | 43 ++++ 9 files changed, 607 insertions(+), 29 deletions(-) create mode 100644 pipeline/backend/kubernetes/secrets.go create mode 100644 pipeline/backend/kubernetes/secrets_test.go diff --git a/pipeline/backend/kubernetes/backend_options.go b/pipeline/backend/kubernetes/backend_options.go index da3644670..fa64da9f1 100644 --- a/pipeline/backend/kubernetes/backend_options.go +++ b/pipeline/backend/kubernetes/backend_options.go @@ -16,6 +16,7 @@ type BackendOptions struct { NodeSelector map[string]string `mapstructure:"nodeSelector"` Tolerations []Toleration `mapstructure:"tolerations"` SecurityContext *SecurityContext `mapstructure:"securityContext"` + Secrets []SecretRef `mapstructure:"secrets"` } // Resources defines two maps for kubernetes resource definitions. @@ -65,6 +66,19 @@ type SecProfile struct { type SecProfileType string +// SecretRef defines Kubernetes secret reference. +type SecretRef struct { + Name string `mapstructure:"name"` + Key string `mapstructure:"key"` + Target SecretTarget `mapstructure:"target"` +} + +// SecretTarget defines secret mount target. +type SecretTarget struct { + Env string `mapstructure:"env"` + File string `mapstructure:"file"` +} + const ( SecProfileTypeRuntimeDefault SecProfileType = "RuntimeDefault" SecProfileTypeLocalhost SecProfileType = "Localhost" diff --git a/pipeline/backend/kubernetes/backend_options_test.go b/pipeline/backend/kubernetes/backend_options_test.go index 3d5a73f32..3d868ced1 100644 --- a/pipeline/backend/kubernetes/backend_options_test.go +++ b/pipeline/backend/kubernetes/backend_options_test.go @@ -44,6 +44,22 @@ func Test_parseBackendOptions(t *testing.T) { "localhostProfile": "k8s-apparmor-example-deny-write", }, }, + "secrets": []map[string]any{ + { + "name": "aws", + "key": "access-key", + "target": map[string]any{ + "env": "AWS_SECRET_ACCESS_KEY", + }, + }, + { + "name": "reg-cred", + "key": ".dockerconfigjson", + "target": map[string]any{ + "file": "~/.docker/config.json", + }, + }, + }, }, }, }) @@ -73,5 +89,17 @@ func Test_parseBackendOptions(t *testing.T) { LocalhostProfile: "k8s-apparmor-example-deny-write", }, }, + Secrets: []SecretRef{ + { + Name: "aws", + Key: "access-key", + Target: SecretTarget{Env: "AWS_SECRET_ACCESS_KEY"}, + }, + { + Name: "reg-cred", + Key: ".dockerconfigjson", + Target: SecretTarget{File: "~/.docker/config.json"}, + }, + }, }, got) } diff --git a/pipeline/backend/kubernetes/flags.go b/pipeline/backend/kubernetes/flags.go index 219595f09..7b69ff540 100644 --- a/pipeline/backend/kubernetes/flags.go +++ b/pipeline/backend/kubernetes/flags.go @@ -14,9 +14,7 @@ package kubernetes -import ( - "github.com/urfave/cli/v2" -) +import "github.com/urfave/cli/v2" var Flags = []cli.Flag{ &cli.StringFlag{ @@ -84,4 +82,10 @@ var Flags = []cli.Flag{ Usage: "backend k8s pull secret names for private registries", Value: cli.NewStringSlice("regcred"), }, + &cli.BoolFlag{ + EnvVars: []string{"WOODPECKER_BACKEND_K8S_ALLOW_NATIVE_SECRETS"}, + Name: "backend-k8s-allow-native-secrets", + Usage: "whether to allow existing Kubernetes secrets to be referenced from steps", + Value: false, + }, } diff --git a/pipeline/backend/kubernetes/kubernetes.go b/pipeline/backend/kubernetes/kubernetes.go index dc04403ff..e0dc18b0d 100644 --- a/pipeline/backend/kubernetes/kubernetes.go +++ b/pipeline/backend/kubernetes/kubernetes.go @@ -65,6 +65,7 @@ type config struct { PodNodeSelector map[string]string ImagePullSecretNames []string SecurityContext SecurityContextConfig + NativeSecretsAllowFromStep bool } type SecurityContextConfig struct { RunAsNonRoot bool @@ -97,6 +98,7 @@ func configFromCliContext(ctx context.Context) (*config, error) { SecurityContext: SecurityContextConfig{ RunAsNonRoot: c.Bool("backend-k8s-secctx-nonroot"), // cspell:words secctx nonroot }, + NativeSecretsAllowFromStep: c.Bool("backend-k8s-allow-native-secrets"), } // TODO: remove in next major if len(config.ImagePullSecretNames) == 1 && config.ImagePullSecretNames[0] == "regcred" { diff --git a/pipeline/backend/kubernetes/pod.go b/pipeline/backend/kubernetes/pod.go index b240d794a..679543470 100644 --- a/pipeline/backend/kubernetes/pod.go +++ b/pipeline/backend/kubernetes/pod.go @@ -38,17 +38,23 @@ const ( func mkPod(step *types.Step, config *config, podName, goos string, options BackendOptions) (*v1.Pod, error) { var err error + nsp := newNativeSecretsProcessor(config, options.Secrets) + err = nsp.process() + if err != nil { + return nil, err + } + meta, err := podMeta(step, config, options, podName) if err != nil { return nil, err } - spec, err := podSpec(step, config, options) + spec, err := podSpec(step, config, options, nsp) if err != nil { return nil, err } - container, err := podContainer(step, podName, goos, options) + container, err := podContainer(step, podName, goos, options, nsp) if err != nil { return nil, err } @@ -146,27 +152,31 @@ func podAnnotations(config *config, options BackendOptions, podName string) map[ return annotations } -func podSpec(step *types.Step, config *config, options BackendOptions) (v1.PodSpec, error) { +func podSpec(step *types.Step, config *config, options BackendOptions, nsp nativeSecretsProcessor) (v1.PodSpec, error) { var err error spec := v1.PodSpec{ RestartPolicy: v1.RestartPolicyNever, RuntimeClassName: options.RuntimeClassName, ServiceAccountName: options.ServiceAccountName, - ImagePullSecrets: imagePullSecretsReferences(config.ImagePullSecretNames), HostAliases: hostAliases(step.ExtraHosts), NodeSelector: nodeSelector(options.NodeSelector, config.PodNodeSelector, step.Environment["CI_SYSTEM_PLATFORM"]), Tolerations: tolerations(options.Tolerations), SecurityContext: podSecurityContext(options.SecurityContext, config.SecurityContext, step.Privileged), } - spec.Volumes, err = volumes(step.Volumes) + spec.Volumes, err = pvcVolumes(step.Volumes) if err != nil { return spec, err } + log.Trace().Msgf("using the image pull secrets: %v", config.ImagePullSecretNames) + spec.ImagePullSecrets = secretsReferences(config.ImagePullSecretNames) + + spec.Volumes = append(spec.Volumes, nsp.volumes...) + return spec, nil } -func podContainer(step *types.Step, podName, goos string, options BackendOptions) (v1.Container, error) { +func podContainer(step *types.Step, podName, goos string, options BackendOptions, nsp nativeSecretsProcessor) (v1.Container, error) { var err error container := v1.Container{ Name: podName, @@ -201,10 +211,14 @@ func podContainer(step *types.Step, podName, goos string, options BackendOptions return container, err } + container.EnvFrom = append(container.EnvFrom, nsp.envFromSources...) + container.Env = append(container.Env, nsp.envVars...) + container.VolumeMounts = append(container.VolumeMounts, nsp.mounts...) + return container, nil } -func volumes(volumes []string) ([]v1.Volume, error) { +func pvcVolumes(volumes []string) ([]v1.Volume, error) { var vols []v1.Volume for _, v := range volumes { @@ -212,13 +226,13 @@ func volumes(volumes []string) ([]v1.Volume, error) { if err != nil { return nil, err } - vols = append(vols, volume(volumeName)) + vols = append(vols, pvcVolume(volumeName)) } return vols, nil } -func volume(name string) v1.Volume { +func pvcVolume(name string) v1.Volume { pvcSource := v1.PersistentVolumeClaimVolumeSource{ ClaimName: name, ReadOnly: false, @@ -285,22 +299,6 @@ func hostAlias(extraHost types.HostAlias) v1.HostAlias { } } -func imagePullSecretsReferences(imagePullSecretNames []string) []v1.LocalObjectReference { - log.Trace().Msgf("using the image pull secrets: %v", imagePullSecretNames) - - secretReferences := make([]v1.LocalObjectReference, len(imagePullSecretNames)) - for i, imagePullSecretName := range imagePullSecretNames { - secretReferences[i] = imagePullSecretsReference(imagePullSecretName) - } - return secretReferences -} - -func imagePullSecretsReference(imagePullSecretName string) v1.LocalObjectReference { - return v1.LocalObjectReference{ - Name: imagePullSecretName, - } -} - func resourceRequirements(resources Resources) (v1.ResourceRequirements, error) { var err error requirements := v1.ResourceRequirements{} diff --git a/pipeline/backend/kubernetes/pod_test.go b/pipeline/backend/kubernetes/pod_test.go index db0da7488..8b8241545 100644 --- a/pipeline/backend/kubernetes/pod_test.go +++ b/pipeline/backend/kubernetes/pod_test.go @@ -467,6 +467,124 @@ func TestScratchPod(t *testing.T) { assert.NoError(t, err) ja := jsonassert.New(t) - t.Log(string(podJSON)) + ja.Assertf(string(podJSON), expected) +} + +func TestSecrets(t *testing.T) { + expected := ` + { + "metadata": { + "name": "wp-3kgk0qj36d2me01he8bebctabr-0", + "namespace": "woodpecker", + "creationTimestamp": null, + "labels": { + "step": "test-secrets" + } + }, + "spec": { + "volumes": [ + { + "name": "workspace", + "persistentVolumeClaim": { + "claimName": "workspace" + } + }, + { + "name": "reg-cred", + "secret": { + "secretName": "reg-cred" + } + } + ], + "containers": [ + { + "name": "wp-3kgk0qj36d2me01he8bebctabr-0", + "image": "alpine", + "envFrom": [ + { + "secretRef": { + "name": "ghcr-push-secret" + } + } + ], + "env": [ + { + "name": "CGO", + "value": "0" + }, + { + "name": "AWS_ACCESS_KEY_ID", + "valueFrom": { + "secretKeyRef": { + "name": "aws-ecr", + "key": "AWS_ACCESS_KEY_ID" + } + } + }, + { + "name": "AWS_SECRET_ACCESS_KEY", + "valueFrom": { + "secretKeyRef": { + "name": "aws-ecr", + "key": "access-key" + } + } + } + ], + "resources": {}, + "volumeMounts": [ + { + "name": "workspace", + "mountPath": "/woodpecker/src" + }, + { + "name": "reg-cred", + "mountPath": "~/.docker/config.json", + "subPath": ".dockerconfigjson", + "readOnly": true + } + ] + } + ], + "restartPolicy": "Never" + }, + "status": {} + }` + + pod, err := mkPod(&types.Step{ + Name: "test-secrets", + Image: "alpine", + Environment: map[string]string{"CGO": "0"}, + Volumes: []string{"workspace:/woodpecker/src"}, + }, &config{ + Namespace: "woodpecker", + NativeSecretsAllowFromStep: true, + }, "wp-3kgk0qj36d2me01he8bebctabr-0", "linux/amd64", BackendOptions{ + Secrets: []SecretRef{ + { + Name: "ghcr-push-secret", + }, + { + Name: "aws-ecr", + Key: "AWS_ACCESS_KEY_ID", + }, + { + Name: "aws-ecr", + Key: "access-key", + Target: SecretTarget{Env: "AWS_SECRET_ACCESS_KEY"}, + }, + { + Name: "reg-cred", + Key: ".dockerconfigjson", + Target: SecretTarget{File: "~/.docker/config.json"}, + }, + }, + }) + assert.NoError(t, err) + + podJSON, err := json.Marshal(pod) + assert.NoError(t, err) + + ja := jsonassert.New(t) ja.Assertf(string(podJSON), expected) } diff --git a/pipeline/backend/kubernetes/secrets.go b/pipeline/backend/kubernetes/secrets.go new file mode 100644 index 000000000..47fb401da --- /dev/null +++ b/pipeline/backend/kubernetes/secrets.go @@ -0,0 +1,191 @@ +// Copyright 2024 Woodpecker Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package kubernetes + +import ( + "fmt" + "strings" + + "github.com/rs/zerolog/log" + v1 "k8s.io/api/core/v1" +) + +type nativeSecretsProcessor struct { + config *config + secrets []SecretRef + envFromSources []v1.EnvFromSource + envVars []v1.EnvVar + volumes []v1.Volume + mounts []v1.VolumeMount +} + +func newNativeSecretsProcessor(config *config, secrets []SecretRef) nativeSecretsProcessor { + return nativeSecretsProcessor{ + config: config, + secrets: secrets, + } +} + +func (nsp *nativeSecretsProcessor) isEnabled() bool { + return nsp.config.NativeSecretsAllowFromStep +} + +func (nsp *nativeSecretsProcessor) process() error { + if len(nsp.secrets) > 0 { + if !nsp.isEnabled() { + log.Debug().Msg("Secret names were defined in backend options, but secret access is disallowed by instance configuration.") + return nil + } + } else { + return nil + } + + for _, secret := range nsp.secrets { + switch { + case secret.isSimple(): + simpleSecret, err := secret.toEnvFromSource() + if err != nil { + return err + } + nsp.envFromSources = append(nsp.envFromSources, simpleSecret) + case secret.isAdvanced(): + advancedSecret, err := secret.toEnvVar() + if err != nil { + return err + } + nsp.envVars = append(nsp.envVars, advancedSecret) + case secret.isFile(): + volume, err := secret.toVolume() + if err != nil { + return err + } + nsp.volumes = append(nsp.volumes, volume) + + mount, err := secret.toVolumeMount() + if err != nil { + return err + } + nsp.mounts = append(nsp.mounts, mount) + } + } + + return nil +} + +func (sr SecretRef) isSimple() bool { + return len(sr.Key) == 0 && len(sr.Target.Env) == 0 && !sr.isFile() +} + +func (sr SecretRef) isAdvanced() bool { + return (len(sr.Key) > 0 || len(sr.Target.Env) > 0) && !sr.isFile() +} + +func (sr SecretRef) isFile() bool { + return len(sr.Target.File) > 0 +} + +func (sr SecretRef) toEnvFromSource() (v1.EnvFromSource, error) { + env := v1.EnvFromSource{} + + if !sr.isSimple() { + return env, fmt.Errorf("secret '%s' is not simple reference", sr.Name) + } + + env = v1.EnvFromSource{ + SecretRef: &v1.SecretEnvSource{ + LocalObjectReference: secretReference(sr.Name), + }, + } + + return env, nil +} + +func (sr SecretRef) toEnvVar() (v1.EnvVar, error) { + envVar := v1.EnvVar{} + + if !sr.isAdvanced() { + return envVar, fmt.Errorf("secret '%s' is not advanced reference", sr.Name) + } + + envVar.ValueFrom = &v1.EnvVarSource{ + SecretKeyRef: &v1.SecretKeySelector{ + LocalObjectReference: secretReference(sr.Name), + Key: sr.Key, + }, + } + + if len(sr.Target.Env) > 0 { + envVar.Name = sr.Target.Env + } else { + envVar.Name = strings.ToUpper(sr.Key) + } + + return envVar, nil +} + +func (sr SecretRef) toVolume() (v1.Volume, error) { + var err error + volume := v1.Volume{} + + if !sr.isFile() { + return volume, fmt.Errorf("secret '%s' is not file reference", sr.Name) + } + + volume.Name, err = volumeName(sr.Name) + if err != nil { + return volume, err + } + + volume.Secret = &v1.SecretVolumeSource{ + SecretName: sr.Name, + } + + return volume, nil +} + +func (sr SecretRef) toVolumeMount() (v1.VolumeMount, error) { + var err error + mount := v1.VolumeMount{ + ReadOnly: true, + } + + if !sr.isFile() { + return mount, fmt.Errorf("secret '%s' is not file reference", sr.Name) + } + + mount.Name, err = volumeName(sr.Name) + if err != nil { + return mount, err + } + + mount.MountPath = sr.Target.File + mount.SubPath = sr.Key + + return mount, nil +} + +func secretsReferences(names []string) []v1.LocalObjectReference { + secretReferences := make([]v1.LocalObjectReference, len(names)) + for i, imagePullSecretName := range names { + secretReferences[i] = secretReference(imagePullSecretName) + } + return secretReferences +} + +func secretReference(name string) v1.LocalObjectReference { + return v1.LocalObjectReference{ + Name: name, + } +} diff --git a/pipeline/backend/kubernetes/secrets_test.go b/pipeline/backend/kubernetes/secrets_test.go new file mode 100644 index 000000000..fe0c76097 --- /dev/null +++ b/pipeline/backend/kubernetes/secrets_test.go @@ -0,0 +1,180 @@ +// Copyright 2024 Woodpecker Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package kubernetes + +import ( + "testing" + + "github.com/stretchr/testify/assert" + v1 "k8s.io/api/core/v1" +) + +func TestNativeSecretsEnabled(t *testing.T) { + nsp := newNativeSecretsProcessor(&config{ + NativeSecretsAllowFromStep: true, + }, nil) + assert.Equal(t, true, nsp.isEnabled()) +} + +func TestNativeSecretsDisabled(t *testing.T) { + nsp := newNativeSecretsProcessor(&config{ + NativeSecretsAllowFromStep: false, + }, []SecretRef{ + { + Name: "env-simple", + }, + { + Name: "env-advanced", + Key: "key", + Target: SecretTarget{ + Env: "ENV_VAR", + }, + }, + { + Name: "env-file", + Key: "cert", + Target: SecretTarget{ + File: "/etc/ca/x3.cert", + }, + }, + }) + assert.Equal(t, false, nsp.isEnabled()) + + err := nsp.process() + assert.NoError(t, err) + assert.Empty(t, nsp.envFromSources) + assert.Empty(t, nsp.envVars) + assert.Empty(t, nsp.volumes) + assert.Empty(t, nsp.mounts) +} + +func TestSimpleSecret(t *testing.T) { + nsp := newNativeSecretsProcessor(&config{ + NativeSecretsAllowFromStep: true, + }, []SecretRef{ + { + Name: "test-secret", + }, + }) + + err := nsp.process() + assert.NoError(t, err) + assert.Empty(t, nsp.envVars) + assert.Empty(t, nsp.volumes) + assert.Empty(t, nsp.mounts) + assert.Equal(t, []v1.EnvFromSource{ + { + SecretRef: &v1.SecretEnvSource{ + LocalObjectReference: v1.LocalObjectReference{Name: "test-secret"}, + }, + }, + }, nsp.envFromSources) +} + +func TestSecretWithKey(t *testing.T) { + nsp := newNativeSecretsProcessor(&config{ + NativeSecretsAllowFromStep: true, + }, []SecretRef{ + { + Name: "test-secret", + Key: "access_key", + }, + }) + + err := nsp.process() + assert.NoError(t, err) + assert.Empty(t, nsp.envFromSources) + assert.Empty(t, nsp.volumes) + assert.Empty(t, nsp.mounts) + assert.Equal(t, []v1.EnvVar{ + { + Name: "ACCESS_KEY", + ValueFrom: &v1.EnvVarSource{ + SecretKeyRef: &v1.SecretKeySelector{ + LocalObjectReference: v1.LocalObjectReference{Name: "test-secret"}, + Key: "access_key", + }, + }, + }, + }, nsp.envVars) +} + +func TestSecretWithKeyMapping(t *testing.T) { + nsp := newNativeSecretsProcessor(&config{ + NativeSecretsAllowFromStep: true, + }, []SecretRef{ + { + Name: "test-secret", + Key: "aws-secret", + Target: SecretTarget{ + Env: "AWS_SECRET_ACCESS_KEY", + }, + }, + }) + + err := nsp.process() + assert.NoError(t, err) + assert.Empty(t, nsp.envFromSources) + assert.Empty(t, nsp.volumes) + assert.Empty(t, nsp.mounts) + assert.Equal(t, []v1.EnvVar{ + { + Name: "AWS_SECRET_ACCESS_KEY", + ValueFrom: &v1.EnvVarSource{ + SecretKeyRef: &v1.SecretKeySelector{ + LocalObjectReference: v1.LocalObjectReference{Name: "test-secret"}, + Key: "aws-secret", + }, + }, + }, + }, nsp.envVars) +} + +func TestFileSecret(t *testing.T) { + nsp := newNativeSecretsProcessor(&config{ + NativeSecretsAllowFromStep: true, + }, []SecretRef{ + { + Name: "reg-cred", + Key: ".dockerconfigjson", + Target: SecretTarget{ + File: "~/.docker/config.json", + }, + }, + }) + + err := nsp.process() + assert.NoError(t, err) + assert.Empty(t, nsp.envFromSources) + assert.Empty(t, nsp.envVars) + assert.Equal(t, []v1.Volume{ + { + Name: "reg-cred", + VolumeSource: v1.VolumeSource{ + Secret: &v1.SecretVolumeSource{ + SecretName: "reg-cred", + }, + }, + }, + }, nsp.volumes) + assert.Equal(t, []v1.VolumeMount{ + { + Name: "reg-cred", + ReadOnly: true, + MountPath: "~/.docker/config.json", + SubPath: ".dockerconfigjson", + }, + }, nsp.mounts) +} diff --git a/pipeline/frontend/yaml/linter/schema/schema.json b/pipeline/frontend/yaml/linter/schema/schema.json index f735c0ea3..79f5cb21c 100644 --- a/pipeline/frontend/yaml/linter/schema/schema.json +++ b/pipeline/frontend/yaml/linter/schema/schema.json @@ -730,6 +730,14 @@ "runtimeClassName": { "description": "Read more: https://woodpecker-ci.org/docs/administration/backends/kubernetes#runtimeclassname", "type": "string" + }, + "secrets": { + "description": "The secrets section defines a list of references to the native Kubernetes secrets", + "type": "array", + "items": { + "$ref": "#/definitions/step_kubernetes_secret" + }, + "minLength": 1 } } }, @@ -810,6 +818,41 @@ "type": "string" } }, + "step_kubernetes_secret": { + "description": "A reference to a native Kubernetes secret", + "type": "object", + "additionalProperties": false, + "properties": { + "name": { + "description": "The name of the secret. Can be used if using the array style secrets list.", + "type": "string" + }, + "key": { + "description": "The key of the secret to select from.", + "type": "string" + }, + "target": { + "$ref": "#/definitions/step_kubernetes_secret_target" + } + } + }, + "step_kubernetes_secret_target": { + "description": "A target which a native Kubernetes secret maps to.", + "oneOf": [ + { + "env": { + "description": "The name of the environment variable which secret maps to.", + "type": "string" + } + }, + { + "file": { + "description": "The filename (path) which secret maps to.", + "type": "string" + } + } + ] + }, "services": { "description": "Read more: https://woodpecker-ci.org/docs/usage/services", "oneOf": [