mirror of
https://github.com/woodpecker-ci/woodpecker.git
synced 2024-11-26 11:51:02 +00:00
Agent-wide node selector (#3608)
Co-authored-by: qwerty287 <80460567+qwerty287@users.noreply.github.com>
This commit is contained in:
parent
c53a05b6fb
commit
065eebd306
5 changed files with 33 additions and 4 deletions
|
@ -79,7 +79,8 @@ And then overwrite the `nodeSelector` in the `backend_options` section of the st
|
||||||
kubernetes.io/arch: "${ARCH}"
|
kubernetes.io/arch: "${ARCH}"
|
||||||
```
|
```
|
||||||
|
|
||||||
You can use [PodNodeSelector](https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers/#podnodeselector) admission controller if you want to set the node selector by per-namespace basis.
|
You can use [WOODPECKER_BACKEND_K8S_POD_NODE_SELECTOR](#woodpecker_backend_k8s_pod_node_selector) if you want to set the node selector per Agent
|
||||||
|
or [PodNodeSelector](https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers/#podnodeselector) admission controller if you want to set the node selector by per-namespace basis.
|
||||||
|
|
||||||
### Tolerations
|
### Tolerations
|
||||||
|
|
||||||
|
@ -279,6 +280,12 @@ Additional annotations to apply to worker Pods. Must be a YAML object, e.g. `{"e
|
||||||
|
|
||||||
Determines if Pod annotations can be defined from a step's backend options.
|
Determines if Pod annotations can be defined from a step's backend options.
|
||||||
|
|
||||||
|
### `WOODPECKER_BACKEND_K8S_POD_NODE_SELECTOR`
|
||||||
|
|
||||||
|
> Default: empty
|
||||||
|
|
||||||
|
Additional node selector to apply to worker pods. Must be a YAML object, e.g. `{"topology.kubernetes.io/region":"eu-central-1"}`.
|
||||||
|
|
||||||
### `WOODPECKER_BACKEND_K8S_SECCTX_NONROOT`
|
### `WOODPECKER_BACKEND_K8S_SECCTX_NONROOT`
|
||||||
|
|
||||||
> Default: `false`
|
> Default: `false`
|
||||||
|
|
|
@ -61,6 +61,12 @@ var Flags = []cli.Flag{
|
||||||
Usage: "backend k8s additional Agent-wide worker pod annotations",
|
Usage: "backend k8s additional Agent-wide worker pod annotations",
|
||||||
Value: "",
|
Value: "",
|
||||||
},
|
},
|
||||||
|
&cli.StringFlag{
|
||||||
|
EnvVars: []string{"WOODPECKER_BACKEND_K8S_POD_NODE_SELECTOR"},
|
||||||
|
Name: "backend-k8s-pod-node-selector",
|
||||||
|
Usage: "backend k8s Agent-wide worker pod node selector",
|
||||||
|
Value: "",
|
||||||
|
},
|
||||||
&cli.BoolFlag{
|
&cli.BoolFlag{
|
||||||
EnvVars: []string{"WOODPECKER_BACKEND_K8S_POD_ANNOTATIONS_ALLOW_FROM_STEP"},
|
EnvVars: []string{"WOODPECKER_BACKEND_K8S_POD_ANNOTATIONS_ALLOW_FROM_STEP"},
|
||||||
Name: "backend-k8s-pod-annotations-allow-from-step",
|
Name: "backend-k8s-pod-annotations-allow-from-step",
|
||||||
|
|
|
@ -62,6 +62,7 @@ type config struct {
|
||||||
PodLabelsAllowFromStep bool
|
PodLabelsAllowFromStep bool
|
||||||
PodAnnotations map[string]string
|
PodAnnotations map[string]string
|
||||||
PodAnnotationsAllowFromStep bool
|
PodAnnotationsAllowFromStep bool
|
||||||
|
PodNodeSelector map[string]string
|
||||||
ImagePullSecretNames []string
|
ImagePullSecretNames []string
|
||||||
SecurityContext SecurityContextConfig
|
SecurityContext SecurityContextConfig
|
||||||
}
|
}
|
||||||
|
@ -91,6 +92,7 @@ func configFromCliContext(ctx context.Context) (*config, error) {
|
||||||
PodLabelsAllowFromStep: c.Bool("backend-k8s-pod-labels-allow-from-step"),
|
PodLabelsAllowFromStep: c.Bool("backend-k8s-pod-labels-allow-from-step"),
|
||||||
PodAnnotations: make(map[string]string), // just init empty map to prevent nil panic
|
PodAnnotations: make(map[string]string), // just init empty map to prevent nil panic
|
||||||
PodAnnotationsAllowFromStep: c.Bool("backend-k8s-pod-annotations-allow-from-step"),
|
PodAnnotationsAllowFromStep: c.Bool("backend-k8s-pod-annotations-allow-from-step"),
|
||||||
|
PodNodeSelector: make(map[string]string), // just init empty map to prevent nil panic
|
||||||
ImagePullSecretNames: c.StringSlice("backend-k8s-pod-image-pull-secret-names"),
|
ImagePullSecretNames: c.StringSlice("backend-k8s-pod-image-pull-secret-names"),
|
||||||
SecurityContext: SecurityContextConfig{
|
SecurityContext: SecurityContextConfig{
|
||||||
RunAsNonRoot: c.Bool("backend-k8s-secctx-nonroot"), // cspell:words secctx nonroot
|
RunAsNonRoot: c.Bool("backend-k8s-secctx-nonroot"), // cspell:words secctx nonroot
|
||||||
|
@ -113,6 +115,12 @@ func configFromCliContext(ctx context.Context) (*config, error) {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if nodeSelector := c.String("backend-k8s-pod-node-selector"); nodeSelector != "" {
|
||||||
|
if err := yaml.Unmarshal([]byte(nodeSelector), &config.PodNodeSelector); err != nil {
|
||||||
|
log.Error().Err(err).Msgf("could not unmarshal pod node selector '%s'", nodeSelector)
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
}
|
||||||
return &config, nil
|
return &config, nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -173,6 +181,7 @@ func (e *kube) getConfig() *config {
|
||||||
c := *e.config
|
c := *e.config
|
||||||
c.PodLabels = maps.Clone(e.config.PodLabels)
|
c.PodLabels = maps.Clone(e.config.PodLabels)
|
||||||
c.PodAnnotations = maps.Clone(e.config.PodAnnotations)
|
c.PodAnnotations = maps.Clone(e.config.PodAnnotations)
|
||||||
|
c.PodNodeSelector = maps.Clone(e.config.PodNodeSelector)
|
||||||
c.ImagePullSecretNames = slices.Clone(e.config.ImagePullSecretNames)
|
c.ImagePullSecretNames = slices.Clone(e.config.ImagePullSecretNames)
|
||||||
return &c
|
return &c
|
||||||
}
|
}
|
||||||
|
|
|
@ -154,7 +154,7 @@ func podSpec(step *types.Step, config *config, options BackendOptions) (v1.PodSp
|
||||||
ServiceAccountName: options.ServiceAccountName,
|
ServiceAccountName: options.ServiceAccountName,
|
||||||
ImagePullSecrets: imagePullSecretsReferences(config.ImagePullSecretNames),
|
ImagePullSecrets: imagePullSecretsReferences(config.ImagePullSecretNames),
|
||||||
HostAliases: hostAliases(step.ExtraHosts),
|
HostAliases: hostAliases(step.ExtraHosts),
|
||||||
NodeSelector: nodeSelector(options.NodeSelector, step.Environment["CI_SYSTEM_PLATFORM"]),
|
NodeSelector: nodeSelector(options.NodeSelector, config.PodNodeSelector, step.Environment["CI_SYSTEM_PLATFORM"]),
|
||||||
Tolerations: tolerations(options.Tolerations),
|
Tolerations: tolerations(options.Tolerations),
|
||||||
SecurityContext: podSecurityContext(options.SecurityContext, config.SecurityContext, step.Privileged),
|
SecurityContext: podSecurityContext(options.SecurityContext, config.SecurityContext, step.Privileged),
|
||||||
}
|
}
|
||||||
|
@ -331,7 +331,7 @@ func resourceList(resources map[string]string) (v1.ResourceList, error) {
|
||||||
return requestResources, nil
|
return requestResources, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func nodeSelector(backendNodeSelector map[string]string, platform string) map[string]string {
|
func nodeSelector(backendNodeSelector, configNodeSelector map[string]string, platform string) map[string]string {
|
||||||
nodeSelector := make(map[string]string)
|
nodeSelector := make(map[string]string)
|
||||||
|
|
||||||
if platform != "" {
|
if platform != "" {
|
||||||
|
@ -340,6 +340,11 @@ func nodeSelector(backendNodeSelector map[string]string, platform string) map[st
|
||||||
log.Trace().Msgf("using the node selector from the Agent's platform: %v", nodeSelector)
|
log.Trace().Msgf("using the node selector from the Agent's platform: %v", nodeSelector)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if len(configNodeSelector) > 0 {
|
||||||
|
log.Trace().Msgf("appending labels to the node selector from the configuration: %v", configNodeSelector)
|
||||||
|
maps.Copy(nodeSelector, configNodeSelector)
|
||||||
|
}
|
||||||
|
|
||||||
if len(backendNodeSelector) > 0 {
|
if len(backendNodeSelector) > 0 {
|
||||||
log.Trace().Msgf("appending labels to the node selector from the backend options: %v", backendNodeSelector)
|
log.Trace().Msgf("appending labels to the node selector from the backend options: %v", backendNodeSelector)
|
||||||
maps.Copy(nodeSelector, backendNodeSelector)
|
maps.Copy(nodeSelector, backendNodeSelector)
|
||||||
|
|
|
@ -240,7 +240,8 @@ func TestFullPod(t *testing.T) {
|
||||||
],
|
],
|
||||||
"restartPolicy": "Never",
|
"restartPolicy": "Never",
|
||||||
"nodeSelector": {
|
"nodeSelector": {
|
||||||
"storage": "ssd"
|
"storage": "ssd",
|
||||||
|
"topology.kubernetes.io/region": "eu-central-1"
|
||||||
},
|
},
|
||||||
"runtimeClassName": "runc",
|
"runtimeClassName": "runc",
|
||||||
"serviceAccountName": "wp-svc-acc",
|
"serviceAccountName": "wp-svc-acc",
|
||||||
|
@ -331,6 +332,7 @@ func TestFullPod(t *testing.T) {
|
||||||
PodLabelsAllowFromStep: true,
|
PodLabelsAllowFromStep: true,
|
||||||
PodAnnotations: map[string]string{"apps.kubernetes.io/pod-index": "0"},
|
PodAnnotations: map[string]string{"apps.kubernetes.io/pod-index": "0"},
|
||||||
PodAnnotationsAllowFromStep: true,
|
PodAnnotationsAllowFromStep: true,
|
||||||
|
PodNodeSelector: map[string]string{"topology.kubernetes.io/region": "eu-central-1"},
|
||||||
SecurityContext: SecurityContextConfig{RunAsNonRoot: false},
|
SecurityContext: SecurityContextConfig{RunAsNonRoot: false},
|
||||||
}, "wp-01he8bebctabr3kgk0qj36d2me-0", "linux/amd64", BackendOptions{
|
}, "wp-01he8bebctabr3kgk0qj36d2me-0", "linux/amd64", BackendOptions{
|
||||||
Labels: map[string]string{"part-of": "woodpecker-ci"},
|
Labels: map[string]string{"part-of": "woodpecker-ci"},
|
||||||
|
|
Loading…
Reference in a new issue