mirrors/woodpecker
1
0
Fork 0
mirror of https://github.com/woodpecker-ci/woodpecker.git synced 2024-11-29 05:11:04 +00:00
Code Issues Projects Releases Wiki Activity

Allow setting resources for kubernetes on a per-step basis (#1767)

Browse source 
This add a simple implementation of requests/limits for individual
steps. There is no validation of what the resource actually is beyond
checking that it can successfully be converted to a Quantity, so it can
be used for things other than just memory/CPU.

close #1809
This commit is contained in:
Stephen Muth 2023-06-03 18:50:08 -04:00 committed by GitHub
parent 14177635b6
commit 2941e508b3
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
10 changed files with 273 additions and 131 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
docs/docs/30-administration/15-agent-config.md
View file

@ -123,15 +123,19 @@ Configures if the gRPC server certificate should be verified, only valid when `W
### `WOODPECKER_BACKEND` ### `WOODPECKER_BACKEND`
> Default: `auto-detect` > Default: `auto-detect`
Configures the backend engine to run pipelines on. Possible values are `auto-detect`, `docker`, `local` or `ssh`. Configures the backend engine to run pipelines on. Possible values are `auto-detect`, `docker`, `local`, `ssh` or `kubernetes`.
### `WOODPECKER_BACKEND_DOCKER_*` ### `WOODPECKER_BACKEND_DOCKER_*`
See [Docker backend configuration](backends/docker/#configuration) See [Docker backend configuration](./22-backends/10-docker.md#configuration)
### `WOODPECKER_BACKEND_SSH_*` ### `WOODPECKER_BACKEND_SSH_*`
See [SSH backend configuration](backends/ssh/#configuration) See [SSH backend configuration](./22-backends/30-ssh.md#configuration)
### `WOODPECKER_BACKEND_K8S_*`
See [Kubernetes backend configuration](./22-backends/40-kubernetes.md#configuration)
## Advanced Settings ## Advanced Settings

66
docs/docs/30-administration/22-backends/40-kubernetes.md Normal file
View file

@ -0,0 +1,66 @@
# Kubernetes backend
:::caution
Kubernetes support is still experimental and not all pipeline features are fully supported yet.
Check the [current state](https://github.com/woodpecker-ci/woodpecker/issues/1513)
:::
The kubernetes backend executes each step inside a newly created pod. A PVC is also created for the lifetime of the pipeline, for transferring files between steps.
## Configuration
### `WOODPECKER_BACKEND_K8S_NAMESPACE`
> Default: `woodpecker`
The namespace to create worker pods in.
### `WOODPECKER_BACKEND_K8S_VOLUME_SIZE`
> Default: `10G`
The volume size of the pipeline volume.
### `WOODPECKER_BACKEND_K8S_STORAGE_CLASS`
> Default: empty
The storage class to use for the pipeline volume.
### `WOODPECKER_BACKEND_K8S_STORAGE_RWX`
> Default: `true`
Determines if `RWX` should be used for the pipeline volume's [access mode](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#access-modes). If false, `RWO` is used instead.
### `WOODPECKER_BACKEND_K8S_POD_LABELS`
> Default: empty
Additional labels to apply to worker pods. Must be a YAML object, e.g. `{"example.com/test-label":"test-value"}`.
### `WOODPECKER_BACKEND_K8S_POD_ANNOTATIONS`
> Default: empty
Additional annotations to apply to worker pods. Must be a YAML object, e.g. `{"example.com/test-annotation":"test-value"}`.
## Resources
The kubernetes backend also allows for specifying requests and limits on a per-step basic, most commonly for CPU and memory.
Example pipeline configuration:
```yaml
pipeline:
build:
image: golang
commands:
- go get
- go build
- go test
backend_options:
kubernetes:
resources:
requests:
memory: 128Mi
cpu: 1000m
limits:
memory: 256Mi
```
See the [kubernetes documentation](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/) for more information on using resources.

2
docs/docs/30-administration/80-kubernetes.md
View file

@ -1,6 +1,6 @@
# Kubernetes # Kubernetes
Woodpecker does support Kubernetes as a backend. Woodpecker does support Kubernetes as a backend. See the [Kubernetes backend configuration](./22-backends/40-kubernetes.md#configuration) for backend-specific options.
:::caution :::caution
Kubernetes support is still experimental and not all pipeline features are fully supported yet. Kubernetes support is still experimental and not all pipeline features are fully supported yet.

47
pipeline/backend/kubernetes/pod.go
View file

@ -1,6 +1,7 @@
package kubernetes package kubernetes
import ( import (
"fmt"
"strings" "strings"
"github.com/woodpecker-ci/woodpecker/pipeline/backend/common" "github.com/woodpecker-ci/woodpecker/pipeline/backend/common"
@ -62,35 +63,21 @@ func Pod(namespace string, step *types.Step, labels, annotations map[string]stri
hostAliases = append(hostAliases, v1.HostAlias{IP: host[1], Hostnames: []string{host[0]}}) hostAliases = append(hostAliases, v1.HostAlias{IP: host[1], Hostnames: []string{host[0]}})
} }
// TODO: add support for resource limits resourceRequirements := v1.ResourceRequirements{Requests: v1.ResourceList{}, Limits: v1.ResourceList{}}
// if step.Resources.CPULimit == "" { var err error
// step.Resources.CPULimit = "2" for key, val := range step.BackendOptions.Kubernetes.Resources.Requests {
// } resourceKey := v1.ResourceName(key)
// if step.Resources.MemoryLimit == "" { resourceRequirements.Requests[resourceKey], err = resource.ParseQuantity(val)
// step.Resources.MemoryLimit = "2G" if err != nil {
// } return nil, fmt.Errorf("resource request '%v' quantity '%v': %w", key, val, err)
// memoryLimit := resource.MustParse(step.Resources.MemoryLimit) }
// CPULimit := resource.MustParse(step.Resources.CPULimit) }
for key, val := range step.BackendOptions.Kubernetes.Resources.Limits {
memoryLimit := resource.MustParse("2G") resourceKey := v1.ResourceName(key)
CPULimit := resource.MustParse("2") resourceRequirements.Limits[resourceKey], err = resource.ParseQuantity(val)
if err != nil {
memoryLimitValue, _ := memoryLimit.AsInt64() return nil, fmt.Errorf("resource limit '%v' quantity '%v': %w", key, val, err)
CPULimitValue, _ := CPULimit.AsInt64() }
loadfactor := 0.5
memoryRequest := resource.NewQuantity(int64(float64(memoryLimitValue)*loadfactor), resource.DecimalSI)
CPURequest := resource.NewQuantity(int64(float64(CPULimitValue)*loadfactor), resource.DecimalSI)
resources := v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceMemory: *memoryRequest,
v1.ResourceCPU: *CPURequest,
},
Limits: v1.ResourceList{
v1.ResourceMemory: memoryLimit,
v1.ResourceCPU: CPULimit,
},
} }
podName, err := dnsName(step.Name) podName, err := dnsName(step.Name)
@ -130,7 +117,7 @@ func Pod(namespace string, step *types.Step, labels, annotations map[string]stri
WorkingDir: step.WorkingDir, WorkingDir: step.WorkingDir,
Env: mapToEnvVars(step.Environment), Env: mapToEnvVars(step.Environment),
VolumeMounts: volMounts, VolumeMounts: volMounts,
Resources: resources, Resources: resourceRequirements,
SecurityContext: &v1.SecurityContext{ SecurityContext: &v1.SecurityContext{
Privileged: &step.Privileged, Privileged: &step.Privileged,
}, },

6
pipeline/backend/types/backend.go Normal file
View file

@ -0,0 +1,6 @@
package types
// BackendOptions defines advanced options for specific backends
type BackendOptions struct {
Kubernetes KubernetesBackendOptions `json:"kubernetes,omitempty"`
}

12
pipeline/backend/types/backend_kubernetes.go Normal file
View file

@ -0,0 +1,12 @@
package types
// KubernetesBackendOptions defines all the advanced options for the kubernetes backend
type KubernetesBackendOptions struct {
Resources Resources `json:"resouces,omitempty"`
}
// Resources defines two maps for kubernetes resource definitions
type Resources struct {
Requests map[string]string `json:"requests,omitempty"`
Limits map[string]string `json:"limits,omitempty"`
}

63
pipeline/backend/types/step.go
View file

@ -2,35 +2,36 @@ package types
// Step defines a container process. // Step defines a container process.
type Step struct { type Step struct {
Name string `json:"name"` Name string `json:"name"`
Alias string `json:"alias,omitempty"` Alias string `json:"alias,omitempty"`
Image string `json:"image,omitempty"` Image string `json:"image,omitempty"`
Pull bool `json:"pull,omitempty"` Pull bool `json:"pull,omitempty"`
Detached bool `json:"detach,omitempty"` Detached bool `json:"detach,omitempty"`
Privileged bool `json:"privileged,omitempty"` Privileged bool `json:"privileged,omitempty"`
WorkingDir string `json:"working_dir,omitempty"` WorkingDir string `json:"working_dir,omitempty"`
Environment map[string]string `json:"environment,omitempty"` Environment map[string]string `json:"environment,omitempty"`
Labels map[string]string `json:"labels,omitempty"` Labels map[string]string `json:"labels,omitempty"`
Entrypoint []string `json:"entrypoint,omitempty"` Entrypoint []string `json:"entrypoint,omitempty"`
Commands []string `json:"commands,omitempty"` Commands []string `json:"commands,omitempty"`
ExtraHosts []string `json:"extra_hosts,omitempty"` ExtraHosts []string `json:"extra_hosts,omitempty"`
Volumes []string `json:"volumes,omitempty"` Volumes []string `json:"volumes,omitempty"`
Tmpfs []string `json:"tmpfs,omitempty"` Tmpfs []string `json:"tmpfs,omitempty"`
Devices []string `json:"devices,omitempty"` Devices []string `json:"devices,omitempty"`
Networks []Conn `json:"networks,omitempty"` Networks []Conn `json:"networks,omitempty"`
DNS []string `json:"dns,omitempty"` DNS []string `json:"dns,omitempty"`
DNSSearch []string `json:"dns_search,omitempty"` DNSSearch []string `json:"dns_search,omitempty"`
MemSwapLimit int64 `json:"memswap_limit,omitempty"` MemSwapLimit int64 `json:"memswap_limit,omitempty"`
MemLimit int64 `json:"mem_limit,omitempty"` MemLimit int64 `json:"mem_limit,omitempty"`
ShmSize int64 `json:"shm_size,omitempty"` ShmSize int64 `json:"shm_size,omitempty"`
CPUQuota int64 `json:"cpu_quota,omitempty"` CPUQuota int64 `json:"cpu_quota,omitempty"`
CPUShares int64 `json:"cpu_shares,omitempty"` CPUShares int64 `json:"cpu_shares,omitempty"`
CPUSet string `json:"cpu_set,omitempty"` CPUSet string `json:"cpu_set,omitempty"`
OnFailure bool `json:"on_failure,omitempty"` OnFailure bool `json:"on_failure,omitempty"`
OnSuccess bool `json:"on_success,omitempty"` OnSuccess bool `json:"on_success,omitempty"`
Failure string `json:"failure,omitempty"` Failure string `json:"failure,omitempty"`
AuthConfig Auth `json:"auth_config,omitempty"` AuthConfig Auth `json:"auth_config,omitempty"`
NetworkMode string `json:"network_mode,omitempty"` NetworkMode string `json:"network_mode,omitempty"`
IpcMode string `json:"ipc_mode,omitempty"` IpcMode string `json:"ipc_mode,omitempty"`
Sysctls map[string]string `json:"sysctls,omitempty"` Sysctls map[string]string `json:"sysctls,omitempty"`
BackendOptions BackendOptions `json:"backend_options,omitempty"`
} }

71
pipeline/frontend/yaml/compiler/convert.go
View file

@ -110,6 +110,16 @@ func (c *Compiler) createProcess(name string, container *yaml.Container, section
} }
} }
// Kubernetes advanced settings
backendOptions := backend.BackendOptions{
Kubernetes: backend.KubernetesBackendOptions{
Resources: backend.Resources{
Limits: container.BackendOptions.Kubernetes.Resources.Limits,
Requests: container.BackendOptions.Kubernetes.Resources.Requests,
},
},
}
memSwapLimit := int64(container.MemSwapLimit) memSwapLimit := int64(container.MemSwapLimit)
if c.reslimit.MemSwapLimit != 0 { if c.reslimit.MemSwapLimit != 0 {
memSwapLimit = c.reslimit.MemSwapLimit memSwapLimit = c.reslimit.MemSwapLimit
@ -146,36 +156,37 @@ func (c *Compiler) createProcess(name string, container *yaml.Container, section
} }
return &backend.Step{ return &backend.Step{
Name: name, Name: name,
Alias: container.Name, Alias: container.Name,
Image: container.Image, Image: container.Image,
Pull: container.Pull, Pull: container.Pull,
Detached: detached, Detached: detached,
Privileged: privileged, Privileged: privileged,
WorkingDir: workingdir, WorkingDir: workingdir,
Environment: environment, Environment: environment,
Labels: container.Labels, Labels: container.Labels,
Commands: container.Commands, Commands: container.Commands,
ExtraHosts: container.ExtraHosts, ExtraHosts: container.ExtraHosts,
Volumes: volumes, Volumes: volumes,
Tmpfs: container.Tmpfs, Tmpfs: container.Tmpfs,
Devices: container.Devices, Devices: container.Devices,
Networks: networks, Networks: networks,
DNS: container.DNS, DNS: container.DNS,
DNSSearch: container.DNSSearch, DNSSearch: container.DNSSearch,
MemSwapLimit: memSwapLimit, MemSwapLimit: memSwapLimit,
MemLimit: memLimit, MemLimit: memLimit,
ShmSize: shmSize, ShmSize: shmSize,
Sysctls: container.Sysctls, Sysctls: container.Sysctls,
CPUQuota: cpuQuota, CPUQuota: cpuQuota,
CPUShares: cpuShares, CPUShares: cpuShares,
CPUSet: cpuSet, CPUSet: cpuSet,
AuthConfig: authConfig, AuthConfig: authConfig,
OnSuccess: onSuccess, OnSuccess: onSuccess,
OnFailure: onFailure, OnFailure: onFailure,
Failure: failure, Failure: failure,
NetworkMode: networkMode, NetworkMode: networkMode,
IpcMode: ipcMode, IpcMode: ipcMode,
BackendOptions: backendOptions,
} }
} }

87
pipeline/frontend/yaml/container.go
View file

@ -19,6 +19,20 @@ type (
Email string Email string
} }
// Advanced backend options
BackendOptions struct {
Kubernetes KubernetesBackendOptions `yaml:"kubernetes,omitempty"`
}
KubernetesBackendOptions struct {
Resources Resources `yaml:"resources,omitempty"`
}
Resources struct {
Requests map[string]string `yaml:"requests,omitempty"`
Limits map[string]string `yaml:"limits,omitempty"`
}
// Containers denotes an ordered collection of containers. // Containers denotes an ordered collection of containers.
Containers struct { Containers struct {
Containers []*Container Containers []*Container
@ -26,42 +40,43 @@ type (
// Container defines a container. // Container defines a container.
Container struct { Container struct {
AuthConfig AuthConfig `yaml:"auth_config,omitempty"` AuthConfig AuthConfig `yaml:"auth_config,omitempty"`
CapAdd []string `yaml:"cap_add,omitempty"` CapAdd []string `yaml:"cap_add,omitempty"`
CapDrop []string `yaml:"cap_drop,omitempty"` CapDrop []string `yaml:"cap_drop,omitempty"`
Commands types.StringOrSlice `yaml:"commands,omitempty"` Commands types.StringOrSlice `yaml:"commands,omitempty"`
CPUQuota types.StringorInt `yaml:"cpu_quota,omitempty"` CPUQuota types.StringorInt `yaml:"cpu_quota,omitempty"`
CPUSet string `yaml:"cpuset,omitempty"` CPUSet string `yaml:"cpuset,omitempty"`
CPUShares types.StringorInt `yaml:"cpu_shares,omitempty"` CPUShares types.StringorInt `yaml:"cpu_shares,omitempty"`
Detached bool `yaml:"detach,omitempty"` Detached bool `yaml:"detach,omitempty"`
Devices []string `yaml:"devices,omitempty"` Devices []string `yaml:"devices,omitempty"`
Tmpfs []string `yaml:"tmpfs,omitempty"` Tmpfs []string `yaml:"tmpfs,omitempty"`
DNS types.StringOrSlice `yaml:"dns,omitempty"` DNS types.StringOrSlice `yaml:"dns,omitempty"`
DNSSearch types.StringOrSlice `yaml:"dns_search,omitempty"` DNSSearch types.StringOrSlice `yaml:"dns_search,omitempty"`
Directory string `yaml:"directory,omitempty"` Directory string `yaml:"directory,omitempty"`
Environment types.SliceorMap `yaml:"environment,omitempty"` Environment types.SliceorMap `yaml:"environment,omitempty"`
ExtraHosts []string `yaml:"extra_hosts,omitempty"` ExtraHosts []string `yaml:"extra_hosts,omitempty"`
Group string `yaml:"group,omitempty"` Group string `yaml:"group,omitempty"`
Image string `yaml:"image,omitempty"` Image string `yaml:"image,omitempty"`
Failure string `yaml:"failure,omitempty"` Failure string `yaml:"failure,omitempty"`
Isolation string `yaml:"isolation,omitempty"` Isolation string `yaml:"isolation,omitempty"`
Labels types.SliceorMap `yaml:"labels,omitempty"` Labels types.SliceorMap `yaml:"labels,omitempty"`
MemLimit types.MemStringorInt `yaml:"mem_limit,omitempty"` MemLimit types.MemStringorInt `yaml:"mem_limit,omitempty"`
MemSwapLimit types.MemStringorInt `yaml:"memswap_limit,omitempty"` MemSwapLimit types.MemStringorInt `yaml:"memswap_limit,omitempty"`
MemSwappiness types.MemStringorInt `yaml:"mem_swappiness,omitempty"` MemSwappiness types.MemStringorInt `yaml:"mem_swappiness,omitempty"`
Name string `yaml:"name,omitempty"` Name string `yaml:"name,omitempty"`
NetworkMode string `yaml:"network_mode,omitempty"` NetworkMode string `yaml:"network_mode,omitempty"`
IpcMode string `yaml:"ipc_mode,omitempty"` IpcMode string `yaml:"ipc_mode,omitempty"`
Networks types.Networks `yaml:"networks,omitempty"` Networks types.Networks `yaml:"networks,omitempty"`
Privileged bool `yaml:"privileged,omitempty"` Privileged bool `yaml:"privileged,omitempty"`
Pull bool `yaml:"pull,omitempty"` Pull bool `yaml:"pull,omitempty"`
ShmSize types.MemStringorInt `yaml:"shm_size,omitempty"` ShmSize types.MemStringorInt `yaml:"shm_size,omitempty"`
Ulimits types.Ulimits `yaml:"ulimits,omitempty"` Ulimits types.Ulimits `yaml:"ulimits,omitempty"`
Volumes types.Volumes `yaml:"volumes,omitempty"` Volumes types.Volumes `yaml:"volumes,omitempty"`
Secrets Secrets `yaml:"secrets,omitempty"` Secrets Secrets `yaml:"secrets,omitempty"`
Sysctls types.SliceorMap `yaml:"sysctls,omitempty"` Sysctls types.SliceorMap `yaml:"sysctls,omitempty"`
When constraint.When `yaml:"when,omitempty"` When constraint.When `yaml:"when,omitempty"`
Settings map[string]interface{} `yaml:"settings"` Settings map[string]interface{} `yaml:"settings"`
BackendOptions BackendOptions `yaml:"backend_options,omitempty"`
} }
) )

40
pipeline/schema/schema.json
View file

@ -243,6 +243,9 @@
"type": "string", "type": "string",
"enum": ["fail", "ignore"], "enum": ["fail", "ignore"],
"default": "fail" "default": "fail"
},
"backend_options": {
"$ref": "#/definitions/step_backend_options"
} }
} }
}, },
@ -480,6 +483,43 @@
"description": "Read more: https://woodpecker-ci.org/docs/usage/pipeline-syntax#directory", "description": "Read more: https://woodpecker-ci.org/docs/usage/pipeline-syntax#directory",
"type": "string" "type": "string"
}, },
"step_backend_options": {
"description": "Advanced options for the different agent backends",
"type": "object",
"properties": {
"kubernetes" :{
"$ref": "#/definitions/step_backend_kubernetes_resources"
}
}
},
"step_backend_kubernetes": {
"description": "Advanced options for the kubernetes agent backends",
"type": "object",
"properties": {
"resources" :{
"$ref": "#/definitions/step_backend_kubernetes_resources"
}
}
},
"step_backend_kubernetes_resources": {
"description": "Resources for the kubernetes backend. Read more: https://woodpecker-ci.org/docs/administration/backends/kubernetes",
"type": "object",
"properties": {
"requests": {
"$ref": "#/definitions/step_kubernetes_resources_object"
},
"limits": {
"$ref": "#/definitions/step_kubernetes_resources_object"
}
}
},
"step_kubernetes_resources_object": {
"description": "A list of kubernetes resource mappings",
"type": "object",
"additionalProperties": {
"type": "string"
}
},
"services": { "services": {
"description": "Read more: https://woodpecker-ci.org/docs/usage/services", "description": "Read more: https://woodpecker-ci.org/docs/usage/services",
"type": "object", "type": "object",