mirrors/woodpecker
1
0
Fork 0
mirror of https://github.com/woodpecker-ci/woodpecker.git synced 2024-11-25 19:31:05 +00:00
Code Issues Projects Releases Wiki Activity

Allow setting resources for kubernetes on a per-step basis (#1767)

Browse source 
This add a simple implementation of requests/limits for individual
steps. There is no validation of what the resource actually is beyond
checking that it can successfully be converted to a Quantity, so it can
be used for things other than just memory/CPU.

close #1809
This commit is contained in:
Stephen Muth 2023-06-03 18:50:08 -04:00 committed by GitHub
parent 14177635b6
commit 2941e508b3
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
10 changed files with 273 additions and 131 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
docs/docs/30-administration/15-agent-config.md
View file

@ -123,15 +123,19 @@ Configures if the gRPC server certificate should be verified, only valid when `W
### `WOODPECKER_BACKEND`
> Default: `auto-detect`
Configures the backend engine to run pipelines on. Possible values are `auto-detect`, `docker`, `local` or `ssh`.
Configures the backend engine to run pipelines on. Possible values are `auto-detect`, `docker`, `local`, `ssh` or `kubernetes`.
### `WOODPECKER_BACKEND_DOCKER_*`
See [Docker backend configuration](backends/docker/#configuration)
See [Docker backend configuration](./22-backends/10-docker.md#configuration)
### `WOODPECKER_BACKEND_SSH_*`
See [SSH backend configuration](backends/ssh/#configuration)
See [SSH backend configuration](./22-backends/30-ssh.md#configuration)
### `WOODPECKER_BACKEND_K8S_*`
See [Kubernetes backend configuration](./22-backends/40-kubernetes.md#configuration)
## Advanced Settings

66
docs/docs/30-administration/22-backends/40-kubernetes.md Normal file
View file

@ -0,0 +1,66 @@
# Kubernetes backend
:::caution
Kubernetes support is still experimental and not all pipeline features are fully supported yet.
Check the [current state](https://github.com/woodpecker-ci/woodpecker/issues/1513)
:::
The kubernetes backend executes each step inside a newly created pod. A PVC is also created for the lifetime of the pipeline, for transferring files between steps.
## Configuration
### `WOODPECKER_BACKEND_K8S_NAMESPACE`
> Default: `woodpecker`
The namespace to create worker pods in.
### `WOODPECKER_BACKEND_K8S_VOLUME_SIZE`
> Default: `10G`
The volume size of the pipeline volume.
### `WOODPECKER_BACKEND_K8S_STORAGE_CLASS`
> Default: empty
The storage class to use for the pipeline volume.
### `WOODPECKER_BACKEND_K8S_STORAGE_RWX`
> Default: `true`
Determines if `RWX` should be used for the pipeline volume's [access mode](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#access-modes). If false, `RWO` is used instead.
### `WOODPECKER_BACKEND_K8S_POD_LABELS`
> Default: empty
Additional labels to apply to worker pods. Must be a YAML object, e.g. `{"example.com/test-label":"test-value"}`.
### `WOODPECKER_BACKEND_K8S_POD_ANNOTATIONS`
> Default: empty
Additional annotations to apply to worker pods. Must be a YAML object, e.g. `{"example.com/test-annotation":"test-value"}`.
## Resources
The kubernetes backend also allows for specifying requests and limits on a per-step basic, most commonly for CPU and memory.
Example pipeline configuration:
```yaml
pipeline:
build:
image: golang
commands:
- go get
- go build
- go test
backend_options:
kubernetes:
resources:
requests:
memory: 128Mi
cpu: 1000m
limits:
memory: 256Mi
```
See the [kubernetes documentation](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/) for more information on using resources.

2
docs/docs/30-administration/80-kubernetes.md
View file

@ -1,6 +1,6 @@
# Kubernetes
Woodpecker does support Kubernetes as a backend.
Woodpecker does support Kubernetes as a backend. See the [Kubernetes backend configuration](./22-backends/40-kubernetes.md#configuration) for backend-specific options.
:::caution
Kubernetes support is still experimental and not all pipeline features are fully supported yet.

47
pipeline/backend/kubernetes/pod.go
View file

@ -1,6 +1,7 @@
package kubernetes
import (
"fmt"
"strings"
"github.com/woodpecker-ci/woodpecker/pipeline/backend/common"
@ -62,35 +63,21 @@ func Pod(namespace string, step *types.Step, labels, annotations map[string]stri
hostAliases = append(hostAliases, v1.HostAlias{IP: host[1], Hostnames: []string{host[0]}})
}
// TODO: add support for resource limits
// if step.Resources.CPULimit == "" {
// step.Resources.CPULimit = "2"
// }
// if step.Resources.MemoryLimit == "" {
// step.Resources.MemoryLimit = "2G"
// }
// memoryLimit := resource.MustParse(step.Resources.MemoryLimit)
// CPULimit := resource.MustParse(step.Resources.CPULimit)
memoryLimit := resource.MustParse("2G")
CPULimit := resource.MustParse("2")
memoryLimitValue, _ := memoryLimit.AsInt64()
CPULimitValue, _ := CPULimit.AsInt64()
loadfactor := 0.5
memoryRequest := resource.NewQuantity(int64(float64(memoryLimitValue)*loadfactor), resource.DecimalSI)
CPURequest := resource.NewQuantity(int64(float64(CPULimitValue)*loadfactor), resource.DecimalSI)
resources := v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceMemory: *memoryRequest,
v1.ResourceCPU: *CPURequest,
},
Limits: v1.ResourceList{
v1.ResourceMemory: memoryLimit,
v1.ResourceCPU: CPULimit,
},
resourceRequirements := v1.ResourceRequirements{Requests: v1.ResourceList{}, Limits: v1.ResourceList{}}
var err error
for key, val := range step.BackendOptions.Kubernetes.Resources.Requests {
resourceKey := v1.ResourceName(key)
resourceRequirements.Requests[resourceKey], err = resource.ParseQuantity(val)
if err != nil {
return nil, fmt.Errorf("resource request '%v' quantity '%v': %w", key, val, err)
}
}
for key, val := range step.BackendOptions.Kubernetes.Resources.Limits {
resourceKey := v1.ResourceName(key)
resourceRequirements.Limits[resourceKey], err = resource.ParseQuantity(val)
if err != nil {
return nil, fmt.Errorf("resource limit '%v' quantity '%v': %w", key, val, err)
}
}
podName, err := dnsName(step.Name)
@ -130,7 +117,7 @@ func Pod(namespace string, step *types.Step, labels, annotations map[string]stri
WorkingDir: step.WorkingDir,
Env: mapToEnvVars(step.Environment),
VolumeMounts: volMounts,
Resources: resources,
Resources: resourceRequirements,
SecurityContext: &v1.SecurityContext{
Privileged: &step.Privileged,
},

6
pipeline/backend/types/backend.go Normal file
View file

@ -0,0 +1,6 @@
package types
// BackendOptions defines advanced options for specific backends
type BackendOptions struct {
Kubernetes KubernetesBackendOptions `json:"kubernetes,omitempty"`
}

12
pipeline/backend/types/backend_kubernetes.go Normal file
View file

@ -0,0 +1,12 @@
package types
// KubernetesBackendOptions defines all the advanced options for the kubernetes backend
type KubernetesBackendOptions struct {
Resources Resources `json:"resouces,omitempty"`
}
// Resources defines two maps for kubernetes resource definitions
type Resources struct {
Requests map[string]string `json:"requests,omitempty"`
Limits map[string]string `json:"limits,omitempty"`
}

63
pipeline/backend/types/step.go
View file

@ -2,35 +2,36 @@ package types
// Step defines a container process.
type Step struct {
Name string `json:"name"`
Alias string `json:"alias,omitempty"`
Image string `json:"image,omitempty"`
Pull bool `json:"pull,omitempty"`
Detached bool `json:"detach,omitempty"`
Privileged bool `json:"privileged,omitempty"`
WorkingDir string `json:"working_dir,omitempty"`
Environment map[string]string `json:"environment,omitempty"`
Labels map[string]string `json:"labels,omitempty"`
Entrypoint []string `json:"entrypoint,omitempty"`
Commands []string `json:"commands,omitempty"`
ExtraHosts []string `json:"extra_hosts,omitempty"`
Volumes []string `json:"volumes,omitempty"`
Tmpfs []string `json:"tmpfs,omitempty"`
Devices []string `json:"devices,omitempty"`
Networks []Conn `json:"networks,omitempty"`
DNS []string `json:"dns,omitempty"`
DNSSearch []string `json:"dns_search,omitempty"`
MemSwapLimit int64 `json:"memswap_limit,omitempty"`
MemLimit int64 `json:"mem_limit,omitempty"`
ShmSize int64 `json:"shm_size,omitempty"`
CPUQuota int64 `json:"cpu_quota,omitempty"`
CPUShares int64 `json:"cpu_shares,omitempty"`
CPUSet string `json:"cpu_set,omitempty"`
OnFailure bool `json:"on_failure,omitempty"`
OnSuccess bool `json:"on_success,omitempty"`
Failure string `json:"failure,omitempty"`
AuthConfig Auth `json:"auth_config,omitempty"`
NetworkMode string `json:"network_mode,omitempty"`
IpcMode string `json:"ipc_mode,omitempty"`
Sysctls map[string]string `json:"sysctls,omitempty"`
Name string `json:"name"`
Alias string `json:"alias,omitempty"`
Image string `json:"image,omitempty"`
Pull bool `json:"pull,omitempty"`
Detached bool `json:"detach,omitempty"`
Privileged bool `json:"privileged,omitempty"`
WorkingDir string `json:"working_dir,omitempty"`
Environment map[string]string `json:"environment,omitempty"`
Labels map[string]string `json:"labels,omitempty"`
Entrypoint []string `json:"entrypoint,omitempty"`
Commands []string `json:"commands,omitempty"`
ExtraHosts []string `json:"extra_hosts,omitempty"`
Volumes []string `json:"volumes,omitempty"`
Tmpfs []string `json:"tmpfs,omitempty"`
Devices []string `json:"devices,omitempty"`
Networks []Conn `json:"networks,omitempty"`
DNS []string `json:"dns,omitempty"`
DNSSearch []string `json:"dns_search,omitempty"`
MemSwapLimit int64 `json:"memswap_limit,omitempty"`
MemLimit int64 `json:"mem_limit,omitempty"`
ShmSize int64 `json:"shm_size,omitempty"`
CPUQuota int64 `json:"cpu_quota,omitempty"`
CPUShares int64 `json:"cpu_shares,omitempty"`
CPUSet string `json:"cpu_set,omitempty"`
OnFailure bool `json:"on_failure,omitempty"`
OnSuccess bool `json:"on_success,omitempty"`
Failure string `json:"failure,omitempty"`
AuthConfig Auth `json:"auth_config,omitempty"`
NetworkMode string `json:"network_mode,omitempty"`
IpcMode string `json:"ipc_mode,omitempty"`
Sysctls map[string]string `json:"sysctls,omitempty"`
BackendOptions BackendOptions `json:"backend_options,omitempty"`
}

71
pipeline/frontend/yaml/compiler/convert.go
View file

@ -110,6 +110,16 @@ func (c *Compiler) createProcess(name string, container *yaml.Container, section
}
}
// Kubernetes advanced settings
backendOptions := backend.BackendOptions{
Kubernetes: backend.KubernetesBackendOptions{
Resources: backend.Resources{
Limits: container.BackendOptions.Kubernetes.Resources.Limits,
Requests: container.BackendOptions.Kubernetes.Resources.Requests,
},
},
}
memSwapLimit := int64(container.MemSwapLimit)
if c.reslimit.MemSwapLimit != 0 {
memSwapLimit = c.reslimit.MemSwapLimit
@ -146,36 +156,37 @@ func (c *Compiler) createProcess(name string, container *yaml.Container, section
}
return &backend.Step{
Name: name,
Alias: container.Name,
Image: container.Image,
Pull: container.Pull,
Detached: detached,
Privileged: privileged,
WorkingDir: workingdir,
Environment: environment,
Labels: container.Labels,
Commands: container.Commands,
ExtraHosts: container.ExtraHosts,
Volumes: volumes,
Tmpfs: container.Tmpfs,
Devices: container.Devices,
Networks: networks,
DNS: container.DNS,
DNSSearch: container.DNSSearch,
MemSwapLimit: memSwapLimit,
MemLimit: memLimit,
ShmSize: shmSize,
Sysctls: container.Sysctls,
CPUQuota: cpuQuota,
CPUShares: cpuShares,
CPUSet: cpuSet,
AuthConfig: authConfig,
OnSuccess: onSuccess,
OnFailure: onFailure,
Failure: failure,
NetworkMode: networkMode,
IpcMode: ipcMode,
Name: name,
Alias: container.Name,
Image: container.Image,
Pull: container.Pull,
Detached: detached,
Privileged: privileged,
WorkingDir: workingdir,
Environment: environment,
Labels: container.Labels,
Commands: container.Commands,
ExtraHosts: container.ExtraHosts,
Volumes: volumes,
Tmpfs: container.Tmpfs,
Devices: container.Devices,
Networks: networks,
DNS: container.DNS,
DNSSearch: container.DNSSearch,
MemSwapLimit: memSwapLimit,
MemLimit: memLimit,
ShmSize: shmSize,
Sysctls: container.Sysctls,
CPUQuota: cpuQuota,
CPUShares: cpuShares,
CPUSet: cpuSet,
AuthConfig: authConfig,
OnSuccess: onSuccess,
OnFailure: onFailure,
Failure: failure,
NetworkMode: networkMode,
IpcMode: ipcMode,
BackendOptions: backendOptions,
}
}

87
pipeline/frontend/yaml/container.go
View file

@ -19,6 +19,20 @@ type (
Email string
}
// Advanced backend options
BackendOptions struct {
Kubernetes KubernetesBackendOptions `yaml:"kubernetes,omitempty"`
}
KubernetesBackendOptions struct {
Resources Resources `yaml:"resources,omitempty"`
}
Resources struct {
Requests map[string]string `yaml:"requests,omitempty"`
Limits map[string]string `yaml:"limits,omitempty"`
}
// Containers denotes an ordered collection of containers.
Containers struct {
Containers []*Container
@ -26,42 +40,43 @@ type (
// Container defines a container.
Container struct {
AuthConfig AuthConfig `yaml:"auth_config,omitempty"`
CapAdd []string `yaml:"cap_add,omitempty"`
CapDrop []string `yaml:"cap_drop,omitempty"`
Commands types.StringOrSlice `yaml:"commands,omitempty"`
CPUQuota types.StringorInt `yaml:"cpu_quota,omitempty"`
CPUSet string `yaml:"cpuset,omitempty"`
CPUShares types.StringorInt `yaml:"cpu_shares,omitempty"`
Detached bool `yaml:"detach,omitempty"`
Devices []string `yaml:"devices,omitempty"`
Tmpfs []string `yaml:"tmpfs,omitempty"`
DNS types.StringOrSlice `yaml:"dns,omitempty"`
DNSSearch types.StringOrSlice `yaml:"dns_search,omitempty"`
Directory string `yaml:"directory,omitempty"`
Environment types.SliceorMap `yaml:"environment,omitempty"`
ExtraHosts []string `yaml:"extra_hosts,omitempty"`
Group string `yaml:"group,omitempty"`
Image string `yaml:"image,omitempty"`
Failure string `yaml:"failure,omitempty"`
Isolation string `yaml:"isolation,omitempty"`
Labels types.SliceorMap `yaml:"labels,omitempty"`
MemLimit types.MemStringorInt `yaml:"mem_limit,omitempty"`
MemSwapLimit types.MemStringorInt `yaml:"memswap_limit,omitempty"`
MemSwappiness types.MemStringorInt `yaml:"mem_swappiness,omitempty"`
Name string `yaml:"name,omitempty"`
NetworkMode string `yaml:"network_mode,omitempty"`
IpcMode string `yaml:"ipc_mode,omitempty"`
Networks types.Networks `yaml:"networks,omitempty"`
Privileged bool `yaml:"privileged,omitempty"`
Pull bool `yaml:"pull,omitempty"`
ShmSize types.MemStringorInt `yaml:"shm_size,omitempty"`
Ulimits types.Ulimits `yaml:"ulimits,omitempty"`
Volumes types.Volumes `yaml:"volumes,omitempty"`
Secrets Secrets `yaml:"secrets,omitempty"`
Sysctls types.SliceorMap `yaml:"sysctls,omitempty"`
When constraint.When `yaml:"when,omitempty"`
Settings map[string]interface{} `yaml:"settings"`
AuthConfig AuthConfig `yaml:"auth_config,omitempty"`
CapAdd []string `yaml:"cap_add,omitempty"`
CapDrop []string `yaml:"cap_drop,omitempty"`
Commands types.StringOrSlice `yaml:"commands,omitempty"`
CPUQuota types.StringorInt `yaml:"cpu_quota,omitempty"`
CPUSet string `yaml:"cpuset,omitempty"`
CPUShares types.StringorInt `yaml:"cpu_shares,omitempty"`
Detached bool `yaml:"detach,omitempty"`
Devices []string `yaml:"devices,omitempty"`
Tmpfs []string `yaml:"tmpfs,omitempty"`
DNS types.StringOrSlice `yaml:"dns,omitempty"`
DNSSearch types.StringOrSlice `yaml:"dns_search,omitempty"`
Directory string `yaml:"directory,omitempty"`
Environment types.SliceorMap `yaml:"environment,omitempty"`
ExtraHosts []string `yaml:"extra_hosts,omitempty"`
Group string `yaml:"group,omitempty"`
Image string `yaml:"image,omitempty"`
Failure string `yaml:"failure,omitempty"`
Isolation string `yaml:"isolation,omitempty"`
Labels types.SliceorMap `yaml:"labels,omitempty"`
MemLimit types.MemStringorInt `yaml:"mem_limit,omitempty"`
MemSwapLimit types.MemStringorInt `yaml:"memswap_limit,omitempty"`
MemSwappiness types.MemStringorInt `yaml:"mem_swappiness,omitempty"`
Name string `yaml:"name,omitempty"`
NetworkMode string `yaml:"network_mode,omitempty"`
IpcMode string `yaml:"ipc_mode,omitempty"`
Networks types.Networks `yaml:"networks,omitempty"`
Privileged bool `yaml:"privileged,omitempty"`
Pull bool `yaml:"pull,omitempty"`
ShmSize types.MemStringorInt `yaml:"shm_size,omitempty"`
Ulimits types.Ulimits `yaml:"ulimits,omitempty"`
Volumes types.Volumes `yaml:"volumes,omitempty"`
Secrets Secrets `yaml:"secrets,omitempty"`
Sysctls types.SliceorMap `yaml:"sysctls,omitempty"`
When constraint.When `yaml:"when,omitempty"`
Settings map[string]interface{} `yaml:"settings"`
BackendOptions BackendOptions `yaml:"backend_options,omitempty"`
}
)

40
pipeline/schema/schema.json
View file

@ -243,6 +243,9 @@
"type": "string",
"enum": ["fail", "ignore"],
"default": "fail"
},
"backend_options": {
"$ref": "#/definitions/step_backend_options"
}
}
},
@ -480,6 +483,43 @@
"description": "Read more: https://woodpecker-ci.org/docs/usage/pipeline-syntax#directory",
"type": "string"
},
"step_backend_options": {
"description": "Advanced options for the different agent backends",
"type": "object",
"properties": {
"kubernetes" :{
"$ref": "#/definitions/step_backend_kubernetes_resources"
}
}
},
"step_backend_kubernetes": {
"description": "Advanced options for the kubernetes agent backends",
"type": "object",
"properties": {
"resources" :{
"$ref": "#/definitions/step_backend_kubernetes_resources"
}
}
},
"step_backend_kubernetes_resources": {
"description": "Resources for the kubernetes backend. Read more: https://woodpecker-ci.org/docs/administration/backends/kubernetes",
"type": "object",
"properties": {
"requests": {
"$ref": "#/definitions/step_kubernetes_resources_object"
},
"limits": {
"$ref": "#/definitions/step_kubernetes_resources_object"
}
}
},
"step_kubernetes_resources_object": {
"description": "A list of kubernetes resource mappings",
"type": "object",
"additionalProperties": {
"type": "string"
}
},
"services": {
"description": "Read more: https://woodpecker-ci.org/docs/usage/services",
"type": "object",