mirror of
https://github.com/woodpecker-ci/woodpecker.git
synced 2025-01-09 00:55:32 +00:00
Merge branch 'origin/main' into 'next-release/main'
This commit is contained in:
commit
f9e542c1c6
22 changed files with 221 additions and 293 deletions
|
@ -295,36 +295,8 @@ var flags = append([]cli.Flag{
|
||||||
Usage: "How many retries of fetching the Woodpecker configuration from a forge are done before we fail",
|
Usage: "How many retries of fetching the Woodpecker configuration from a forge are done before we fail",
|
||||||
Value: 3,
|
Value: 3,
|
||||||
},
|
},
|
||||||
&cli.IntFlag{
|
//
|
||||||
Sources: cli.EnvVars("WOODPECKER_LIMIT_MEM_SWAP"),
|
// generic forge settings
|
||||||
Name: "limit-mem-swap",
|
|
||||||
Usage: "maximum memory used for swap in bytes",
|
|
||||||
},
|
|
||||||
&cli.IntFlag{
|
|
||||||
Sources: cli.EnvVars("WOODPECKER_LIMIT_MEM"),
|
|
||||||
Name: "limit-mem",
|
|
||||||
Usage: "maximum memory allowed in bytes",
|
|
||||||
},
|
|
||||||
&cli.IntFlag{
|
|
||||||
Sources: cli.EnvVars("WOODPECKER_LIMIT_SHM_SIZE"),
|
|
||||||
Name: "limit-shm-size",
|
|
||||||
Usage: "docker compose /dev/shm allowed in bytes",
|
|
||||||
},
|
|
||||||
&cli.IntFlag{
|
|
||||||
Sources: cli.EnvVars("WOODPECKER_LIMIT_CPU_QUOTA"),
|
|
||||||
Name: "limit-cpu-quota",
|
|
||||||
Usage: "impose a cpu quota",
|
|
||||||
},
|
|
||||||
&cli.IntFlag{
|
|
||||||
Sources: cli.EnvVars("WOODPECKER_LIMIT_CPU_SHARES"),
|
|
||||||
Name: "limit-cpu-shares",
|
|
||||||
Usage: "change the cpu shares",
|
|
||||||
},
|
|
||||||
&cli.StringFlag{
|
|
||||||
Sources: cli.EnvVars("WOODPECKER_LIMIT_CPU_SET"),
|
|
||||||
Name: "limit-cpu-set",
|
|
||||||
Usage: "set the cpus allowed to execute containers",
|
|
||||||
},
|
|
||||||
//
|
//
|
||||||
&cli.StringFlag{
|
&cli.StringFlag{
|
||||||
Name: "forge-url",
|
Name: "forge-url",
|
||||||
|
|
|
@ -178,14 +178,6 @@ func setupEvilGlobals(ctx context.Context, c *cli.Command, s store.Store) error
|
||||||
server.Config.Pipeline.DefaultTimeout = c.Int("default-pipeline-timeout")
|
server.Config.Pipeline.DefaultTimeout = c.Int("default-pipeline-timeout")
|
||||||
server.Config.Pipeline.MaxTimeout = c.Int("max-pipeline-timeout")
|
server.Config.Pipeline.MaxTimeout = c.Int("max-pipeline-timeout")
|
||||||
|
|
||||||
// limits
|
|
||||||
server.Config.Pipeline.Limits.MemSwapLimit = c.Int("limit-mem-swap")
|
|
||||||
server.Config.Pipeline.Limits.MemLimit = c.Int("limit-mem")
|
|
||||||
server.Config.Pipeline.Limits.ShmSize = c.Int("limit-shm-size")
|
|
||||||
server.Config.Pipeline.Limits.CPUQuota = c.Int("limit-cpu-quota")
|
|
||||||
server.Config.Pipeline.Limits.CPUShares = c.Int("limit-cpu-shares")
|
|
||||||
server.Config.Pipeline.Limits.CPUSet = c.String("limit-cpu-set")
|
|
||||||
|
|
||||||
// backend options for pipeline compiler
|
// backend options for pipeline compiler
|
||||||
server.Config.Pipeline.Proxy.No = c.String("backend-no-proxy")
|
server.Config.Pipeline.Proxy.No = c.String("backend-no-proxy")
|
||||||
server.Config.Pipeline.Proxy.HTTP = c.String("backend-http-proxy")
|
server.Config.Pipeline.Proxy.HTTP = c.String("backend-http-proxy")
|
||||||
|
|
|
@ -476,44 +476,6 @@ Supported variables:
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
### `WOODPECKER_LIMIT_MEM_SWAP`
|
|
||||||
|
|
||||||
> Default: `0`
|
|
||||||
|
|
||||||
The maximum amount of memory a single pipeline container is allowed to swap to disk, configured in bytes. There is no limit if `0`.
|
|
||||||
|
|
||||||
### `WOODPECKER_LIMIT_MEM`
|
|
||||||
|
|
||||||
> Default: `0`
|
|
||||||
|
|
||||||
The maximum amount of memory a single pipeline container can use, configured in bytes. There is no limit if `0`.
|
|
||||||
|
|
||||||
### `WOODPECKER_LIMIT_SHM_SIZE`
|
|
||||||
|
|
||||||
> Default: `0`
|
|
||||||
|
|
||||||
The maximum amount of memory of `/dev/shm` allowed in bytes. There is no limit if `0`.
|
|
||||||
|
|
||||||
### `WOODPECKER_LIMIT_CPU_QUOTA`
|
|
||||||
|
|
||||||
> Default: `0`
|
|
||||||
|
|
||||||
The number of microseconds per CPU period that the container is limited to before throttled. There is no limit if `0`.
|
|
||||||
|
|
||||||
### `WOODPECKER_LIMIT_CPU_SHARES`
|
|
||||||
|
|
||||||
> Default: `0`
|
|
||||||
|
|
||||||
The relative weight vs. other containers.
|
|
||||||
|
|
||||||
### `WOODPECKER_LIMIT_CPU_SET`
|
|
||||||
|
|
||||||
> Default: empty
|
|
||||||
|
|
||||||
Comma-separated list to limit the specific CPUs or cores a pipeline container can use.
|
|
||||||
|
|
||||||
Example: `WOODPECKER_LIMIT_CPU_SET=1,2`
|
|
||||||
|
|
||||||
### `WOODPECKER_CONFIG_SERVICE_ENDPOINT`
|
### `WOODPECKER_CONFIG_SERVICE_ENDPOINT`
|
||||||
|
|
||||||
> Default: empty
|
> Default: empty
|
||||||
|
|
|
@ -64,3 +64,41 @@ Enable IPv6 for the networks used by pipeline containers (steps). Make sure you
|
||||||
|
|
||||||
List of default volumes separated by comma to be mounted to all pipeline containers (steps). For example to use custom CA
|
List of default volumes separated by comma to be mounted to all pipeline containers (steps). For example to use custom CA
|
||||||
certificates installed on host and host timezone use `/etc/ssl/certs:/etc/ssl/certs:ro,/etc/timezone:/etc/timezone`.
|
certificates installed on host and host timezone use `/etc/ssl/certs:/etc/ssl/certs:ro,/etc/timezone:/etc/timezone`.
|
||||||
|
|
||||||
|
### `WOODPECKER_BACKEND_DOCKER_LIMIT_MEM_SWAP`
|
||||||
|
|
||||||
|
> Default: `0`
|
||||||
|
|
||||||
|
The maximum amount of memory a single pipeline container is allowed to swap to disk, configured in bytes. There is no limit if `0`.
|
||||||
|
|
||||||
|
### `WOODPECKER_BACKEND_DOCKER_LIMIT_MEM`
|
||||||
|
|
||||||
|
> Default: `0`
|
||||||
|
|
||||||
|
The maximum amount of memory a single pipeline container can use, configured in bytes. There is no limit if `0`.
|
||||||
|
|
||||||
|
### `WOODPECKER_BACKEND_DOCKER_LIMIT_SHM_SIZE`
|
||||||
|
|
||||||
|
> Default: `0`
|
||||||
|
|
||||||
|
The maximum amount of memory of `/dev/shm` allowed in bytes. There is no limit if `0`.
|
||||||
|
|
||||||
|
### `WOODPECKER_BACKEND_DOCKER_LIMIT_CPU_QUOTA`
|
||||||
|
|
||||||
|
> Default: `0`
|
||||||
|
|
||||||
|
The number of microseconds per CPU period that the container is limited to before throttled. There is no limit if `0`.
|
||||||
|
|
||||||
|
### `WOODPECKER_BACKEND_DOCKER_LIMIT_CPU_SHARES`
|
||||||
|
|
||||||
|
> Default: `0`
|
||||||
|
|
||||||
|
The relative weight vs. other containers.
|
||||||
|
|
||||||
|
### `WOODPECKER_BACKEND_DOCKER_LIMIT_CPU_SET`
|
||||||
|
|
||||||
|
> Default: empty
|
||||||
|
|
||||||
|
Comma-separated list to limit the specific CPUs or cores a pipeline container can use.
|
||||||
|
|
||||||
|
Example: `WOODPECKER_BACKEND_DOCKER_LIMIT_CPU_SET=1,2`
|
||||||
|
|
|
@ -4,13 +4,14 @@ Some versions need some changes to the server configuration or the pipeline conf
|
||||||
|
|
||||||
## `next`
|
## `next`
|
||||||
|
|
||||||
- Set `/woodpecker` as defautl workdir for the **woodpecker-cli** container
|
|
||||||
- Removed built-in environment variables:
|
- Removed built-in environment variables:
|
||||||
- `CI_COMMIT_URL` use `CI_PIPELINE_FORGE_URL`
|
- `CI_COMMIT_URL` use `CI_PIPELINE_FORGE_URL`
|
||||||
- `CI_STEP_FINISHED` as empty during execution
|
- `CI_STEP_FINISHED` as empty during execution
|
||||||
- `CI_PIPELINE_FINISHED` as empty during execution
|
- `CI_PIPELINE_FINISHED` as empty during execution
|
||||||
- `CI_PIPELINE_STATUS` was always `success`
|
- `CI_PIPELINE_STATUS` was always `success`
|
||||||
- `CI_STEP_STATUS` was always `success`
|
- `CI_STEP_STATUS` was always `success`
|
||||||
|
- Set `/woodpecker` as defautl workdir for the **woodpecker-cli** container
|
||||||
|
- Move docker resource limit settings from server into agent configuration
|
||||||
- Rename server environment variable `WOODPECKER_ESCALATE` to `WOODPECKER_PLUGINS_PRIVILEGED`
|
- Rename server environment variable `WOODPECKER_ESCALATE` to `WOODPECKER_PLUGINS_PRIVILEGED`
|
||||||
- All default privileged plugins (like `woodpeckerci/plugin-docker-buildx`) were removed. Please carefully [re-add those plugins](./30-administration/10-server-config.md#woodpecker_plugins_privileged) you trust and rely on.
|
- All default privileged plugins (like `woodpeckerci/plugin-docker-buildx`) were removed. Please carefully [re-add those plugins](./30-administration/10-server-config.md#woodpecker_plugins_privileged) you trust and rely on.
|
||||||
- `WOODPECKER_DEFAULT_CLONE_IMAGE` got depricated use `WOODPECKER_DEFAULT_CLONE_PLUGIN`
|
- `WOODPECKER_DEFAULT_CLONE_IMAGE` got depricated use `WOODPECKER_DEFAULT_CLONE_PLUGIN`
|
||||||
|
|
71
pipeline/backend/docker/config.go
Normal file
71
pipeline/backend/docker/config.go
Normal file
|
@ -0,0 +1,71 @@
|
||||||
|
// Copyright 2024 Woodpecker Authors
|
||||||
|
//
|
||||||
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
// you may not use this file except in compliance with the License.
|
||||||
|
// You may obtain a copy of the License at
|
||||||
|
//
|
||||||
|
// http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
//
|
||||||
|
// Unless required by applicable law or agreed to in writing, software
|
||||||
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
// See the License for the specific language governing permissions and
|
||||||
|
// limitations under the License.
|
||||||
|
|
||||||
|
package docker
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/rs/zerolog/log"
|
||||||
|
"github.com/urfave/cli/v3"
|
||||||
|
)
|
||||||
|
|
||||||
|
type config struct {
|
||||||
|
enableIPv6 bool
|
||||||
|
network string
|
||||||
|
volumes []string
|
||||||
|
resourceLimit resourceLimit
|
||||||
|
}
|
||||||
|
|
||||||
|
type resourceLimit struct {
|
||||||
|
MemSwapLimit int64
|
||||||
|
MemLimit int64
|
||||||
|
ShmSize int64
|
||||||
|
CPUQuota int64
|
||||||
|
CPUShares int64
|
||||||
|
CPUSet string
|
||||||
|
}
|
||||||
|
|
||||||
|
func configFromCli(c *cli.Command) (config, error) {
|
||||||
|
conf := config{
|
||||||
|
enableIPv6: c.Bool("backend-docker-ipv6"),
|
||||||
|
network: c.String("backend-docker-network"),
|
||||||
|
resourceLimit: resourceLimit{
|
||||||
|
MemSwapLimit: c.Int("backend-docker-limit-mem-swap"),
|
||||||
|
MemLimit: c.Int("backend-docker-limit-mem"),
|
||||||
|
ShmSize: c.Int("backend-docker-limit-shm-size"),
|
||||||
|
CPUQuota: c.Int("backend-docker-limit-cpu-quota"),
|
||||||
|
CPUShares: c.Int("backend-docker-limit-cpu-shares"),
|
||||||
|
CPUSet: c.String("backend-docker-limit-cpu-set"),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
volumes := strings.Split(c.String("backend-docker-volumes"), ",")
|
||||||
|
conf.volumes = make([]string, 0, len(volumes))
|
||||||
|
// Validate provided volume definitions
|
||||||
|
for _, v := range volumes {
|
||||||
|
if v == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
parts, err := splitVolumeParts(v)
|
||||||
|
if err != nil {
|
||||||
|
log.Error().Err(err).Msgf("can not parse volume config")
|
||||||
|
return conf, fmt.Errorf("invalid volume '%s' provided in WOODPECKER_BACKEND_DOCKER_VOLUMES: %w", v, err)
|
||||||
|
}
|
||||||
|
conf.volumes = append(conf.volumes, strings.Join(parts, ":"))
|
||||||
|
}
|
||||||
|
|
||||||
|
return conf, nil
|
||||||
|
}
|
|
@ -68,20 +68,20 @@ func toContainerName(step *types.Step) string {
|
||||||
}
|
}
|
||||||
|
|
||||||
// returns a container host configuration.
|
// returns a container host configuration.
|
||||||
func toHostConfig(step *types.Step) *container.HostConfig {
|
func toHostConfig(step *types.Step, conf *config) *container.HostConfig {
|
||||||
config := &container.HostConfig{
|
config := &container.HostConfig{
|
||||||
Resources: container.Resources{
|
Resources: container.Resources{
|
||||||
CPUQuota: step.CPUQuota,
|
CPUQuota: conf.resourceLimit.CPUQuota,
|
||||||
CPUShares: step.CPUShares,
|
CPUShares: conf.resourceLimit.CPUShares,
|
||||||
CpusetCpus: step.CPUSet,
|
CpusetCpus: conf.resourceLimit.CPUSet,
|
||||||
Memory: step.MemLimit,
|
Memory: conf.resourceLimit.MemLimit,
|
||||||
MemorySwap: step.MemSwapLimit,
|
MemorySwap: conf.resourceLimit.MemSwapLimit,
|
||||||
},
|
},
|
||||||
|
ShmSize: conf.resourceLimit.ShmSize,
|
||||||
LogConfig: container.LogConfig{
|
LogConfig: container.LogConfig{
|
||||||
Type: "json-file",
|
Type: "json-file",
|
||||||
},
|
},
|
||||||
Privileged: step.Privileged,
|
Privileged: step.Privileged,
|
||||||
ShmSize: step.ShmSize,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(step.NetworkMode) != 0 {
|
if len(step.NetworkMode) != 0 {
|
||||||
|
|
|
@ -196,7 +196,19 @@ func TestToConfigSmall(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestToConfigFull(t *testing.T) {
|
func TestToConfigFull(t *testing.T) {
|
||||||
engine := docker{info: system.Info{OSType: "linux/riscv64"}}
|
engine := docker{
|
||||||
|
info: system.Info{OSType: "linux/riscv64"},
|
||||||
|
config: config{
|
||||||
|
enableIPv6: true,
|
||||||
|
resourceLimit: resourceLimit{
|
||||||
|
MemSwapLimit: 12,
|
||||||
|
MemLimit: 13,
|
||||||
|
ShmSize: 14,
|
||||||
|
CPUQuota: 15,
|
||||||
|
CPUShares: 16,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
conf := engine.toConfig(&backend.Step{
|
conf := engine.toConfig(&backend.Step{
|
||||||
Name: "test",
|
Name: "test",
|
||||||
|
@ -216,11 +228,6 @@ func TestToConfigFull(t *testing.T) {
|
||||||
Networks: []backend.Conn{{Name: "extra-net", Aliases: []string{"extra.net"}}},
|
Networks: []backend.Conn{{Name: "extra-net", Aliases: []string{"extra.net"}}},
|
||||||
DNS: []string{"9.9.9.9", "8.8.8.8"},
|
DNS: []string{"9.9.9.9", "8.8.8.8"},
|
||||||
DNSSearch: nil,
|
DNSSearch: nil,
|
||||||
MemSwapLimit: 12,
|
|
||||||
MemLimit: 13,
|
|
||||||
ShmSize: 14,
|
|
||||||
CPUQuota: 15,
|
|
||||||
CPUShares: 16,
|
|
||||||
OnFailure: true,
|
OnFailure: true,
|
||||||
OnSuccess: true,
|
OnSuccess: true,
|
||||||
Failure: "fail",
|
Failure: "fail",
|
||||||
|
|
|
@ -41,10 +41,8 @@ import (
|
||||||
|
|
||||||
type docker struct {
|
type docker struct {
|
||||||
client client.APIClient
|
client client.APIClient
|
||||||
enableIPv6 bool
|
|
||||||
network string
|
|
||||||
volumes []string
|
|
||||||
info system.Info
|
info system.Info
|
||||||
|
config config
|
||||||
}
|
}
|
||||||
|
|
||||||
const (
|
const (
|
||||||
|
@ -132,22 +130,9 @@ func (e *docker) Load(ctx context.Context) (*backend.BackendInfo, error) {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
e.enableIPv6 = c.Bool("backend-docker-ipv6")
|
e.config, err = configFromCli(c)
|
||||||
e.network = c.String("backend-docker-network")
|
|
||||||
|
|
||||||
volumes := strings.Split(c.String("backend-docker-volumes"), ",")
|
|
||||||
e.volumes = make([]string, 0, len(volumes))
|
|
||||||
// Validate provided volume definitions
|
|
||||||
for _, v := range volumes {
|
|
||||||
if v == "" {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
parts, err := splitVolumeParts(v)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Error().Err(err).Msgf("invalid volume '%s' provided in WOODPECKER_BACKEND_DOCKER_VOLUMES", v)
|
return nil, err
|
||||||
continue
|
|
||||||
}
|
|
||||||
e.volumes = append(e.volumes, strings.Join(parts, ":"))
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return &backend.BackendInfo{
|
return &backend.BackendInfo{
|
||||||
|
@ -175,7 +160,7 @@ func (e *docker) SetupWorkflow(ctx context.Context, conf *backend.Config, taskUU
|
||||||
for _, n := range conf.Networks {
|
for _, n := range conf.Networks {
|
||||||
_, err := e.client.NetworkCreate(ctx, n.Name, network.CreateOptions{
|
_, err := e.client.NetworkCreate(ctx, n.Name, network.CreateOptions{
|
||||||
Driver: networkDriver,
|
Driver: networkDriver,
|
||||||
EnableIPv6: &e.enableIPv6,
|
EnableIPv6: &e.config.enableIPv6,
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
|
@ -188,7 +173,7 @@ func (e *docker) StartStep(ctx context.Context, step *backend.Step, taskUUID str
|
||||||
log.Trace().Str("taskUUID", taskUUID).Msgf("start step %s", step.Name)
|
log.Trace().Str("taskUUID", taskUUID).Msgf("start step %s", step.Name)
|
||||||
|
|
||||||
config := e.toConfig(step)
|
config := e.toConfig(step)
|
||||||
hostConfig := toHostConfig(step)
|
hostConfig := toHostConfig(step, &e.config)
|
||||||
containerName := toContainerName(step)
|
containerName := toContainerName(step)
|
||||||
|
|
||||||
// create pull options with encoded authorization credentials.
|
// create pull options with encoded authorization credentials.
|
||||||
|
@ -217,7 +202,7 @@ func (e *docker) StartStep(ctx context.Context, step *backend.Step, taskUUID str
|
||||||
}
|
}
|
||||||
|
|
||||||
// add default volumes to the host configuration
|
// add default volumes to the host configuration
|
||||||
hostConfig.Binds = utils.DeduplicateStrings(append(hostConfig.Binds, e.volumes...))
|
hostConfig.Binds = utils.DeduplicateStrings(append(hostConfig.Binds, e.config.volumes...))
|
||||||
|
|
||||||
_, err := e.client.ContainerCreate(ctx, config, hostConfig, nil, nil, containerName)
|
_, err := e.client.ContainerCreate(ctx, config, hostConfig, nil, nil, containerName)
|
||||||
if client.IsErrNotFound(err) {
|
if client.IsErrNotFound(err) {
|
||||||
|
@ -251,8 +236,8 @@ func (e *docker) StartStep(ctx context.Context, step *backend.Step, taskUUID str
|
||||||
}
|
}
|
||||||
|
|
||||||
// join the container to an existing network
|
// join the container to an existing network
|
||||||
if e.network != "" {
|
if e.config.network != "" {
|
||||||
err = e.client.NetworkConnect(ctx, e.network, containerName, &network.EndpointSettings{})
|
err = e.client.NetworkConnect(ctx, e.config.network, containerName, &network.EndpointSettings{})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
|
@ -56,4 +56,37 @@ var Flags = []cli.Flag{
|
||||||
Name: "backend-docker-volumes",
|
Name: "backend-docker-volumes",
|
||||||
Usage: "backend docker volumes (comma separated)",
|
Usage: "backend docker volumes (comma separated)",
|
||||||
},
|
},
|
||||||
|
//
|
||||||
|
// resource limit parameters
|
||||||
|
//
|
||||||
|
&cli.IntFlag{
|
||||||
|
Sources: cli.EnvVars("WOODPECKER_BACKEND_DOCKER_LIMIT_MEM_SWAP", "WOODPECKER_LIMIT_MEM_SWAP"),
|
||||||
|
Name: "backend-docker-limit-mem-swap",
|
||||||
|
Usage: "maximum memory used for swap in bytes",
|
||||||
|
},
|
||||||
|
&cli.IntFlag{
|
||||||
|
Sources: cli.EnvVars("WOODPECKER_BACKEND_DOCKER_LIMIT_MEM", "WOODPECKER_LIMIT_MEM"),
|
||||||
|
Name: "backend-docker-limit-mem",
|
||||||
|
Usage: "maximum memory allowed in bytes",
|
||||||
|
},
|
||||||
|
&cli.IntFlag{
|
||||||
|
Sources: cli.EnvVars("WOODPECKER_BACKEND_DOCKER_LIMIT_SHM_SIZE", "WOODPECKER_LIMIT_SHM_SIZE"),
|
||||||
|
Name: "backend-docker-limit-shm-size",
|
||||||
|
Usage: "docker /dev/shm allowed in bytes",
|
||||||
|
},
|
||||||
|
&cli.IntFlag{
|
||||||
|
Sources: cli.EnvVars("WOODPECKER_BACKEND_DOCKER_LIMIT_CPU_QUOTA", "WOODPECKER_LIMIT_CPU_QUOTA"),
|
||||||
|
Name: "backend-docker-limit-cpu-quota",
|
||||||
|
Usage: "impose a cpu quota",
|
||||||
|
},
|
||||||
|
&cli.IntFlag{
|
||||||
|
Sources: cli.EnvVars("WOODPECKER_BACKEND_DOCKER_LIMIT_CPU_SHARES", "WOODPECKER_LIMIT_CPU_SHARES"),
|
||||||
|
Name: "backend-docker-limit-cpu-shares",
|
||||||
|
Usage: "change the cpu shares",
|
||||||
|
},
|
||||||
|
&cli.StringFlag{
|
||||||
|
Sources: cli.EnvVars("WOODPECKER_BACKEND_DOCKER_LIMIT_CPU_SET", "WOODPECKER_LIMIT_CPU_SET"),
|
||||||
|
Name: "backend-docker-limit-cpu-set",
|
||||||
|
Usage: "set the cpus allowed to execute containers",
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
|
@ -34,12 +34,6 @@ type Step struct {
|
||||||
Networks []Conn `json:"networks,omitempty"`
|
Networks []Conn `json:"networks,omitempty"`
|
||||||
DNS []string `json:"dns,omitempty"`
|
DNS []string `json:"dns,omitempty"`
|
||||||
DNSSearch []string `json:"dns_search,omitempty"`
|
DNSSearch []string `json:"dns_search,omitempty"`
|
||||||
MemSwapLimit int64 `json:"memswap_limit,omitempty"`
|
|
||||||
MemLimit int64 `json:"mem_limit,omitempty"`
|
|
||||||
ShmSize int64 `json:"shm_size,omitempty"`
|
|
||||||
CPUQuota int64 `json:"cpu_quota,omitempty"`
|
|
||||||
CPUShares int64 `json:"cpu_shares,omitempty"`
|
|
||||||
CPUSet string `json:"cpu_set,omitempty"`
|
|
||||||
OnFailure bool `json:"on_failure,omitempty"`
|
OnFailure bool `json:"on_failure,omitempty"`
|
||||||
OnSuccess bool `json:"on_success,omitempty"`
|
OnSuccess bool `json:"on_success,omitempty"`
|
||||||
Failure string `json:"failure,omitempty"`
|
Failure string `json:"failure,omitempty"`
|
||||||
|
|
|
@ -81,15 +81,6 @@ func (s *Secret) Match(event string) bool {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
type ResourceLimit struct {
|
|
||||||
MemSwapLimit int64
|
|
||||||
MemLimit int64
|
|
||||||
ShmSize int64
|
|
||||||
CPUQuota int64
|
|
||||||
CPUShares int64
|
|
||||||
CPUSet string
|
|
||||||
}
|
|
||||||
|
|
||||||
// Compiler compiles the yaml.
|
// Compiler compiles the yaml.
|
||||||
type Compiler struct {
|
type Compiler struct {
|
||||||
local bool
|
local bool
|
||||||
|
@ -104,7 +95,6 @@ type Compiler struct {
|
||||||
metadata metadata.Metadata
|
metadata metadata.Metadata
|
||||||
registries []Registry
|
registries []Registry
|
||||||
secrets map[string]Secret
|
secrets map[string]Secret
|
||||||
reslimit ResourceLimit
|
|
||||||
defaultClonePlugin string
|
defaultClonePlugin string
|
||||||
trustedClonePlugins []string
|
trustedClonePlugins []string
|
||||||
trustedPipeline bool
|
trustedPipeline bool
|
||||||
|
|
|
@ -151,31 +151,6 @@ func (c *Compiler) createProcess(container *yaml_types.Container, stepType backe
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
memSwapLimit := int64(container.MemSwapLimit)
|
|
||||||
if c.reslimit.MemSwapLimit != 0 {
|
|
||||||
memSwapLimit = c.reslimit.MemSwapLimit
|
|
||||||
}
|
|
||||||
memLimit := int64(container.MemLimit)
|
|
||||||
if c.reslimit.MemLimit != 0 {
|
|
||||||
memLimit = c.reslimit.MemLimit
|
|
||||||
}
|
|
||||||
shmSize := int64(container.ShmSize)
|
|
||||||
if c.reslimit.ShmSize != 0 {
|
|
||||||
shmSize = c.reslimit.ShmSize
|
|
||||||
}
|
|
||||||
cpuQuota := int64(container.CPUQuota)
|
|
||||||
if c.reslimit.CPUQuota != 0 {
|
|
||||||
cpuQuota = c.reslimit.CPUQuota
|
|
||||||
}
|
|
||||||
cpuShares := int64(container.CPUShares)
|
|
||||||
if c.reslimit.CPUShares != 0 {
|
|
||||||
cpuShares = c.reslimit.CPUShares
|
|
||||||
}
|
|
||||||
cpuSet := container.CPUSet
|
|
||||||
if c.reslimit.CPUSet != "" {
|
|
||||||
cpuSet = c.reslimit.CPUSet
|
|
||||||
}
|
|
||||||
|
|
||||||
var ports []backend_types.Port
|
var ports []backend_types.Port
|
||||||
for _, portDef := range container.Ports {
|
for _, portDef := range container.Ports {
|
||||||
port, err := convertPort(portDef)
|
port, err := convertPort(portDef)
|
||||||
|
@ -214,12 +189,6 @@ func (c *Compiler) createProcess(container *yaml_types.Container, stepType backe
|
||||||
Networks: networks,
|
Networks: networks,
|
||||||
DNS: container.DNS,
|
DNS: container.DNS,
|
||||||
DNSSearch: container.DNSSearch,
|
DNSSearch: container.DNSSearch,
|
||||||
MemSwapLimit: memSwapLimit,
|
|
||||||
MemLimit: memLimit,
|
|
||||||
ShmSize: shmSize,
|
|
||||||
CPUQuota: cpuQuota,
|
|
||||||
CPUShares: cpuShares,
|
|
||||||
CPUSet: cpuSet,
|
|
||||||
AuthConfig: authConfig,
|
AuthConfig: authConfig,
|
||||||
OnSuccess: onSuccess,
|
OnSuccess: onSuccess,
|
||||||
OnFailure: onFailure,
|
OnFailure: onFailure,
|
||||||
|
|
|
@ -157,21 +157,6 @@ func WithNetworks(networks ...string) Option {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// WithResourceLimit configures the compiler with default resource limits that
|
|
||||||
// are applied each container in the pipeline.
|
|
||||||
func WithResourceLimit(swap, mem, shmSize, cpuQuota, cpuShares int64, cpuSet string) Option {
|
|
||||||
return func(compiler *Compiler) {
|
|
||||||
compiler.reslimit = ResourceLimit{
|
|
||||||
MemSwapLimit: swap,
|
|
||||||
MemLimit: mem,
|
|
||||||
ShmSize: shmSize,
|
|
||||||
CPUQuota: cpuQuota,
|
|
||||||
CPUShares: cpuShares,
|
|
||||||
CPUSet: cpuSet,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func WithDefaultClonePlugin(cloneImage string) Option {
|
func WithDefaultClonePlugin(cloneImage string) Option {
|
||||||
return func(compiler *Compiler) {
|
return func(compiler *Compiler) {
|
||||||
compiler.defaultClonePlugin = cloneImage
|
compiler.defaultClonePlugin = cloneImage
|
||||||
|
|
|
@ -67,25 +67,6 @@ func TestWithNetworks(t *testing.T) {
|
||||||
assert.Equal(t, "overlay_bar", compiler.networks[1])
|
assert.Equal(t, "overlay_bar", compiler.networks[1])
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestWithResourceLimit(t *testing.T) {
|
|
||||||
compiler := New(
|
|
||||||
WithResourceLimit(
|
|
||||||
1,
|
|
||||||
2,
|
|
||||||
3,
|
|
||||||
4,
|
|
||||||
5,
|
|
||||||
"0,2-5",
|
|
||||||
),
|
|
||||||
)
|
|
||||||
assert.EqualValues(t, 1, compiler.reslimit.MemSwapLimit)
|
|
||||||
assert.EqualValues(t, 2, compiler.reslimit.MemLimit)
|
|
||||||
assert.EqualValues(t, 3, compiler.reslimit.ShmSize)
|
|
||||||
assert.EqualValues(t, 4, compiler.reslimit.CPUQuota)
|
|
||||||
assert.EqualValues(t, 5, compiler.reslimit.CPUShares)
|
|
||||||
assert.Equal(t, "0,2-5", compiler.reslimit.CPUSet)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestWithPrefix(t *testing.T) {
|
func TestWithPrefix(t *testing.T) {
|
||||||
assert.Equal(t, "someprefix_", New(WithPrefix("someprefix_")).prefix)
|
assert.Equal(t, "someprefix_", New(WithPrefix("someprefix_")).prefix)
|
||||||
}
|
}
|
||||||
|
|
|
@ -207,9 +207,6 @@ func (l *Linter) lintTrusted(config *WorkflowConfig, c *types.Container, area st
|
||||||
if c.Privileged {
|
if c.Privileged {
|
||||||
errors = append(errors, "Insufficient privileges to use privileged mode")
|
errors = append(errors, "Insufficient privileges to use privileged mode")
|
||||||
}
|
}
|
||||||
if c.ShmSize != 0 {
|
|
||||||
errors = append(errors, "Insufficient privileges to override shm_size")
|
|
||||||
}
|
|
||||||
if len(c.DNS) != 0 {
|
if len(c.DNS) != 0 {
|
||||||
errors = append(errors, "Insufficient privileges to use custom dns")
|
errors = append(errors, "Insufficient privileges to use custom dns")
|
||||||
}
|
}
|
||||||
|
|
|
@ -120,10 +120,6 @@ func TestLintErrors(t *testing.T) {
|
||||||
from: "steps: { build: { image: golang, privileged: true } }",
|
from: "steps: { build: { image: golang, privileged: true } }",
|
||||||
want: "Insufficient privileges to use privileged mode",
|
want: "Insufficient privileges to use privileged mode",
|
||||||
},
|
},
|
||||||
{
|
|
||||||
from: "steps: { build: { image: golang, shm_size: 10gb } }",
|
|
||||||
want: "Insufficient privileges to override shm_size",
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
from: "steps: { build: { image: golang, dns: [ 8.8.8.8 ] } }",
|
from: "steps: { build: { image: golang, dns: [ 8.8.8.8 ] } }",
|
||||||
want: "Insufficient privileges to use custom dns",
|
want: "Insufficient privileges to use custom dns",
|
||||||
|
|
|
@ -54,17 +54,11 @@ type (
|
||||||
Privileged bool `yaml:"privileged,omitempty"`
|
Privileged bool `yaml:"privileged,omitempty"`
|
||||||
|
|
||||||
// Undocumented
|
// Undocumented
|
||||||
CPUQuota base.StringOrInt `yaml:"cpu_quota,omitempty"`
|
|
||||||
CPUSet string `yaml:"cpuset,omitempty"`
|
|
||||||
CPUShares base.StringOrInt `yaml:"cpu_shares,omitempty"`
|
|
||||||
Devices []string `yaml:"devices,omitempty"`
|
Devices []string `yaml:"devices,omitempty"`
|
||||||
DNSSearch base.StringOrSlice `yaml:"dns_search,omitempty"`
|
DNSSearch base.StringOrSlice `yaml:"dns_search,omitempty"`
|
||||||
DNS base.StringOrSlice `yaml:"dns,omitempty"`
|
DNS base.StringOrSlice `yaml:"dns,omitempty"`
|
||||||
ExtraHosts []string `yaml:"extra_hosts,omitempty"`
|
ExtraHosts []string `yaml:"extra_hosts,omitempty"`
|
||||||
MemLimit base.MemStringOrInt `yaml:"mem_limit,omitempty"`
|
|
||||||
MemSwapLimit base.MemStringOrInt `yaml:"memswap_limit,omitempty"`
|
|
||||||
NetworkMode string `yaml:"network_mode,omitempty"`
|
NetworkMode string `yaml:"network_mode,omitempty"`
|
||||||
ShmSize base.MemStringOrInt `yaml:"shm_size,omitempty"`
|
|
||||||
Tmpfs []string `yaml:"tmpfs,omitempty"`
|
Tmpfs []string `yaml:"tmpfs,omitempty"`
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
|
@ -30,9 +30,6 @@ image: golang:latest
|
||||||
commands:
|
commands:
|
||||||
- go build
|
- go build
|
||||||
- go test
|
- go test
|
||||||
cpu_quota: 11
|
|
||||||
cpuset: 1,2
|
|
||||||
cpu_shares: 99
|
|
||||||
detach: true
|
detach: true
|
||||||
devices:
|
devices:
|
||||||
- /dev/ttyUSB0:/dev/ttyUSB0
|
- /dev/ttyUSB0:/dev/ttyUSB0
|
||||||
|
@ -54,9 +51,6 @@ networks:
|
||||||
- other-network
|
- other-network
|
||||||
pull: true
|
pull: true
|
||||||
privileged: true
|
privileged: true
|
||||||
shm_size: 1kb
|
|
||||||
mem_limit: 1kb
|
|
||||||
memswap_limit: 1kb
|
|
||||||
volumes:
|
volumes:
|
||||||
- /var/lib/mysql
|
- /var/lib/mysql
|
||||||
- /opt/data:/var/lib/mysql
|
- /opt/data:/var/lib/mysql
|
||||||
|
@ -79,9 +73,6 @@ ports:
|
||||||
func TestUnmarshalContainer(t *testing.T) {
|
func TestUnmarshalContainer(t *testing.T) {
|
||||||
want := Container{
|
want := Container{
|
||||||
Commands: base.StringOrSlice{"go build", "go test"},
|
Commands: base.StringOrSlice{"go build", "go test"},
|
||||||
CPUQuota: base.StringOrInt(11),
|
|
||||||
CPUSet: "1,2",
|
|
||||||
CPUShares: base.StringOrInt(99),
|
|
||||||
Detached: true,
|
Detached: true,
|
||||||
Devices: []string{"/dev/ttyUSB0:/dev/ttyUSB0"},
|
Devices: []string{"/dev/ttyUSB0:/dev/ttyUSB0"},
|
||||||
Directory: "example/",
|
Directory: "example/",
|
||||||
|
@ -91,13 +82,10 @@ func TestUnmarshalContainer(t *testing.T) {
|
||||||
Environment: map[string]any{"RACK_ENV": "development", "SHOW": true},
|
Environment: map[string]any{"RACK_ENV": "development", "SHOW": true},
|
||||||
ExtraHosts: []string{"somehost:162.242.195.82", "otherhost:50.31.209.229", "ipv6:2001:db8::10"},
|
ExtraHosts: []string{"somehost:162.242.195.82", "otherhost:50.31.209.229", "ipv6:2001:db8::10"},
|
||||||
Image: "golang:latest",
|
Image: "golang:latest",
|
||||||
MemLimit: base.MemStringOrInt(1024),
|
|
||||||
MemSwapLimit: base.MemStringOrInt(1024),
|
|
||||||
Name: "my-build-container",
|
Name: "my-build-container",
|
||||||
NetworkMode: "bridge",
|
NetworkMode: "bridge",
|
||||||
Pull: true,
|
Pull: true,
|
||||||
Privileged: true,
|
Privileged: true,
|
||||||
ShmSize: base.MemStringOrInt(1024),
|
|
||||||
Tmpfs: base.StringOrSlice{"/var/lib/test"},
|
Tmpfs: base.StringOrSlice{"/var/lib/test"},
|
||||||
Volumes: Volumes{
|
Volumes: Volumes{
|
||||||
Volumes: []*Volume{
|
Volumes: []*Volume{
|
||||||
|
|
|
@ -66,7 +66,6 @@ var Config = struct {
|
||||||
DefaultCancelPreviousPipelineEvents []model.WebhookEvent
|
DefaultCancelPreviousPipelineEvents []model.WebhookEvent
|
||||||
DefaultClonePlugin string
|
DefaultClonePlugin string
|
||||||
TrustedClonePlugins []string
|
TrustedClonePlugins []string
|
||||||
Limits model.ResourceLimit
|
|
||||||
Volumes []string
|
Volumes []string
|
||||||
Networks []string
|
Networks []string
|
||||||
PrivilegedPlugins []string
|
PrivilegedPlugins []string
|
||||||
|
|
|
@ -1,25 +0,0 @@
|
||||||
// Copyright 2018 Drone.IO Inc.
|
|
||||||
//
|
|
||||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
||||||
// you may not use this file except in compliance with the License.
|
|
||||||
// You may obtain a copy of the License at
|
|
||||||
//
|
|
||||||
// http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
//
|
|
||||||
// Unless required by applicable law or agreed to in writing, software
|
|
||||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
||||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
||||||
// See the License for the specific language governing permissions and
|
|
||||||
// limitations under the License.
|
|
||||||
|
|
||||||
package model
|
|
||||||
|
|
||||||
// ResourceLimit is the resource limit to set on pipeline steps.
|
|
||||||
type ResourceLimit struct {
|
|
||||||
MemSwapLimit int64
|
|
||||||
MemLimit int64
|
|
||||||
ShmSize int64
|
|
||||||
CPUQuota int64
|
|
||||||
CPUShares int64
|
|
||||||
CPUSet string
|
|
||||||
}
|
|
|
@ -270,7 +270,6 @@ func (b *StepBuilder) toInternalRepresentation(parsed *yaml_types.Workflow, envi
|
||||||
compiler.WithEnviron(b.Envs),
|
compiler.WithEnviron(b.Envs),
|
||||||
// TODO: server deps should be moved into StepBuilder fields and set on StepBuilder creation
|
// TODO: server deps should be moved into StepBuilder fields and set on StepBuilder creation
|
||||||
compiler.WithEscalated(server.Config.Pipeline.PrivilegedPlugins...),
|
compiler.WithEscalated(server.Config.Pipeline.PrivilegedPlugins...),
|
||||||
compiler.WithResourceLimit(server.Config.Pipeline.Limits.MemSwapLimit, server.Config.Pipeline.Limits.MemLimit, server.Config.Pipeline.Limits.ShmSize, server.Config.Pipeline.Limits.CPUQuota, server.Config.Pipeline.Limits.CPUShares, server.Config.Pipeline.Limits.CPUSet),
|
|
||||||
compiler.WithVolumes(server.Config.Pipeline.Volumes...),
|
compiler.WithVolumes(server.Config.Pipeline.Volumes...),
|
||||||
compiler.WithNetworks(server.Config.Pipeline.Networks...),
|
compiler.WithNetworks(server.Config.Pipeline.Networks...),
|
||||||
compiler.WithLocal(false),
|
compiler.WithLocal(false),
|
||||||
|
|
Loading…
Reference in a new issue