mirrors/woodpecker
1
0
Fork 0
mirror of https://github.com/woodpecker-ci/woodpecker.git synced 2024-05-31 22:58:35 +00:00
Code Issues Projects Releases Wiki Activity

Merge branch 'origin/main' into 'next-release/main'

Browse source
This commit is contained in:
oauth 2024-05-13 18:14:05 +00:00
parent 8e11e8e873 a700b5855e
commit 3b921ccde6
11 changed files with 203 additions and 34 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.cspell.json
View file

@ -112,6 +112,8 @@
"*.excalidraw",
"*.svg",
"Makefile",
"flake.nix",
"flake.lock",
// TODO: remove the following
"CHANGELOG.md",
".woodpecker/",

6
cmd/server/flags.go
View file

@ -273,6 +273,12 @@ var flags = append([]cli.Flag{
Usage: "how many seconds before timeout when fetching the Woodpecker configuration from a Forge",
Value: time.Second * 3,
},
&cli.UintFlag{
EnvVars: []string{"WOODPECKER_FORGE_RETRY"},
Name: "forge-retry",
Usage: "How many retries of fetching the Woodpecker configuration from a forge are done before we fail",
Value: 3,
},
&cli.Int64Flag{
EnvVars: []string{"WOODPECKER_LIMIT_MEM_SWAP"},
Name: "limit-mem-swap",

6
docs/docs/30-administration/10-server-config.md
View file

@ -525,6 +525,12 @@ Specify a configuration service endpoint, see [Configuration Extension](./100-ex
Specify timeout when fetching the Woodpecker configuration from forge. See <https://pkg.go.dev/time#ParseDuration> for syntax reference.
### `WOODPECKER_FORGE_RETRY`
> Default: 3
Specify how many retries of fetching the Woodpecker configuration from a forge are done before we fail.
### `WOODPECKER_ENABLE_SWAGGER`
> Default: true

68
docs/docs/30-administration/22-backends/40-kubernetes.md
View file

@ -4,19 +4,19 @@ toc_max_heading_level: 2
# Kubernetes backend
The kubernetes backend executes steps inside standalone pods. A temporary PVC is created for the lifetime of the pipeline to transfer files between steps.
The Kubernetes backend executes steps inside standalone Pods. A temporary PVC is created for the lifetime of the pipeline to transfer files between steps.
## Images from private registries
In order to pull private container images defined in your pipeline YAML you must provide [registry credentials in Kubernetes secret](https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/).
As the secret is Agent-wide, it has to be placed in namespace defined by `WOODPECKER_BACKEND_K8S_NAMESPACE`.
Besides, you need to provide the secret name to Agent via `WOODPECKER_BACKEND_K8S_PULL_SECRET_NAMES`.
In order to pull private container images defined in your pipeline YAML you must provide [registry credentials in Kubernetes Secret](https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/).
As the Secret is Agent-wide, it has to be placed in namespace defined by `WOODPECKER_BACKEND_K8S_NAMESPACE`.
Besides, you need to provide the Secret name to Agent via `WOODPECKER_BACKEND_K8S_PULL_SECRET_NAMES`.
## Job specific configuration
### Resources
The kubernetes backend also allows for specifying requests and limits on a per-step basic, most commonly for CPU and memory.
The Kubernetes backend also allows for specifying requests and limits on a per-step basic, most commonly for CPU and memory.
We recommend to add a `resources` definition to all steps to ensure efficient scheduling.
Here is an example definition with an arbitrary `resources` definition below the `backend_options` section:
@ -42,13 +42,13 @@ You can use [Limit Ranges](https://kubernetes.io/docs/concepts/policy/limit-rang
### Runtime class
`runtimeClassName` specifies the name of the RuntimeClass which will be used to run this pod. If no `runtimeClassName` is specified, the default RuntimeHandler will be used.
See the [kubernetes documentation](https://kubernetes.io/docs/concepts/containers/runtime-class/) for more information on specifying runtime classes.
`runtimeClassName` specifies the name of the RuntimeClass which will be used to run this Pod. If no `runtimeClassName` is specified, the default RuntimeHandler will be used.
See the [Kubernetes documentation](https://kubernetes.io/docs/concepts/containers/runtime-class/) for more information on specifying runtime classes.
### Service account
`serviceAccountName` specifies the name of the ServiceAccount which the pod will mount. This service account must be created externally.
See the [kubernetes documentation](https://kubernetes.io/docs/concepts/security/service-accounts/) for more information on using service accounts.
`serviceAccountName` specifies the name of the ServiceAccount which the Pod will mount. This service account must be created externally.
See the [Kubernetes documentation](https://kubernetes.io/docs/concepts/security/service-accounts/) for more information on using service accounts.
### Node selector
@ -83,8 +83,8 @@ You can use [PodNodeSelector](https://kubernetes.io/docs/reference/access-authn-
### Tolerations
When you use `nodeSelector` and the node pool is configured with Taints, you need to specify the Tolerations. Tolerations allow the scheduler to schedule pods with matching taints.
See the [kubernetes documentation](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/) for more information on using tolerations.
When you use `nodeSelector` and the node pool is configured with Taints, you need to specify the Tolerations. Tolerations allow the scheduler to schedule Pods with matching taints.
See the [Kubernetes documentation](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/) for more information on using tolerations.
Example pipeline configuration:
@ -117,7 +117,7 @@ steps:
### Volumes
To mount volumes a persistent volume (PV) and persistent volume claim (PVC) are needed on the cluster which can be referenced in steps via the `volumes` option.
To mount volumes a PersistentVolume (PV) and PersistentVolumeClaim (PVC) are needed on the cluster which can be referenced in steps via the `volumes` option.
Assuming a PVC named `woodpecker-cache` exists, it can be referenced as follows in a step:
```yaml
@ -134,7 +134,7 @@ steps:
### Security context
Use the following configuration to set the [Security Context](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/) for the pod/container running a given pipeline step:
Use the following configuration to set the [Security Context](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/) for the Pod/container running a given pipeline step:
```yaml
steps:
@ -151,9 +151,9 @@ steps:
[...]
```
Note that the `backend_options.kubernetes.securityContext` object allows you to set both pod and container level security context options in one object.
By default, the properties will be set at the pod level. Properties that are only supported on the container level will be set there instead. So, the
configuration shown above will result in something like the following pod spec:
Note that the `backend_options.kubernetes.securityContext` object allows you to set both Pod and container level security context options in one object.
By default, the properties will be set at the Pod level. Properties that are only supported on the container level will be set there instead. So, the
configuration shown above will result in something like the following Pod spec:
```yaml
kind: Pod
@ -195,6 +195,24 @@ backend_options:
AppArmor syntax follows [KEP-24](https://github.com/kubernetes/enhancements/blob/fddcbb9cbf3df39ded03bad71228265ac6e5215f/keps/sig-node/24-apparmor/README.md).
:::
### Annotations and labels
You can specify arbitrary [annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/) and [labels](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/) to be set on the Pod definition for a given workflow step using the following configuration:
```yaml
backend_options:
kubernetes:
annotations:
workflow-group: alpha
io.kubernetes.cri-o.Devices: /dev/fuse
labels:
environment: ci
app.kubernetes.io/name: builder
```
In order to enable this configuration you need to set the appropriate environment variables to `true` on the woodpecker agent:
[WOODPECKER_BACKEND_K8S_POD_ANNOTATIONS_ALLOW_FROM_STEP](#woodpecker_backend_k8s_pod_annotations_allow_from_step) and/or [WOODPECKER_BACKEND_K8S_POD_LABELS_ALLOW_FROM_STEP](#woodpecker_backend_k8s_pod_labels_allow_from_step).
## Tips and tricks
### CRI-O
@ -217,7 +235,7 @@ These env vars can be set in the `env:` sections of the agent.
> Default: `woodpecker`
The namespace to create worker pods in.
The namespace to create worker Pods in.
### `WOODPECKER_BACKEND_K8S_VOLUME_SIZE`
@ -241,13 +259,25 @@ Determines if `RWX` should be used for the pipeline volume's [access mode](https
> Default: empty
Additional labels to apply to worker pods. Must be a YAML object, e.g. `{"example.com/test-label":"test-value"}`.
Additional labels to apply to worker Pods. Must be a YAML object, e.g. `{"example.com/test-label":"test-value"}`.
### `WOODPECKER_BACKEND_K8S_POD_LABELS_ALLOW_FROM_STEP`
> Default: `false`
Determines if additional Pod labels can be defined from a step's backend options.
### `WOODPECKER_BACKEND_K8S_POD_ANNOTATIONS`
> Default: empty
Additional annotations to apply to worker pods. Must be a YAML object, e.g. `{"example.com/test-annotation":"test-value"}`.
Additional annotations to apply to worker Pods. Must be a YAML object, e.g. `{"example.com/test-annotation":"test-value"}`.
### `WOODPECKER_BACKEND_K8S_POD_ANNOTATIONS_ALLOW_FROM_STEP`
> Default: `false`
Determines if Pod annotations can be defined from a step's backend options.
### `WOODPECKER_BACKEND_K8S_SECCTX_NONROOT`

76
flake.lock Normal file
View file

@ -0,0 +1,76 @@
{
"nodes": {
"flake-utils": {
"inputs": {
"systems": "systems"
},
"locked": {
"lastModified": 1710146030,
"narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=",
"owner": "numtide",
"repo": "flake-utils",
"rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a",
"type": "github"
},
"original": {
"id": "flake-utils",
"type": "indirect"
}
},
"nixpkgs": {
"locked": {
"lastModified": 1715614915,
"narHash": "sha256-O6sqpppOtlfgx6PK5bnkAvBudK1rpjP7ig0dj7HvIl0=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "d2ed14aa4f912254c578fc19b842f2910c9146be",
"type": "github"
},
"original": {
"owner": "NixOS",
"ref": "master",
"repo": "nixpkgs",
"type": "github"
}
},
"root": {
"inputs": {
"flake-utils": "flake-utils",
"nixpkgs": "nixpkgs",
"systems": "systems_2"
}
},
"systems": {
"locked": {
"lastModified": 1681028828,
"narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
"owner": "nix-systems",
"repo": "default",
"rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
"type": "github"
},
"original": {
"owner": "nix-systems",
"repo": "default",
"type": "github"
}
},
"systems_2": {
"locked": {
"lastModified": 1681028828,
"narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
"owner": "nix-systems",
"repo": "default",
"rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
"type": "github"
},
"original": {
"owner": "nix-systems",
"repo": "default",
"type": "github"
}
}
},
"root": "root",
"version": 7
}

41
flake.nix Normal file
View file

@ -0,0 +1,41 @@
{
# Override nixpkgs to use the latest set of node packages
inputs.nixpkgs.url = "github:NixOS/nixpkgs/master";
inputs.systems.url = "github:nix-systems/default";
outputs =
{
self,
nixpkgs,
flake-utils,
systems,
}:
flake-utils.lib.eachSystem (import systems) (
system:
let
pkgs = import nixpkgs { inherit system; };
in
{
devShells.default = pkgs.mkShell {
buildInputs = with pkgs; [
# generic
gnumake
gnutar
# frontend
nodejs
nodePackages.pnpm
nodePackages.typescript
nodePackages.typescript-language-server
# backend
go
gofumpt
golangci-lint
go-mockery
protobuf
];
};
}
);
}

2
server/services/config/combined_test.go
View file

@ -221,7 +221,7 @@ func TestFetchFromConfigService(t *testing.T) {
f.On("Netrc", mock.Anything, mock.Anything).Return(&model.Netrc{Machine: "mock", Login: "mock", Password: "mock"}, nil)
forgeFetcher := config.NewForge(time.Second * 3)
forgeFetcher := config.NewForge(time.Second*3, 3)
configFetcher := config.NewCombined(forgeFetcher, httpFetcher)
files, err := configFetcher.Fetch(
context.Background(),

14
server/services/config/forge.go
View file

@ -29,17 +29,15 @@ import (
"go.woodpecker-ci.org/woodpecker/v2/shared/constant"
)
const (
forgeFetchingRetryCount = 3
)
type forgeFetcher struct {
timeout time.Duration
timeout time.Duration
retryCount uint
}
func NewForge(timeout time.Duration) Service {
func NewForge(timeout time.Duration, retries uint) Service {
return &forgeFetcher{
timeout: timeout,
timeout: timeout,
retryCount: retries,
}
}
@ -58,7 +56,7 @@ func (f *forgeFetcher) Fetch(ctx context.Context, forge forge.Forge, user *model
}
// try to fetch multiple times
for i := 0; i < forgeFetchingRetryCount; i++ {
for i := 0; i < int(f.retryCount); i++ {
files, err = ffc.fetch(ctx, strings.TrimSpace(repo.Config))
if err != nil {
log.Trace().Err(err).Msgf("%d. try failed", i+1)

3
server/services/config/forge_test.go
View file

@ -306,7 +306,8 @@ func TestFetch(t *testing.T) {
f.On("Dir", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil, fmt.Errorf("directory not found"))
configFetcher := config.NewForge(
time.Second * 3,
time.Second*3,
3,
)
files, err := configFetcher.Fetch(
context.Background(),

7
server/services/manager.go
View file

@ -72,13 +72,18 @@ func NewManager(c *cli.Context, store store.Store, setupForge SetupForge) (Manag
return nil, err
}
configService, err := setupConfigService(c, signaturePrivateKey)
if err != nil {
return nil, err
}
return &manager{
signaturePrivateKey: signaturePrivateKey,
signaturePublicKey: signaturePublicKey,
store: store,
secret: setupSecretService(store),
registry: setupRegistryService(store, c.String("docker-config")),
config: setupConfigService(c, signaturePrivateKey),
config: configService,
environment: environment.Parse(c.StringSlice("environment")),
forgeCache: ttlcache.New(ttlcache.WithDisableTouchOnHit[int64, forge.Forge]()),
setupForge: setupForge,

12
server/services/setup.go
View file

@ -56,16 +56,20 @@ func setupSecretService(store store.Store) secret.Service {
return secret.NewDB(store)
}
func setupConfigService(c *cli.Context, privateSignatureKey crypto.PrivateKey) config.Service {
func setupConfigService(c *cli.Context, privateSignatureKey crypto.PrivateKey) (config.Service, error) {
timeout := c.Duration("forge-timeout")
configFetcher := config.NewForge(timeout)
retries := c.Uint("forge-retry")
if retries == 0 {
return nil, fmt.Errorf("WOODPECKER_FORGE_RETRY can not be 0")
}
configFetcher := config.NewForge(timeout, retries)
if endpoint := c.String("config-service-endpoint"); endpoint != "" {
httpFetcher := config.NewHTTP(endpoint, privateSignatureKey)
return config.NewCombined(configFetcher, httpFetcher)
return config.NewCombined(configFetcher, httpFetcher), nil
}
return configFetcher
return configFetcher, nil
}
// setupSignatureKeys generate or load key pair to sign webhooks requests (i.e. used for service extensions)