mirror of
https://github.com/woodpecker-ci/woodpecker.git
synced 2024-10-31 22:38:49 +00:00
Merge branch 'origin/main' into 'next-release/main'
This commit is contained in:
commit
3b921ccde6
11 changed files with 203 additions and 34 deletions
|
@ -112,6 +112,8 @@
|
|||
"*.excalidraw",
|
||||
"*.svg",
|
||||
"Makefile",
|
||||
"flake.nix",
|
||||
"flake.lock",
|
||||
// TODO: remove the following
|
||||
"CHANGELOG.md",
|
||||
".woodpecker/",
|
||||
|
|
|
@ -273,6 +273,12 @@ var flags = append([]cli.Flag{
|
|||
Usage: "how many seconds before timeout when fetching the Woodpecker configuration from a Forge",
|
||||
Value: time.Second * 3,
|
||||
},
|
||||
&cli.UintFlag{
|
||||
EnvVars: []string{"WOODPECKER_FORGE_RETRY"},
|
||||
Name: "forge-retry",
|
||||
Usage: "How many retries of fetching the Woodpecker configuration from a forge are done before we fail",
|
||||
Value: 3,
|
||||
},
|
||||
&cli.Int64Flag{
|
||||
EnvVars: []string{"WOODPECKER_LIMIT_MEM_SWAP"},
|
||||
Name: "limit-mem-swap",
|
||||
|
|
|
@ -525,6 +525,12 @@ Specify a configuration service endpoint, see [Configuration Extension](./100-ex
|
|||
|
||||
Specify timeout when fetching the Woodpecker configuration from forge. See <https://pkg.go.dev/time#ParseDuration> for syntax reference.
|
||||
|
||||
### `WOODPECKER_FORGE_RETRY`
|
||||
|
||||
> Default: 3
|
||||
|
||||
Specify how many retries of fetching the Woodpecker configuration from a forge are done before we fail.
|
||||
|
||||
### `WOODPECKER_ENABLE_SWAGGER`
|
||||
|
||||
> Default: true
|
||||
|
|
|
@ -4,19 +4,19 @@ toc_max_heading_level: 2
|
|||
|
||||
# Kubernetes backend
|
||||
|
||||
The kubernetes backend executes steps inside standalone pods. A temporary PVC is created for the lifetime of the pipeline to transfer files between steps.
|
||||
The Kubernetes backend executes steps inside standalone Pods. A temporary PVC is created for the lifetime of the pipeline to transfer files between steps.
|
||||
|
||||
## Images from private registries
|
||||
|
||||
In order to pull private container images defined in your pipeline YAML you must provide [registry credentials in Kubernetes secret](https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/).
|
||||
As the secret is Agent-wide, it has to be placed in namespace defined by `WOODPECKER_BACKEND_K8S_NAMESPACE`.
|
||||
Besides, you need to provide the secret name to Agent via `WOODPECKER_BACKEND_K8S_PULL_SECRET_NAMES`.
|
||||
In order to pull private container images defined in your pipeline YAML you must provide [registry credentials in Kubernetes Secret](https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/).
|
||||
As the Secret is Agent-wide, it has to be placed in namespace defined by `WOODPECKER_BACKEND_K8S_NAMESPACE`.
|
||||
Besides, you need to provide the Secret name to Agent via `WOODPECKER_BACKEND_K8S_PULL_SECRET_NAMES`.
|
||||
|
||||
## Job specific configuration
|
||||
|
||||
### Resources
|
||||
|
||||
The kubernetes backend also allows for specifying requests and limits on a per-step basic, most commonly for CPU and memory.
|
||||
The Kubernetes backend also allows for specifying requests and limits on a per-step basic, most commonly for CPU and memory.
|
||||
We recommend to add a `resources` definition to all steps to ensure efficient scheduling.
|
||||
|
||||
Here is an example definition with an arbitrary `resources` definition below the `backend_options` section:
|
||||
|
@ -42,13 +42,13 @@ You can use [Limit Ranges](https://kubernetes.io/docs/concepts/policy/limit-rang
|
|||
|
||||
### Runtime class
|
||||
|
||||
`runtimeClassName` specifies the name of the RuntimeClass which will be used to run this pod. If no `runtimeClassName` is specified, the default RuntimeHandler will be used.
|
||||
See the [kubernetes documentation](https://kubernetes.io/docs/concepts/containers/runtime-class/) for more information on specifying runtime classes.
|
||||
`runtimeClassName` specifies the name of the RuntimeClass which will be used to run this Pod. If no `runtimeClassName` is specified, the default RuntimeHandler will be used.
|
||||
See the [Kubernetes documentation](https://kubernetes.io/docs/concepts/containers/runtime-class/) for more information on specifying runtime classes.
|
||||
|
||||
### Service account
|
||||
|
||||
`serviceAccountName` specifies the name of the ServiceAccount which the pod will mount. This service account must be created externally.
|
||||
See the [kubernetes documentation](https://kubernetes.io/docs/concepts/security/service-accounts/) for more information on using service accounts.
|
||||
`serviceAccountName` specifies the name of the ServiceAccount which the Pod will mount. This service account must be created externally.
|
||||
See the [Kubernetes documentation](https://kubernetes.io/docs/concepts/security/service-accounts/) for more information on using service accounts.
|
||||
|
||||
### Node selector
|
||||
|
||||
|
@ -83,8 +83,8 @@ You can use [PodNodeSelector](https://kubernetes.io/docs/reference/access-authn-
|
|||
|
||||
### Tolerations
|
||||
|
||||
When you use `nodeSelector` and the node pool is configured with Taints, you need to specify the Tolerations. Tolerations allow the scheduler to schedule pods with matching taints.
|
||||
See the [kubernetes documentation](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/) for more information on using tolerations.
|
||||
When you use `nodeSelector` and the node pool is configured with Taints, you need to specify the Tolerations. Tolerations allow the scheduler to schedule Pods with matching taints.
|
||||
See the [Kubernetes documentation](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/) for more information on using tolerations.
|
||||
|
||||
Example pipeline configuration:
|
||||
|
||||
|
@ -117,7 +117,7 @@ steps:
|
|||
|
||||
### Volumes
|
||||
|
||||
To mount volumes a persistent volume (PV) and persistent volume claim (PVC) are needed on the cluster which can be referenced in steps via the `volumes` option.
|
||||
To mount volumes a PersistentVolume (PV) and PersistentVolumeClaim (PVC) are needed on the cluster which can be referenced in steps via the `volumes` option.
|
||||
Assuming a PVC named `woodpecker-cache` exists, it can be referenced as follows in a step:
|
||||
|
||||
```yaml
|
||||
|
@ -134,7 +134,7 @@ steps:
|
|||
|
||||
### Security context
|
||||
|
||||
Use the following configuration to set the [Security Context](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/) for the pod/container running a given pipeline step:
|
||||
Use the following configuration to set the [Security Context](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/) for the Pod/container running a given pipeline step:
|
||||
|
||||
```yaml
|
||||
steps:
|
||||
|
@ -151,9 +151,9 @@ steps:
|
|||
[...]
|
||||
```
|
||||
|
||||
Note that the `backend_options.kubernetes.securityContext` object allows you to set both pod and container level security context options in one object.
|
||||
By default, the properties will be set at the pod level. Properties that are only supported on the container level will be set there instead. So, the
|
||||
configuration shown above will result in something like the following pod spec:
|
||||
Note that the `backend_options.kubernetes.securityContext` object allows you to set both Pod and container level security context options in one object.
|
||||
By default, the properties will be set at the Pod level. Properties that are only supported on the container level will be set there instead. So, the
|
||||
configuration shown above will result in something like the following Pod spec:
|
||||
|
||||
```yaml
|
||||
kind: Pod
|
||||
|
@ -195,6 +195,24 @@ backend_options:
|
|||
AppArmor syntax follows [KEP-24](https://github.com/kubernetes/enhancements/blob/fddcbb9cbf3df39ded03bad71228265ac6e5215f/keps/sig-node/24-apparmor/README.md).
|
||||
:::
|
||||
|
||||
### Annotations and labels
|
||||
|
||||
You can specify arbitrary [annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/) and [labels](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/) to be set on the Pod definition for a given workflow step using the following configuration:
|
||||
|
||||
```yaml
|
||||
backend_options:
|
||||
kubernetes:
|
||||
annotations:
|
||||
workflow-group: alpha
|
||||
io.kubernetes.cri-o.Devices: /dev/fuse
|
||||
labels:
|
||||
environment: ci
|
||||
app.kubernetes.io/name: builder
|
||||
```
|
||||
|
||||
In order to enable this configuration you need to set the appropriate environment variables to `true` on the woodpecker agent:
|
||||
[WOODPECKER_BACKEND_K8S_POD_ANNOTATIONS_ALLOW_FROM_STEP](#woodpecker_backend_k8s_pod_annotations_allow_from_step) and/or [WOODPECKER_BACKEND_K8S_POD_LABELS_ALLOW_FROM_STEP](#woodpecker_backend_k8s_pod_labels_allow_from_step).
|
||||
|
||||
## Tips and tricks
|
||||
|
||||
### CRI-O
|
||||
|
@ -217,7 +235,7 @@ These env vars can be set in the `env:` sections of the agent.
|
|||
|
||||
> Default: `woodpecker`
|
||||
|
||||
The namespace to create worker pods in.
|
||||
The namespace to create worker Pods in.
|
||||
|
||||
### `WOODPECKER_BACKEND_K8S_VOLUME_SIZE`
|
||||
|
||||
|
@ -241,13 +259,25 @@ Determines if `RWX` should be used for the pipeline volume's [access mode](https
|
|||
|
||||
> Default: empty
|
||||
|
||||
Additional labels to apply to worker pods. Must be a YAML object, e.g. `{"example.com/test-label":"test-value"}`.
|
||||
Additional labels to apply to worker Pods. Must be a YAML object, e.g. `{"example.com/test-label":"test-value"}`.
|
||||
|
||||
### `WOODPECKER_BACKEND_K8S_POD_LABELS_ALLOW_FROM_STEP`
|
||||
|
||||
> Default: `false`
|
||||
|
||||
Determines if additional Pod labels can be defined from a step's backend options.
|
||||
|
||||
### `WOODPECKER_BACKEND_K8S_POD_ANNOTATIONS`
|
||||
|
||||
> Default: empty
|
||||
|
||||
Additional annotations to apply to worker pods. Must be a YAML object, e.g. `{"example.com/test-annotation":"test-value"}`.
|
||||
Additional annotations to apply to worker Pods. Must be a YAML object, e.g. `{"example.com/test-annotation":"test-value"}`.
|
||||
|
||||
### `WOODPECKER_BACKEND_K8S_POD_ANNOTATIONS_ALLOW_FROM_STEP`
|
||||
|
||||
> Default: `false`
|
||||
|
||||
Determines if Pod annotations can be defined from a step's backend options.
|
||||
|
||||
### `WOODPECKER_BACKEND_K8S_SECCTX_NONROOT`
|
||||
|
||||
|
|
76
flake.lock
Normal file
76
flake.lock
Normal file
|
@ -0,0 +1,76 @@
|
|||
{
|
||||
"nodes": {
|
||||
"flake-utils": {
|
||||
"inputs": {
|
||||
"systems": "systems"
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1710146030,
|
||||
"narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=",
|
||||
"owner": "numtide",
|
||||
"repo": "flake-utils",
|
||||
"rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"id": "flake-utils",
|
||||
"type": "indirect"
|
||||
}
|
||||
},
|
||||
"nixpkgs": {
|
||||
"locked": {
|
||||
"lastModified": 1715614915,
|
||||
"narHash": "sha256-O6sqpppOtlfgx6PK5bnkAvBudK1rpjP7ig0dj7HvIl0=",
|
||||
"owner": "NixOS",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "d2ed14aa4f912254c578fc19b842f2910c9146be",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "NixOS",
|
||||
"ref": "master",
|
||||
"repo": "nixpkgs",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"root": {
|
||||
"inputs": {
|
||||
"flake-utils": "flake-utils",
|
||||
"nixpkgs": "nixpkgs",
|
||||
"systems": "systems_2"
|
||||
}
|
||||
},
|
||||
"systems": {
|
||||
"locked": {
|
||||
"lastModified": 1681028828,
|
||||
"narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
|
||||
"owner": "nix-systems",
|
||||
"repo": "default",
|
||||
"rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "nix-systems",
|
||||
"repo": "default",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"systems_2": {
|
||||
"locked": {
|
||||
"lastModified": 1681028828,
|
||||
"narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
|
||||
"owner": "nix-systems",
|
||||
"repo": "default",
|
||||
"rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "nix-systems",
|
||||
"repo": "default",
|
||||
"type": "github"
|
||||
}
|
||||
}
|
||||
},
|
||||
"root": "root",
|
||||
"version": 7
|
||||
}
|
41
flake.nix
Normal file
41
flake.nix
Normal file
|
@ -0,0 +1,41 @@
|
|||
{
|
||||
# Override nixpkgs to use the latest set of node packages
|
||||
inputs.nixpkgs.url = "github:NixOS/nixpkgs/master";
|
||||
inputs.systems.url = "github:nix-systems/default";
|
||||
|
||||
outputs =
|
||||
{
|
||||
self,
|
||||
nixpkgs,
|
||||
flake-utils,
|
||||
systems,
|
||||
}:
|
||||
flake-utils.lib.eachSystem (import systems) (
|
||||
system:
|
||||
let
|
||||
pkgs = import nixpkgs { inherit system; };
|
||||
in
|
||||
{
|
||||
devShells.default = pkgs.mkShell {
|
||||
buildInputs = with pkgs; [
|
||||
# generic
|
||||
gnumake
|
||||
gnutar
|
||||
|
||||
# frontend
|
||||
nodejs
|
||||
nodePackages.pnpm
|
||||
nodePackages.typescript
|
||||
nodePackages.typescript-language-server
|
||||
|
||||
# backend
|
||||
go
|
||||
gofumpt
|
||||
golangci-lint
|
||||
go-mockery
|
||||
protobuf
|
||||
];
|
||||
};
|
||||
}
|
||||
);
|
||||
}
|
|
@ -221,7 +221,7 @@ func TestFetchFromConfigService(t *testing.T) {
|
|||
|
||||
f.On("Netrc", mock.Anything, mock.Anything).Return(&model.Netrc{Machine: "mock", Login: "mock", Password: "mock"}, nil)
|
||||
|
||||
forgeFetcher := config.NewForge(time.Second * 3)
|
||||
forgeFetcher := config.NewForge(time.Second*3, 3)
|
||||
configFetcher := config.NewCombined(forgeFetcher, httpFetcher)
|
||||
files, err := configFetcher.Fetch(
|
||||
context.Background(),
|
||||
|
|
|
@ -29,17 +29,15 @@ import (
|
|||
"go.woodpecker-ci.org/woodpecker/v2/shared/constant"
|
||||
)
|
||||
|
||||
const (
|
||||
forgeFetchingRetryCount = 3
|
||||
)
|
||||
|
||||
type forgeFetcher struct {
|
||||
timeout time.Duration
|
||||
timeout time.Duration
|
||||
retryCount uint
|
||||
}
|
||||
|
||||
func NewForge(timeout time.Duration) Service {
|
||||
func NewForge(timeout time.Duration, retries uint) Service {
|
||||
return &forgeFetcher{
|
||||
timeout: timeout,
|
||||
timeout: timeout,
|
||||
retryCount: retries,
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -58,7 +56,7 @@ func (f *forgeFetcher) Fetch(ctx context.Context, forge forge.Forge, user *model
|
|||
}
|
||||
|
||||
// try to fetch multiple times
|
||||
for i := 0; i < forgeFetchingRetryCount; i++ {
|
||||
for i := 0; i < int(f.retryCount); i++ {
|
||||
files, err = ffc.fetch(ctx, strings.TrimSpace(repo.Config))
|
||||
if err != nil {
|
||||
log.Trace().Err(err).Msgf("%d. try failed", i+1)
|
||||
|
|
|
@ -306,7 +306,8 @@ func TestFetch(t *testing.T) {
|
|||
f.On("Dir", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil, fmt.Errorf("directory not found"))
|
||||
|
||||
configFetcher := config.NewForge(
|
||||
time.Second * 3,
|
||||
time.Second*3,
|
||||
3,
|
||||
)
|
||||
files, err := configFetcher.Fetch(
|
||||
context.Background(),
|
||||
|
|
|
@ -72,13 +72,18 @@ func NewManager(c *cli.Context, store store.Store, setupForge SetupForge) (Manag
|
|||
return nil, err
|
||||
}
|
||||
|
||||
configService, err := setupConfigService(c, signaturePrivateKey)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &manager{
|
||||
signaturePrivateKey: signaturePrivateKey,
|
||||
signaturePublicKey: signaturePublicKey,
|
||||
store: store,
|
||||
secret: setupSecretService(store),
|
||||
registry: setupRegistryService(store, c.String("docker-config")),
|
||||
config: setupConfigService(c, signaturePrivateKey),
|
||||
config: configService,
|
||||
environment: environment.Parse(c.StringSlice("environment")),
|
||||
forgeCache: ttlcache.New(ttlcache.WithDisableTouchOnHit[int64, forge.Forge]()),
|
||||
setupForge: setupForge,
|
||||
|
|
|
@ -56,16 +56,20 @@ func setupSecretService(store store.Store) secret.Service {
|
|||
return secret.NewDB(store)
|
||||
}
|
||||
|
||||
func setupConfigService(c *cli.Context, privateSignatureKey crypto.PrivateKey) config.Service {
|
||||
func setupConfigService(c *cli.Context, privateSignatureKey crypto.PrivateKey) (config.Service, error) {
|
||||
timeout := c.Duration("forge-timeout")
|
||||
configFetcher := config.NewForge(timeout)
|
||||
retries := c.Uint("forge-retry")
|
||||
if retries == 0 {
|
||||
return nil, fmt.Errorf("WOODPECKER_FORGE_RETRY can not be 0")
|
||||
}
|
||||
configFetcher := config.NewForge(timeout, retries)
|
||||
|
||||
if endpoint := c.String("config-service-endpoint"); endpoint != "" {
|
||||
httpFetcher := config.NewHTTP(endpoint, privateSignatureKey)
|
||||
return config.NewCombined(configFetcher, httpFetcher)
|
||||
return config.NewCombined(configFetcher, httpFetcher), nil
|
||||
}
|
||||
|
||||
return configFetcher
|
||||
return configFetcher, nil
|
||||
}
|
||||
|
||||
// setupSignatureKeys generate or load key pair to sign webhooks requests (i.e. used for service extensions)
|
||||
|
|
Loading…
Reference in a new issue