mirror of
https://github.com/woodpecker-ci/woodpecker.git
synced 2024-12-23 08:56:29 +00:00
Support go plugins for forges and agent backends (#2751)
As of #2520 Support to load new forges and agent backends at runtime using go's plugin system. (https://pkg.go.dev/plugin) I also added a simple example addon (a new forge which just prints log statements), it should be removed later of course, but you can see an example. --------- Co-authored-by: Michalis Zampetakis <mzampetakis@gmail.com> Co-authored-by: Anbraten <anton@ju60.de>
This commit is contained in:
parent
6432109daf
commit
dfc2c265b1
11 changed files with 276 additions and 8 deletions
|
@ -43,6 +43,8 @@ import (
|
|||
"go.woodpecker-ci.org/woodpecker/v2/pipeline/backend"
|
||||
"go.woodpecker-ci.org/woodpecker/v2/pipeline/backend/types"
|
||||
"go.woodpecker-ci.org/woodpecker/v2/pipeline/rpc"
|
||||
"go.woodpecker-ci.org/woodpecker/v2/shared/addon"
|
||||
addonTypes "go.woodpecker-ci.org/woodpecker/v2/shared/addon/types"
|
||||
"go.woodpecker-ci.org/woodpecker/v2/shared/utils"
|
||||
"go.woodpecker-ci.org/woodpecker/v2/version"
|
||||
)
|
||||
|
@ -149,21 +151,23 @@ func run(c *cli.Context) error {
|
|||
return err
|
||||
}
|
||||
|
||||
backendCtx := context.WithValue(ctx, types.CliContext, c)
|
||||
backend.Init(backendCtx)
|
||||
|
||||
var wg sync.WaitGroup
|
||||
parallel := c.Int("max-workflows")
|
||||
wg.Add(parallel)
|
||||
|
||||
// new backend
|
||||
backendEngine, err := backend.FindBackend(backendCtx, c.String("backend-engine"))
|
||||
// new engine
|
||||
backendCtx := context.WithValue(ctx, types.CliContext, c)
|
||||
backendEngine, err := getBackendEngine(backendCtx, c.String("backend-engine"), c.StringSlice("addons"))
|
||||
if err != nil {
|
||||
log.Error().Err(err).Msgf("cannot find backend engine '%s'", c.String("backend-engine"))
|
||||
return err
|
||||
}
|
||||
|
||||
// load backend (e.g. init api client)
|
||||
if !backendEngine.IsAvailable(backendCtx) {
|
||||
log.Error().Str("engine", backendEngine.Name()).Msg("selected backend engine is unavailable")
|
||||
return fmt.Errorf("selected backend engine %s is unavailable", backendEngine.Name())
|
||||
}
|
||||
|
||||
// load engine (e.g. init api client)
|
||||
engInfo, err := backendEngine.Load(backendCtx)
|
||||
if err != nil {
|
||||
log.Error().Err(err).Msg("cannot load backend engine")
|
||||
|
@ -247,6 +251,25 @@ func run(c *cli.Context) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
func getBackendEngine(backendCtx context.Context, backendName string, addons []string) (types.Backend, error) {
|
||||
addonBackend, err := addon.Load[types.Backend](addons, addonTypes.TypeBackend)
|
||||
if err != nil {
|
||||
log.Error().Err(err).Msg("cannot load backend addon")
|
||||
return nil, err
|
||||
}
|
||||
if addonBackend != nil {
|
||||
return addonBackend.Value, nil
|
||||
}
|
||||
|
||||
backend.Init(backendCtx)
|
||||
engine, err := backend.FindBackend(backendCtx, backendName)
|
||||
if err != nil {
|
||||
log.Error().Err(err).Msgf("cannot find backend engine '%s'", backendName)
|
||||
return nil, err
|
||||
}
|
||||
return engine, nil
|
||||
}
|
||||
|
||||
func runWithRetry(context *cli.Context) error {
|
||||
retryCount := context.Int("connect-retry-count")
|
||||
retryDelay := context.Duration("connect-retry-delay")
|
||||
|
|
|
@ -97,4 +97,9 @@ var flags = []cli.Flag{
|
|||
Usage: "backend to run pipelines on",
|
||||
Value: "auto-detect",
|
||||
},
|
||||
&cli.StringSliceFlag{
|
||||
EnvVars: []string{"WOODPECKER_ADDONS"},
|
||||
Name: "addons",
|
||||
Usage: "list of addon files",
|
||||
},
|
||||
}
|
||||
|
|
|
@ -251,6 +251,11 @@ var flags = append([]cli.Flag{
|
|||
Name: "enable-swagger",
|
||||
Value: true,
|
||||
},
|
||||
&cli.StringSliceFlag{
|
||||
EnvVars: []string{"WOODPECKER_ADDONS"},
|
||||
Name: "addons",
|
||||
Usage: "list of addon files",
|
||||
},
|
||||
//
|
||||
// backend options for pipeline compiler
|
||||
//
|
||||
|
|
|
@ -49,6 +49,8 @@ import (
|
|||
"go.woodpecker-ci.org/woodpecker/v2/server/store"
|
||||
"go.woodpecker-ci.org/woodpecker/v2/server/store/datastore"
|
||||
"go.woodpecker-ci.org/woodpecker/v2/server/store/types"
|
||||
"go.woodpecker-ci.org/woodpecker/v2/shared/addon"
|
||||
addonTypes "go.woodpecker-ci.org/woodpecker/v2/shared/addon/types"
|
||||
)
|
||||
|
||||
func setupStore(c *cli.Context) (store.Store, error) {
|
||||
|
@ -130,8 +132,16 @@ func setupMembershipService(_ *cli.Context, r forge.Forge) cache.MembershipServi
|
|||
return cache.NewMembershipService(r)
|
||||
}
|
||||
|
||||
// setupForge helper function to setup the forge from the CLI arguments.
|
||||
// setupForge helper function to set up the forge from the CLI arguments.
|
||||
func setupForge(c *cli.Context) (forge.Forge, error) {
|
||||
addonForge, err := addon.Load[forge.Forge](c.StringSlice("addons"), addonTypes.TypeForge)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if addonForge != nil {
|
||||
return addonForge.Value, nil
|
||||
}
|
||||
|
||||
switch {
|
||||
case c.Bool("github"):
|
||||
return setupGitHub(c)
|
||||
|
|
|
@ -521,6 +521,12 @@ Supported variables:
|
|||
- `owner`: the repo's owner
|
||||
- `repo`: the repo's name
|
||||
|
||||
### WOODPECKER_ADDONS
|
||||
|
||||
> Default: empty
|
||||
|
||||
List of addon files. See [addons](./75-addons/00-overview.md).
|
||||
|
||||
---
|
||||
|
||||
### `WOODPECKER_LIMIT_MEM_SWAP`
|
||||
|
|
|
@ -178,6 +178,12 @@ Configures if the gRPC server certificate should be verified, only valid when `W
|
|||
|
||||
Configures the backend engine to run pipelines on. Possible values are `auto-detect`, `docker`, `local` or `kubernetes`.
|
||||
|
||||
### WOODPECKER_ADDONS
|
||||
|
||||
> Default: empty
|
||||
|
||||
List of addon files. See [addons](./75-addons/00-overview.md).
|
||||
|
||||
### `WOODPECKER_BACKEND_DOCKER_*`
|
||||
|
||||
See [Docker backend configuration](./22-backends/10-docker.md#configuration)
|
||||
|
|
48
docs/docs/30-administration/75-addons/00-overview.md
Normal file
48
docs/docs/30-administration/75-addons/00-overview.md
Normal file
|
@ -0,0 +1,48 @@
|
|||
# Addons
|
||||
|
||||
:::warning
|
||||
Addons are still experimental. Their implementation can change and break at any time.
|
||||
:::
|
||||
|
||||
:::danger
|
||||
You need to trust the author of the addons you use. Depending on their type, addons can access forge authentication codes, your secrets or other sensitive information.
|
||||
:::
|
||||
|
||||
To adapt Woodpecker to your needs beyond the [configuration](../10-server-config.md), Woodpecker has its own **addon** system, built ontop of [Go's internal plugin system](https://go.dev/pkg/plugin).
|
||||
|
||||
Addons can be used for:
|
||||
|
||||
- Forges
|
||||
- Agent backends
|
||||
|
||||
## Restrictions
|
||||
|
||||
Addons are restricted by how Go plugins work. This includes the following restrictions:
|
||||
|
||||
- only supported on Linux, FreeBSD and macOS
|
||||
- addons must have been built for the correct Woodpecker version. If an addon is not provided specifically for this version, you likely won't be able to use it.
|
||||
|
||||
## Usage
|
||||
|
||||
To use an addon, download the addon version built for your Woodpecker version. Then, you can add the following to your configuration:
|
||||
|
||||
```diff
|
||||
# docker-compose.yml
|
||||
version: '3'
|
||||
|
||||
services:
|
||||
woodpecker-server:
|
||||
[...]
|
||||
environment:
|
||||
+ - WOODPECKER_ADDONS=/path/to/your/addon/file.so
|
||||
```
|
||||
|
||||
In case you run Woodpecker as container, you probably want to mount the addon binaries to `/opt/addons/`.
|
||||
|
||||
You can list multiple addons, Woodpecker will automatically determine their type. If you specify multiple addons with the same type, only the first one will be used.
|
||||
|
||||
Using an addon always overwrites Woodpecker's internal setup. This means, that a forge addon will be used if specified, no matter what's configured for the forges natively supported by Woodpecker.
|
||||
|
||||
### Bug reports
|
||||
|
||||
If you experience bugs, please check which component has the issue. If it's the addon, **do not raise an issue in the main repository**, but rather use the separate addon repositories. To check which component is responsible for the bug, look at the logs. Logs from addons are marked with a special field `addon` containing their addon file name.
|
88
docs/docs/30-administration/75-addons/20-creating-addons.md
Normal file
88
docs/docs/30-administration/75-addons/20-creating-addons.md
Normal file
|
@ -0,0 +1,88 @@
|
|||
# Creating addons
|
||||
|
||||
Addons are written in Go.
|
||||
|
||||
## Writing your code
|
||||
|
||||
An addon consists of two variables/functions in Go.
|
||||
|
||||
1. The `Type` variable. Specifies the type of the addon and must be directly accessed from `shared/addons/types/types.go`.
|
||||
2. The `Addon` function which is the main point of your addon.
|
||||
This function takes two arguments:
|
||||
|
||||
1. The zerolog logger you should use to log errors, warnings etc.
|
||||
2. A slice of strings with the environment variables used as configuration.
|
||||
|
||||
It returns two values:
|
||||
|
||||
1. The actual addon. For type reference see [table below](#return-types).
|
||||
2. An error. If this error is not `nil`, Woodpecker exits.
|
||||
|
||||
Directly import Woodpecker's Go package (`go.woodpecker-ci.org/woodpecker/woodpecker/v2`) and use the interfaces and types defined there.
|
||||
|
||||
### Return types
|
||||
|
||||
| Addon type | Return type |
|
||||
| ---------- | -------------------------------------------------------------------------------- |
|
||||
| `Forge` | `"go.woodpecker-ci.org/woodpecker/woodpecker/v2/server/forge".Forge` |
|
||||
| `Backend` | `"go.woodpecker-ci.org/woodpecker/woodpecker/v2/pipeline/backend/types".Backend` |
|
||||
|
||||
## Compiling
|
||||
|
||||
After you write your addon code, compile your addon:
|
||||
|
||||
```sh
|
||||
go build -buildmode plugin
|
||||
```
|
||||
|
||||
The output file is your addon which is now ready to be used.
|
||||
|
||||
## Restrictions
|
||||
|
||||
Addons must directly depend on Woodpecker's core (`go.woodpecker-ci.org/woodpecker/woodpecker/v2`).
|
||||
The addon must have been built with **excatly the same code** as the Woodpecker instance you'd like to use it on. This means: If you build your addon with a specific commit from Woodpecker `next`, you can likely only use it with the Woodpecker version compiled from this commit.
|
||||
Also, if you change something inside of Woodpecker without committing, it might fail because you need to recompile your addon with this code first.
|
||||
|
||||
In addition to this, addons are only supported on Linux, FreeBSD and macOS.
|
||||
|
||||
:::info
|
||||
It is recommended to at least support the latest released version of Woodpecker.
|
||||
:::
|
||||
|
||||
### Compile for different versions
|
||||
|
||||
As long as there are no changes to Woodpecker's interfaces or they are backwards-compatible, you can easily compile the addon for multiple version by changing the version of `go.woodpecker-ci.org/woodpecker/woodpecker/v2` using `go get` before compiling.
|
||||
|
||||
## Logging
|
||||
|
||||
The entrypoint receives a `zerolog.Logger` as input. **Do not use any other logging solution.** This logger follows the configuration of the Woodpecker instance and adds a special field `addon` to the log entries which allows users to find out which component is writing the log messages.
|
||||
|
||||
## Example structure
|
||||
|
||||
```go
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
|
||||
"github.com/rs/zerolog"
|
||||
"go.woodpecker-ci.org/woodpecker/woodpecker/v2/server/forge"
|
||||
forge_types "go.woodpecker-ci.org/woodpecker/woodpecker/v2/server/forge/types"
|
||||
"go.woodpecker-ci.org/woodpecker/woodpecker/v2/server/model"
|
||||
addon_types "go.woodpecker-ci.org/woodpecker/woodpecker/v2/shared/addon/types"
|
||||
)
|
||||
|
||||
var Type = addon_types.TypeForge
|
||||
|
||||
func Addon(logger zerolog.Logger, env []string) (forge.Forge, error) {
|
||||
logger.Info().Msg("hello world from addon")
|
||||
return &config{l: logger}, nil
|
||||
}
|
||||
|
||||
type config struct {
|
||||
l zerolog.Logger
|
||||
}
|
||||
|
||||
// ... in this case, `config` must implement `forge.Forge`. You must directly use Woodpecker's packages - see imports above.
|
||||
```
|
6
docs/docs/30-administration/75-addons/_category_.yml
Normal file
6
docs/docs/30-administration/75-addons/_category_.yml
Normal file
|
@ -0,0 +1,6 @@
|
|||
label: 'Addons'
|
||||
collapsible: true
|
||||
collapsed: true
|
||||
link:
|
||||
type: 'doc'
|
||||
id: 'overview'
|
63
shared/addon/addon.go
Normal file
63
shared/addon/addon.go
Normal file
|
@ -0,0 +1,63 @@
|
|||
package addon
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"plugin"
|
||||
|
||||
"github.com/rs/zerolog"
|
||||
"github.com/rs/zerolog/log"
|
||||
|
||||
"go.woodpecker-ci.org/woodpecker/v2/shared/addon/types"
|
||||
)
|
||||
|
||||
var pluginCache = map[string]*plugin.Plugin{}
|
||||
|
||||
type Addon[T any] struct {
|
||||
Type types.Type
|
||||
Value T
|
||||
}
|
||||
|
||||
func Load[T any](files []string, t types.Type) (*Addon[T], error) {
|
||||
for _, file := range files {
|
||||
if _, has := pluginCache[file]; !has {
|
||||
p, err := plugin.Open(file)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
pluginCache[file] = p
|
||||
}
|
||||
|
||||
typeLookup, err := pluginCache[file].Lookup("Type")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if addonType, is := typeLookup.(*types.Type); !is {
|
||||
return nil, errors.New("addon type is incorrect")
|
||||
} else if *addonType != t {
|
||||
continue
|
||||
}
|
||||
|
||||
mainLookup, err := pluginCache[file].Lookup("Addon")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
main, is := mainLookup.(func(zerolog.Logger, []string) (T, error))
|
||||
if !is {
|
||||
return nil, errors.New("addon main function has incorrect type")
|
||||
}
|
||||
|
||||
logger := log.Logger.With().Str("addon", file).Logger()
|
||||
|
||||
mainOut, err := main(logger, os.Environ())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &Addon[T]{
|
||||
Type: t,
|
||||
Value: mainOut,
|
||||
}, nil
|
||||
}
|
||||
|
||||
return nil, nil
|
||||
}
|
8
shared/addon/types/types.go
Normal file
8
shared/addon/types/types.go
Normal file
|
@ -0,0 +1,8 @@
|
|||
package types
|
||||
|
||||
type Type string
|
||||
|
||||
const (
|
||||
TypeForge Type = "forge"
|
||||
TypeBackend Type = "backend"
|
||||
)
|
Loading…
Reference in a new issue