Define WOODPECKER_FORGE_TIMEOUT server config (#1558)

When a server such as Codeberg has unusually high response time, three
seconds may not be enough to fetch the configuration.

Signed-off-by: Earl Warren <contact@earl-warren.org>
Co-authored-by: 6543 <6543@obermui.de>
This commit is contained in:
Earl Warren 2023-02-01 18:53:19 +01:00 committed by GitHub
parent e13792bff6
commit 7835a632e4
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 24 additions and 7 deletions

View file

@ -209,6 +209,12 @@ var flags = []cli.Flag{
// //
// resource limit parameters // resource limit parameters
// //
&cli.DurationFlag{
EnvVars: []string{"WOODPECKER_FORGE_TIMEOUT"},
Name: "forge-timeout",
Usage: "how many seconds before timeout when fetching the Woodpecker configuration from a Forge",
Value: time.Second * 3,
},
&cli.Int64Flag{ &cli.Int64Flag{
EnvVars: []string{"WOODPECKER_LIMIT_MEM_SWAP"}, EnvVars: []string{"WOODPECKER_LIMIT_MEM_SWAP"},
Name: "limit-mem-swap", Name: "limit-mem-swap",

View file

@ -269,6 +269,7 @@ func setupEvilGlobals(c *cli.Context, v store.Store, f forge.Forge) {
// forge // forge
server.Config.Services.Forge = f server.Config.Services.Forge = f
server.Config.Services.Timeout = c.Duration("forge-timeout")
// services // services
server.Config.Services.Queue = setupQueue(c, v) server.Config.Services.Queue = setupQueue(c, v)

View file

@ -377,6 +377,13 @@ Example: `WOODPECKER_LIMIT_CPU_SET=1,2`
Specify a configuration service endpoint, see [Configuration Extension](./100-external-configuration-api.md) Specify a configuration service endpoint, see [Configuration Extension](./100-external-configuration-api.md)
### `WOODPECKER_FORGE_TIMEOUT`
> Default: 3sec
Specify how many seconds before timeout when fetching the Woodpecker configuration from a Forge
--- ---
### `WOODPECKER_GITHUB_...` ### `WOODPECKER_GITHUB_...`

View file

@ -39,6 +39,7 @@ var Config = struct {
Registries model.RegistryService Registries model.RegistryService
Environ model.EnvironService Environ model.EnvironService
Forge forge.Forge Forge forge.Forge
Timeout time.Duration
Membership cache.MembershipService Membership cache.MembershipService
ConfigService config.Extension ConfigService config.Extension
SignaturePrivateKey crypto.PrivateKey SignaturePrivateKey crypto.PrivateKey

View file

@ -39,28 +39,27 @@ type configFetcher struct {
repo *model.Repo repo *model.Repo
pipeline *model.Pipeline pipeline *model.Pipeline
configExtension config.Extension configExtension config.Extension
timeout time.Duration
} }
func NewConfigFetcher(forge Forge, configExtension config.Extension, user *model.User, repo *model.Repo, pipeline *model.Pipeline) ConfigFetcher { func NewConfigFetcher(forge Forge, timeout time.Duration, configExtension config.Extension, user *model.User, repo *model.Repo, pipeline *model.Pipeline) ConfigFetcher {
return &configFetcher{ return &configFetcher{
forge: forge, forge: forge,
user: user, user: user,
repo: repo, repo: repo,
pipeline: pipeline, pipeline: pipeline,
configExtension: configExtension, configExtension: configExtension,
timeout: timeout,
} }
} }
// configFetchTimeout determine seconds the configFetcher wait until cancel fetch process
var configFetchTimeout = time.Second * 3
// Fetch pipeline config from source forge // Fetch pipeline config from source forge
func (cf *configFetcher) Fetch(ctx context.Context) (files []*types.FileMeta, err error) { func (cf *configFetcher) Fetch(ctx context.Context) (files []*types.FileMeta, err error) {
log.Trace().Msgf("Start Fetching config for '%s'", cf.repo.FullName) log.Trace().Msgf("Start Fetching config for '%s'", cf.repo.FullName)
// try to fetch 3 times // try to fetch 3 times
for i := 0; i < 3; i++ { for i := 0; i < 3; i++ {
files, err = cf.fetch(ctx, configFetchTimeout, strings.TrimSpace(cf.repo.Config)) files, err = cf.fetch(ctx, time.Second*cf.timeout, strings.TrimSpace(cf.repo.Config))
if err != nil { if err != nil {
log.Trace().Err(err).Msgf("%d. try failed", i+1) log.Trace().Err(err).Msgf("%d. try failed", i+1)
} }
@ -69,7 +68,7 @@ func (cf *configFetcher) Fetch(ctx context.Context) (files []*types.FileMeta, er
} }
if cf.configExtension != nil && cf.configExtension.IsConfigured() { if cf.configExtension != nil && cf.configExtension.IsConfigured() {
fetchCtx, cancel := context.WithTimeout(ctx, configFetchTimeout) fetchCtx, cancel := context.WithTimeout(ctx, cf.timeout)
defer cancel() // ok here as we only try http fetching once, returning on fail and success defer cancel() // ok here as we only try http fetching once, returning on fail and success
log.Trace().Msgf("ConfigFetch[%s]: getting config from external http service", cf.repo.FullName) log.Trace().Msgf("ConfigFetch[%s]: getting config from external http service", cf.repo.FullName)

View file

@ -25,6 +25,7 @@ import (
"net/http/httptest" "net/http/httptest"
"path/filepath" "path/filepath"
"testing" "testing"
"time"
"github.com/go-ap/httpsig" "github.com/go-ap/httpsig"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
@ -314,6 +315,7 @@ func TestFetch(t *testing.T) {
configFetcher := forge.NewConfigFetcher( configFetcher := forge.NewConfigFetcher(
f, f,
time.Second*3,
config.NewHTTP("", ""), config.NewHTTP("", ""),
&model.User{Token: "xxx"}, &model.User{Token: "xxx"},
repo, repo,
@ -519,6 +521,7 @@ func TestFetchFromConfigService(t *testing.T) {
configFetcher := forge.NewConfigFetcher( configFetcher := forge.NewConfigFetcher(
f, f,
time.Second*3,
configAPI, configAPI,
&model.User{Token: "xxx"}, &model.User{Token: "xxx"},
repo, repo,

View file

@ -60,7 +60,7 @@ func Create(ctx context.Context, _store store.Store, repo *model.Repo, pipeline
) )
// fetch the pipeline file from the forge // fetch the pipeline file from the forge
configFetcher := forge.NewConfigFetcher(server.Config.Services.Forge, server.Config.Services.ConfigService, repoUser, repo, pipeline) configFetcher := forge.NewConfigFetcher(server.Config.Services.Forge, server.Config.Services.Timeout, server.Config.Services.ConfigService, repoUser, repo, pipeline)
forgeYamlConfigs, configFetchErr = configFetcher.Fetch(ctx) forgeYamlConfigs, configFetchErr = configFetcher.Fetch(ctx)
if configFetchErr == nil { if configFetchErr == nil {
filtered, parseErr = checkIfFiltered(pipeline, forgeYamlConfigs) filtered, parseErr = checkIfFiltered(pipeline, forgeYamlConfigs)