woodpecker/server/shared/configFetcher.go

161 lines
4.7 KiB
Go
Raw Normal View History

package shared
import (
"context"
"errors"
"fmt"
2019-06-24 07:04:30 +00:00
"strings"
"time"
"github.com/rs/zerolog/log"
"github.com/woodpecker-ci/woodpecker/server/plugins/config"
"github.com/woodpecker-ci/woodpecker/server/model"
"github.com/woodpecker-ci/woodpecker/server/remote"
)
type ConfigFetcher interface {
Fetch(ctx context.Context) (files []*remote.FileMeta, err error)
}
// TODO(974) move to new package
type configFetcher struct {
remote remote.Remote
user *model.User
repo *model.Repo
build *model.Build
configExtension config.Extension
}
func NewConfigFetcher(remote remote.Remote, configExtension config.Extension, user *model.User, repo *model.Repo, build *model.Build) ConfigFetcher {
return &configFetcher{
remote: remote,
user: user,
repo: repo,
build: build,
configExtension: configExtension,
}
}
// configFetchTimeout determine seconds the configFetcher wait until cancel fetch process
Add support for pipeline configuration service (#804) * Add configuration extension flags to server Add httpsignatures dependency Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Add http fetching to config fetcher Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Refetch config on rebuild Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * - Ensure multipipeline compatiblity - Send original config in http request Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Basic tests of config api Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Simple docs page Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Better flag naming Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Rename usages of the term yaml Rename ConfigAPI struct Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Doc adjustments Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * More docs touchups Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Fix env vars in docs Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * fix json tags for api calls Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Add example config service Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Consistent naming for configService Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Docs: Change example repository location Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Fix tests after response field rename Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Revert accidential unrelated change in api hook Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Update server flag descriptions Co-authored-by: Anbraten <anton@ju60.de> Co-authored-by: Anbraten <anton@ju60.de>
2022-02-28 09:56:23 +00:00
var configFetchTimeout = time.Second * 3
// Fetch pipeline config from source forge
func (cf *configFetcher) Fetch(ctx context.Context) (files []*remote.FileMeta, err error) {
log.Trace().Msgf("Start Fetching config for '%s'", cf.repo.FullName)
Add support for pipeline configuration service (#804) * Add configuration extension flags to server Add httpsignatures dependency Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Add http fetching to config fetcher Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Refetch config on rebuild Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * - Ensure multipipeline compatiblity - Send original config in http request Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Basic tests of config api Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Simple docs page Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Better flag naming Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Rename usages of the term yaml Rename ConfigAPI struct Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Doc adjustments Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * More docs touchups Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Fix env vars in docs Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * fix json tags for api calls Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Add example config service Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Consistent naming for configService Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Docs: Change example repository location Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Fix tests after response field rename Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Revert accidential unrelated change in api hook Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Update server flag descriptions Co-authored-by: Anbraten <anton@ju60.de> Co-authored-by: Anbraten <anton@ju60.de>
2022-02-28 09:56:23 +00:00
// try to fetch 3 times
for i := 0; i < 3; i++ {
Add support for pipeline configuration service (#804) * Add configuration extension flags to server Add httpsignatures dependency Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Add http fetching to config fetcher Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Refetch config on rebuild Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * - Ensure multipipeline compatiblity - Send original config in http request Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Basic tests of config api Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Simple docs page Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Better flag naming Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Rename usages of the term yaml Rename ConfigAPI struct Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Doc adjustments Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * More docs touchups Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Fix env vars in docs Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * fix json tags for api calls Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Add example config service Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Consistent naming for configService Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Docs: Change example repository location Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Fix tests after response field rename Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Revert accidential unrelated change in api hook Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Update server flag descriptions Co-authored-by: Anbraten <anton@ju60.de> Co-authored-by: Anbraten <anton@ju60.de>
2022-02-28 09:56:23 +00:00
files, err = cf.fetch(ctx, configFetchTimeout, strings.TrimSpace(cf.repo.Config))
if err != nil {
log.Trace().Err(err).Msgf("%d. try failed", i+1)
}
if errors.Is(err, context.DeadlineExceeded) {
continue
}
Add support for pipeline configuration service (#804) * Add configuration extension flags to server Add httpsignatures dependency Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Add http fetching to config fetcher Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Refetch config on rebuild Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * - Ensure multipipeline compatiblity - Send original config in http request Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Basic tests of config api Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Simple docs page Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Better flag naming Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Rename usages of the term yaml Rename ConfigAPI struct Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Doc adjustments Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * More docs touchups Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Fix env vars in docs Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * fix json tags for api calls Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Add example config service Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Consistent naming for configService Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Docs: Change example repository location Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Fix tests after response field rename Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Revert accidential unrelated change in api hook Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Update server flag descriptions Co-authored-by: Anbraten <anton@ju60.de> Co-authored-by: Anbraten <anton@ju60.de>
2022-02-28 09:56:23 +00:00
2022-06-02 10:49:12 +00:00
if cf.configExtension != nil && cf.configExtension.IsConfigured() {
Add support for pipeline configuration service (#804) * Add configuration extension flags to server Add httpsignatures dependency Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Add http fetching to config fetcher Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Refetch config on rebuild Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * - Ensure multipipeline compatiblity - Send original config in http request Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Basic tests of config api Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Simple docs page Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Better flag naming Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Rename usages of the term yaml Rename ConfigAPI struct Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Doc adjustments Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * More docs touchups Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Fix env vars in docs Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * fix json tags for api calls Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Add example config service Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Consistent naming for configService Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Docs: Change example repository location Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Fix tests after response field rename Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Revert accidential unrelated change in api hook Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Update server flag descriptions Co-authored-by: Anbraten <anton@ju60.de> Co-authored-by: Anbraten <anton@ju60.de>
2022-02-28 09:56:23 +00:00
fetchCtx, cancel := context.WithTimeout(ctx, configFetchTimeout)
defer cancel() // ok here as we only try http fetching once, returning on fail and success
log.Trace().Msgf("ConfigFetch[%s]: getting config from external http service", cf.repo.FullName)
newConfigs, useOld, err := cf.configExtension.FetchConfig(fetchCtx, cf.repo, cf.build, files)
Add support for pipeline configuration service (#804) * Add configuration extension flags to server Add httpsignatures dependency Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Add http fetching to config fetcher Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Refetch config on rebuild Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * - Ensure multipipeline compatiblity - Send original config in http request Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Basic tests of config api Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Simple docs page Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Better flag naming Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Rename usages of the term yaml Rename ConfigAPI struct Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Doc adjustments Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * More docs touchups Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Fix env vars in docs Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * fix json tags for api calls Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Add example config service Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Consistent naming for configService Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Docs: Change example repository location Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Fix tests after response field rename Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Revert accidential unrelated change in api hook Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Update server flag descriptions Co-authored-by: Anbraten <anton@ju60.de> Co-authored-by: Anbraten <anton@ju60.de>
2022-02-28 09:56:23 +00:00
if err != nil {
2022-03-09 00:44:08 +00:00
log.Error().Msg("Got error " + err.Error())
Add support for pipeline configuration service (#804) * Add configuration extension flags to server Add httpsignatures dependency Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Add http fetching to config fetcher Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Refetch config on rebuild Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * - Ensure multipipeline compatiblity - Send original config in http request Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Basic tests of config api Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Simple docs page Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Better flag naming Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Rename usages of the term yaml Rename ConfigAPI struct Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Doc adjustments Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * More docs touchups Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Fix env vars in docs Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * fix json tags for api calls Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Add example config service Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Consistent naming for configService Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Docs: Change example repository location Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Fix tests after response field rename Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Revert accidential unrelated change in api hook Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at> * Update server flag descriptions Co-authored-by: Anbraten <anton@ju60.de> Co-authored-by: Anbraten <anton@ju60.de>
2022-02-28 09:56:23 +00:00
return nil, fmt.Errorf("On Fetching config via http : %s", err)
}
if !useOld {
return newConfigs, nil
}
}
return
}
return
}
// fetch config by timeout
// TODO: deduplicate code
func (cf *configFetcher) fetch(c context.Context, timeout time.Duration, config string) ([]*remote.FileMeta, error) {
ctx, cancel := context.WithTimeout(c, timeout)
defer cancel()
if len(config) > 0 {
log.Trace().Msgf("ConfigFetch[%s]: use user config '%s'", cf.repo.FullName, config)
// either a file
if !strings.HasSuffix(config, "/") {
file, err := cf.remote.File(ctx, cf.user, cf.repo, cf.build, config)
if err == nil && len(file) != 0 {
log.Trace().Msgf("ConfigFetch[%s]: found file '%s'", cf.repo.FullName, config)
return []*remote.FileMeta{{
Name: config,
Data: file,
}}, nil
}
}
// or a folder
files, err := cf.remote.Dir(ctx, cf.user, cf.repo, cf.build, strings.TrimSuffix(config, "/"))
if err == nil && len(files) != 0 {
log.Trace().Msgf("ConfigFetch[%s]: found %d files in '%s'", cf.repo.FullName, len(files), config)
return filterPipelineFiles(files), nil
}
return nil, fmt.Errorf("config '%s' not found: %s", config, err)
}
log.Trace().Msgf("ConfigFetch[%s]: user did not defined own config follow default procedure", cf.repo.FullName)
// no user defined config so try .woodpecker/*.yml -> .woodpecker.yml -> .drone.yml
// test .woodpecker/ folder
// if folder is not supported we will get a "Not implemented" error and continue
config = ".woodpecker"
files, err := cf.remote.Dir(ctx, cf.user, cf.repo, cf.build, config)
files = filterPipelineFiles(files)
if err == nil && len(files) != 0 {
log.Trace().Msgf("ConfigFetch[%s]: found %d files in '%s'", cf.repo.FullName, len(files), config)
return files, nil
}
config = ".woodpecker.yml"
file, err := cf.remote.File(ctx, cf.user, cf.repo, cf.build, config)
if err == nil && len(file) != 0 {
log.Trace().Msgf("ConfigFetch[%s]: found file '%s'", cf.repo.FullName, config)
return []*remote.FileMeta{{
Name: config,
Data: file,
}}, nil
}
config = ".drone.yml"
file, err = cf.remote.File(ctx, cf.user, cf.repo, cf.build, config)
if err == nil && len(file) != 0 {
log.Trace().Msgf("ConfigFetch[%s]: found file '%s'", cf.repo.FullName, config)
return []*remote.FileMeta{{
Name: config,
Data: file,
}}, nil
}
select {
case <-ctx.Done():
return nil, ctx.Err()
default:
return []*remote.FileMeta{}, fmt.Errorf("ConfigFetcher: Fallback did not found config: %s", err)
}
}
func filterPipelineFiles(files []*remote.FileMeta) []*remote.FileMeta {
var res []*remote.FileMeta
for _, file := range files {
if strings.HasSuffix(file.Name, ".yml") {
res = append(res, file)
}
}
return res
}