mirrors/woodpecker
1
0
Fork 0
mirror of https://github.com/woodpecker-ci/woodpecker.git synced 2024-12-27 10:50:33 +00:00
Code Issues Projects Releases Wiki Activity

Fetching multiple pipelines from Github, skipping some logic for now

Browse source
This commit is contained in:
Laszlo Fogas 2019-06-04 15:04:18 +02:00
parent 75d30dea09
commit 5cc4dca56f
6 changed files with 130 additions and 63 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
model/config.go
View file

@ -16,8 +16,8 @@ package model
// ConfigStore persists pipeline configuration to storage.
type ConfigStore interface {
ConfigLoad(int64) (*Config, error)
ConfigFind(*Repo, string) (*Config, error)
ConfigLoad(ID int64) (*Config, error)
ConfigFind(repo *Repo, sha string) (*Config, error)
ConfigFindApproved(*Config) (bool, error)
ConfigCreate(*Config) error
}

49
remote/github/github.go
View file

@ -23,6 +23,7 @@ import (
"regexp"
"strconv"
"strings"
"sync"
"github.com/laszlocph/drone-oss-08/model"
"github.com/laszlocph/drone-oss-08/remote"
@ -233,6 +234,9 @@ func (c *client) File(u *model.User, r *model.Repo, b *model.Build, f string) ([
if err != nil {
return nil, err
}
if data == nil {
return nil, fmt.Errorf("%s is a folder not a file use Dir(..)", f)
}
return data.Decode()
}
@ -246,18 +250,45 @@ func (c *client) Dir(u *model.User, r *model.Repo, b *model.Build, f string) ([]
return nil, err
}
var files []*remote.FileMeta
fc := make(chan *remote.FileMeta)
errc := make(chan error)
wg := &sync.WaitGroup{}
wg.Add(len(data))
for _, file := range data {
data, err := file.Decode()
if err != nil {
return nil, err
}
files = append(files, &remote.FileMeta{
Name: *file.Name,
Data: data,
})
go func(path string) {
content, err := c.File(u, r, b, path)
if err != nil {
errc <- err
}
fc <- &remote.FileMeta{
Name: path,
Data: content,
}
}(f + "/" + *file.Name)
}
var files []*remote.FileMeta
var errors []error
go func() {
for {
select {
case err := <-errc:
errors = append(errors, err)
wg.Done()
case fileMeta := <-fc:
files = append(files, fileMeta)
wg.Done()
}
}
}()
wg.Wait()
close(fc)
close(errc)
return files, nil
}

29
remote/remote.go
View file

@ -18,7 +18,6 @@ package remote
import (
"net/http"
"time"
"github.com/laszlocph/drone-oss-08/model"
@ -161,31 +160,3 @@ func Refresh(c context.Context, u *model.User) (bool, error) {
}
return refresher.Refresh(u)
}
// FileBackoff fetches the file using an exponential backoff.
func FileBackoff(remote Remote, u *model.User, r *model.Repo, b *model.Build, f string) (out []byte, err error) {
for i := 0; i < 5; i++ {
select {
case <-time.After(time.Second * time.Duration(i)):
out, err = remote.File(u, r, b, f)
if err == nil {
return
}
}
}
return
}
// DirBackoff fetches the folder using an exponential backoff.
func DirBackoff(remote Remote, u *model.User, r *model.Repo, b *model.Build, f string) (out []*FileMeta, err error) {
for i := 0; i < 5; i++ {
select {
case <-time.After(time.Second * time.Duration(i)):
out, err = remote.Dir(u, r, b, f)
if err == nil {
return
}
}
}
return
}

37
server/configFetcher.go Normal file
View file

@ -0,0 +1,37 @@
package server
import (
"time"
"github.com/laszlocph/drone-oss-08/model"
"github.com/laszlocph/drone-oss-08/remote"
)
type configFetcher struct {
remote_ remote.Remote
user *model.User
repo *model.Repo
build *model.Build
}
func (cf *configFetcher) Fetch() ([]*remote.FileMeta, error) {
for i := 0; i < 5; i++ {
select {
case <-time.After(time.Second * time.Duration(i)):
file, err := cf.remote_.File(cf.user, cf.repo, cf.build, cf.repo.Config) // either a file
if err == nil {
return []*remote.FileMeta{&remote.FileMeta{
Name: cf.repo.Config,
Data: file,
}}, nil
}
dir, err := cf.remote_.Dir(cf.user, cf.repo, cf.build, cf.repo.Config) // or a folder
if err != nil {
return nil, err
}
return dir, nil
}
}
return []*remote.FileMeta{}, nil
}

22
server/configFetcher_test.go Normal file
View file

@ -0,0 +1,22 @@
package server
import (
"testing"
"github.com/laszlocph/drone-oss-08/model"
"github.com/laszlocph/drone-oss-08/remote/github"
)
func TestFetchGithub(t *testing.T) {
github, err := github.New(github.Opts{URL: "https://github.com"})
if err != nil {
t.Fatal(err)
}
configFetcher := &configFetcher{
remote_: github,
user: &model.User{Token: "xxx"},
repo: &model.Repo{Owner: "laszlocph", Name: "drone-multipipeline", Config: ".drone"},
build: &model.Build{Commit: "89ab7b2d6bfb347144ac7c557e638ab402848fee"},
}
configFetcher.Fetch()
}

52
server/hook.go
View file

@ -33,7 +33,6 @@ import (
"github.com/laszlocph/drone-oss-08/shared/token"
"github.com/laszlocph/drone-oss-08/store"
"github.com/laszlocph/drone-oss-08/cncd/pipeline/pipeline/frontend/yaml"
"github.com/laszlocph/drone-oss-08/cncd/pipeline/pipeline/rpc"
"github.com/laszlocph/drone-oss-08/cncd/pubsub"
"github.com/laszlocph/drone-oss-08/cncd/queue"
@ -143,35 +142,37 @@ func PostHook(c *gin.Context) {
}
// fetch the build file from the remote
remoteYamlConfig, err := remote.FileBackoff(remote_, user, repo, build, repo.Config)
configFetcher := &configFetcher{remote_: remote_, user: user, repo: repo, build: build}
remoteYamlConfigs, err := configFetcher.Fetch()
if err != nil {
logrus.Errorf("error: %s: cannot find %s in %s: %s", repo.FullName, repo.Config, build.Ref, err)
c.AbortWithError(404, err)
return
}
conf, err := findOrPersistPipelineConfig(repo, remoteYamlConfig)
if err != nil {
logrus.Errorf("failure to find or persist build config for %s. %s", repo.FullName, err)
c.AbortWithError(500, err)
return
}
build.ConfigID = conf.ID
// persist the build config for historical correctness, restarts, etc
// conf, err := findOrPersistPipelineConfig(repo, remoteYamlConfig)
// if err != nil {
// logrus.Errorf("failure to find or persist build config for %s. %s", repo.FullName, err)
// c.AbortWithError(500, err)
// return
// }
// build.ConfigID = conf.ID
// verify that pipeline can be built at all
parsedPipelineConfig, err := yaml.ParseString(conf.Data)
if err == nil {
if !parsedPipelineConfig.Branches.Match(build.Branch) && build.Event != model.EventTag && build.Event != model.EventDeploy {
c.String(200, "Branch does not match restrictions defined in yaml")
return
}
}
// parsedPipelineConfig, err := yaml.ParseString(conf.Data)
// if err == nil {
// if !parsedPipelineConfig.Branches.Match(build.Branch) && build.Event != model.EventTag && build.Event != model.EventDeploy {
// c.String(200, "Branch does not match restrictions defined in yaml")
// return
// }
// }
if repo.IsGated {
allowed, _ := Config.Services.Senders.SenderAllowed(user, repo, build, conf)
if !allowed {
build.Status = model.StatusBlocked
}
}
// if repo.IsGated {
// allowed, _ := Config.Services.Senders.SenderAllowed(user, repo, build, conf)
// if !allowed {
// build.Status = model.StatusBlocked
// }
// }
// update some build fields
build.RepoID = repo.ID
@ -226,6 +227,11 @@ func PostHook(c *gin.Context) {
}
}()
var yamls []string
for _, y := range remoteYamlConfigs {
yamls = append(yamls, string(y.Data))
}
b := procBuilder{
Repo: repo,
Curr: build,
@ -235,7 +241,7 @@ func PostHook(c *gin.Context) {
Regs: regs,
Envs: envs,
Link: httputil.GetURL(c.Request),
Yamls: []string{conf.Data},
Yamls: yamls,
}
buildItems, err := b.Build()
if err != nil {