diff --git a/model/config.go b/model/config.go index 8eae6e34c..e81a97242 100644 --- a/model/config.go +++ b/model/config.go @@ -16,8 +16,8 @@ package model // ConfigStore persists pipeline configuration to storage. type ConfigStore interface { - ConfigLoad(int64) (*Config, error) - ConfigFind(*Repo, string) (*Config, error) + ConfigLoad(ID int64) (*Config, error) + ConfigFind(repo *Repo, sha string) (*Config, error) ConfigFindApproved(*Config) (bool, error) ConfigCreate(*Config) error } diff --git a/remote/github/github.go b/remote/github/github.go index 355a8accc..694cf9535 100644 --- a/remote/github/github.go +++ b/remote/github/github.go @@ -23,6 +23,7 @@ import ( "regexp" "strconv" "strings" + "sync" "github.com/laszlocph/drone-oss-08/model" "github.com/laszlocph/drone-oss-08/remote" @@ -233,6 +234,9 @@ func (c *client) File(u *model.User, r *model.Repo, b *model.Build, f string) ([ if err != nil { return nil, err } + if data == nil { + return nil, fmt.Errorf("%s is a folder not a file use Dir(..)", f) + } return data.Decode() } @@ -246,18 +250,45 @@ func (c *client) Dir(u *model.User, r *model.Repo, b *model.Build, f string) ([] return nil, err } - var files []*remote.FileMeta + fc := make(chan *remote.FileMeta) + errc := make(chan error) + + wg := &sync.WaitGroup{} + wg.Add(len(data)) + for _, file := range data { - data, err := file.Decode() - if err != nil { - return nil, err - } - files = append(files, &remote.FileMeta{ - Name: *file.Name, - Data: data, - }) + go func(path string) { + content, err := c.File(u, r, b, path) + if err != nil { + errc <- err + } + fc <- &remote.FileMeta{ + Name: path, + Data: content, + } + }(f + "/" + *file.Name) } + var files []*remote.FileMeta + var errors []error + + go func() { + for { + select { + case err := <-errc: + errors = append(errors, err) + wg.Done() + case fileMeta := <-fc: + files = append(files, fileMeta) + wg.Done() + } + } + }() + + wg.Wait() + close(fc) + close(errc) + return files, nil } diff --git a/remote/remote.go b/remote/remote.go index 070882d6e..7804cd97b 100644 --- a/remote/remote.go +++ b/remote/remote.go @@ -18,7 +18,6 @@ package remote import ( "net/http" - "time" "github.com/laszlocph/drone-oss-08/model" @@ -161,31 +160,3 @@ func Refresh(c context.Context, u *model.User) (bool, error) { } return refresher.Refresh(u) } - -// FileBackoff fetches the file using an exponential backoff. -func FileBackoff(remote Remote, u *model.User, r *model.Repo, b *model.Build, f string) (out []byte, err error) { - for i := 0; i < 5; i++ { - select { - case <-time.After(time.Second * time.Duration(i)): - out, err = remote.File(u, r, b, f) - if err == nil { - return - } - } - } - return -} - -// DirBackoff fetches the folder using an exponential backoff. -func DirBackoff(remote Remote, u *model.User, r *model.Repo, b *model.Build, f string) (out []*FileMeta, err error) { - for i := 0; i < 5; i++ { - select { - case <-time.After(time.Second * time.Duration(i)): - out, err = remote.Dir(u, r, b, f) - if err == nil { - return - } - } - } - return -} diff --git a/server/configFetcher.go b/server/configFetcher.go new file mode 100644 index 000000000..6ca48a4de --- /dev/null +++ b/server/configFetcher.go @@ -0,0 +1,37 @@ +package server + +import ( + "time" + + "github.com/laszlocph/drone-oss-08/model" + "github.com/laszlocph/drone-oss-08/remote" +) + +type configFetcher struct { + remote_ remote.Remote + user *model.User + repo *model.Repo + build *model.Build +} + +func (cf *configFetcher) Fetch() ([]*remote.FileMeta, error) { + for i := 0; i < 5; i++ { + select { + case <-time.After(time.Second * time.Duration(i)): + file, err := cf.remote_.File(cf.user, cf.repo, cf.build, cf.repo.Config) // either a file + if err == nil { + return []*remote.FileMeta{&remote.FileMeta{ + Name: cf.repo.Config, + Data: file, + }}, nil + } + + dir, err := cf.remote_.Dir(cf.user, cf.repo, cf.build, cf.repo.Config) // or a folder + if err != nil { + return nil, err + } + return dir, nil + } + } + return []*remote.FileMeta{}, nil +} diff --git a/server/configFetcher_test.go b/server/configFetcher_test.go new file mode 100644 index 000000000..1ac3f6825 --- /dev/null +++ b/server/configFetcher_test.go @@ -0,0 +1,22 @@ +package server + +import ( + "testing" + + "github.com/laszlocph/drone-oss-08/model" + "github.com/laszlocph/drone-oss-08/remote/github" +) + +func TestFetchGithub(t *testing.T) { + github, err := github.New(github.Opts{URL: "https://github.com"}) + if err != nil { + t.Fatal(err) + } + configFetcher := &configFetcher{ + remote_: github, + user: &model.User{Token: "xxx"}, + repo: &model.Repo{Owner: "laszlocph", Name: "drone-multipipeline", Config: ".drone"}, + build: &model.Build{Commit: "89ab7b2d6bfb347144ac7c557e638ab402848fee"}, + } + configFetcher.Fetch() +} diff --git a/server/hook.go b/server/hook.go index 09ef1ff24..5a71ab9d4 100644 --- a/server/hook.go +++ b/server/hook.go @@ -33,7 +33,6 @@ import ( "github.com/laszlocph/drone-oss-08/shared/token" "github.com/laszlocph/drone-oss-08/store" - "github.com/laszlocph/drone-oss-08/cncd/pipeline/pipeline/frontend/yaml" "github.com/laszlocph/drone-oss-08/cncd/pipeline/pipeline/rpc" "github.com/laszlocph/drone-oss-08/cncd/pubsub" "github.com/laszlocph/drone-oss-08/cncd/queue" @@ -143,35 +142,37 @@ func PostHook(c *gin.Context) { } // fetch the build file from the remote - remoteYamlConfig, err := remote.FileBackoff(remote_, user, repo, build, repo.Config) + configFetcher := &configFetcher{remote_: remote_, user: user, repo: repo, build: build} + remoteYamlConfigs, err := configFetcher.Fetch() if err != nil { logrus.Errorf("error: %s: cannot find %s in %s: %s", repo.FullName, repo.Config, build.Ref, err) c.AbortWithError(404, err) return } - conf, err := findOrPersistPipelineConfig(repo, remoteYamlConfig) - if err != nil { - logrus.Errorf("failure to find or persist build config for %s. %s", repo.FullName, err) - c.AbortWithError(500, err) - return - } - build.ConfigID = conf.ID + // persist the build config for historical correctness, restarts, etc + // conf, err := findOrPersistPipelineConfig(repo, remoteYamlConfig) + // if err != nil { + // logrus.Errorf("failure to find or persist build config for %s. %s", repo.FullName, err) + // c.AbortWithError(500, err) + // return + // } + // build.ConfigID = conf.ID // verify that pipeline can be built at all - parsedPipelineConfig, err := yaml.ParseString(conf.Data) - if err == nil { - if !parsedPipelineConfig.Branches.Match(build.Branch) && build.Event != model.EventTag && build.Event != model.EventDeploy { - c.String(200, "Branch does not match restrictions defined in yaml") - return - } - } + // parsedPipelineConfig, err := yaml.ParseString(conf.Data) + // if err == nil { + // if !parsedPipelineConfig.Branches.Match(build.Branch) && build.Event != model.EventTag && build.Event != model.EventDeploy { + // c.String(200, "Branch does not match restrictions defined in yaml") + // return + // } + // } - if repo.IsGated { - allowed, _ := Config.Services.Senders.SenderAllowed(user, repo, build, conf) - if !allowed { - build.Status = model.StatusBlocked - } - } + // if repo.IsGated { + // allowed, _ := Config.Services.Senders.SenderAllowed(user, repo, build, conf) + // if !allowed { + // build.Status = model.StatusBlocked + // } + // } // update some build fields build.RepoID = repo.ID @@ -226,6 +227,11 @@ func PostHook(c *gin.Context) { } }() + var yamls []string + for _, y := range remoteYamlConfigs { + yamls = append(yamls, string(y.Data)) + } + b := procBuilder{ Repo: repo, Curr: build, @@ -235,7 +241,7 @@ func PostHook(c *gin.Context) { Regs: regs, Envs: envs, Link: httputil.GetURL(c.Request), - Yamls: []string{conf.Data}, + Yamls: yamls, } buildItems, err := b.Build() if err != nil {