Fetching multiple pipelines from Github, skipping some logic for now

model/config.go

@@ -16,8 +16,8 @@ package model
 
 // ConfigStore persists pipeline configuration to storage.
 type ConfigStore interface {
-	ConfigLoad(int64) (*Config, error)
+	ConfigLoad(ID int64) (*Config, error)
-	ConfigFind(*Repo, string) (*Config, error)
+	ConfigFind(repo *Repo, sha string) (*Config, error)
 	ConfigFindApproved(*Config) (bool, error)
 	ConfigCreate(*Config) error
 }

remote/github/github.go

@@ -23,6 +23,7 @@ import (
 	"regexp"
 	"strconv"
 	"strings"
+	"sync"
 
 	"github.com/laszlocph/drone-oss-08/model"
 	"github.com/laszlocph/drone-oss-08/remote"
@@ -233,6 +234,9 @@ func (c *client) File(u *model.User, r *model.Repo, b *model.Build, f string) ([
 	if err != nil {
 		return nil, err
 	}
+	if data == nil {
+		return nil, fmt.Errorf("%s is a folder not a file use Dir(..)", f)
+	}
 	return data.Decode()
 }
 
@@ -246,18 +250,45 @@ func (c *client) Dir(u *model.User, r *model.Repo, b *model.Build, f string) ([]
 		return nil, err
 	}
 
-	var files []*remote.FileMeta
+	fc := make(chan *remote.FileMeta)
+	errc := make(chan error)
+
+	wg := &sync.WaitGroup{}
+	wg.Add(len(data))
+
 	for _, file := range data {
-		data, err := file.Decode()
+		go func(path string) {
-		if err != nil {
+			content, err := c.File(u, r, b, path)
-			return nil, err
+			if err != nil {
-		}
+				errc <- err
-		files = append(files, &remote.FileMeta{
+			}
-			Name: *file.Name,
+			fc <- &remote.FileMeta{
-			Data: data,
+				Name: path,
-		})
+				Data: content,
+			}
+		}(f + "/" + *file.Name)
 	}
 
+	var files []*remote.FileMeta
+	var errors []error
+
+	go func() {
+		for {
+			select {
+			case err := <-errc:
+				errors = append(errors, err)
+				wg.Done()
+			case fileMeta := <-fc:
+				files = append(files, fileMeta)
+				wg.Done()
+			}
+		}
+	}()
+
+	wg.Wait()
+	close(fc)
+	close(errc)
+
 	return files, nil
 }
 

remote/remote.go

@@ -18,7 +18,6 @@ package remote
 
 import (
 	"net/http"
-	"time"
 
 	"github.com/laszlocph/drone-oss-08/model"
 
@@ -161,31 +160,3 @@ func Refresh(c context.Context, u *model.User) (bool, error) {
 	}
 	return refresher.Refresh(u)
 }
-
-// FileBackoff fetches the file using an exponential backoff.
-func FileBackoff(remote Remote, u *model.User, r *model.Repo, b *model.Build, f string) (out []byte, err error) {
-	for i := 0; i < 5; i++ {
-		select {
-		case <-time.After(time.Second * time.Duration(i)):
-			out, err = remote.File(u, r, b, f)
-			if err == nil {
-				return
-			}
-		}
-	}
-	return
-}
-
-// DirBackoff fetches the folder using an exponential backoff.
-func DirBackoff(remote Remote, u *model.User, r *model.Repo, b *model.Build, f string) (out []*FileMeta, err error) {
-	for i := 0; i < 5; i++ {
-		select {
-		case <-time.After(time.Second * time.Duration(i)):
-			out, err = remote.Dir(u, r, b, f)
-			if err == nil {
-				return
-			}
-		}
-	}
-	return
-}

server/configFetcher.go

@@ -0,0 +1,37 @@
+package server
+
+import (
+	"time"
+
+	"github.com/laszlocph/drone-oss-08/model"
+	"github.com/laszlocph/drone-oss-08/remote"
+)
+
+type configFetcher struct {
+	remote_ remote.Remote
+	user    *model.User
+	repo    *model.Repo
+	build   *model.Build
+}
+
+func (cf *configFetcher) Fetch() ([]*remote.FileMeta, error) {
+	for i := 0; i < 5; i++ {
+		select {
+		case <-time.After(time.Second * time.Duration(i)):
+			file, err := cf.remote_.File(cf.user, cf.repo, cf.build, cf.repo.Config) // either a file
+			if err == nil {
+				return []*remote.FileMeta{&remote.FileMeta{
+					Name: cf.repo.Config,
+					Data: file,
+				}}, nil
+			}
+
+			dir, err := cf.remote_.Dir(cf.user, cf.repo, cf.build, cf.repo.Config) // or a folder
+			if err != nil {
+				return nil, err
+			}
+			return dir, nil
+		}
+	}
+	return []*remote.FileMeta{}, nil
+}

server/configFetcher_test.go

@@ -0,0 +1,22 @@
+package server
+
+import (
+	"testing"
+
+	"github.com/laszlocph/drone-oss-08/model"
+	"github.com/laszlocph/drone-oss-08/remote/github"
+)
+
+func TestFetchGithub(t *testing.T) {
+	github, err := github.New(github.Opts{URL: "https://github.com"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	configFetcher := &configFetcher{
+		remote_: github,
+		user:    &model.User{Token: "xxx"},
+		repo:    &model.Repo{Owner: "laszlocph", Name: "drone-multipipeline", Config: ".drone"},
+		build:   &model.Build{Commit: "89ab7b2d6bfb347144ac7c557e638ab402848fee"},
+	}
+	configFetcher.Fetch()
+}

server/hook.go

@@ -33,7 +33,6 @@ import (
 	"github.com/laszlocph/drone-oss-08/shared/token"
 	"github.com/laszlocph/drone-oss-08/store"
 
-	"github.com/laszlocph/drone-oss-08/cncd/pipeline/pipeline/frontend/yaml"
 	"github.com/laszlocph/drone-oss-08/cncd/pipeline/pipeline/rpc"
 	"github.com/laszlocph/drone-oss-08/cncd/pubsub"
 	"github.com/laszlocph/drone-oss-08/cncd/queue"
@@ -143,35 +142,37 @@ func PostHook(c *gin.Context) {
 	}
 
 	// fetch the build file from the remote
-	remoteYamlConfig, err := remote.FileBackoff(remote_, user, repo, build, repo.Config)
+	configFetcher := &configFetcher{remote_: remote_, user: user, repo: repo, build: build}
+	remoteYamlConfigs, err := configFetcher.Fetch()
 	if err != nil {
 		logrus.Errorf("error: %s: cannot find %s in %s: %s", repo.FullName, repo.Config, build.Ref, err)
 		c.AbortWithError(404, err)
 		return
 	}
-	conf, err := findOrPersistPipelineConfig(repo, remoteYamlConfig)
+	// persist the build config for historical correctness, restarts, etc
-	if err != nil {
+	// conf, err := findOrPersistPipelineConfig(repo, remoteYamlConfig)
-		logrus.Errorf("failure to find or persist build config for %s. %s", repo.FullName, err)
+	// if err != nil {
-		c.AbortWithError(500, err)
+	// 	logrus.Errorf("failure to find or persist build config for %s. %s", repo.FullName, err)
-		return
+	// 	c.AbortWithError(500, err)
-	}
+	// 	return
-	build.ConfigID = conf.ID
+	// }
+	// build.ConfigID = conf.ID
 
 	// verify that pipeline can be built at all
-	parsedPipelineConfig, err := yaml.ParseString(conf.Data)
+	// parsedPipelineConfig, err := yaml.ParseString(conf.Data)
-	if err == nil {
+	// if err == nil {
-		if !parsedPipelineConfig.Branches.Match(build.Branch) && build.Event != model.EventTag && build.Event != model.EventDeploy {
+	// 	if !parsedPipelineConfig.Branches.Match(build.Branch) && build.Event != model.EventTag && build.Event != model.EventDeploy {
-			c.String(200, "Branch does not match restrictions defined in yaml")
+	// 		c.String(200, "Branch does not match restrictions defined in yaml")
-			return
+	// 		return
-		}
+	// 	}
-	}
+	// }
 
-	if repo.IsGated {
+	// if repo.IsGated {
-		allowed, _ := Config.Services.Senders.SenderAllowed(user, repo, build, conf)
+	// 	allowed, _ := Config.Services.Senders.SenderAllowed(user, repo, build, conf)
-		if !allowed {
+	// 	if !allowed {
-			build.Status = model.StatusBlocked
+	// 		build.Status = model.StatusBlocked
-		}
+	// 	}
-	}
+	// }
 
 	// update some build fields
 	build.RepoID = repo.ID
@@ -226,6 +227,11 @@ func PostHook(c *gin.Context) {
 		}
 	}()
 
+	var yamls []string
+	for _, y := range remoteYamlConfigs {
+		yamls = append(yamls, string(y.Data))
+	}
+
 	b := procBuilder{
 		Repo:  repo,
 		Curr:  build,
@@ -235,7 +241,7 @@ func PostHook(c *gin.Context) {
 		Regs:  regs,
 		Envs:  envs,
 		Link:  httputil.GetURL(c.Request),
-		Yamls: []string{conf.Data},
+		Yamls: yamls,
 	}
 	buildItems, err := b.Build()
 	if err != nil {