From 130c623a355b721e39e9e07d14da183169de8750 Mon Sep 17 00:00:00 2001 From: Brad Rydzewski Date: Sat, 18 Mar 2017 19:25:53 +0800 Subject: [PATCH] compare yaml files --- remote/bitbucket/bitbucket.go | 7 +- remote/bitbucketserver/bitbucketserver.go | 15 ++- remote/gerrit/gerrit.go | 5 + remote/github/github.go | 9 +- remote/gitlab/client/project.go | 28 ++++- remote/gitlab/gitlab.go | 15 +++ remote/gogs/gogs.go | 5 + remote/mock/remote.go | 73 ++++++++---- remote/remote.go | 8 +- server/build.go | 134 ++++++++++++++++++---- server/hook.go | 59 +++++++--- 11 files changed, 278 insertions(+), 80 deletions(-) diff --git a/remote/bitbucket/bitbucket.go b/remote/bitbucket/bitbucket.go index a272fa884..736f8882a 100644 --- a/remote/bitbucket/bitbucket.go +++ b/remote/bitbucket/bitbucket.go @@ -183,7 +183,12 @@ func (c *config) Perm(u *model.User, owner, name string) (*model.Perm, error) { // File fetches the file from the Bitbucket repository and returns its contents. func (c *config) File(u *model.User, r *model.Repo, b *model.Build, f string) ([]byte, error) { - config, err := c.newClient(u).FindSource(r.Owner, r.Name, b.Commit, f) + return c.FileRef(u, r, b.Commit, f) +} + +// FileRef fetches the file from the Bitbucket repository and returns its contents. +func (c *config) FileRef(u *model.User, r *model.Repo, ref, f string) ([]byte, error) { + config, err := c.newClient(u).FindSource(r.Owner, r.Name, ref, f) if err != nil { return nil, err } diff --git a/remote/bitbucketserver/bitbucketserver.go b/remote/bitbucketserver/bitbucketserver.go index 6068e1051..6e44baa3f 100644 --- a/remote/bitbucketserver/bitbucketserver.go +++ b/remote/bitbucketserver/bitbucketserver.go @@ -9,14 +9,15 @@ import ( "crypto/x509" "encoding/pem" "fmt" - "github.com/drone/drone/model" - "github.com/drone/drone/remote" - "github.com/drone/drone/remote/bitbucketserver/internal" - "github.com/mrjones/oauth" "io/ioutil" "net/http" "net/url" "strings" + + "github.com/drone/drone/model" + "github.com/drone/drone/remote" + "github.com/drone/drone/remote/bitbucketserver/internal" + "github.com/mrjones/oauth" ) const ( @@ -164,6 +165,12 @@ func (c *Config) File(u *model.User, r *model.Repo, b *model.Build, f string) ([ return client.FindFileForRepo(r.Owner, r.Name, f, b.Ref) } +func (c *Config) FileRef(u *model.User, r *model.Repo, ref, f string) ([]byte, error) { + client := internal.NewClientWithToken(c.URL, c.Consumer, u.Token) + + return client.FindFileForRepo(r.Owner, r.Name, f, ref) +} + // Status is not supported by the bitbucketserver driver. func (c *Config) Status(u *model.User, r *model.Repo, b *model.Build, link string) error { status := internal.BuildStatus{ diff --git a/remote/gerrit/gerrit.go b/remote/gerrit/gerrit.go index 6b8be5162..4554e3ffe 100644 --- a/remote/gerrit/gerrit.go +++ b/remote/gerrit/gerrit.go @@ -94,6 +94,11 @@ func (c *client) File(u *model.User, r *model.Repo, b *model.Build, f string) ([ return nil, nil } +// File is not supported by the Gerrit driver. +func (c *client) FileRef(u *model.User, r *model.Repo, ref, f string) ([]byte, error) { + return nil, nil +} + // Status is not supported by the Gogs driver. func (c *client) Status(u *model.User, r *model.Repo, b *model.Build, link string) error { return nil diff --git a/remote/github/github.go b/remote/github/github.go index 7b38278b1..ba3468f61 100644 --- a/remote/github/github.go +++ b/remote/github/github.go @@ -219,12 +219,17 @@ func (c *client) Perm(u *model.User, owner, name string) (*model.Perm, error) { return convertPerm(repo), nil } -// File fetches the file from the Bitbucket repository and returns its contents. +// File fetches the file from the GitHub repository and returns its contents. func (c *client) File(u *model.User, r *model.Repo, b *model.Build, f string) ([]byte, error) { + return c.FileRef(u, r, b.Commit, f) +} + +// FileRef fetches the file from the GitHub repository and returns its contents. +func (c *client) FileRef(u *model.User, r *model.Repo, ref, f string) ([]byte, error) { client := c.newClientToken(u.Token) opts := new(github.RepositoryContentGetOptions) - opts.Ref = b.Commit + opts.Ref = ref data, _, _, err := client.Repositories.GetContents(r.Owner, r.Name, f, opts) if err != nil { return nil, err diff --git a/remote/gitlab/client/project.go b/remote/gitlab/client/project.go index 6140ab32e..287655e22 100644 --- a/remote/gitlab/client/project.go +++ b/remote/gitlab/client/project.go @@ -7,11 +7,12 @@ import ( ) const ( - searchUrl = "/projects/search/:query" - projectsUrl = "/projects" - projectUrl = "/projects/:id" - repoUrlRawFile = "/projects/:id/repository/blobs/:sha" - commitStatusUrl = "/projects/:id/statuses/:sha" + searchUrl = "/projects/search/:query" + projectsUrl = "/projects" + projectUrl = "/projects/:id" + repoUrlRawFile = "/projects/:id/repository/blobs/:sha" + repoUrlRawFileRef = "/projects/:id/repository/files" + commitStatusUrl = "/projects/:id/statuses/:sha" ) // Get a list of all projects owned by the authenticated user. @@ -96,6 +97,23 @@ func (c *Client) RepoRawFile(id, sha, filepath string) ([]byte, error) { return contents, err } +func (c *Client) RepoRawFileRef(id, ref, filepath string) ([]byte, error) { + url, opaque := c.ResourceUrl( + repoUrlRawFileRef, + QMap{ + ":id": id, + }, + QMap{ + "filepath": filepath, + "ref": strings.TrimPrefix(ref, "refs/heads/"), + }, + ) + + contents, err := c.Do("GET", url, opaque, nil) + + return contents, err +} + // func (c *Client) SetStatus(id, sha, state, desc, ref, link string) error { url, opaque := c.ResourceUrl( diff --git a/remote/gitlab/gitlab.go b/remote/gitlab/gitlab.go index 728d8d8ea..97e5eb772 100644 --- a/remote/gitlab/gitlab.go +++ b/remote/gitlab/gitlab.go @@ -321,6 +321,21 @@ func (g *Gitlab) File(user *model.User, repo *model.Repo, build *model.Build, f return out, err } +// FileRef fetches the file from the GitHub repository and returns its contents. +func (g *Gitlab) FileRef(u *model.User, r *model.Repo, ref, f string) ([]byte, error) { + var client = NewClient(g.URL, u.Token, g.SkipVerify) + id, err := GetProjectId(g, client, r.Owner, r.Name) + if err != nil { + return nil, err + } + + out, err := client.RepoRawFileRef(id, ref, f) + if err != nil { + return nil, err + } + return out, err +} + // NOTE Currently gitlab doesn't support status for commits and events, // also if we want get MR status in gitlab we need implement a special plugin for gitlab, // gitlab uses API to fetch build status on client side. But for now we skip this. diff --git a/remote/gogs/gogs.go b/remote/gogs/gogs.go index e5e77df2f..aa5b922e6 100644 --- a/remote/gogs/gogs.go +++ b/remote/gogs/gogs.go @@ -190,6 +190,11 @@ func (c *client) File(u *model.User, r *model.Repo, b *model.Build, f string) ([ return cfg, err } +// FileRef fetches the file from the Gogs repository and returns its contents. +func (c *client) FileRef(u *model.User, r *model.Repo, ref, f string) ([]byte, error) { + return c.newClientToken(u.Token).GetFile(r.Owner, r.Name, ref, f) +} + // Status is not supported by the Gogs driver. func (c *client) Status(u *model.User, r *model.Repo, b *model.Build, link string) error { return nil diff --git a/remote/mock/remote.go b/remote/mock/remote.go index d44194710..0a6d41964 100644 --- a/remote/mock/remote.go +++ b/remote/mock/remote.go @@ -1,4 +1,4 @@ -package mock +package mocks import ( "net/http" @@ -7,7 +7,7 @@ import ( "github.com/stretchr/testify/mock" ) -// This is an autogenerated mock type for the Remote type +// Remote is an autogenerated mock type for the Remote type type Remote struct { mock.Mock } @@ -84,6 +84,29 @@ func (_m *Remote) File(u *model.User, r *model.Repo, b *model.Build, f string) ( return r0, r1 } +// FileRef provides a mock function with given fields: u, r, ref, f +func (_m *Remote) FileRef(u *model.User, r *model.Repo, ref string, f string) ([]byte, error) { + ret := _m.Called(u, r, ref, f) + + var r0 []byte + if rf, ok := ret.Get(0).(func(*model.User, *model.Repo, string, string) []byte); ok { + r0 = rf(u, r, ref, f) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]byte) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(*model.User, *model.Repo, string, string) error); ok { + r1 = rf(u, r, ref, f) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // Hook provides a mock function with given fields: r func (_m *Remote) Hook(r *http.Request) (*model.Repo, *model.Build, error) { ret := _m.Called(r) @@ -245,29 +268,6 @@ func (_m *Remote) Status(u *model.User, r *model.Repo, b *model.Build, link stri return r0 } -// Teams provides a mock function with given fields: u -func (_m *Remote) Teams(u *model.User) ([]*model.Team, error) { - ret := _m.Called(u) - - var r0 []*model.Team - if rf, ok := ret.Get(0).(func(*model.User) []*model.Team); ok { - r0 = rf(u) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]*model.Team) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func(*model.User) error); ok { - r1 = rf(u) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - // TeamPerm provides a mock function with given fields: u, org func (_m *Remote) TeamPerm(u *model.User, org string) (*model.Perm, error) { ret := _m.Called(u, org) @@ -290,3 +290,26 @@ func (_m *Remote) TeamPerm(u *model.User, org string) (*model.Perm, error) { return r0, r1 } + +// Teams provides a mock function with given fields: u +func (_m *Remote) Teams(u *model.User) ([]*model.Team, error) { + ret := _m.Called(u) + + var r0 []*model.Team + if rf, ok := ret.Get(0).(func(*model.User) []*model.Team); ok { + r0 = rf(u) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*model.Team) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(*model.User) error); ok { + r1 = rf(u) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} diff --git a/remote/remote.go b/remote/remote.go index aac6aa53a..ea7d0ec06 100644 --- a/remote/remote.go +++ b/remote/remote.go @@ -41,6 +41,10 @@ type Remote interface { // format. File(u *model.User, r *model.Repo, b *model.Build, f string) ([]byte, error) + // FileRef fetches a file from the remote repository for the given ref + // and returns in string format. + FileRef(u *model.User, r *model.Repo, ref, f string) ([]byte, error) + // Status sends the commit status to the remote system. // An example would be the GitHub pull request status. Status(u *model.User, r *model.Repo, b *model.Build, link string) error @@ -109,12 +113,12 @@ func Perm(c context.Context, u *model.User, owner, repo string) (*model.Perm, er // File fetches a file from the remote repository and returns in string format. func File(c context.Context, u *model.User, r *model.Repo, b *model.Build, f string) (out []byte, err error) { - for i:=0;i<5;i++ { + for i := 0; i < 5; i++ { out, err = FromContext(c).File(u, r, b, f) if err == nil { return } - time.Sleep(1*time.Second) + time.Sleep(1 * time.Second) } return } diff --git a/server/build.go b/server/build.go index 2f37396ae..0028e95d0 100644 --- a/server/build.go +++ b/server/build.go @@ -158,9 +158,10 @@ func DeleteBuild(c *gin.Context) { func PostApproval(c *gin.Context) { var ( - repo = session.Repo(c) - user = session.User(c) - num, _ = strconv.Atoi( + remote_ = remote.FromContext(c) + repo = session.Repo(c) + user = session.User(c) + num, _ = strconv.Atoi( c.Params.ByName("number"), ) ) @@ -178,16 +179,115 @@ func PostApproval(c *gin.Context) { build.Reviewed = time.Now().Unix() build.Reviewer = user.Login - if err := store.UpdateBuild(c, build); err != nil { - c.String(500, "error updating build. %s", err) + // + // + // This code is copied pasted until I have a chance + // to refactor into a proper function. Lots of changes + // and technical debt. No judgement please! + // + // + + // fetch the build file from the database + cfg := ToConfig(c) + raw, err := remote_.File(user, repo, build, cfg.Yaml) + if err != nil { + logrus.Errorf("failure to get build config for %s. %s", repo.FullName, err) + c.AbortWithError(404, err) return } - // - // TODO start build - // + netrc, err := remote_.Netrc(user, repo) + if err != nil { + c.String(500, "Failed to generate netrc file. %s", err) + return + } + + if uerr := store.UpdateBuild(c, build); err != nil { + c.String(500, "error updating build. %s", uerr) + return + } c.JSON(200, build) + + // get the previous build so that we can send + // on status change notifications + last, _ := store.GetBuildLastBefore(c, repo, build.Branch, build.ID) + secs, err := store.GetMergedSecretList(c, repo) + if err != nil { + logrus.Debugf("Error getting secrets for %s#%d. %s", repo.FullName, build.Number, err) + } + + defer func() { + uri := fmt.Sprintf("%s/%s/%d", httputil.GetURL(c.Request), repo.FullName, build.Number) + err = remote_.Status(user, repo, build, uri) + if err != nil { + logrus.Errorf("error setting commit status for %s/%d", repo.FullName, build.Number) + } + }() + + b := builder{ + Repo: repo, + Curr: build, + Last: last, + Netrc: netrc, + Secs: secs, + Link: httputil.GetURL(c.Request), + Yaml: string(raw), + } + items, err := b.Build() + if err != nil { + build.Status = model.StatusError + build.Started = time.Now().Unix() + build.Finished = build.Started + build.Error = err.Error() + store.UpdateBuild(c, build) + return + } + + for _, item := range items { + build.Jobs = append(build.Jobs, item.Job) + store.CreateJob(c, item.Job) + // TODO err + } + + // + // publish topic + // + message := pubsub.Message{ + Labels: map[string]string{ + "repo": repo.FullName, + "private": strconv.FormatBool(repo.IsPrivate), + }, + } + message.Data, _ = json.Marshal(model.Event{ + Type: model.Enqueued, + Repo: *repo, + Build: *build, + }) + // TODO remove global reference + config.pubsub.Publish(c, "topic/events", message) + // + // end publish topic + // + + for _, item := range items { + task := new(queue.Task) + task.ID = fmt.Sprint(item.Job.ID) + task.Labels = map[string]string{} + task.Labels["platform"] = item.Platform + for k, v := range item.Labels { + task.Labels[k] = v + } + + task.Data, _ = json.Marshal(rpc.Pipeline{ + ID: fmt.Sprint(item.Job.ID), + Config: item.Config, + Timeout: b.Repo.Timeout, + }) + + config.logger.Open(context.Background(), task.ID) + config.queue.Push(context.Background(), task) + } } func PostDecline(c *gin.Context) { @@ -219,20 +319,10 @@ func PostDecline(c *gin.Context) { return } - owner, err := store.GetUser(c, repo.UserID) - if err == nil { - if refresher, ok := remote_.(remote.Refresher); ok { - ok, _ := refresher.Refresh(user) - if ok { - store.UpdateUser(c, user) - } - } - - uri := fmt.Sprintf("%s/%s/%d", httputil.GetURL(c.Request), repo.FullName, build.Number) - err = remote_.Status(owner, repo, build, uri) - if err != nil { - logrus.Errorf("error setting commit status for %s/%d", repo.FullName, build.Number) - } + uri := fmt.Sprintf("%s/%s/%d", httputil.GetURL(c.Request), repo.FullName, build.Number) + err = remote_.Status(user, repo, build, uri) + if err != nil { + logrus.Errorf("error setting commit status for %s/%d", repo.FullName, build.Number) } c.JSON(200, build) diff --git a/server/hook.go b/server/hook.go index 6ac9f6a49..e0d01c2a8 100644 --- a/server/hook.go +++ b/server/hook.go @@ -1,6 +1,7 @@ package server import ( + "bytes" "context" "encoding/json" "fmt" @@ -9,7 +10,6 @@ import ( "time" "github.com/gin-gonic/gin" - "github.com/square/go-jose" "github.com/Sirupsen/logrus" "github.com/drone/drone/model" @@ -139,10 +139,6 @@ func PostHook(c *gin.Context) { c.AbortWithError(404, err) return } - sec, err := remote_.File(user, repo, build, cfg.Shasum) - if err != nil { - logrus.Debugf("cannot find yaml signature for %s. %s", repo.FullName, err) - } netrc, err := remote_.Netrc(user, repo) if err != nil { @@ -159,26 +155,47 @@ func PostHook(c *gin.Context) { } } - signature, err := jose.ParseSigned(string(sec)) - if err != nil { - logrus.Debugf("cannot parse .drone.yml.sig file. %s", err) - } else if len(sec) == 0 { - logrus.Debugf("cannot parse .drone.yml.sig file. empty file") - } else { - build.Signed = true - output, verr := signature.Verify([]byte(repo.Hash)) - if verr != nil { - logrus.Debugf("cannot verify .drone.yml.sig file. %s", verr) - } else if string(output) != string(raw) { - logrus.Debugf("cannot verify .drone.yml.sig file. no match") + // TODO default logic should avoid the approval if all + // secrets have skip-verify flag + + if build.Event == model.EventPull { + old, ferr := remote_.FileRef(user, repo, build.Ref, cfg.Yaml) + if ferr != nil { + build.Status = model.StatusBlocked + } else if bytes.Equal(old, raw) { + build.Status = model.StatusPending } else { - build.Verified = true + // this block is executed if the target yaml file + // does not match the base yaml. + + // TODO unfortunately we have no good way to get the + // sender repository permissions unless the user is + // a registered drone user. + sender, uerr := store.GetUserLogin(c, build.Sender) + if uerr != nil { + build.Status = model.StatusBlocked + } else { + if refresher, ok := remote_.(remote.Refresher); ok { + ok, _ := refresher.Refresh(sender) + if ok { + store.UpdateUser(c, sender) + } + } + // if the sender does not have push access to the + // repository the pull request should be blocked. + perm, perr := remote_.Perm(sender, repo.Owner, repo.Name) + if perr != nil || perm.Push == false { + build.Status = model.StatusBlocked + } + } } + } else { + build.Status = model.StatusPending } // update some build fields - build.Status = model.StatusPending build.RepoID = repo.ID + build.Verified = true if err := store.CreateBuild(c, build, build.Jobs...); err != nil { logrus.Errorf("failure to save commit for %s. %s", repo.FullName, err) @@ -188,6 +205,10 @@ func PostHook(c *gin.Context) { c.JSON(200, build) + if build.Status == model.StatusBlocked { + return + } + // get the previous build so that we can send // on status change notifications last, _ := store.GetBuildLastBefore(c, repo, build.Branch, build.ID)