diff --git a/README.md b/README.md index 3c5a91099..4aaa524d4 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,13 @@ Drone documentation is organized into several categories: * [CLI Reference](http://readme.drone.io/devs/cli/) * [API Reference](http://readme.drone.io/devs/api/builds) +### Documentation for 0.5 (unstable) + +If you are using the 0.5 unstable release (master branch) please see the updated documentation: + +* [Setup Guide](http://readme.drone.io/0.5/manage/server/) +* [Build Guide](http://readme.drone.io/0.5/usage/overview/) + ### Community, Help Contributions, questions, and comments are welcomed and encouraged. Drone developers hang out in the [drone/drone](https://gitter.im/drone/drone) room on gitter. We ask that you please post your questions to [gitter](https://gitter.im/drone/drone) before creating an issue. @@ -51,10 +58,4 @@ make gen # Generate code make build # Build the binary ``` -If you are seeing slow compile times please install the following: - -```sh -go install github.com/mattn/go-sqlite3 -``` - If you are having trouble building this project please reference its `.drone.yml` file. Everything you need to know about building Drone is defined in that file. diff --git a/drone/agent/agent.go b/drone/agent/agent.go index 5f499e845..5bf306621 100644 --- a/drone/agent/agent.go +++ b/drone/agent/agent.go @@ -168,7 +168,7 @@ func start(c *cli.Context) { for { if err := r.run(); err != nil { dur := c.Duration("backoff") - logrus.Warnf("Attempting to reconnect in %v", dur) + logrus.Warnf("reconnect in %v. %s", dur, err.Error()) time.Sleep(dur) } } diff --git a/drone/daemon.go b/drone/daemon.go index 91b271ccb..30d9048bb 100644 --- a/drone/daemon.go +++ b/drone/daemon.go @@ -320,141 +320,6 @@ func start(c *cli.Context) error { ) } -// -// func setupCache(c *cli.Context) cache.Cache { -// return cache.NewTTL( -// c.Duration("cache-ttl"), -// ) -// } -// -// func setupBus(c *cli.Context) bus.Bus { -// return bus.New() -// } -// -// func setupQueue(c *cli.Context) queue.Queue { -// return queue.New() -// } -// -// func setupStream(c *cli.Context) stream.Stream { -// return stream.New() -// } -// -// func setupStore(c *cli.Context) store.Store { -// return datastore.New( -// c.String("driver"), -// c.String("datasource"), -// ) -// } -// -// func setupRemote(c *cli.Context) remote.Remote { -// var remote remote.Remote -// var err error -// switch { -// case c.Bool("github"): -// remote, err = setupGithub(c) -// case c.Bool("gitlab"): -// remote, err = setupGitlab(c) -// case c.Bool("bitbucket"): -// remote, err = setupBitbucket(c) -// case c.Bool("stash"): -// remote, err = setupStash(c) -// case c.Bool("gogs"): -// remote, err = setupGogs(c) -// default: -// err = fmt.Errorf("version control system not configured") -// } -// if err != nil { -// logrus.Fatalln(err) -// } -// return remote -// } -// -// func setupBitbucket(c *cli.Context) (remote.Remote, error) { -// return bitbucket.New( -// c.String("bitbucket-client"), -// c.String("bitbucket-server"), -// ), nil -// } -// -// func setupGogs(c *cli.Context) (remote.Remote, error) { -// return gogs.New(gogs.Opts{ -// URL: c.String("gogs-server"), -// Username: c.String("gogs-git-username"), -// Password: c.String("gogs-git-password"), -// PrivateMode: c.Bool("gogs-private-mode"), -// SkipVerify: c.Bool("gogs-skip-verify"), -// }) -// } -// -// func setupStash(c *cli.Context) (remote.Remote, error) { -// return bitbucketserver.New(bitbucketserver.Opts{ -// URL: c.String("stash-server"), -// Username: c.String("stash-git-username"), -// Password: c.String("stash-git-password"), -// ConsumerKey: c.String("stash-consumer-key"), -// ConsumerRSA: c.String("stash-consumer-rsa"), -// SkipVerify: c.Bool("stash-skip-verify"), -// }) -// } -// -// func setupGitlab(c *cli.Context) (remote.Remote, error) { -// return gitlab.New(gitlab.Opts{ -// URL: c.String("gitlab-server"), -// Client: c.String("gitlab-client"), -// Secret: c.String("gitlab-sercret"), -// Username: c.String("gitlab-git-username"), -// Password: c.String("gitlab-git-password"), -// PrivateMode: c.Bool("gitlab-private-mode"), -// SkipVerify: c.Bool("gitlab-skip-verify"), -// }) -// } -// -// func setupGithub(c *cli.Context) (remote.Remote, error) { -// return github.New( -// c.String("github-server"), -// c.String("github-client"), -// c.String("github-sercret"), -// c.StringSlice("github-scope"), -// c.Bool("github-private-mode"), -// c.Bool("github-skip-verify"), -// c.BoolT("github-merge-ref"), -// ) -// } -// -// func setupConfig(c *cli.Context) *server.Config { -// return &server.Config{ -// Open: c.Bool("open"), -// Yaml: c.String("yaml"), -// Secret: c.String("agent-secret"), -// Admins: sliceToMap(c.StringSlice("admin")), -// Orgs: sliceToMap(c.StringSlice("orgs")), -// } -// } -// -// func sliceToMap(s []string) map[string]bool { -// v := map[string]bool{} -// for _, ss := range s { -// v[ss] = true -// } -// return v -// } -// -// func printSecret(c *cli.Context) error { -// secret := c.String("agent-secret") -// if secret == "" { -// return fmt.Errorf("missing DRONE_AGENT_SECRET configuration parameter") -// } -// t := token.New(secret, "") -// s, err := t.Sign(secret) -// if err != nil { -// return fmt.Errorf("invalid value for DRONE_AGENT_SECRET. %s", s) -// } -// -// logrus.Infof("using agent secret %s", secret) -// logrus.Warnf("agents can connect with token %s", s) -// return nil -// } - var agreement = ` --- diff --git a/remote/gerrit/gerrit.go b/remote/gerrit/gerrit.go new file mode 100644 index 000000000..33963ff0d --- /dev/null +++ b/remote/gerrit/gerrit.go @@ -0,0 +1,115 @@ +package gerrit + +import ( + "fmt" + "net" + "net/http" + "net/url" + + "github.com/drone/drone/model" + "github.com/drone/drone/remote" +) + +// IMPORTANT Gerrit support is not yet implemented. This file is a placeholder +// for future implementaiton. + +// Opts defines configuration options. +type Opts struct { + URL string // Gerrit server url. + Username string // Optional machine account username. + Password string // Optional machine account password. + PrivateMode bool // Gerrit is running in private mode. + SkipVerify bool // Skip ssl verification. +} + +type client struct { + URL string + Machine string + Username string + Password string + PrivateMode bool + SkipVerify bool +} + +// New returns a Remote implementation that integrates with Getter, an open +// source Git hosting service and code review system. +func New(opts Opts) (remote.Remote, error) { + url, err := url.Parse(opts.URL) + if err != nil { + return nil, err + } + host, _, err := net.SplitHostPort(url.Host) + if err == nil { + url.Host = host + } + return &client{ + URL: opts.URL, + Machine: url.Host, + Username: opts.Username, + Password: opts.Password, + PrivateMode: opts.PrivateMode, + SkipVerify: opts.SkipVerify, + }, nil +} + +// Login authenticates an account with Gerrit using oauth authenticaiton. The +// Gerrit account details are returned when the user is successfully authenticated. +func (c *client) Login(res http.ResponseWriter, req *http.Request) (*model.User, error) { + return nil, nil +} + +// Auth is not supported by the Gerrit driver. +func (c *client) Auth(token, secret string) (string, error) { + return "", fmt.Errorf("Not Implemented") +} + +// Teams is not supported by the Gerrit driver. +func (c *client) Teams(u *model.User) ([]*model.Team, error) { + var empty []*model.Team + return empty, nil +} + +// Repo is not supported by the Gerrit driver. +func (c *client) Repo(u *model.User, owner, name string) (*model.Repo, error) { + return nil, nil +} + +// Repos is not supported by the Gerrit driver. +func (c *client) Repos(u *model.User) ([]*model.RepoLite, error) { + return nil, nil +} + +// Perm is not supported by the Gerrit driver. +func (c *client) Perm(u *model.User, owner, name string) (*model.Perm, error) { + return nil, nil +} + +// File is not supported by the Gerrit driver. +func (c *client) File(u *model.User, r *model.Repo, b *model.Build, f string) ([]byte, error) { + return nil, nil +} + +// Status is not supported by the Gogs driver. +func (c *client) Status(u *model.User, r *model.Repo, b *model.Build, link string) error { + return nil +} + +// Netrc is not supported by the Gerrit driver. +func (c *client) Netrc(u *model.User, r *model.Repo) (*model.Netrc, error) { + return nil, nil +} + +// Activate is not supported by the Gerrit driver. +func (c *client) Activate(u *model.User, r *model.Repo, link string) error { + return nil +} + +// Deactivate is not supported by the Gogs driver. +func (c *client) Deactivate(u *model.User, r *model.Repo, link string) error { + return nil +} + +// Hook is not supported by the Gerrit driver. +func (c *client) Hook(r *http.Request) (*model.Repo, *model.Build, error) { + return nil, nil, nil +} diff --git a/remote/github/convert.go b/remote/github/convert.go new file mode 100644 index 000000000..a4e768253 --- /dev/null +++ b/remote/github/convert.go @@ -0,0 +1,126 @@ +package github + +import ( + "github.com/drone/drone/model" + + "github.com/google/go-github/github" +) + +const defaultBranch = "master" + +const ( + statusPending = "pending" + statusSuccess = "success" + statusFailure = "failure" + statusError = "error" +) + +const ( + descPending = "this build is pending" + descSuccess = "the build was successful" + descFailure = "the build failed" + descError = "oops, something went wrong" +) + +// convertStatus is a helper function used to convert a Drone status to a +// GitHub commit status. +func convertStatus(status string) string { + switch status { + case model.StatusPending, model.StatusRunning: + return statusPending + case model.StatusSuccess: + return statusSuccess + case model.StatusFailure: + return statusFailure + default: + return statusError + } +} + +// convertDesc is a helper function used to convert a Drone status to a +// GitHub status description. +func convertDesc(status string) string { + switch status { + case model.StatusPending, model.StatusRunning: + return descPending + case model.StatusSuccess: + return descSuccess + case model.StatusFailure: + return descFailure + default: + return descError + } +} + +// convertRepo is a helper function used to convert a GitHub repository +// structure to the common Drone repository structure. +func convertRepo(from *github.Repository, private bool) *model.Repo { + repo := &model.Repo{ + Owner: *from.Owner.Login, + Name: *from.Name, + FullName: *from.FullName, + Link: *from.HTMLURL, + IsPrivate: *from.Private, + Clone: *from.CloneURL, + Avatar: *from.Owner.AvatarURL, + Kind: model.RepoGit, + Branch: defaultBranch, + } + if from.DefaultBranch != nil { + repo.Branch = *from.DefaultBranch + } + if private { + repo.IsPrivate = true + } + return repo +} + +// convertPerm is a helper function used to convert a GitHub repository +// permissions to the common Drone permissions structure. +func convertPerm(from *github.Repository) *model.Perm { + return &model.Perm{ + Admin: (*from.Permissions)["admin"], + Push: (*from.Permissions)["push"], + Pull: (*from.Permissions)["pull"], + } +} + +// convertRepoList is a helper function used to convert a GitHub repository +// list to the common Drone repository structure. +func convertRepoList(from []github.Repository) []*model.RepoLite { + var repos []*model.RepoLite + for _, repo := range from { + repos = append(repos, convertRepoLite(repo)) + } + return repos +} + +// convertRepoLite is a helper function used to convert a GitHub repository +// structure to the common Drone repository structure. +func convertRepoLite(from github.Repository) *model.RepoLite { + return &model.RepoLite{ + Owner: *from.Owner.Login, + Name: *from.Name, + FullName: *from.FullName, + Avatar: *from.Owner.AvatarURL, + } +} + +// convertTeamList is a helper function used to convert a GitHub team list to +// the common Drone repository structure. +func convertTeamList(from []github.Organization) []*model.Team { + var teams []*model.Team + for _, team := range from { + teams = append(teams, convertTeam(team)) + } + return teams +} + +// convertTeam is a helper function used to convert a GitHub team structure +// to the common Drone repository structure. +func convertTeam(from github.Organization) *model.Team { + return &model.Team{ + Login: *from.Login, + Avatar: *from.AvatarURL, + } +} diff --git a/remote/github/convert_test.go b/remote/github/convert_test.go new file mode 100644 index 000000000..9a240dd37 --- /dev/null +++ b/remote/github/convert_test.go @@ -0,0 +1,171 @@ +package github + +import ( + "testing" + + "github.com/drone/drone/model" + "github.com/google/go-github/github" + + "github.com/franela/goblin" +) + +func Test_helper(t *testing.T) { + + g := goblin.Goblin(t) + g.Describe("GitHub converter", func() { + + g.It("should convert passing status", func() { + g.Assert(convertStatus(model.StatusSuccess)).Equal(statusSuccess) + }) + + g.It("should convert pending status", func() { + g.Assert(convertStatus(model.StatusPending)).Equal(statusPending) + g.Assert(convertStatus(model.StatusRunning)).Equal(statusPending) + }) + + g.It("should convert failing status", func() { + g.Assert(convertStatus(model.StatusFailure)).Equal(statusFailure) + }) + + g.It("should convert error status", func() { + g.Assert(convertStatus(model.StatusKilled)).Equal(statusError) + g.Assert(convertStatus(model.StatusError)).Equal(statusError) + }) + + g.It("should convert passing desc", func() { + g.Assert(convertDesc(model.StatusSuccess)).Equal(descSuccess) + }) + + g.It("should convert pending desc", func() { + g.Assert(convertDesc(model.StatusPending)).Equal(descPending) + g.Assert(convertDesc(model.StatusRunning)).Equal(descPending) + }) + + g.It("should convert failing desc", func() { + g.Assert(convertDesc(model.StatusFailure)).Equal(descFailure) + }) + + g.It("should convert error desc", func() { + g.Assert(convertDesc(model.StatusKilled)).Equal(descError) + g.Assert(convertDesc(model.StatusError)).Equal(descError) + }) + + g.It("should convert repository lite", func() { + from := github.Repository{ + FullName: github.String("octocat/hello-world"), + Name: github.String("hello-world"), + Owner: &github.User{ + AvatarURL: github.String("http://..."), + Login: github.String("octocat"), + }, + } + + to := convertRepoLite(from) + g.Assert(to.Avatar).Equal("http://...") + g.Assert(to.FullName).Equal("octocat/hello-world") + g.Assert(to.Owner).Equal("octocat") + g.Assert(to.Name).Equal("hello-world") + }) + + g.It("should convert repository list", func() { + from := []github.Repository{ + { + FullName: github.String("octocat/hello-world"), + Name: github.String("hello-world"), + Owner: &github.User{ + AvatarURL: github.String("http://..."), + Login: github.String("octocat"), + }, + }, + } + + to := convertRepoList(from) + g.Assert(to[0].Avatar).Equal("http://...") + g.Assert(to[0].FullName).Equal("octocat/hello-world") + g.Assert(to[0].Owner).Equal("octocat") + g.Assert(to[0].Name).Equal("hello-world") + }) + + g.It("should convert repository", func() { + from := github.Repository{ + FullName: github.String("octocat/hello-world"), + Name: github.String("hello-world"), + HTMLURL: github.String("https://github.com/octocat/hello-world"), + CloneURL: github.String("https://github.com/octocat/hello-world.git"), + DefaultBranch: github.String("develop"), + Private: github.Bool(true), + Owner: &github.User{ + AvatarURL: github.String("http://..."), + Login: github.String("octocat"), + }, + } + + to := convertRepo(&from, false) + g.Assert(to.Avatar).Equal("http://...") + g.Assert(to.FullName).Equal("octocat/hello-world") + g.Assert(to.Owner).Equal("octocat") + g.Assert(to.Name).Equal("hello-world") + g.Assert(to.Branch).Equal("develop") + g.Assert(to.Kind).Equal("git") + g.Assert(to.IsPrivate).IsTrue() + g.Assert(to.Clone).Equal("https://github.com/octocat/hello-world.git") + g.Assert(to.Link).Equal("https://github.com/octocat/hello-world") + }) + + g.It("should convert repository permissions", func() { + from := &github.Repository{ + Permissions: &map[string]bool{ + "admin": true, + "push": true, + "pull": true, + }, + } + + to := convertPerm(from) + g.Assert(to.Push).IsTrue() + g.Assert(to.Pull).IsTrue() + g.Assert(to.Admin).IsTrue() + }) + + g.It("should convert team", func() { + from := github.Organization{ + Login: github.String("octocat"), + AvatarURL: github.String("http://..."), + } + to := convertTeam(from) + g.Assert(to.Login).Equal("octocat") + g.Assert(to.Avatar).Equal("http://...") + }) + + g.It("should convert team list", func() { + from := []github.Organization{ + { + Login: github.String("octocat"), + AvatarURL: github.String("http://..."), + }, + } + to := convertTeamList(from) + g.Assert(to[0].Login).Equal("octocat") + g.Assert(to[0].Avatar).Equal("http://...") + }) + + // + // g.It("should convert user", func() { + // token := &oauth2.Token{ + // AccessToken: "foo", + // RefreshToken: "bar", + // Expiry: time.Now(), + // } + // user := &internal.Account{Login: "octocat"} + // user.Links.Avatar.Href = "http://..." + // + // result := convertUser(user, token) + // g.Assert(result.Avatar).Equal(user.Links.Avatar.Href) + // g.Assert(result.Login).Equal(user.Login) + // g.Assert(result.Token).Equal(token.AccessToken) + // g.Assert(result.Token).Equal(token.AccessToken) + // g.Assert(result.Secret).Equal(token.RefreshToken) + // g.Assert(result.Expiry).Equal(token.Expiry.UTC().Unix()) + // }) + }) +} diff --git a/remote/github/github.go b/remote/github/github.go index ac9b1a6aa..a7a92a086 100644 --- a/remote/github/github.go +++ b/remote/github/github.go @@ -2,8 +2,9 @@ package github import ( "crypto/tls" - "encoding/json" + "encoding/base32" "fmt" + "net" "net/http" "net/url" "regexp" @@ -13,104 +14,111 @@ import ( "github.com/drone/drone/model" "github.com/drone/drone/remote" "github.com/drone/drone/shared/httputil" - "github.com/drone/drone/shared/oauth2" + "github.com/gorilla/securecookie" "github.com/google/go-github/github" + "golang.org/x/oauth2" ) const ( - DefaultURL = "https://github.com" // Default GitHub URL - DefaultAPI = "https://api.github.com" // Default GitHub API URL + defaultURL = "https://github.com" // Default GitHub URL + defaultAPI = "https://api.github.com" // Default GitHub API URL ) -type Github struct { +// Opts defines configuration options. +type Opts struct { + URL string // GitHub server url. + Client string // GitHub oauth client id. + Secret string // GitHub oauth client secret. + Scopes []string // GitHub oauth scopes + Username string // Optional machine account username. + Password string // Optional machine account password. + PrivateMode bool // GitHub is running in private mode. + SkipVerify bool // Skip ssl verification. + MergeRef bool // Clone pull requests using the merge ref. +} + +// New returns a Remote implementation that integrates with a GitHub Cloud or +// GitHub Enterprise version control hosting provider. +func New(opts Opts) (remote.Remote, error) { + url, err := url.Parse(opts.URL) + if err != nil { + return nil, err + } + host, _, err := net.SplitHostPort(url.Host) + if err == nil { + url.Host = host + } + remote := &client{ + API: defaultAPI, + URL: defaultURL, + Client: opts.Client, + Secret: opts.Secret, + Scope: strings.Join(opts.Scopes, ","), + PrivateMode: opts.PrivateMode, + SkipVerify: opts.SkipVerify, + MergeRef: opts.MergeRef, + Machine: url.Host, + Username: opts.Username, + Password: opts.Password, + } + if opts.URL != defaultURL { + remote.URL = strings.TrimSuffix(opts.URL, "/") + remote.API = remote.URL + "/api/v3/" + } + return remote, nil +} + +type client struct { URL string API string Client string Secret string Scope string - MergeRef string + Machine string + Username string + Password string PrivateMode bool SkipVerify bool -} - -func New(url, client, secret string, scope []string, private, skipverify, mergeref bool) (remote.Remote, error) { - remote := &Github{ - URL: strings.TrimSuffix(url, "/"), - Client: client, - Secret: secret, - Scope: strings.Join(scope, ","), - PrivateMode: private, - SkipVerify: skipverify, - MergeRef: "head", - } - - if remote.URL == DefaultURL { - remote.API = DefaultAPI - } else { - remote.API = remote.URL + "/api/v3/" - } - if mergeref { - remote.MergeRef = "merge" - } - - return remote, nil + MergeRef bool } // Login authenticates the session and returns the remote user details. -func (g *Github) Login(res http.ResponseWriter, req *http.Request) (*model.User, error) { +func (c *client) Login(res http.ResponseWriter, req *http.Request) (*model.User, error) { + config := c.newConfig(httputil.GetURL(req)) - var config = &oauth2.Config{ - ClientId: g.Client, - ClientSecret: g.Secret, - Scope: g.Scope, - AuthURL: fmt.Sprintf("%s/login/oauth/authorize", g.URL), - TokenURL: fmt.Sprintf("%s/login/oauth/access_token", g.URL), - RedirectURL: fmt.Sprintf("%s/authorize", httputil.GetURL(req)), - } - - // get the OAuth code - var code = req.FormValue("code") + code := req.FormValue("code") if len(code) == 0 { - var random = GetRandom() - http.Redirect(res, req, config.AuthCodeURL(random), http.StatusSeeOther) + rand := base32.StdEncoding.EncodeToString(securecookie.GenerateRandomKey(32)) + http.Redirect(res, req, config.AuthCodeURL(rand), http.StatusSeeOther) return nil, nil } - var trans = &oauth2.Transport{ - Config: config, - } - if g.SkipVerify { - trans.Transport = &http.Transport{ - Proxy: http.ProxyFromEnvironment, - TLSClientConfig: &tls.Config{ - InsecureSkipVerify: true, - }, - } - } - var token, err = trans.Exchange(code) + // TODO(bradrydzewski) what is the best way to provide a SkipVerify flag + // when exchanging the token? + + token, err := config.Exchange(oauth2.NoContext, code) if err != nil { - return nil, fmt.Errorf("Error exchanging token. %s", err) + return nil, err } - var client = NewClient(g.API, token.AccessToken, g.SkipVerify) - var useremail, errr = GetUserEmail(client) - if errr != nil { - return nil, fmt.Errorf("Error retrieving user or verified email. %s", errr) + client := c.newClientToken(token.AccessToken) + useremail, err := GetUserEmail(client) + if err != nil { + return nil, err } - user := model.User{} - user.Login = *useremail.Login - user.Email = *useremail.Email - user.Token = token.AccessToken - user.Avatar = *useremail.AvatarURL - return &user, nil + return &model.User{ + Login: *useremail.Login, + Email: *useremail.Email, + Token: token.AccessToken, + Avatar: *useremail.AvatarURL, + }, nil } -// Auth authenticates the session and returns the remote user -// login for the given token and secret -func (g *Github) Auth(token, secret string) (string, error) { - client := NewClient(g.API, token, g.SkipVerify) +// Auth returns the GitHub user login for the given access token. +func (c *client) Auth(token, secret string) (string, error) { + client := c.newClientToken(token) user, _, err := client.Users.Get("") if err != nil { return "", err @@ -118,117 +126,150 @@ func (g *Github) Auth(token, secret string) (string, error) { return *user.Login, nil } -func (g *Github) Teams(u *model.User) ([]*model.Team, error) { - client := NewClient(g.API, u.Token, g.SkipVerify) - orgs, err := GetOrgs(client) - if err != nil { - return nil, err - } +// Teams returns a list of all team membership for the GitHub account. +func (c *client) Teams(u *model.User) ([]*model.Team, error) { + client := c.newClientToken(u.Token) + + opts := new(github.ListOptions) + opts.Page = 1 var teams []*model.Team - for _, org := range orgs { - teams = append(teams, &model.Team{ - Login: *org.Login, - Avatar: *org.AvatarURL, - }) + for opts.Page > 0 { + list, resp, err := client.Organizations.List("", opts) + if err != nil { + return nil, err + } + teams = append(teams, convertTeamList(list)...) + opts.Page = resp.NextPage } return teams, nil } -// Repo fetches the named repository from the remote system. -func (g *Github) Repo(u *model.User, owner, name string) (*model.Repo, error) { - client := NewClient(g.API, u.Token, g.SkipVerify) - r, err := GetRepo(client, owner, name) +// Repo returns the named GitHub repository. +func (c *client) Repo(u *model.User, owner, name string) (*model.Repo, error) { + client := c.newClientToken(u.Token) + repo, _, err := client.Repositories.Get(owner, name) if err != nil { return nil, err } - - repo := &model.Repo{ - Owner: owner, - Name: name, - FullName: *r.FullName, - Link: *r.HTMLURL, - IsPrivate: *r.Private, - Clone: *r.CloneURL, - Avatar: *r.Owner.AvatarURL, - Kind: model.RepoGit, - } - - if r.DefaultBranch != nil { - repo.Branch = *r.DefaultBranch - } else { - repo.Branch = "master" - } - - if g.PrivateMode { - repo.IsPrivate = true - } - - return repo, err + return convertRepo(repo, c.PrivateMode), nil } -// Repos fetches a list of repos from the remote system. -func (g *Github) Repos(u *model.User) ([]*model.RepoLite, error) { - client := NewClient(g.API, u.Token, g.SkipVerify) +// Repos returns a list of all repositories for GitHub account, including +// organization repositories. +func (c *client) Repos(u *model.User) ([]*model.RepoLite, error) { + client := c.newClientToken(u.Token) - all, err := GetUserRepos(client) + opts := new(github.RepositoryListOptions) + opts.PerPage = 100 + opts.Page = 1 + + var repos []*model.RepoLite + for opts.Page > 0 { + list, resp, err := client.Repositories.List("", opts) + if err != nil { + return nil, err + } + repos = append(repos, convertRepoList(list)...) + opts.Page = resp.NextPage + } + return repos, nil +} + +// Perm returns the user permissions for the named GitHub repository. +func (c *client) Perm(u *model.User, owner, name string) (*model.Perm, error) { + client := c.newClientToken(u.Token) + repo, _, err := client.Repositories.Get(owner, name) if err != nil { return nil, err } - - var repos = []*model.RepoLite{} - for _, repo := range all { - repos = append(repos, &model.RepoLite{ - Owner: *repo.Owner.Login, - Name: *repo.Name, - FullName: *repo.FullName, - Avatar: *repo.Owner.AvatarURL, - }) - } - return repos, err + return convertPerm(repo), nil } -// Perm fetches the named repository permissions from -// the remote system for the specified user. -func (g *Github) Perm(u *model.User, owner, name string) (*model.Perm, error) { +// File fetches the file from the Bitbucket repository and returns its contents. +func (c *client) File(u *model.User, r *model.Repo, b *model.Build, f string) ([]byte, error) { + client := c.newClientToken(u.Token) - client := NewClient(g.API, u.Token, g.SkipVerify) - repo, err := GetRepo(client, owner, name) + opts := new(github.RepositoryContentGetOptions) + opts.Ref = b.Commit + data, _, _, err := client.Repositories.GetContents(r.Owner, r.Name, f, opts) if err != nil { return nil, err } - m := &model.Perm{} - m.Admin = (*repo.Permissions)["admin"] - m.Push = (*repo.Permissions)["push"] - m.Pull = (*repo.Permissions)["pull"] - return m, nil + return data.Decode() } -// File fetches a file from the remote repository and returns in string format. -func (g *Github) File(u *model.User, r *model.Repo, b *model.Build, f string) ([]byte, error) { - client := NewClient(g.API, u.Token, g.SkipVerify) - cfg, err := GetFile(client, r.Owner, r.Name, f, b.Commit) - return cfg, err +// Netrc returns a netrc file capable of authenticating GitHub requests and +// cloning GitHub repositories. The netrc will use the global machine account +// when configured. +func (c *client) Netrc(u *model.User, r *model.Repo) (*model.Netrc, error) { + if c.Password != "" { + return &model.Netrc{ + Login: c.Username, + Password: c.Password, + Machine: c.Machine, + }, nil + } + return &model.Netrc{ + Login: u.Token, + Password: "x-oauth-basic", + Machine: c.Machine, + }, nil } +// helper function to return the bitbucket oauth2 config +func (c *client) newConfig(redirect string) *oauth2.Config { + return &oauth2.Config{ + ClientID: c.Client, + ClientSecret: c.Secret, + Endpoint: oauth2.Endpoint{ + AuthURL: fmt.Sprintf("%s/login/oauth/authorize", c.URL), + TokenURL: fmt.Sprintf("%s/login/oauth/access_token", c.URL), + }, + RedirectURL: fmt.Sprintf("%s/authorize", redirect), + } +} + +// helper function to return the bitbucket oauth2 client +func (c *client) newClientToken(token string) *github.Client { + ts := oauth2.StaticTokenSource( + &oauth2.Token{AccessToken: token}, + ) + tc := oauth2.NewClient(oauth2.NoContext, ts) + if c.SkipVerify { + tc.Transport.(*oauth2.Transport).Base = &http.Transport{ + Proxy: http.ProxyFromEnvironment, + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: true, + }, + } + } + github := github.NewClient(tc) + github.BaseURL, _ = url.Parse(c.API) + return github +} + +// +// TODO(bradrydzewski) refactor below functions +// + // Status sends the commit status to the remote system. // An example would be the GitHub pull request status. -func (g *Github) Status(u *model.User, r *model.Repo, b *model.Build, link string) error { - client := NewClient(g.API, u.Token, g.SkipVerify) - if b.Event == "deployment" { +func (c *client) Status(u *model.User, r *model.Repo, b *model.Build, link string) error { + client := c.newClientToken(u.Token) + switch b.Event { + case "deployment": return deploymentStatus(client, r, b, link) - } else { + default: return repoStatus(client, r, b, link) } } func repoStatus(client *github.Client, r *model.Repo, b *model.Build, link string) error { - status := getStatus(b.Status) - desc := getDesc(b.Status) data := github.RepoStatus{ Context: github.String("continuous-integration/drone"), - State: github.String(status), - Description: github.String(desc), + State: github.String(convertStatus(b.Status)), + Description: github.String(convertDesc(b.Status)), TargetURL: github.String(link), } _, _, err := client.Repositories.CreateStatus(r.Owner, r.Name, b.Commit, &data) @@ -239,283 +280,46 @@ var reDeploy = regexp.MustCompile(".+/deployments/(\\d+)") func deploymentStatus(client *github.Client, r *model.Repo, b *model.Build, link string) error { matches := reDeploy.FindStringSubmatch(b.Link) - // if the deployment was not triggered from github, don't send a deployment status if len(matches) != 2 { return nil } - // the deployment ID is only available in the the link to the build as the last element in the URL id, _ := strconv.Atoi(matches[1]) - status := getStatus(b.Status) - desc := getDesc(b.Status) + data := github.DeploymentStatusRequest{ - State: github.String(status), - Description: github.String(desc), + State: github.String(convertStatus(b.Status)), + Description: github.String(convertDesc(b.Status)), TargetURL: github.String(link), } _, _, err := client.Repositories.CreateDeploymentStatus(r.Owner, r.Name, id, &data) return err } -// Netrc returns a .netrc file that can be used to clone -// private repositories from a remote system. -func (g *Github) Netrc(u *model.User, r *model.Repo) (*model.Netrc, error) { - url_, err := url.Parse(g.URL) - if err != nil { - return nil, err - } - netrc := &model.Netrc{} - netrc.Login = u.Token - netrc.Password = "x-oauth-basic" - netrc.Machine = url_.Host - return netrc, nil -} - // Activate activates a repository by creating the post-commit hook and // adding the SSH deploy key, if applicable. -func (g *Github) Activate(u *model.User, r *model.Repo, link string) error { - client := NewClient(g.API, u.Token, g.SkipVerify) +func (c *client) Activate(u *model.User, r *model.Repo, link string) error { + client := c.newClientToken(u.Token) _, err := CreateUpdateHook(client, r.Owner, r.Name, link) return err } // Deactivate removes a repository by removing all the post-commit hooks // which are equal to link and removing the SSH deploy key. -func (g *Github) Deactivate(u *model.User, r *model.Repo, link string) error { - client := NewClient(g.API, u.Token, g.SkipVerify) +func (c *client) Deactivate(u *model.User, r *model.Repo, link string) error { + client := c.newClientToken(u.Token) return DeleteHook(client, r.Owner, r.Name, link) } // Hook parses the post-commit hook from the Request body // and returns the required data in a standard format. -func (g *Github) Hook(r *http.Request) (*model.Repo, *model.Build, error) { - +func (c *client) Hook(r *http.Request) (*model.Repo, *model.Build, error) { switch r.Header.Get("X-Github-Event") { case "pull_request": - return g.pullRequest(r) + return c.pullRequest(r) case "push": - return g.push(r) + return c.push(r) case "deployment": - return g.deployment(r) + return c.deployment(r) default: return nil, nil, nil } } - -// push parses a hook with event type `push` and returns -// the commit data. -func (g *Github) push(r *http.Request) (*model.Repo, *model.Build, error) { - payload := GetPayload(r) - hook := &pushHook{} - err := json.Unmarshal(payload, hook) - if err != nil { - return nil, nil, err - } - if hook.Deleted { - return nil, nil, err - } - - repo := &model.Repo{} - repo.Owner = hook.Repo.Owner.Login - if len(repo.Owner) == 0 { - repo.Owner = hook.Repo.Owner.Name - } - repo.Name = hook.Repo.Name - // Generating rather than using hook.Repo.FullName as it's - // not always present - repo.FullName = fmt.Sprintf("%s/%s", repo.Owner, repo.Name) - repo.Link = hook.Repo.HTMLURL - repo.IsPrivate = hook.Repo.Private - repo.Clone = hook.Repo.CloneURL - repo.Branch = hook.Repo.DefaultBranch - repo.Kind = model.RepoGit - - build := &model.Build{} - build.Event = model.EventPush - build.Commit = hook.Head.ID - build.Ref = hook.Ref - build.Link = hook.Head.URL - build.Branch = strings.Replace(build.Ref, "refs/heads/", "", -1) - build.Message = hook.Head.Message - // build.Timestamp = hook.Head.Timestamp - build.Email = hook.Head.Author.Email - build.Avatar = hook.Sender.Avatar - build.Author = hook.Sender.Login - build.Remote = hook.Repo.CloneURL - - if len(build.Author) == 0 { - build.Author = hook.Head.Author.Username - // default gravatar? - } - - if strings.HasPrefix(build.Ref, "refs/tags/") { - // just kidding, this is actually a tag event - build.Event = model.EventTag - } - - return repo, build, nil -} - -// pullRequest parses a hook with event type `pullRequest` -// and returns the commit data. -func (g *Github) pullRequest(r *http.Request) (*model.Repo, *model.Build, error) { - payload := GetPayload(r) - hook := &struct { - Action string `json:"action"` - PullRequest *github.PullRequest `json:"pull_request"` - Repo *github.Repository `json:"repository"` - }{} - err := json.Unmarshal(payload, hook) - if err != nil { - return nil, nil, err - } - - // ignore these - if hook.Action != "opened" && hook.Action != "synchronize" { - return nil, nil, nil - } - if *hook.PullRequest.State != "open" { - return nil, nil, nil - } - - repo := &model.Repo{} - repo.Owner = *hook.Repo.Owner.Login - repo.Name = *hook.Repo.Name - repo.FullName = *hook.Repo.FullName - repo.Link = *hook.Repo.HTMLURL - repo.IsPrivate = *hook.Repo.Private - repo.Clone = *hook.Repo.CloneURL - repo.Kind = model.RepoGit - repo.Branch = "master" - if hook.Repo.DefaultBranch != nil { - repo.Branch = *hook.Repo.DefaultBranch - } - - build := &model.Build{} - build.Event = model.EventPull - build.Commit = *hook.PullRequest.Head.SHA - build.Ref = fmt.Sprintf("refs/pull/%d/%s", *hook.PullRequest.Number, g.MergeRef) - build.Link = *hook.PullRequest.HTMLURL - build.Branch = *hook.PullRequest.Head.Ref - build.Message = *hook.PullRequest.Title - build.Author = *hook.PullRequest.User.Login - build.Avatar = *hook.PullRequest.User.AvatarURL - build.Remote = *hook.PullRequest.Base.Repo.CloneURL - build.Title = *hook.PullRequest.Title - // build.Timestamp = time.Now().UTC().Format("2006-01-02 15:04:05.000000000 +0000 MST") - - return repo, build, nil -} - -func (g *Github) deployment(r *http.Request) (*model.Repo, *model.Build, error) { - payload := GetPayload(r) - hook := &deployHook{} - - err := json.Unmarshal(payload, hook) - if err != nil { - return nil, nil, err - } - - // for older versions of GitHub. Remove. - if hook.Deployment.ID == 0 { - hook.Deployment.ID = hook.ID - hook.Deployment.Sha = hook.Sha - hook.Deployment.Ref = hook.Ref - hook.Deployment.Task = hook.Name - hook.Deployment.Env = hook.Env - hook.Deployment.Desc = hook.Desc - } - - repo := &model.Repo{} - repo.Owner = hook.Repo.Owner.Login - if len(repo.Owner) == 0 { - repo.Owner = hook.Repo.Owner.Name - } - repo.Name = hook.Repo.Name - repo.FullName = fmt.Sprintf("%s/%s", repo.Owner, repo.Name) - repo.Link = hook.Repo.HTMLURL - repo.IsPrivate = hook.Repo.Private - repo.Clone = hook.Repo.CloneURL - repo.Branch = hook.Repo.DefaultBranch - repo.Kind = model.RepoGit - - // ref can be - // branch, tag, or sha - - build := &model.Build{} - build.Event = model.EventDeploy - build.Commit = hook.Deployment.Sha - build.Link = hook.Deployment.Url - build.Message = hook.Deployment.Desc - build.Avatar = hook.Sender.Avatar - build.Author = hook.Sender.Login - build.Ref = hook.Deployment.Ref - build.Branch = hook.Deployment.Ref - build.Deploy = hook.Deployment.Env - - // if the ref is a sha or short sha we need to manually - // construct the ref. - if strings.HasPrefix(build.Commit, build.Ref) || build.Commit == build.Ref { - build.Branch = repo.Branch - build.Ref = fmt.Sprintf("refs/heads/%s", repo.Branch) - - } - // if the ref is a branch we should make sure it has refs/heads prefix - if !strings.HasPrefix(build.Ref, "refs/") { // branch or tag - build.Ref = fmt.Sprintf("refs/heads/%s", build.Branch) - - } - - return repo, build, nil -} - -func (g *Github) String() string { - return "github" -} - -const ( - StatusPending = "pending" - StatusSuccess = "success" - StatusFailure = "failure" - StatusError = "error" -) - -const ( - DescPending = "this build is pending" - DescSuccess = "the build was successful" - DescFailure = "the build failed" - DescError = "oops, something went wrong" -) - -// getStatus is a helper functin that converts a Drone -// status to a GitHub status. -func getStatus(status string) string { - switch status { - case model.StatusPending, model.StatusRunning: - return StatusPending - case model.StatusSuccess: - return StatusSuccess - case model.StatusFailure: - return StatusFailure - case model.StatusError, model.StatusKilled: - return StatusError - default: - return StatusError - } -} - -// getDesc is a helper function that generates a description -// message for the build based on the status. -func getDesc(status string) string { - switch status { - case model.StatusPending, model.StatusRunning: - return DescPending - case model.StatusSuccess: - return DescSuccess - case model.StatusFailure: - return DescFailure - case model.StatusError, model.StatusKilled: - return DescError - default: - return DescError - } -} diff --git a/remote/github/github_test.go b/remote/github/github_test.go index dcf673370..01e5649a5 100644 --- a/remote/github/github_test.go +++ b/remote/github/github_test.go @@ -11,7 +11,7 @@ import ( func TestHook(t *testing.T) { var ( - github Github + github client r *http.Request body *bytes.Buffer ) @@ -20,7 +20,7 @@ func TestHook(t *testing.T) { g.Describe("Hook", func() { g.BeforeEach(func() { - github = Github{} + github = client{} body = bytes.NewBuffer([]byte{}) r, _ = http.NewRequest("POST", "https://drone.com/hook", body) }) @@ -31,11 +31,11 @@ func TestHook(t *testing.T) { }) g.It("Should set build author to the pull request author", func() { - hookJson, err := ioutil.ReadFile("fixtures/pull_request.json") - if err != nil { - panic(err) + hookJSON, ioerr := ioutil.ReadFile("fixtures/pull_request.json") + if ioerr != nil { + panic(ioerr) } - body.Write(hookJson) + body.Write(hookJSON) _, build, err := github.Hook(r) g.Assert(err).Equal(nil) diff --git a/remote/github/helper.go b/remote/github/helper.go index 362df938c..78d0f56f5 100644 --- a/remote/github/helper.go +++ b/remote/github/helper.go @@ -1,40 +1,14 @@ package github import ( - "crypto/tls" - "encoding/base32" "fmt" "io/ioutil" "net/http" - "net/url" "strings" - "github.com/drone/drone/shared/oauth2" "github.com/google/go-github/github" - "github.com/gorilla/securecookie" ) -// NewClient is a helper function that returns a new GitHub -// client using the provided OAuth token. -func NewClient(uri, token string, skipVerify bool) *github.Client { - t := &oauth2.Transport{ - Token: &oauth2.Token{AccessToken: token}, - } - - // this is for GitHub enterprise users that are using - // self-signed certificates. - if skipVerify { - t.Transport = &http.Transport{ - Proxy: http.ProxyFromEnvironment, - TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, - } - } - - c := github.NewClient(t.Client()) - c.BaseURL, _ = url.Parse(uri) - return c -} - // GetUserEmail is a heper function that retrieves the currently // authenticated user from GitHub + Email address. func GetUserEmail(client *github.Client) (*github.User, error) { @@ -58,7 +32,7 @@ func GetUserEmail(client *github.Client) (*github.User, error) { // WARNING, HACK // for out-of-date github enterprise editions the primary // and verified fields won't exist. - if !strings.HasPrefix(*user.URL, DefaultAPI) && len(emails) != 0 { + if !strings.HasPrefix(*user.URL, defaultAPI) && len(emails) != 0 { user.Email = emails[0].Email return user, nil } @@ -66,56 +40,6 @@ func GetUserEmail(client *github.Client) (*github.User, error) { return nil, fmt.Errorf("No verified Email address for GitHub account") } -// GetRepo is a helper function that returns a named repo -func GetRepo(client *github.Client, owner, repo string) (*github.Repository, error) { - r, _, err := client.Repositories.Get(owner, repo) - return r, err -} - -// GetUserRepos is a helper function that returns a list of -// all user repositories. Paginated results are aggregated into -// a single list. -func GetUserRepos(client *github.Client) ([]github.Repository, error) { - var repos []github.Repository - var opts = github.RepositoryListOptions{} - opts.PerPage = 100 - opts.Page = 1 - - // loop through user repository list - for opts.Page > 0 { - list, resp, err := client.Repositories.List("", &opts) - if err != nil { - return nil, err - } - repos = append(repos, list...) - - // increment the next page to retrieve - opts.Page = resp.NextPage - } - - return repos, nil -} - -// GetOrgs is a helper function that returns a list of -// all orgs that a user belongs to. -func GetOrgs(client *github.Client) ([]github.Organization, error) { - var orgs []github.Organization - var opts = github.ListOptions{} - opts.Page = 1 - - for opts.Page > 0 { - list, resp, err := client.Organizations.List("", &opts) - if err != nil { - return nil, err - } - orgs = append(orgs, list...) - - // increment the next page to retrieve - opts.Page = resp.NextPage - } - return orgs, nil -} - // GetHook is a heper function that retrieves a hook by // hostname. To do this, it will retrieve a list of all hooks // and iterate through the list. @@ -136,6 +60,7 @@ func GetHook(client *github.Client, owner, name, url string) (*github.Hook, erro return nil, nil } +// DeleteHook does exactly what you think it does. func DeleteHook(client *github.Client, owner, name, url string) error { hook, err := GetHook(client, owner, name, url) if err != nil { @@ -179,24 +104,6 @@ func CreateUpdateHook(client *github.Client, owner, name, url string) (*github.H return CreateHook(client, owner, name, url) } -// GetFile is a heper function that retrieves a file from -// GitHub and returns its contents in byte array format. -func GetFile(client *github.Client, owner, name, path, ref string) ([]byte, error) { - var opts = new(github.RepositoryContentGetOptions) - opts.Ref = ref - content, _, _, err := client.Repositories.GetContents(owner, name, path, opts) - if err != nil { - return nil, err - } - return content.Decode() -} - -// GetRandom is a helper function that generates a 32-bit random -// key, base32 encoded as a string value. -func GetRandom() string { - return base32.StdEncoding.EncodeToString(securecookie.GenerateRandomKey(32)) -} - // GetPayload is a helper function that will parse the JSON payload. It will // first check for a `payload` parameter in a POST, but can fallback to a // raw JSON body as well. diff --git a/remote/github/parser.go b/remote/github/parser.go new file mode 100644 index 000000000..046d2a7cc --- /dev/null +++ b/remote/github/parser.go @@ -0,0 +1,184 @@ +package github + +import ( + "encoding/json" + "fmt" + "net/http" + "strings" + + "github.com/drone/drone/model" + "github.com/google/go-github/github" +) + +// push parses a hook with event type `push` and returns +// the commit data. +func (c *client) push(r *http.Request) (*model.Repo, *model.Build, error) { + payload := GetPayload(r) + hook := &pushHook{} + err := json.Unmarshal(payload, hook) + if err != nil { + return nil, nil, err + } + if hook.Deleted { + return nil, nil, err + } + + repo := &model.Repo{} + repo.Owner = hook.Repo.Owner.Login + if len(repo.Owner) == 0 { + repo.Owner = hook.Repo.Owner.Name + } + repo.Name = hook.Repo.Name + // Generating rather than using hook.Repo.FullName as it's + // not always present + repo.FullName = fmt.Sprintf("%s/%s", repo.Owner, repo.Name) + repo.Link = hook.Repo.HTMLURL + repo.IsPrivate = hook.Repo.Private + repo.Clone = hook.Repo.CloneURL + repo.Branch = hook.Repo.DefaultBranch + repo.Kind = model.RepoGit + + build := &model.Build{} + build.Event = model.EventPush + build.Commit = hook.Head.ID + build.Ref = hook.Ref + build.Link = hook.Head.URL + build.Branch = strings.Replace(build.Ref, "refs/heads/", "", -1) + build.Message = hook.Head.Message + // build.Timestamp = hook.Head.Timestamp + build.Email = hook.Head.Author.Email + build.Avatar = hook.Sender.Avatar + build.Author = hook.Sender.Login + build.Remote = hook.Repo.CloneURL + + if len(build.Author) == 0 { + build.Author = hook.Head.Author.Username + // default gravatar? + } + + if strings.HasPrefix(build.Ref, "refs/tags/") { + // just kidding, this is actually a tag event + build.Event = model.EventTag + } + + return repo, build, nil +} + +// pullRequest parses a hook with event type `pullRequest` +// and returns the commit data. +func (c *client) pullRequest(r *http.Request) (*model.Repo, *model.Build, error) { + payload := GetPayload(r) + hook := &struct { + Action string `json:"action"` + PullRequest *github.PullRequest `json:"pull_request"` + Repo *github.Repository `json:"repository"` + }{} + err := json.Unmarshal(payload, hook) + if err != nil { + return nil, nil, err + } + + // ignore these + if hook.Action != "opened" && hook.Action != "synchronize" { + return nil, nil, nil + } + if *hook.PullRequest.State != "open" { + return nil, nil, nil + } + + repo := &model.Repo{} + repo.Owner = *hook.Repo.Owner.Login + repo.Name = *hook.Repo.Name + repo.FullName = *hook.Repo.FullName + repo.Link = *hook.Repo.HTMLURL + repo.IsPrivate = *hook.Repo.Private + repo.Clone = *hook.Repo.CloneURL + repo.Kind = model.RepoGit + repo.Branch = "master" + if hook.Repo.DefaultBranch != nil { + repo.Branch = *hook.Repo.DefaultBranch + } + + build := &model.Build{} + build.Event = model.EventPull + build.Commit = *hook.PullRequest.Head.SHA + build.Link = *hook.PullRequest.HTMLURL + build.Branch = *hook.PullRequest.Head.Ref + build.Message = *hook.PullRequest.Title + build.Author = *hook.PullRequest.User.Login + build.Avatar = *hook.PullRequest.User.AvatarURL + build.Remote = *hook.PullRequest.Base.Repo.CloneURL + build.Title = *hook.PullRequest.Title + + if c.MergeRef { + build.Ref = fmt.Sprintf("refs/pull/%d/merge", *hook.PullRequest.Number) + } else { + build.Ref = fmt.Sprintf("refs/pull/%d/head", *hook.PullRequest.Number) + } + + // build.Timestamp = time.Now().UTC().Format("2006-01-02 15:04:05.000000000 +0000 MST") + + return repo, build, nil +} + +func (c *client) deployment(r *http.Request) (*model.Repo, *model.Build, error) { + payload := GetPayload(r) + hook := &deployHook{} + + err := json.Unmarshal(payload, hook) + if err != nil { + return nil, nil, err + } + + // for older versions of GitHub. Remove. + if hook.Deployment.ID == 0 { + hook.Deployment.ID = hook.ID + hook.Deployment.Sha = hook.Sha + hook.Deployment.Ref = hook.Ref + hook.Deployment.Task = hook.Name + hook.Deployment.Env = hook.Env + hook.Deployment.Desc = hook.Desc + } + + repo := &model.Repo{} + repo.Owner = hook.Repo.Owner.Login + if len(repo.Owner) == 0 { + repo.Owner = hook.Repo.Owner.Name + } + repo.Name = hook.Repo.Name + repo.FullName = fmt.Sprintf("%s/%s", repo.Owner, repo.Name) + repo.Link = hook.Repo.HTMLURL + repo.IsPrivate = hook.Repo.Private + repo.Clone = hook.Repo.CloneURL + repo.Branch = hook.Repo.DefaultBranch + repo.Kind = model.RepoGit + + // ref can be + // branch, tag, or sha + + build := &model.Build{} + build.Event = model.EventDeploy + build.Commit = hook.Deployment.Sha + build.Link = hook.Deployment.URL + build.Message = hook.Deployment.Desc + build.Avatar = hook.Sender.Avatar + build.Author = hook.Sender.Login + build.Ref = hook.Deployment.Ref + build.Branch = hook.Deployment.Ref + build.Deploy = hook.Deployment.Env + + // if the ref is a sha or short sha we need to manually + // construct the ref. + if strings.HasPrefix(build.Commit, build.Ref) || build.Commit == build.Ref { + build.Branch = repo.Branch + build.Ref = fmt.Sprintf("refs/heads/%s", repo.Branch) + + } + // if the ref is a branch we should make sure it has refs/heads prefix + if !strings.HasPrefix(build.Ref, "refs/") { // branch or tag + build.Ref = fmt.Sprintf("refs/heads/%s", build.Branch) + + } + + return repo, build, nil +} diff --git a/remote/github/parser_test.go b/remote/github/parser_test.go new file mode 100644 index 000000000..d2e73c266 --- /dev/null +++ b/remote/github/parser_test.go @@ -0,0 +1 @@ +package github diff --git a/remote/github/types.go b/remote/github/types.go index 7671d7023..e3eb45ae7 100644 --- a/remote/github/types.go +++ b/remote/github/types.go @@ -1,8 +1,5 @@ package github -type postHook struct { -} - type pushHook struct { Ref string `json:"ref"` Deleted bool `json:"deleted"` @@ -54,7 +51,7 @@ type deployHook struct { Ref string `json:"ref"` Task string `json:"task"` Env string `json:"environment"` - Url string `json:"url"` + URL string `json:"url"` Desc string `json:"description"` } `json:"deployment"` @@ -78,9 +75,8 @@ type deployHook struct { DefaultBranch string `json:"default_branch"` } `json:"repository"` - // these are legacy fields that have been moded - // to the deployment section. They are here for - // older versions of GitHub and will be removed + // these are legacy fields that have been added to the deployment section. + // They are here for older versions of GitHub and will be removed. ID int64 `json:"id"` Sha string `json:"sha"` diff --git a/router/middleware/remote.go b/router/middleware/remote.go index 58881078f..529f64217 100644 --- a/router/middleware/remote.go +++ b/router/middleware/remote.go @@ -90,13 +90,15 @@ func setupGitlab(c *cli.Context) (remote.Remote, error) { // helper function to setup the GitHub remote from the CLI arguments. func setupGithub(c *cli.Context) (remote.Remote, error) { - return github.New( - c.String("github-server"), - c.String("github-client"), - c.String("github-sercret"), - c.StringSlice("github-scope"), - c.Bool("github-private-mode"), - c.Bool("github-skip-verify"), - c.BoolT("github-merge-ref"), - ) + return github.New(github.Opts{ + URL: c.String("github-server"), + Client: c.String("github-client"), + Secret: c.String("github-sercret"), + Scopes: c.StringSlice("github-scope"), + Username: c.String("github-git-username"), + Password: c.String("github-git-password"), + PrivateMode: c.Bool("github-private-mode"), + SkipVerify: c.Bool("github-skip-verify"), + MergeRef: c.BoolT("github-merge-ref"), + }) } diff --git a/router/middleware/session/agent.go b/router/middleware/session/agent.go index 8f8c722b8..bb0183e3c 100644 --- a/router/middleware/session/agent.go +++ b/router/middleware/session/agent.go @@ -8,14 +8,18 @@ import ( // AuthorizeAgent authorizes requsts from build agents to access the queue. func AuthorizeAgent(c *gin.Context) { secret := c.MustGet("agent").(string) + if secret == "" { + c.String(401, "invalid or empty token.") + return + } parsed, err := token.ParseRequest(c.Request, func(t *token.Token) (string, error) { return secret, nil }) if err != nil { - c.AbortWithError(403, err) + c.String(500, "invalid or empty token. %s", err) } else if parsed.Kind != token.AgentToken { - c.AbortWithStatus(403) + c.String(403, "invalid token. please use an agent token") } else { c.Next() } diff --git a/server/login.go b/server/login.go index 0934155d1..a234d3849 100644 --- a/server/login.go +++ b/server/login.go @@ -1,15 +1,16 @@ package server import ( + "encoding/base32" "net/http" "time" "github.com/drone/drone/model" "github.com/drone/drone/remote" - "github.com/drone/drone/shared/crypto" "github.com/drone/drone/shared/httputil" "github.com/drone/drone/shared/token" "github.com/drone/drone/store" + "github.com/gorilla/securecookie" "github.com/Sirupsen/logrus" "github.com/gin-gonic/gin" @@ -39,12 +40,23 @@ func GetLogin(c *gin.Context) { if err != nil { // if self-registration is disabled we should return a not authorized error - if !config.Open { + if !config.Open && !config.IsAdmin(tmpuser) { logrus.Errorf("cannot register %s. registration closed", tmpuser.Login) c.Redirect(303, "/login?error=access_denied") return } + // if self-registration is enabled for whitelisted organizations we need to + // check the user's organization membership. + if len(config.Orgs) != 0 { + teams, terr := remote.Teams(c, tmpuser) + if terr != nil || config.IsMember(teams) == false { + logrus.Errorf("cannot verify team membership for %s.", u.Login) + c.Redirect(303, "/login?error=access_denied") + return + } + } + // create the user account u = &model.User{ Login: tmpuser.Login, @@ -52,7 +64,9 @@ func GetLogin(c *gin.Context) { Secret: tmpuser.Secret, Email: tmpuser.Email, Avatar: tmpuser.Avatar, - Hash: crypto.Rand(), + Hash: base32.StdEncoding.EncodeToString( + securecookie.GenerateRandomKey(32), + ), } // insert the user into the database @@ -69,33 +83,23 @@ func GetLogin(c *gin.Context) { u.Email = tmpuser.Email u.Avatar = tmpuser.Avatar + // if self-registration is enabled for whitelisted organizations we need to + // check the user's organization membership. + if len(config.Orgs) != 0 { + teams, terr := remote.Teams(c, u) + if terr != nil || config.IsMember(teams) == false { + logrus.Errorf("cannot verify team membership for %s.", u.Login) + c.Redirect(303, "/login?error=access_denied") + return + } + } + if err := store.UpdateUser(c, u); err != nil { logrus.Errorf("cannot update %s. %s", u.Login, err) c.Redirect(303, "/login?error=internal_error") return } - if len(config.Orgs) != 0 { - teams, terr := remote.Teams(c, u) - if terr != nil { - logrus.Errorf("cannot verify team membership for %s. %s.", tmpuser.Login, terr) - c.Redirect(303, "/login?error=access_denied") - return - } - var member bool - for _, team := range teams { - if config.Orgs[team.Login] { - member = true - break - } - } - if !member { - logrus.Errorf("cannot verify team membership for %s. %s.", tmpuser.Login, terr) - c.Redirect(303, "/login?error=access_denied") - return - } - } - exp := time.Now().Add(time.Hour * 72).Unix() token := token.New(token.SessToken, u.Login) tokenstr, err := token.SignExpires(u.Hash, exp) diff --git a/server/repo.go b/server/repo.go index a5768c8cf..08163afb4 100644 --- a/server/repo.go +++ b/server/repo.go @@ -1,15 +1,16 @@ package server import ( + "encoding/base32" "fmt" "net/http" "github.com/gin-gonic/gin" + "github.com/gorilla/securecookie" "github.com/drone/drone/cache" "github.com/drone/drone/remote" "github.com/drone/drone/router/middleware/session" - "github.com/drone/drone/shared/crypto" "github.com/drone/drone/shared/httputil" "github.com/drone/drone/shared/token" "github.com/drone/drone/store" @@ -54,7 +55,9 @@ func PostRepo(c *gin.Context) { r.AllowPush = true r.AllowPull = true r.Timeout = 60 // 1 hour default build time - r.Hash = crypto.Rand() + r.Hash = base32.StdEncoding.EncodeToString( + securecookie.GenerateRandomKey(32), + ) // crates the jwt token used to verify the repository t := token.New(token.HookToken, r.FullName) diff --git a/server/user.go b/server/user.go index 210aa7abf..c984b3384 100644 --- a/server/user.go +++ b/server/user.go @@ -5,9 +5,9 @@ import ( "github.com/gin-gonic/gin" + "github.com/drone/drone-exp/shared/crypto" "github.com/drone/drone/cache" "github.com/drone/drone/router/middleware/session" - "github.com/drone/drone/shared/crypto" "github.com/drone/drone/shared/token" "github.com/drone/drone/store" ) diff --git a/server/users.go b/server/users.go index 3661692f2..dbe003577 100644 --- a/server/users.go +++ b/server/users.go @@ -1,12 +1,13 @@ package server import ( + "encoding/base32" "net/http" "github.com/gin-gonic/gin" + "github.com/gorilla/securecookie" "github.com/drone/drone/model" - "github.com/drone/drone/shared/crypto" "github.com/drone/drone/store" ) @@ -14,9 +15,9 @@ func GetUsers(c *gin.Context) { users, err := store.GetUserList(c) if err != nil { c.String(500, "Error getting user list. %s", err) - } else { - c.JSON(200, users) + return } + c.JSON(200, users) } func GetUser(c *gin.Context) { @@ -41,7 +42,6 @@ func PatchUser(c *gin.Context) { c.AbortWithStatus(http.StatusNotFound) return } - user.Admin = in.Admin user.Active = in.Active err = store.UpdateUser(c, user) @@ -65,7 +65,9 @@ func PostUser(c *gin.Context) { Login: in.Login, Email: in.Email, Avatar: in.Avatar, - Hash: crypto.Rand(), + Hash: base32.StdEncoding.EncodeToString( + securecookie.GenerateRandomKey(32), + ), } if err = store.CreateUser(c, user); err != nil { c.String(http.StatusInternalServerError, err.Error()) diff --git a/shared/crypto/crypto.go b/shared/crypto/crypto.go deleted file mode 100644 index 76fa6b894..000000000 --- a/shared/crypto/crypto.go +++ /dev/null @@ -1,118 +0,0 @@ -package crypto - -import ( - "crypto/rand" - "crypto/rsa" - "crypto/x509" - "encoding/pem" - "io" - - "code.google.com/p/go.crypto/ssh" - "github.com/square/go-jose" -) - -const ( - RSA_BITS = 2048 // Default number of bits in an RSA key - RSA_BITS_MIN = 768 // Minimum number of bits in an RSA key -) - -// standard characters allowed in token string. -var chars = []byte("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789") - -// default token length -var length = 32 - -// Rand generates a 32-bit random string. -func Rand() string { - b := make([]byte, length) - r := make([]byte, length+(length/4)) // storage for random bytes. - clen := byte(len(chars)) - maxrb := byte(256 - (256 % len(chars))) - i := 0 - for { - io.ReadFull(rand.Reader, r) - for _, c := range r { - if c >= maxrb { - // Skip this number to avoid modulo bias. - continue - } - b[i] = chars[c%clen] - i++ - if i == length { - return string(b) - } - } - } -} - -// helper function to generate an RSA Private Key. -func GeneratePrivateKey() (*rsa.PrivateKey, error) { - return rsa.GenerateKey(rand.Reader, RSA_BITS) -} - -// helper function that marshalls an RSA Public Key to an SSH -// .authorized_keys format -func MarshalPublicKey(public *rsa.PublicKey) []byte { - private, err := ssh.NewPublicKey(public) - if err != nil { - return []byte{} - } - - return ssh.MarshalAuthorizedKey(private) -} - -// helper function that marshalls an RSA Private Key to -// a PEM encoded file. -func MarshalPrivateKey(private *rsa.PrivateKey) []byte { - marshaled := x509.MarshalPKCS1PrivateKey(private) - encoded := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Headers: nil, Bytes: marshaled}) - return encoded -} - -// UnmarshalPrivateKey is a helper function that unmarshals a PEM -// bytes to an RSA Private Key -func UnmarshalPrivateKey(private []byte) *rsa.PrivateKey { - decoded, _ := pem.Decode(private) - parsed, err := x509.ParsePKCS1PrivateKey(decoded.Bytes) - if err != nil { - return nil - } - return parsed -} - -// Encrypt encrypts a secret string. -func Encrypt(in, privKey string) (string, error) { - rsaPrivKey, err := decodePrivateKey(privKey) - if err != nil { - return "", err - } - - return encrypt(in, &rsaPrivKey.PublicKey) -} - -// decodePrivateKey is a helper function that unmarshals a PEM -// bytes to an RSA Private Key -func decodePrivateKey(privateKey string) (*rsa.PrivateKey, error) { - derBlock, _ := pem.Decode([]byte(privateKey)) - return x509.ParsePKCS1PrivateKey(derBlock.Bytes) -} - -// encrypt encrypts a plaintext variable using JOSE with -// RSA_OAEP and A128GCM algorithms. -func encrypt(text string, pubKey *rsa.PublicKey) (string, error) { - var encrypted string - var plaintext = []byte(text) - - // Creates a new encrypter using defaults - encrypter, err := jose.NewEncrypter(jose.RSA_OAEP, jose.A128GCM, pubKey) - if err != nil { - return encrypted, err - } - // Encrypts the plaintext value and serializes - // as a JOSE string. - object, err := encrypter.Encrypt(plaintext) - if err != nil { - return encrypted, err - } - return object.CompactSerialize() -} diff --git a/shared/crypto/crypto_test.go b/shared/crypto/crypto_test.go deleted file mode 100644 index 35b7e488d..000000000 --- a/shared/crypto/crypto_test.go +++ /dev/null @@ -1,68 +0,0 @@ -package crypto - -import ( - "testing" - - "github.com/franela/goblin" - "github.com/square/go-jose" -) - -func TestKeys(t *testing.T) { - - g := goblin.Goblin(t) - g.Describe("Generate Key", func() { - - g.It("Generates a private key", func() { - _, err := GeneratePrivateKey() - g.Assert(err == nil).IsTrue() - }) - }) -} - -func Test_Encrypt(t *testing.T) { - - g := goblin.Goblin(t) - g.Describe("Secure", func() { - - g.It("Should encrypt a string", func() { - ciphertext, err := Encrypt("top_secret", fakePriv) - g.Assert(err == nil).IsTrue() - - object, _ := jose.ParseEncrypted(ciphertext) - privKey, _ := decodePrivateKey(fakePriv) - plaintext, _ := object.Decrypt(privKey) - g.Assert(string(plaintext)).Equal("top_secret") - }) - - }) -} - -var fakePriv = ` ------BEGIN RSA PRIVATE KEY----- -MIIEowIBAAKCAQEA71FaA+otDak2rXF/4h69Tz+OxS6NOWaOc/n7dinHXnlo3Toy -ZzvwweJGQKIOfPNBMncz+8h6oLOByFvb95Z1UEM0d+KCFCCutOeN9NNMw4fkUtSZ -7sm6T35wQUkDOiO1YAGy27hQfT7iryhPwA8KmgZmt7toNNf+WymPR8DMwAAYeqHA -5DIEWWsg+RLohOJ0itIk9q6Us9WYhng0sZ9+U+C87FospjKRMyAinSvKx0Uan4ap -YGbLjDQHimWtimfT4XWCGTO1cWno378Vm/newUN6WVaeZ2CSHcWgD2fWcjFixX2A -SvcvfuCo7yZPUPWeiYKrc5d1CC3ncocu43LhSQIDAQABAoIBAQDIbYKM+sfmxAwF -8KOg1gvIXjuNCrK+GxU9LmSajtzpU5cuiHoEGaBGUOJzaQXnQbcds9W2ji2dfxk3 -my87SShRIyfDK9GzV7fZzIAIRhrpO1tOv713zj0aLJOJKcPpIlTZ5jJMcC4A5vTk -q0c3W6GOY8QNJohckXT2FnVoK6GPPiaZnavkwH33cJk0j1vMsbADdKF7Jdfq9FBF -Lx+Za7wo79MQIr68KEqsqMpmrawIf1T3TqOCNbkPCL2tu5EfoyGIItrH33SBOV/B -HbIfe4nJYZMWXhe3kZ/xCFqiRx6/wlc5pGCwCicgHJJe/l8Y9OticDCCyJDQtD8I -6927/j2NAoGBAPNRRY8r5ES5f8ftEktcLwh2zw08PNkcolTeqsEMbWAQspV/v+Ay -4niEXIN3ix2yTnMgrtxRGO7zdPnMaTN8E88FsSDKQ97lm7m3jo7lZtDMz16UxGmd -AOOuXwUtpngz7OrQ25NXhvFYLTgLoPsv3PbFbF1pwbhZqPTttTdg5so3AoGBAPvK -ta/n7DMZd/HptrkdkxxHaGN19ZjBVIqyeORhIDznEYjv9Z90JvzRxCmUriD4fyJC -/XSTytORa34UgmOk1XFtxWusXhnYqCTIHG/MKCy9D4ifzFzii9y/M+EnQIMb658l -+edLyrGFla+t5NS1XAqDYjfqpUFbMvU1kVoDJ/B/AoGBANBQe3o5PMSuAD19tdT5 -Rnc7qMcPFJVZE44P2SdQaW/+u7aM2gyr5AMEZ2RS+7LgDpQ4nhyX/f3OSA75t/PR -PfBXUi/dm8AA2pNlGNM0ihMn1j6GpaY6OiG0DzwSulxdMHBVgjgijrCgKo66Pgfw -EYDgw4cyXR1k/ec8gJK6Dr1/AoGBANvmSY77Kdnm4E4yIxbAsX39DznuBzQFhGQt -Qk+SU6lc1H+Xshg0ROh/+qWl5/17iOzPPLPXb0getJZEKywDBTYu/D/xJa3E/fRB -oDQzRNLtuudDSCPG5wc/JXv53+mhNMKlU/+gvcEUPYpUgIkUavHzlI/pKbJOh86H -ng3Su8rZAn9w/zkoJu+n7sHta/Hp6zPTbvjZ1EijZp0+RygBgiv9UjDZ6D9EGcjR -ZiFwuc8I0g7+GRkgG2NbfqX5Cewb/nbJQpHPO31bqJrcLzU0KurYAwQVx6WGW0He -ERIlTeOMxVo6M0OpI+rH5bOLdLLEVhNtM/4HUFi1Qy6CCMbN2t3H ------END RSA PRIVATE KEY----- -`