added github remote

This commit is contained in:
Brad Rydzewski 2015-04-08 15:00:27 -07:00
parent 42f2ac392a
commit 09bd7cf71a
5 changed files with 838 additions and 0 deletions

74
remote/client/client.go Normal file
View file

@ -0,0 +1,74 @@
package client
import (
"net"
"net/http"
"net/rpc"
"github.com/drone/drone/common"
"github.com/drone/drone/settings"
)
// Client communicates with a Remote plugin using the
// net/rpc protocol.
type Client struct {
*rpc.Client
}
// New returns a new, remote datastore backend that connects
// via tcp and exchanges data using Go's RPC mechanism.
func New(service *settings.Service) (*Client, error) {
conn, err := net.Dial("tcp", service.Address)
if err != nil {
return nil, err
}
client := &Client{
rpc.NewClient(conn),
}
return client, nil
}
func (c *Client) Login(token, secret string) (*common.User, error) {
return nil, nil
}
// Repo fetches the named repository from the remote system.
func (c *Client) Repo(u *common.User, owner, repo string) (*common.Repo, error) {
return nil, nil
}
func (c *Client) Perm(u *common.User, owner, repo string) (*common.Perm, error) {
return nil, nil
}
func (c *Client) Script(u *common.User, r *common.Repo, b *common.Build) ([]byte, error) {
return nil, nil
}
func (c *Client) Status(u *common.User, r *common.Repo, b *common.Build, link string) error {
return nil
}
func (c *Client) Activate(u *common.User, r *common.Repo, k *common.Keypair, link string) error {
return nil
}
func (c *Client) Deactivate(u *common.User, r *common.Repo, link string) error {
return nil
}
func (c *Client) Hook(r *http.Request) (*common.Hook, error) {
hook := new(common.Hook)
header := make(http.Header)
copyHeader(r.Header, header)
return hook, nil
}
func copyHeader(dst, src http.Header) {
for k, vv := range src {
for _, v := range vv {
dst.Add(k, v)
}
}
}

384
remote/github/github.go Normal file
View file

@ -0,0 +1,384 @@
package github
import (
"encoding/json"
"fmt"
"net/http"
"strings"
"github.com/drone/drone/common"
"github.com/drone/drone/settings"
"github.com/hashicorp/golang-lru"
"github.com/google/go-github/github"
)
const (
DefaultAPI = "https://api.github.com/"
DefaultURL = "https://github.com"
DefaultScope = "repo,repo:status,user:email"
)
type GitHub struct {
URL string
API string
Client string
Secret string
PrivateMode bool
SkipVerify bool
Orgs []string
Open bool
cache *lru.Cache
}
func New(service *settings.Service) *GitHub {
var github = GitHub{
API: DefaultAPI,
URL: DefaultURL,
Client: service.OAuth.Client,
Secret: service.OAuth.Secret,
PrivateMode: service.PrivateMode,
SkipVerify: service.SkipVerify,
Orgs: service.Orgs,
Open: service.Open,
}
var err error
github.cache, err = lru.New(1028)
if err != nil {
panic(err)
}
// if GitHub enterprise then ensure we're using the
// appropriate URLs
if !strings.HasPrefix(service.Base, DefaultURL) && len(service.Base) != 0 {
github.URL = service.Base
github.API = service.Base + "/api/v3/"
}
// the API must have a trailing slash
if !strings.HasSuffix(github.API, "/") {
github.API += "/"
}
// the URL must NOT have a trailing slash
if strings.HasSuffix(github.URL, "/") {
github.URL = github.URL[:len(github.URL)-1]
}
return &github
}
func (g *GitHub) Login(token, secret string) (*common.User, error) {
client := NewClient(g.API, token, g.SkipVerify)
login, err := GetUserEmail(client)
if err != nil {
return nil, err
}
user := common.User{}
user.Login = *login.Login
user.Email = *login.Email
user.Name = *login.Name
user.Token = token
user.Secret = secret
return &user, nil
}
// Repo fetches the named repository from the remote system.
func (g *GitHub) Repo(u *common.User, owner, name string) (*common.Repo, error) {
client := NewClient(g.API, u.Token, g.SkipVerify)
repo_, err := GetRepo(client, owner, name)
if err != nil {
return nil, err
}
repo := &common.Repo{}
repo.Owner = owner
repo.Name = name
repo.Language = *repo_.Language
repo.FullName = *repo_.FullName
repo.Link = *repo_.HTMLURL
repo.Private = *repo_.Private
repo.Clone = *repo_.CloneURL
if g.PrivateMode {
repo.Private = true
}
return repo, err
}
// Perm fetches the named repository from the remote system.
func (g *GitHub) Perm(u *common.User, owner, name string) (*common.Perm, error) {
key := fmt.Sprintf("%s/%s/%s", u.Login, owner, name)
val, ok := g.cache.Get(key)
if ok {
return val.(*common.Perm), nil
}
client := NewClient(g.API, u.Token, g.SkipVerify)
repo, err := GetRepo(client, owner, name)
if err != nil {
return nil, err
}
m := &common.Perm{}
m.Login = u.Login
m.Admin = (*repo.Permissions)["admin"]
m.Push = (*repo.Permissions)["push"]
m.Pull = (*repo.Permissions)["pull"]
g.cache.Add(key, m)
return m, nil
}
// Script fetches the build script (.drone.yml) from the remote
// repository and returns in string format.
func (g *GitHub) Script(u *common.User, r *common.Repo, b *common.Build) ([]byte, error) {
client := NewClient(g.API, u.Token, g.SkipVerify)
var sha string
if b.Commit != nil {
sha = b.Commit.Sha
} else {
sha = b.PullRequest.Source.Sha
}
return GetFile(client, r.Owner, r.Name, ".drone.yml", sha)
}
// Activate activates a repository by creating the post-commit hook and
// adding the SSH deploy key, if applicable.
func (g *GitHub) Activate(u *common.User, r *common.Repo, k *common.Keypair, link string) error {
client := NewClient(g.API, u.Token, g.SkipVerify)
title, err := GetKeyTitle(link)
if err != nil {
return err
}
// if the CloneURL is using the SSHURL then we know that
// we need to add an SSH key to GitHub.
if r.Private || g.PrivateMode {
_, err = CreateUpdateKey(client, r.Owner, r.Name, title, k.Public)
if err != nil {
return err
}
}
_, err = CreateUpdateHook(client, r.Owner, r.Name, link)
return err
}
// Deactivate removes a repository by removing all the post-commit hooks
// which are equal to link and removing the SSH deploy key.
func (g *GitHub) Deactivate(u *common.User, r *common.Repo, link string) error {
client := NewClient(g.API, u.Token, g.SkipVerify)
title, err := GetKeyTitle(link)
if err != nil {
return err
}
// remove the deploy-key if it is installed remote.
if r.Private || g.PrivateMode {
if err := DeleteKey(client, r.Owner, r.Name, title); err != nil {
return err
}
}
return DeleteHook(client, r.Owner, r.Name, link)
}
func (g *GitHub) Status(u *common.User, r *common.Repo, b *common.Build, link string) error {
client := NewClient(g.API, u.Token, g.SkipVerify)
var ref string
if b.Commit != nil {
ref = b.Commit.Ref
} else {
ref = b.PullRequest.Source.Ref
}
status := getStatus(b.State)
desc := getDesc(b.State)
data := github.RepoStatus{
Context: github.String("Drone"),
State: github.String(status),
Description: github.String(desc),
TargetURL: github.String(link),
}
_, _, err := client.Repositories.CreateStatus(r.Owner, r.Name, ref, &data)
return err
}
// Hook parses the post-commit hook from the Request body
// and returns the required data in a standard format.
func (g *GitHub) Hook(r *http.Request) (*common.Hook, error) {
switch r.Header.Get("X-Github-Event") {
case "pull_request":
return g.pullRequest(r)
case "push":
return g.push(r)
default:
return nil, nil
}
}
// push parses a hook with event type `push` and returns
// the commit data.
func (g *GitHub) push(r *http.Request) (*common.Hook, error) {
payload := GetPayload(r)
hook := &pushHook{}
err := json.Unmarshal(payload, hook)
if err != nil {
return nil, err
}
repo := &common.Repo{}
repo.Owner = hook.Repo.Owner.Login
repo.Name = hook.Repo.Name
repo.Language = hook.Repo.Language
repo.FullName = hook.Repo.FullName
repo.Link = hook.Repo.HTMLURL
repo.Private = hook.Repo.Private
repo.Clone = hook.Repo.CloneURL
commit := &common.Commit{}
commit.Sha = hook.Head.ID
commit.Ref = hook.Ref
commit.Message = hook.Head.Message
commit.Timestamp = hook.Head.Timestamp
commit.Author = &common.Author{}
commit.Author.Name = hook.Head.Author.Name
commit.Author.Email = hook.Head.Author.Email
commit.Author.Login = hook.Head.Author.Username
// we should ignore github pages
if commit.Ref == "refs/heads/gh-pages" {
return nil, nil
}
return &common.Hook{Repo: repo, Commit: commit}, nil
}
// pullRequest parses a hook with event type `pullRequest`
// and returns the commit data.
func (g *GitHub) pullRequest(r *http.Request) (*common.Hook, error) {
payload := GetPayload(r)
hook := &struct {
Action string `json:"action"`
PullRequest *github.PullRequest `json:"pull_request"`
Repo *github.Repository `json:"repository"`
}{}
err := json.Unmarshal(payload, hook)
if err != nil {
return nil, err
}
// ignore these
if hook.Action != "opened" && hook.Action != "synchronize" {
return nil, nil
}
repo := &common.Repo{}
repo.Owner = *hook.Repo.Owner.Login
repo.Name = *hook.Repo.Name
repo.Language = *hook.Repo.Language
repo.FullName = *hook.Repo.FullName
repo.Link = *hook.Repo.HTMLURL
repo.Private = *hook.Repo.Private
repo.Clone = *hook.Repo.CloneURL
pr := &common.PullRequest{}
pr.Number = *hook.PullRequest.Number
pr.Title = *hook.PullRequest.Title
pr.Source = &common.Commit{}
pr.Source.Sha = *hook.PullRequest.Head.SHA
pr.Source.Ref = *hook.PullRequest.Head.Ref
pr.Source.Author = &common.Author{}
pr.Source.Author.Login = *hook.PullRequest.User.Login
pr.Source.Remote = &common.Remote{}
pr.Source.Remote.Clone = *hook.PullRequest.Head.Repo.CloneURL
pr.Source.Remote.Name = *hook.PullRequest.Head.Repo.Name
pr.Source.Remote.FullName = *hook.PullRequest.Head.Repo.FullName
pr.Target = &common.Commit{}
pr.Target.Sha = *hook.PullRequest.Base.SHA
pr.Target.Ref = *hook.PullRequest.Base.Ref
return &common.Hook{Repo: repo, PullRequest: pr}, nil
}
type pushHook struct {
Ref string `json:"ref"`
Head struct {
ID string `json:"id"`
Message string `json:"message"`
Timestamp string `json:"timestamp"`
Author struct {
Name string `json:"name"`
Email string `json:"name"`
Username string `json:"username"`
} `json:"author"`
Committer struct {
Name string `json:"name"`
Email string `json:"name"`
Username string `json:"username"`
} `json:"committer"`
} `json:"head_commit"`
Repo struct {
Owner struct {
Login string `json:"login"`
} `json:"owner"`
Name string `json:"name"`
FullName string `json:"full_name"`
Language string `json:"language"`
Private bool `json:"private"`
HTMLURL string `json:"html_url"`
CloneURL string `json:"clone_url"`
} `json:"repository"`
}
const (
StatusPending = "pending"
StatusSuccess = "success"
StatusFailure = "failure"
StatusError = "error"
)
const (
DescPending = "this build is pending"
DescSuccess = "the build was successful"
DescFailure = "the build failed"
DescError = "oops, something went wrong"
)
// getStatus is a helper functin that converts a Drone
// status to a GitHub status.
func getStatus(status string) string {
switch status {
case common.StatePending, common.StateRunning:
return StatusPending
case common.StateSuccess:
return StatusSuccess
case common.StateFailure:
return StatusFailure
case common.StateError, common.StateKilled:
return StatusError
default:
return StatusError
}
}
// getDesc is a helper function that generates a description
// message for the build based on the status.
func getDesc(status string) string {
switch status {
case common.StatePending, common.StateRunning:
return DescPending
case common.StateSuccess:
return DescSuccess
case common.StateFailure:
return DescFailure
case common.StateError, common.StateKilled:
return DescError
default:
return DescError
}
}

337
remote/github/helper.go Normal file
View file

@ -0,0 +1,337 @@
package github
import (
"crypto/tls"
"encoding/base32"
"fmt"
"io/ioutil"
"net/http"
"net/url"
"strings"
"github.com/drone/drone/common/oauth2"
"github.com/google/go-github/github"
"github.com/gorilla/securecookie"
)
// NewClient is a helper function that returns a new GitHub
// client using the provided OAuth token.
func NewClient(uri, token string, skipVerify bool) *github.Client {
t := &oauth2.Transport{
Token: &oauth2.Token{AccessToken: token},
}
// this is for GitHub enterprise users that are using
// self-signed certificates.
if skipVerify {
t.Transport = &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
}
}
c := github.NewClient(t.Client())
c.BaseURL, _ = url.Parse(uri)
return c
}
// GetUserEmail is a heper function that retrieves the currently
// authenticated user from GitHub + Email address.
func GetUserEmail(client *github.Client) (*github.User, error) {
user, _, err := client.Users.Get("")
if err != nil {
return nil, err
}
emails, _, err := client.Users.ListEmails(nil)
if err != nil {
return nil, err
}
for _, email := range emails {
if *email.Primary && *email.Verified {
user.Email = email.Email
return user, nil
}
}
// WARNING, HACK
// for out-of-date github enterprise editions the primary
// and verified fields won't exist.
if !strings.HasPrefix(*user.HTMLURL, DefaultURL) && len(emails) != 0 {
user.Email = emails[0].Email
return user, nil
}
return nil, fmt.Errorf("No verified Email address for GitHub account")
}
// GetRepo is a helper function that returns a named repo
func GetRepo(client *github.Client, owner, repo string) (*github.Repository, error) {
r, _, err := client.Repositories.Get(owner, repo)
return r, err
}
// GetAllRepos is a helper function that returns an aggregated list
// of all user and organization repositories.
func GetAllRepos(client *github.Client) ([]github.Repository, error) {
orgs, err := GetOrgs(client)
if err != nil {
return nil, err
}
repos, err := GetUserRepos(client)
if err != nil {
return nil, err
}
for _, org := range orgs {
list, err := GetOrgRepos(client, *org.Login)
if err != nil {
return nil, err
}
repos = append(repos, list...)
}
return repos, nil
}
// GetUserRepos is a helper function that returns a list of
// all user repositories. Paginated results are aggregated into
// a single list.
func GetUserRepos(client *github.Client) ([]github.Repository, error) {
var repos []github.Repository
var opts = github.RepositoryListOptions{}
opts.PerPage = 100
opts.Page = 1
// loop through user repository list
for opts.Page > 0 {
list, resp, err := client.Repositories.List("", &opts)
if err != nil {
return nil, err
}
repos = append(repos, list...)
// increment the next page to retrieve
opts.Page = resp.NextPage
}
return repos, nil
}
// GetOrgRepos is a helper function that returns a list of
// all org repositories. Paginated results are aggregated into
// a single list.
func GetOrgRepos(client *github.Client, org string) ([]github.Repository, error) {
var repos []github.Repository
var opts = github.RepositoryListByOrgOptions{}
opts.PerPage = 100
opts.Page = 1
// loop through user repository list
for opts.Page > 0 {
list, resp, err := client.Repositories.ListByOrg(org, &opts)
if err != nil {
return nil, err
}
repos = append(repos, list...)
// increment the next page to retrieve
opts.Page = resp.NextPage
}
return repos, nil
}
// GetOrgs is a helper function that returns a list of
// all orgs that a user belongs to.
func GetOrgs(client *github.Client) ([]github.Organization, error) {
var orgs []github.Organization
var opts = github.ListOptions{}
opts.Page = 1
for opts.Page > 0 {
list, resp, err := client.Organizations.List("", &opts)
if err != nil {
return nil, err
}
orgs = append(orgs, list...)
// increment the next page to retrieve
opts.Page = resp.NextPage
}
return orgs, nil
}
// GetHook is a heper function that retrieves a hook by
// hostname. To do this, it will retrieve a list of all hooks
// and iterate through the list.
func GetHook(client *github.Client, owner, name, url string) (*github.Hook, error) {
hooks, _, err := client.Repositories.ListHooks(owner, name, nil)
if err != nil {
return nil, err
}
for _, hook := range hooks {
if hook.Config["url"] == url {
return &hook, nil
}
}
return nil, nil
}
func DeleteHook(client *github.Client, owner, name, url string) error {
hook, err := GetHook(client, owner, name, url)
if err != nil {
return err
}
if hook == nil {
return nil
}
_, err = client.Repositories.DeleteHook(owner, name, *hook.ID)
return err
}
// CreateHook is a heper function that creates a post-commit hook
// for the specified repository.
func CreateHook(client *github.Client, owner, name, url string) (*github.Hook, error) {
var hook = new(github.Hook)
hook.Name = github.String("web")
hook.Events = []string{"push", "pull_request"}
hook.Config = map[string]interface{}{}
hook.Config["url"] = url
hook.Config["content_type"] = "form"
created, _, err := client.Repositories.CreateHook(owner, name, hook)
return created, err
}
// CreateUpdateHook is a heper function that creates a post-commit hook
// for the specified repository if it does not already exist, otherwise
// it updates the existing hook
func CreateUpdateHook(client *github.Client, owner, name, url string) (*github.Hook, error) {
var hook, _ = GetHook(client, owner, name, url)
if hook != nil {
hook.Name = github.String("web")
hook.Events = []string{"push", "pull_request"}
hook.Config = map[string]interface{}{}
hook.Config["url"] = url
hook.Config["content_type"] = "form"
var updated, _, err = client.Repositories.EditHook(owner, name, *hook.ID, hook)
return updated, err
}
return CreateHook(client, owner, name, url)
}
// GetKey is a heper function that retrieves a public Key by
// title. To do this, it will retrieve a list of all keys
// and iterate through the list.
func GetKey(client *github.Client, owner, name, title string) (*github.Key, error) {
keys, _, err := client.Repositories.ListKeys(owner, name, nil)
if err != nil {
return nil, err
}
for _, key := range keys {
if *key.Title == title {
return &key, nil
}
}
return nil, nil
}
// GetKeyTitle is a helper function that generates a title for the
// RSA public key based on the username and domain name.
func GetKeyTitle(rawurl string) (string, error) {
var uri, err = url.Parse(rawurl)
if err != nil {
return "", err
}
return fmt.Sprintf("drone@%s", uri.Host), nil
}
// DeleteKey is a helper function that deletes a deploy key
// for the specified repository.
func DeleteKey(client *github.Client, owner, name, title string) error {
var k, err = GetKey(client, owner, name, title)
if err != nil {
return err
}
_, err = client.Repositories.DeleteKey(owner, name, *k.ID)
return err
}
// CreateKey is a helper function that creates a deploy key
// for the specified repository.
func CreateKey(client *github.Client, owner, name, title, key string) (*github.Key, error) {
var k = new(github.Key)
k.Title = github.String(title)
k.Key = github.String(key)
created, _, err := client.Repositories.CreateKey(owner, name, k)
return created, err
}
// CreateUpdateKey is a helper function that creates a deployment key
// for the specified repository if it does not already exist, otherwise
// it updates the existing key
func CreateUpdateKey(client *github.Client, owner, name, title, key string) (*github.Key, error) {
var k, _ = GetKey(client, owner, name, title)
if k != nil {
k.Title = github.String(title)
k.Key = github.String(key)
client.Repositories.DeleteKey(owner, name, *k.ID)
}
return CreateKey(client, owner, name, title, key)
}
// GetFile is a heper function that retrieves a file from
// GitHub and returns its contents in byte array format.
func GetFile(client *github.Client, owner, name, path, ref string) ([]byte, error) {
var opts = new(github.RepositoryContentGetOptions)
opts.Ref = ref
content, _, _, err := client.Repositories.GetContents(owner, name, path, opts)
if err != nil {
return nil, err
}
return content.Decode()
}
// GetRandom is a helper function that generates a 32-bit random
// key, base32 encoded as a string value.
func GetRandom() string {
return base32.StdEncoding.EncodeToString(securecookie.GenerateRandomKey(32))
}
// GetPayload is a helper function that will parse the JSON payload. It will
// first check for a `payload` parameter in a POST, but can fallback to a
// raw JSON body as well.
func GetPayload(req *http.Request) []byte {
var payload = req.FormValue("payload")
if len(payload) == 0 {
defer req.Body.Close()
raw, _ := ioutil.ReadAll(req.Body)
return raw
}
return []byte(payload)
}
// UserBelongsToOrg returns true if the currently authenticated user is a
// member of any of the organizations provided.
func UserBelongsToOrg(client *github.Client, permittedOrgs []string) (bool, error) {
userOrgs, err := GetOrgs(client)
if err != nil {
return false, err
}
userOrgSet := make(map[string]struct{}, len(userOrgs))
for _, org := range userOrgs {
userOrgSet[*org.Login] = struct{}{}
}
for _, org := range permittedOrgs {
if _, ok := userOrgSet[org]; ok {
return true, nil
}
}
return false, nil
}

39
remote/remote.go Normal file
View file

@ -0,0 +1,39 @@
package remote
import (
"net/http"
"github.com/drone/drone/common"
)
type Remote interface {
// Login authenticates the session and returns the
// remote user details.
Login(token, secret string) (*common.User, error)
// Repo fetches the named repository from the remote system.
Repo(u *common.User, owner, repo string) (*common.Repo, error)
// Perm fetches the named repository from the remote system.
Perm(u *common.User, owner, repo string) (*common.Perm, error)
// Script fetches the build script (.drone.yml) from the remote
// repository and returns in string format.
Script(u *common.User, r *common.Repo, b *common.Build) ([]byte, error)
// Status sends the commit status to the remote system.
// An example would be the GitHub pull request status.
Status(u *common.User, r *common.Repo, b *common.Build, link string) error
// Activate activates a repository by creating the post-commit hook and
// adding the SSH deploy key, if applicable.
Activate(u *common.User, r *common.Repo, k *common.Keypair, link string) error
// Deactivate removes a repository by removing all the post-commit hooks
// which are equal to link and removing the SSH deploy key.
Deactivate(u *common.User, r *common.Repo, link string) error
// Hook parses the post-commit hook from the Request body
// and returns the required data in a standard format.
Hook(r *http.Request) (*common.Hook, error)
}

View file

@ -11,6 +11,10 @@ type Service struct {
// may be github, gitlab, bitbucket, or gogs.
Name string `toml:"name"`
// Address defines the address (uri) of the plugin for
// communication via the net/rpc package.
Address string `toml:"address"`
// Base defines the base URL for the service. For example:
// https://github.com
// https://bitbucket.org