Merge branch 'master' into remove_size_limit

This commit is contained in:
Joachim Hill-Grannec 2017-02-04 19:58:51 -08:00
commit 0699138c47
157 changed files with 4518 additions and 2703 deletions

View file

@ -3,7 +3,7 @@ workspace:
path: src/github.com/drone/drone
pipeline:
backend:
test:
image: golang:1.6
environment:
- GO15VENDOREXPERIMENT=1
@ -22,23 +22,34 @@ pipeline:
when:
event: push
publish:
image: s3
archive:
image: plugins/s3
acl: public-read
bucket: downloads.drone.io
source: release/**/*.*
access_key: ${AWS_ACCESS_KEY_ID}
secret_key: ${AWS_SECRET_ACCESS_KEY}
when:
event: push
branch: master
docker:
publish:
image: plugins/docker
repo: drone/drone
tag: [ "0.5.0", "0.5" ]
storage_driver: overlay
username: ${DOCKER_USERNAME}
password: ${DOCKER_PASSWORD}
tag: [ "0.5", "0.5.0", "0.5.0-rc", "latest" ]
when:
branch: master
event: push
notify:
image: plugins/gitter
webhook: ${GITTER_WEBHOOK}
when:
status: [ success, failure ]
event: [ push, pull_request ]
services:
postgres:
image: postgres:9.4.5

View file

@ -1 +1 @@
eyJhbGciOiJIUzI1NiJ9.d29ya3NwYWNlOgogIGJhc2U6IC9nbwogIHBhdGg6IHNyYy9naXRodWIuY29tL2Ryb25lL2Ryb25lCgpwaXBlbGluZToKICBiYWNrZW5kOgogICAgaW1hZ2U6IGdvbGFuZzoxLjYKICAgIGVudmlyb25tZW50OgogICAgICAtIEdPMTVWRU5ET1JFWFBFUklNRU5UPTEKICAgIGNvbW1hbmRzOgogICAgICAtIG1ha2UgZGVwcyBnZW4KICAgICAgLSBtYWtlIHRlc3QgdGVzdF9wb3N0Z3JlcyB0ZXN0X215c3FsCgogIGNvbXBpbGU6CiAgICBpbWFnZTogZ29sYW5nOjEuNgogICAgZW52aXJvbm1lbnQ6CiAgICAgIC0gR08xNVZFTkRPUkVYUEVSSU1FTlQ9MQogICAgICAtIEdPUEFUSD0vZ28KICAgIGNvbW1hbmRzOgogICAgICAtIGV4cG9ydCBQQVRIPSRQQVRIOiRHT1BBVEgvYmluCiAgICAgIC0gbWFrZSBidWlsZAogICAgd2hlbjoKICAgICAgZXZlbnQ6IHB1c2gKCiAgcHVibGlzaDoKICAgIGltYWdlOiBzMwogICAgYWNsOiBwdWJsaWMtcmVhZAogICAgYnVja2V0OiBkb3dubG9hZHMuZHJvbmUuaW8KICAgIHNvdXJjZTogcmVsZWFzZS8qKi8qLioKICAgIHdoZW46CiAgICAgIGV2ZW50OiBwdXNoCiAgICAgIGJyYW5jaDogbWFzdGVyCgogIGRvY2tlcjoKICAgIHJlcG86IGRyb25lL2Ryb25lCiAgICB0YWc6IFsgIjAuNS4wIiwgIjAuNSIgXQogICAgc3RvcmFnZV9kcml2ZXI6IG92ZXJsYXkKICAgIHdoZW46CiAgICAgIGJyYW5jaDogbWFzdGVyCiAgICAgIGV2ZW50OiBwdXNoCgpzZXJ2aWNlczoKICBwb3N0Z3JlczoKICAgIGltYWdlOiBwb3N0Z3Jlczo5LjQuNQogICAgZW52aXJvbm1lbnQ6CiAgICAgIC0gUE9TVEdSRVNfVVNFUj1wb3N0Z3JlcwogIG15c3FsOgogICAgaW1hZ2U6IG15c3FsOjUuNi4yNwogICAgZW52aXJvbm1lbnQ6CiAgICAgIC0gTVlTUUxfREFUQUJBU0U9dGVzdAogICAgICAtIE1ZU1FMX0FMTE9XX0VNUFRZX1BBU1NXT1JEPXllcwo.kQIwqIgs7PnoKIGmzJ6hlbWTbV5zK0w4HVWsux79P3s
eyJhbGciOiJIUzI1NiJ9.d29ya3NwYWNlOgogIGJhc2U6IC9nbwogIHBhdGg6IHNyYy9naXRodWIuY29tL2Ryb25lL2Ryb25lCgpwaXBlbGluZToKICB0ZXN0OgogICAgaW1hZ2U6IGdvbGFuZzoxLjYKICAgIGVudmlyb25tZW50OgogICAgICAtIEdPMTVWRU5ET1JFWFBFUklNRU5UPTEKICAgIGNvbW1hbmRzOgogICAgICAtIG1ha2UgZGVwcyBnZW4KICAgICAgLSBtYWtlIHRlc3QgdGVzdF9wb3N0Z3JlcyB0ZXN0X215c3FsCgogIGNvbXBpbGU6CiAgICBpbWFnZTogZ29sYW5nOjEuNgogICAgZW52aXJvbm1lbnQ6CiAgICAgIC0gR08xNVZFTkRPUkVYUEVSSU1FTlQ9MQogICAgICAtIEdPUEFUSD0vZ28KICAgIGNvbW1hbmRzOgogICAgICAtIGV4cG9ydCBQQVRIPSRQQVRIOiRHT1BBVEgvYmluCiAgICAgIC0gbWFrZSBidWlsZAogICAgd2hlbjoKICAgICAgZXZlbnQ6IHB1c2gKCiAgYXJjaGl2ZToKICAgIGltYWdlOiBwbHVnaW5zL3MzCiAgICBhY2w6IHB1YmxpYy1yZWFkCiAgICBidWNrZXQ6IGRvd25sb2Fkcy5kcm9uZS5pbwogICAgc291cmNlOiByZWxlYXNlLyoqLyouKgogICAgYWNjZXNzX2tleTogJHtBV1NfQUNDRVNTX0tFWV9JRH0KICAgIHNlY3JldF9rZXk6ICR7QVdTX1NFQ1JFVF9BQ0NFU1NfS0VZfQogICAgd2hlbjoKICAgICAgZXZlbnQ6IHB1c2gKICAgICAgYnJhbmNoOiBtYXN0ZXIKCiAgcHVibGlzaDoKICAgIGltYWdlOiBwbHVnaW5zL2RvY2tlcgogICAgcmVwbzogZHJvbmUvZHJvbmUKICAgIHVzZXJuYW1lOiAke0RPQ0tFUl9VU0VSTkFNRX0KICAgIHBhc3N3b3JkOiAke0RPQ0tFUl9QQVNTV09SRH0KICAgIHRhZzogWyAiMC41IiwgIjAuNS4wIiwgIjAuNS4wLXJjIiwgImxhdGVzdCIgXQogICAgd2hlbjoKICAgICAgYnJhbmNoOiBtYXN0ZXIKICAgICAgZXZlbnQ6IHB1c2gKCiAgbm90aWZ5OgogICAgaW1hZ2U6IHBsdWdpbnMvZ2l0dGVyCiAgICB3ZWJob29rOiAke0dJVFRFUl9XRUJIT09LfQogICAgd2hlbjoKICAgICAgc3RhdHVzOiBbIHN1Y2Nlc3MsIGZhaWx1cmUgXQogICAgICBldmVudDogWyBwdXNoLCBwdWxsX3JlcXVlc3QgXQoKc2VydmljZXM6CiAgcG9zdGdyZXM6CiAgICBpbWFnZTogcG9zdGdyZXM6OS40LjUKICAgIGVudmlyb25tZW50OgogICAgICAtIFBPU1RHUkVTX1VTRVI9cG9zdGdyZXMKICBteXNxbDoKICAgIGltYWdlOiBteXNxbDo1LjYuMjcKICAgIGVudmlyb25tZW50OgogICAgICAtIE1ZU1FMX0RBVEFCQVNFPXRlc3QKICAgICAgLSBNWVNRTF9BTExPV19FTVBUWV9QQVNTV09SRD15ZXMK.IK93uHsY4HSQUFESXMxi3UruedD3hrVWg03jjJr2A0I

View file

@ -1,8 +1,14 @@
Thank you for taking the time to use Drone and file an issue or feature request. Before filing an issue please ensure the following boxes are checked, if applicable:
<!-- PLEASE READ BEFORE DELETING
- [ ] I have searched for existing issues
- [ ] I have discussed the issue with the community at https://gitter.im/drone/drone
- [ ] I have provided a sample `.drone.yml` file to help the team reproduce
- [ ] I have provided details from the build logs
- [ ] I have provided details from the server logs by running `docker logs drone`
- [ ] I am not using the issue tracker to ask why my build failed
Bugs or Issues? Please do not open a GitHub issue until you have
discussed and verified on the mailing list:
http://discourse.drone.io/
Failing Builds? Please do not use GitHub issues for generic support
questions. Instead use the mailing list or Stack Overflow:
http://discourse.drone.io/
http://stackoverflow.com/questions/tagged/drone.io
-->

View file

@ -19,6 +19,8 @@ deps_backend:
go get -u golang.org/x/tools/cmd/cover
go get -u github.com/jteeuwen/go-bindata/...
go get -u github.com/elazarl/go-bindata-assetfs/...
go get -u github.com/drone/mq/...
go get -u github.com/tidwall/redlog
gen: gen_template gen_migrations

View file

@ -4,26 +4,15 @@
Drone is a Continuous Integration platform built on container technology. Every build is executed inside an ephemeral Docker container, giving developers complete control over their build environment with guaranteed isolation.
Browse the code at https://sourcegraph.com/github.com/drone/drone
### Goals
Drone's prime directive is to help teams [ship code like GitHub](https://github.com/blog/1241-deploying-at-github#always-be-shipping). Drone is easy to install, setup and maintain and offers a powerful container-based plugin system. Drone aspires to be an industry-wide replacement for Jenkins.
Drone's prime directive is to help teams [ship code like GitHub](https://github.com/blog/1241-deploying-at-github#always-be-shipping). Drone is easy to install, setup and maintain and offers a powerful container-based plugin system. Drone aspires to eventually offer an industry-wide replacement for Jenkins.
### Documentation
Drone documentation is organized into several categories:
* [Setup Guide](http://readme.drone.io/setup/overview)
* [Build Guide](http://readme.drone.io/usage/overview)
* [Plugin Guide](http://readme.drone.io/devs/plugins)
* [CLI Reference](http://readme.drone.io/devs/cli/)
* [API Reference](http://readme.drone.io/devs/api/builds)
### Documentation for 0.5 (unstable)
If you are using the 0.5 unstable release (master branch) please see the updated documentation:
* [Setup Guide](http://readme.drone.io/0.5/installation/server/)
* [Build Guide](http://readme.drone.io/0.5/usage/overview/)
Documentation is published to [readme.drone.io](http://readme.drone.io)
### Community, Help
@ -31,7 +20,7 @@ Contributions, questions, and comments are welcomed and encouraged. Drone develo
### Installation
Please see our [installation guide](http://readme.drone.io/setup/overview) to install the official Docker image.
Please see our [installation guide](http://readme.drone.io/admin/) to install the official Docker image.
### From Source
@ -45,11 +34,9 @@ cd $GOPATH/src/github.com/drone/drone
Commands to build from source:
```sh
export GO15VENDOREXPERIMENT=1
make deps # Download required dependencies
make gen # Generate code
make build # Build the binary
make build_static # Build the binary
```
If you are having trouble building this project please reference its `.drone.yml` file. Everything you need to know about building Drone is defined in that file.

View file

@ -11,11 +11,10 @@ import (
"github.com/drone/drone/build"
"github.com/drone/drone/model"
"github.com/drone/drone/queue"
"github.com/drone/drone/version"
"github.com/drone/drone/yaml"
"github.com/drone/drone/yaml/expander"
"github.com/drone/drone/yaml/transform"
"github.com/drone/envsubst"
)
type Logger interface {
@ -29,6 +28,7 @@ type Agent struct {
Timeout time.Duration
Platform string
Namespace string
Extension []string
Disable []string
Escalate []string
Netrc []string
@ -48,7 +48,7 @@ func (a *Agent) Poll() error {
return nil
}
func (a *Agent) Run(payload *queue.Work, cancel <-chan bool) error {
func (a *Agent) Run(payload *model.Work, cancel <-chan bool) error {
payload.Job.Status = model.StatusRunning
payload.Job.Started = time.Now().Unix()
@ -90,18 +90,44 @@ func (a *Agent) Run(payload *queue.Work, cancel <-chan bool) error {
return err
}
func (a *Agent) prep(w *queue.Work) (*yaml.Config, error) {
func (a *Agent) prep(w *model.Work) (*yaml.Config, error) {
envs := toEnv(w)
w.Yaml = expander.ExpandString(w.Yaml, envs)
envSecrets := map[string]string{}
// list of secrets to interpolate in the yaml
for _, secret := range w.Secrets {
if (w.Verified || secret.SkipVerify) && secret.MatchEvent(w.Build.Event) {
envSecrets[secret.Name] = secret.Value
}
}
var err error
w.Yaml, err = envsubst.Eval(w.Yaml, func(s string) string {
env, ok := envSecrets[s]
if !ok {
env, _ = envs[s]
}
if strings.Contains(env, "\n") {
env = fmt.Sprintf("%q", env)
}
return env
})
if err != nil {
return nil, err
}
// append secrets when verified or when a secret does not require
// verification
var secrets []*model.Secret
for _, secret := range w.Secrets {
if w.Verified || secret.SkipVerify {
secrets = append(secrets, secret)
}
}
// inject the netrc file into the clone plugin if the repository is
// private and requires authentication.
var secrets []*model.Secret
if w.Verified {
secrets = append(secrets, w.Secrets...)
}
if w.Repo.IsPrivate {
secrets = append(secrets, &model.Secret{
Name: "DRONE_NETRC_USERNAME",
@ -155,8 +181,6 @@ func (a *Agent) prep(w *queue.Work) (*yaml.Config, error) {
transform.CommandTransform(conf)
transform.ImagePull(conf, a.Pull)
transform.ImageTag(conf)
transform.ImageName(conf)
transform.ImageNamespace(conf, a.Namespace)
if err := transform.ImageEscalate(conf, a.Escalate); err != nil {
return nil, err
}
@ -168,11 +192,14 @@ func (a *Agent) prep(w *queue.Work) (*yaml.Config, error) {
}
transform.Pod(conf, a.Platform)
if err := transform.RemoteTransform(conf, a.Extension); err != nil {
return nil, err
}
return conf, nil
}
func (a *Agent) exec(spec *yaml.Config, payload *queue.Work, cancel <-chan bool) error {
func (a *Agent) exec(spec *yaml.Config, payload *model.Work, cancel <-chan bool) error {
conf := build.Config{
Engine: a.Engine,
@ -187,6 +214,7 @@ func (a *Agent) exec(spec *yaml.Config, payload *queue.Work, cancel <-chan bool)
return err
}
replacer := NewSecretReplacer(payload.Secrets)
timeout := time.After(time.Duration(payload.Repo.Timeout) * time.Minute)
for {
@ -226,12 +254,13 @@ func (a *Agent) exec(spec *yaml.Config, payload *queue.Work, cancel <-chan bool)
pipeline.Exec()
}
case line := <-pipeline.Pipe():
line.Out = replacer.Replace(line.Out)
a.Logger(line)
}
}
}
func toEnv(w *queue.Work) map[string]string {
func toEnv(w *model.Work) map[string]string {
envs := map[string]string{
"CI": "drone",
"DRONE": "true",
@ -248,6 +277,7 @@ func toEnv(w *queue.Work) map[string]string {
"DRONE_REMOTE_URL": w.Repo.Clone,
"DRONE_COMMIT_SHA": w.Build.Commit,
"DRONE_COMMIT_REF": w.Build.Ref,
"DRONE_COMMIT_REFSPEC": w.Build.Refspec,
"DRONE_COMMIT_BRANCH": w.Build.Branch,
"DRONE_COMMIT_LINK": w.Build.Link,
"DRONE_COMMIT_MESSAGE": w.Build.Message,

46
agent/secret.go Normal file
View file

@ -0,0 +1,46 @@
package agent
import (
"strings"
"github.com/drone/drone/model"
)
// SecretReplacer hides secrets from being exposed by the build output.
type SecretReplacer interface {
// Replace conceals instances of secrets found in s.
Replace(s string) string
}
// NewSecretReplacer creates a SecretReplacer based on whether any value in
// secrets requests it be hidden.
func NewSecretReplacer(secrets []*model.Secret) SecretReplacer {
var r []string
for _, s := range secrets {
if s.Conceal {
r = append(r, s.Value, "*****")
}
}
if len(r) == 0 {
return &noopReplacer{}
}
return &secretReplacer{
replacer: strings.NewReplacer(r...),
}
}
type noopReplacer struct{}
func (*noopReplacer) Replace(s string) string {
return s
}
type secretReplacer struct {
replacer *strings.Replacer
}
func (r *secretReplacer) Replace(s string) string {
return r.replacer.Replace(s)
}

39
agent/secret_test.go Normal file
View file

@ -0,0 +1,39 @@
package agent
import (
"testing"
"github.com/drone/drone/model"
"github.com/franela/goblin"
)
const testString = "This is SECRET: secret_value"
func TestSecret(t *testing.T) {
g := goblin.Goblin(t)
g.Describe("SecretReplacer", func() {
g.It("Should conceal secret", func() {
secrets := []*model.Secret{
{
Name: "SECRET",
Value: "secret_value",
Conceal: true,
},
}
r := NewSecretReplacer(secrets)
g.Assert(r.Replace(testString)).Equal("This is SECRET: *****")
})
g.It("Should not conceal secret", func() {
secrets := []*model.Secret{
{
Name: "SECRET",
Value: "secret_value",
Conceal: false,
},
}
r := NewSecretReplacer(secrets)
g.Assert(r.Replace(testString)).Equal(testString)
})
})
}

View file

@ -1,25 +1,22 @@
package agent
import (
"encoding/json"
"fmt"
"io"
"sync"
"time"
"github.com/Sirupsen/logrus"
"github.com/drone/drone/build"
"github.com/drone/drone/client"
"github.com/drone/drone/queue"
"github.com/drone/drone/model"
"github.com/drone/mq/logger"
"github.com/drone/mq/stomp"
)
// UpdateFunc handles buid pipeline status updates.
type UpdateFunc func(*queue.Work)
type UpdateFunc func(*model.Work)
// LoggerFunc handles buid pipeline logging updates.
type LoggerFunc func(*build.Line)
var NoopUpdateFunc = func(*queue.Work) {}
var NoopUpdateFunc = func(*model.Work) {}
var TermLoggerFunc = func(line *build.Line) {
fmt.Println(line)
@ -27,65 +24,44 @@ var TermLoggerFunc = func(line *build.Line) {
// NewClientUpdater returns an updater that sends updated build details
// to the drone server.
func NewClientUpdater(client client.Client) UpdateFunc {
return func(w *queue.Work) {
for {
err := client.Push(w)
if err == nil {
return
}
logrus.Errorf("Error updating %s/%s#%d.%d. Retry in 30s. %s",
func NewClientUpdater(client *stomp.Client) UpdateFunc {
return func(w *model.Work) {
err := client.SendJSON("/queue/updates", w)
if err != nil {
logger.Warningf("Error updating %s/%s#%d.%d. %s",
w.Repo.Owner, w.Repo.Name, w.Build.Number, w.Job.Number, err)
logrus.Infof("Retry update in 30s")
time.Sleep(time.Second * 30)
}
if w.Job.Status != model.StatusRunning {
var dest = fmt.Sprintf("/topic/logs.%d", w.Job.ID)
var opts = []stomp.MessageOption{
stomp.WithHeader("eof", "true"),
stomp.WithRetain("all"),
}
if err := client.Send(dest, []byte("eof"), opts...); err != nil {
logger.Warningf("Error sending eof %s/%s#%d.%d. %s",
w.Repo.Owner, w.Repo.Name, w.Build.Number, w.Job.Number, err)
}
}
}
}
func NewStreamLogger(stream client.StreamWriter, w io.Writer, limit int64) LoggerFunc {
var err error
var size int64
return func(line *build.Line) {
func NewClientLogger(client *stomp.Client, id int64, limit int64) LoggerFunc {
var size int64
var dest = fmt.Sprintf("/topic/logs.%d", id)
var opts = []stomp.MessageOption{
stomp.WithRetain("all"),
}
return func(line *build.Line) {
if size > limit {
return
}
// TODO remove this double-serialization
linejson, _ := json.Marshal(line)
w.Write(linejson)
w.Write([]byte{'\n'})
if err = stream.WriteJSON(line); err != nil {
if err := client.SendJSON(dest, line, opts...); err != nil {
logrus.Errorf("Error streaming build logs. %s", err)
}
size += int64(len(line.Out))
}
}
func NewClientLogger(client client.Client, id int64, rc io.ReadCloser, wc io.WriteCloser, limit int64) LoggerFunc {
var once sync.Once
var size int64
return func(line *build.Line) {
// annoying hack to only start streaming once the first line is written
once.Do(func() {
go func() {
err := client.Stream(id, rc)
if err != nil && err != io.ErrClosedPipe {
logrus.Errorf("Error streaming build logs. %s", err)
}
}()
})
if size > limit {
return
}
linejson, _ := json.Marshal(line)
wc.Write(linejson)
wc.Write([]byte{'\n'})
size += int64(len(line.Out))
}
}

View file

@ -22,6 +22,7 @@ func toContainerConfig(c *yaml.Container) *dockerclient.ContainerConfig {
Privileged: c.Privileged,
NetworkMode: c.Network,
Memory: c.MemLimit,
ShmSize: c.ShmSize,
CpuShares: c.CPUShares,
CpuQuota: c.CPUQuota,
CpusetCpus: c.CPUSet,

View file

@ -21,7 +21,7 @@ type ExitError struct {
Code int
}
// Error reteurns the error message in string format.
// Error returns the error message in string format.
func (e *ExitError) Error() string {
return fmt.Sprintf("%s : exit code %d", e.Name, e.Code)
}
@ -31,7 +31,7 @@ type OomError struct {
Name string
}
// Error reteurns the error message in string format.
// Error returns the error message in string format.
func (e *OomError) Error() string {
return fmt.Sprintf("%s : received oom kill", e.Name)
}

View file

@ -175,7 +175,6 @@ func (p *Pipeline) exec(c *yaml.Container) error {
}
p.containers = append(p.containers, name)
logrus.Debugf("wait.add(1) for %s logs", name)
p.wait.Add(1)
go func() {
defer func() {
@ -183,7 +182,6 @@ func (p *Pipeline) exec(c *yaml.Container) error {
logrus.Errorln("recover writing build output", r)
}
logrus.Debugf("wait.done() for %s logs", name)
p.wait.Done()
}()
@ -217,7 +215,6 @@ func (p *Pipeline) exec(c *yaml.Container) error {
return err
}
logrus.Debugf("wait.add(1) for %s exit code", name)
p.wait.Add(1)
go func() {
defer func() {
@ -225,7 +222,6 @@ func (p *Pipeline) exec(c *yaml.Container) error {
logrus.Errorln("recover writing exit code to output", r)
}
p.wait.Done()
logrus.Debugf("wait.done() for %s exit code", name)
}()
p.pipe <- &Line{

View file

@ -1,40 +0,0 @@
package bus
//go:generate mockery -name Bus -output mock -case=underscore
import "golang.org/x/net/context"
// Bus represents an event bus implementation that
// allows a publisher to broadcast Event notifications
// to a list of subscribers.
type Bus interface {
// Publish broadcasts an event to all subscribers.
Publish(*Event)
// Subscribe adds the channel to the list of
// subscribers. Each subscriber in the list will
// receive broadcast events.
Subscribe(chan *Event)
// Unsubscribe removes the channel from the list
// of subscribers.
Unsubscribe(chan *Event)
}
// Publish broadcasts an event to all subscribers.
func Publish(c context.Context, event *Event) {
FromContext(c).Publish(event)
}
// Subscribe adds the channel to the list of
// subscribers. Each subscriber in the list will
// receive broadcast events.
func Subscribe(c context.Context, eventc chan *Event) {
FromContext(c).Subscribe(eventc)
}
// Unsubscribe removes the channel from the
// list of subscribers.
func Unsubscribe(c context.Context, eventc chan *Event) {
FromContext(c).Unsubscribe(eventc)
}

View file

@ -1,46 +0,0 @@
package bus
import (
"sync"
)
type eventbus struct {
sync.Mutex
subs map[chan *Event]bool
}
// New creates a simple event bus that manages a list of
// subscribers to which events are published.
func New() Bus {
return newEventbus()
}
func newEventbus() *eventbus {
return &eventbus{
subs: make(map[chan *Event]bool),
}
}
func (b *eventbus) Subscribe(c chan *Event) {
b.Lock()
b.subs[c] = true
b.Unlock()
}
func (b *eventbus) Unsubscribe(c chan *Event) {
b.Lock()
delete(b.subs, c)
b.Unlock()
}
func (b *eventbus) Publish(event *Event) {
b.Lock()
defer b.Unlock()
for s := range b.subs {
go func(c chan *Event) {
defer recover()
c <- event
}(s)
}
}

View file

@ -1,73 +0,0 @@
package bus
import (
"sync"
"testing"
"github.com/drone/drone/model"
. "github.com/franela/goblin"
"github.com/gin-gonic/gin"
)
func TestBus(t *testing.T) {
g := Goblin(t)
g.Describe("Event bus", func() {
g.It("Should unsubscribe", func() {
c := new(gin.Context)
b := newEventbus()
ToContext(c, b)
c1 := make(chan *Event)
c2 := make(chan *Event)
Subscribe(c, c1)
Subscribe(c, c2)
g.Assert(len(b.subs)).Equal(2)
})
g.It("Should subscribe", func() {
c := new(gin.Context)
b := newEventbus()
ToContext(c, b)
c1 := make(chan *Event)
c2 := make(chan *Event)
Subscribe(c, c1)
Subscribe(c, c2)
g.Assert(len(b.subs)).Equal(2)
Unsubscribe(c, c1)
Unsubscribe(c, c2)
g.Assert(len(b.subs)).Equal(0)
})
g.It("Should publish", func() {
c := new(gin.Context)
b := New()
ToContext(c, b)
e1 := NewEvent(Started, &model.Repo{}, &model.Build{}, &model.Job{})
e2 := NewEvent(Started, &model.Repo{}, &model.Build{}, &model.Job{})
c1 := make(chan *Event)
Subscribe(c, c1)
var wg sync.WaitGroup
wg.Add(1)
var r1, r2 *Event
go func() {
r1 = <-c1
r2 = <-c1
wg.Done()
}()
Publish(c, e1)
Publish(c, e2)
wg.Wait()
})
})
}

View file

@ -1,21 +0,0 @@
package bus
import "golang.org/x/net/context"
const key = "bus"
// Setter defines a context that enables setting values.
type Setter interface {
Set(string, interface{})
}
// FromContext returns the Bus associated with this context.
func FromContext(c context.Context) Bus {
return c.Value(key).(Bus)
}
// ToContext adds the Bus to this context if it supports
// the Setter interface.
func ToContext(c Setter, b Bus) {
c.Set(key, b)
}

24
cache/helper.go vendored
View file

@ -8,7 +8,7 @@ import (
"golang.org/x/net/context"
)
// GetPerm returns the user permissions repositories from the cache
// GetPerms returns the user permissions repositories from the cache
// associated with the current repository.
func GetPerms(c context.Context, user *model.User, owner, name string) (*model.Perm, error) {
key := fmt.Sprintf("perms:%s:%s/%s",
@ -31,6 +31,28 @@ func GetPerms(c context.Context, user *model.User, owner, name string) (*model.P
return perm, nil
}
// GetTeamPerms returns the user permissions from the cache
// associated with the current organization.
func GetTeamPerms(c context.Context, user *model.User, org string) (*model.Perm, error) {
key := fmt.Sprintf("perms:%s:%s",
user.Login,
org,
)
// if we fetch from the cache we can return immediately
val, err := Get(c, key)
if err == nil {
return val.(*model.Perm), nil
}
// else we try to grab from the remote system and
// populate our cache.
perm, err := remote.TeamPerm(c, user, org)
if err != nil {
return nil, err
}
Set(c, key, perm)
return perm, nil
}
// GetRepos returns the list of user repositories from the cache
// associated with the current context.
func GetRepos(c context.Context, user *model.User) ([]*model.RepoLite, error) {

View file

@ -4,7 +4,6 @@ import (
"io"
"github.com/drone/drone/model"
"github.com/drone/drone/queue"
)
// Client is used to communicate with a Drone server.
@ -70,6 +69,15 @@ type Client interface {
// TeamSecretDel deletes a named team secret.
TeamSecretDel(string, string) error
// GlobalSecretList returns a list of global secrets.
GlobalSecretList() ([]*model.Secret, error)
// GlobalSecretPost create or updates a global secret.
GlobalSecretPost(secret *model.Secret) error
// GlobalSecretDel deletes a named global secret.
GlobalSecretDel(secret string) error
// Build returns a repository build by number.
Build(string, string, int) (*model.Build, error)
@ -103,27 +111,4 @@ type Client interface {
// AgentList returns a list of build agents.
AgentList() ([]*model.Agent, error)
//
// below items for Queue (internal use only)
//
// Pull pulls work from the server queue.
Pull(os, arch string) (*queue.Work, error)
// Push pushes an update to the server.
Push(*queue.Work) error
// Stream streams the build logs to the server.
Stream(int64, io.ReadCloser) error
LogStream(int64) (StreamWriter, error)
LogPost(int64, io.ReadCloser) error
// Wait waits for the job to the complete.
Wait(int64) *Wait
// Ping the server
Ping() error
}

View file

@ -9,13 +9,11 @@ import (
"io/ioutil"
"net/http"
"net/url"
"os"
"strconv"
"github.com/drone/drone/model"
"github.com/drone/drone/queue"
"github.com/gorilla/websocket"
"golang.org/x/net/context"
"golang.org/x/net/context/ctxhttp"
"golang.org/x/net/proxy"
"golang.org/x/oauth2"
)
@ -44,6 +42,8 @@ const (
pathRepoSecret = "%s/api/repos/%s/%s/secrets/%s"
pathTeamSecrets = "%s/api/teams/%s/secrets"
pathTeamSecret = "%s/api/teams/%s/secrets/%s"
pathGlobalSecrets = "%s/api/global/secrets"
pathGlobalSecret = "%s/api/global/secrets/%s"
pathNodes = "%s/api/nodes"
pathNode = "%s/api/nodes/%d"
pathUsers = "%s/api/users"
@ -73,18 +73,30 @@ func NewClientToken(uri, token string) Client {
// NewClientTokenTLS returns a client at the specified url that authenticates
// all outbound requests with the given token and tls.Config if provided.
func NewClientTokenTLS(uri, token string, c *tls.Config) Client {
func NewClientTokenTLS(uri, token string, c *tls.Config) (Client, error) {
config := new(oauth2.Config)
auther := config.Client(oauth2.NoContext, &oauth2.Token{AccessToken: token})
if c != nil {
if trans, ok := auther.Transport.(*oauth2.Transport); ok {
if os.Getenv("SOCKS_PROXY") != "" {
dialer, err := proxy.SOCKS5("tcp", os.Getenv("SOCKS_PROXY"), nil, proxy.Direct)
if err != nil {
return nil, err
}
trans.Base = &http.Transport{
TLSClientConfig: c,
Proxy: http.ProxyFromEnvironment,
Dial: dialer.Dial,
}
} else {
trans.Base = &http.Transport{
TLSClientConfig: c,
Proxy: http.ProxyFromEnvironment,
}
}
}
return &client{client: auther, base: uri, token: token}
}
return &client{client: auther, base: uri, token: token}, nil
}
// Self returns the currently authenticated user.
@ -284,7 +296,7 @@ func (c *client) SecretDel(owner, name, secret string) error {
return c.delete(uri)
}
// TeamSecretList returns a list of a repository secrets.
// TeamSecretList returns a list of organizational secrets.
func (c *client) TeamSecretList(team string) ([]*model.Secret, error) {
var out []*model.Secret
uri := fmt.Sprintf(pathTeamSecrets, c.base, team)
@ -292,18 +304,38 @@ func (c *client) TeamSecretList(team string) ([]*model.Secret, error) {
return out, err
}
// TeamSecretPost create or updates a repository secret.
// TeamSecretPost create or updates a organizational secret.
func (c *client) TeamSecretPost(team string, secret *model.Secret) error {
uri := fmt.Sprintf(pathTeamSecrets, c.base, team)
return c.post(uri, secret, nil)
}
// TeamSecretDel deletes a named repository secret.
// TeamSecretDel deletes a named orgainization secret.
func (c *client) TeamSecretDel(team, secret string) error {
uri := fmt.Sprintf(pathTeamSecret, c.base, team, secret)
return c.delete(uri)
}
// GlobalSecretList returns a list of global secrets.
func (c *client) GlobalSecretList() ([]*model.Secret, error) {
var out []*model.Secret
uri := fmt.Sprintf(pathGlobalSecrets, c.base)
err := c.get(uri, &out)
return out, err
}
// GlobalSecretPost create or updates a global secret.
func (c *client) GlobalSecretPost(secret *model.Secret) error {
uri := fmt.Sprintf(pathGlobalSecrets, c.base)
return c.post(uri, secret, nil)
}
// GlobalSecretDel deletes a named global secret.
func (c *client) GlobalSecretDel(secret string) error {
uri := fmt.Sprintf(pathGlobalSecret, c.base, secret)
return c.delete(uri)
}
// Sign returns a cryptographic signature for the input string.
func (c *client) Sign(owner, name string, in []byte) ([]byte, error) {
uri := fmt.Sprintf(pathSign, c.base, owner, name)
@ -323,110 +355,6 @@ func (c *client) AgentList() ([]*model.Agent, error) {
return out, err
}
//
// below items for Queue (internal use only)
//
// Pull pulls work from the server queue.
func (c *client) Pull(os, arch string) (*queue.Work, error) {
out := new(queue.Work)
uri := fmt.Sprintf(pathPull, c.base, os, arch)
err := c.post(uri, nil, out)
return out, err
}
// Push pushes an update to the server.
func (c *client) Push(p *queue.Work) error {
uri := fmt.Sprintf(pathPush, c.base, p.Job.ID)
err := c.post(uri, p, nil)
return err
}
// Ping pings the server.
func (c *client) Ping() error {
uri := fmt.Sprintf(pathPing, c.base)
err := c.post(uri, nil, nil)
return err
}
// Stream streams the build logs to the server.
func (c *client) Stream(id int64, rc io.ReadCloser) error {
uri := fmt.Sprintf(pathStream, c.base, id)
err := c.post(uri, rc, nil)
return err
}
// LogPost sends the full build logs to the server.
func (c *client) LogPost(id int64, rc io.ReadCloser) error {
uri := fmt.Sprintf(pathLogs, c.base, id)
return c.post(uri, rc, nil)
}
// StreamWriter implements a special writer for streaming log entries to the
// central Drone server. The standard implementation is the gorilla.Connection.
type StreamWriter interface {
Close() error
WriteJSON(interface{}) error
}
// LogStream streams the build logs to the server.
func (c *client) LogStream(id int64) (StreamWriter, error) {
rawurl := fmt.Sprintf(pathLogsAuth, c.base, id, c.token)
uri, err := url.Parse(rawurl)
if err != nil {
return nil, err
}
if uri.Scheme == "https" {
uri.Scheme = "wss"
} else {
uri.Scheme = "ws"
}
// TODO need TLS client configuration
conn, _, err := websocket.DefaultDialer.Dial(uri.String(), nil)
return conn, err
}
// Wait watches and waits for the build to cancel or finish.
func (c *client) Wait(id int64) *Wait {
ctx, cancel := context.WithCancel(context.Background())
return &Wait{id, c, ctx, cancel}
}
type Wait struct {
id int64
client *client
ctx context.Context
cancel context.CancelFunc
}
func (w *Wait) Done() (*model.Job, error) {
uri := fmt.Sprintf(pathWait, w.client.base, w.id)
req, err := w.client.createRequest(uri, "POST", nil)
if err != nil {
return nil, err
}
res, err := ctxhttp.Do(w.ctx, w.client.client, req)
if err != nil {
return nil, err
}
defer res.Body.Close()
job := &model.Job{}
err = json.NewDecoder(res.Body).Decode(&job)
if err != nil {
return nil, err
}
return job, nil
}
func (w *Wait) Cancel() {
w.cancel()
}
//
// http request helper functions
//

View file

@ -3,17 +3,19 @@ package agent
import (
"os"
"os/signal"
"strings"
"sync"
"syscall"
"time"
"github.com/drone/drone/client"
"github.com/drone/drone/shared/token"
"github.com/samalba/dockerclient"
"github.com/drone/drone/model"
"github.com/drone/mq/logger"
"github.com/drone/mq/stomp"
"github.com/tidwall/redlog"
"github.com/Sirupsen/logrus"
"github.com/codegangsta/cli"
"strings"
"github.com/samalba/dockerclient"
)
// AgentCmd is the exported command for starting the drone agent.
@ -25,7 +27,7 @@ var AgentCmd = cli.Command{
cli.StringFlag{
EnvVar: "DOCKER_HOST",
Name: "docker-host",
Usage: "docker deamon address",
Usage: "docker daemon address",
Value: "unix:///var/run/docker.sock",
},
cli.BoolFlag{
@ -57,17 +59,11 @@ var AgentCmd = cli.Command{
Usage: "docker architecture system",
Value: "amd64",
},
cli.StringFlag{
EnvVar: "DRONE_STORAGE_DRIVER",
Name: "drone-storage-driver",
Usage: "docker storage driver",
Value: "overlay",
},
cli.StringFlag{
EnvVar: "DRONE_SERVER",
Name: "drone-server",
Usage: "drone server address",
Value: "http://localhost:8000",
Value: "ws://localhost:8000/ws/broker",
},
cli.StringFlag{
EnvVar: "DRONE_TOKEN",
@ -100,7 +96,12 @@ var AgentCmd = cli.Command{
EnvVar: "DRONE_TIMEOUT",
Name: "timeout",
Usage: "drone timeout due to log inactivity",
Value: time.Minute * 5,
Value: time.Minute * 15,
},
cli.StringFlag{
EnvVar: "DRONE_FILTER",
Name: "filter",
Usage: "filter jobs processed by this agent",
},
cli.IntFlag{
EnvVar: "DRONE_MAX_LOGS",
@ -132,35 +133,41 @@ var AgentCmd = cli.Command{
Name: "pull",
Usage: "always pull latest plugin images",
},
cli.StringSliceFlag{
EnvVar: "DRONE_YAML_EXTENSION",
Name: "extension",
Usage: "custom plugin extension endpoint",
},
},
}
func start(c *cli.Context) {
log := redlog.New(os.Stderr)
log.SetLevel(0)
logger.SetLogger(log)
// debug level if requested by user
if c.Bool("debug") {
logrus.SetLevel(logrus.DebugLevel)
log.SetLevel(1)
} else {
logrus.SetLevel(logrus.WarnLevel)
}
var accessToken string
if c.String("drone-secret") != "" {
secretToken := c.String("drone-secret")
accessToken, _ = token.New(token.AgentToken, "").Sign(secretToken)
// secretToken := c.String("drone-secret")
accessToken = c.String("drone-secret")
// accessToken, _ = token.New(token.AgentToken, "").Sign(secretToken)
} else {
accessToken = c.String("drone-token")
}
logrus.Infof("Connecting to %s with token %s",
c.String("drone-server"),
accessToken,
)
logger.Noticef("connecting to server %s", c.String("drone-server"))
client := client.NewClientToken(
strings.TrimRight(c.String("drone-server"), "/"),
accessToken,
)
server := strings.TrimRight(c.String("drone-server"), "/")
tls, err := dockerclient.TLSConfigFromCertPath(c.String("docker-cert-path"))
if err == nil {
@ -171,19 +178,15 @@ func start(c *cli.Context) {
logrus.Fatal(err)
}
go func() {
for {
if err := client.Ping(); err != nil {
logrus.Warnf("unable to ping the server. %s", err.Error())
}
time.Sleep(c.Duration("ping"))
}
var client *stomp.Client
handler := func(m *stomp.Message) {
running.Add(1)
defer func() {
running.Done()
client.Ack(m.Ack)
}()
var wg sync.WaitGroup
for i := 0; i < c.Int("docker-max-procs"); i++ {
wg.Add(1)
go func() {
r := pipeline{
drone: client,
docker: docker,
@ -194,19 +197,58 @@ func start(c *cli.Context) {
privileged: c.StringSlice("privileged"),
pull: c.BoolT("pull"),
logs: int64(c.Int("max-log-size")) * 1000000,
extension: c.StringSlice("extension"),
},
}
for {
if err := r.run(); err != nil {
dur := c.Duration("backoff")
logrus.Warnf("reconnect in %v. %s", dur, err.Error())
time.Sleep(dur)
}
}
}()
work := new(model.Work)
m.Unmarshal(work)
r.run(work)
}
handleSignals()
wg.Wait()
backoff := c.Duration("backoff")
for {
// dial the drone server to establish a TCP connection.
client, err = stomp.Dial(server)
if err != nil {
logger.Warningf("connection failed, retry in %v. %s", backoff, err)
<-time.After(backoff)
continue
}
opts := []stomp.MessageOption{
stomp.WithCredentials("x-token", accessToken),
}
// initialize the stomp session and authenticate.
if err = client.Connect(opts...); err != nil {
logger.Warningf("session failed, retry in %v. %s", backoff, err)
<-time.After(backoff)
continue
}
opts = []stomp.MessageOption{
stomp.WithAck("client"),
stomp.WithPrefetch(
c.Int("docker-max-procs"),
),
}
if filter := c.String("filter"); filter != "" {
opts = append(opts, stomp.WithSelector(filter))
}
// subscribe to the pending build queue.
client.Subscribe("/queue/pending", stomp.HandlerFunc(func(m *stomp.Message) {
go handler(m) // HACK until we a channel based Subscribe implementation
}), opts...)
logger.Noticef("connection established, ready to process builds.")
<-client.Done()
logger.Warningf("connection interrupted, attempting to reconnect.")
}
}
// tracks running builds
@ -220,10 +262,10 @@ func handleSignals() {
go func() {
<-c
logrus.Debugln("SIGTERM received.")
logrus.Debugln("wait for running builds to finish.")
logger.Warningf("SIGTERM received.")
logger.Warningf("wait for running builds to finish.")
running.Wait()
logrus.Debugln("done.")
logger.Warningf("done.")
os.Exit(0)
}()
}

View file

@ -1,14 +1,13 @@
package agent
import (
"bytes"
"io/ioutil"
"time"
"github.com/Sirupsen/logrus"
"github.com/drone/drone/agent"
"github.com/drone/drone/build/docker"
"github.com/drone/drone/client"
"github.com/drone/drone/model"
"github.com/drone/mq/stomp"
"github.com/samalba/dockerclient"
)
@ -20,23 +19,20 @@ type config struct {
pull bool
logs int64
timeout time.Duration
extension []string
}
type pipeline struct {
drone client.Client
drone *stomp.Client
docker dockerclient.Client
config config
}
func (r *pipeline) run() error {
w, err := r.drone.Pull("linux", "amd64")
if err != nil {
return err
}
running.Add(1)
defer func() {
running.Done()
}()
func (r *pipeline) run(w *model.Work) {
// defer func() {
// // r.drone.Ack(id, opts)
// }()
logrus.Infof("Starting build %s/%s#%d.%d",
w.Repo.Owner, w.Repo.Name, w.Build.Number, w.Job.Number)
@ -44,53 +40,46 @@ func (r *pipeline) run() error {
cancel := make(chan bool, 1)
engine := docker.NewClient(r.docker)
// streaming the logs
// rc, wc := io.Pipe()
// defer func() {
// wc.Close()
// rc.Close()
// }()
var buf bytes.Buffer
stream, err := r.drone.LogStream(w.Job.ID)
if err != nil {
return err
}
a := agent.Agent{
Update: agent.NewClientUpdater(r.drone),
// Logger: agent.NewClientLogger(r.drone, w.Job.ID, rc, wc, r.config.logs),
Logger: agent.NewStreamLogger(stream, &buf, r.config.logs),
Logger: agent.NewClientLogger(r.drone, w.Job.ID, r.config.logs),
Engine: engine,
Timeout: r.config.timeout,
Platform: r.config.platform,
Namespace: r.config.namespace,
Escalate: r.config.privileged,
Extension: r.config.extension,
Pull: r.config.pull,
}
// signal for canceling the build.
wait := r.drone.Wait(w.Job.ID)
defer wait.Cancel()
go func() {
if _, err := wait.Done(); err == nil {
cancelFunc := func(m *stomp.Message) {
defer m.Release()
id := m.Header.GetInt64("job-id")
if id == w.Job.ID {
cancel <- true
logrus.Infof("Cancel build %s/%s#%d.%d",
w.Repo.Owner, w.Repo.Name, w.Build.Number, w.Job.Number)
}
}
// signal for canceling the build.
sub, err := r.drone.Subscribe("/topic/cancel", stomp.HandlerFunc(cancelFunc))
if err != nil {
logrus.Errorf("Error subscribing to /topic/cancel. %s", err)
}
defer func() {
r.drone.Unsubscribe(sub)
}()
a.Run(w, cancel)
if err := r.drone.LogPost(w.Job.ID, ioutil.NopCloser(&buf)); err != nil {
logrus.Errorf("Error sending logs for %s/%s#%d.%d",
w.Repo.Owner, w.Repo.Name, w.Build.Number, w.Job.Number)
}
stream.Close()
// if err := r.drone.LogPost(w.Job.ID, ioutil.NopCloser(&buf)); err != nil {
// logrus.Errorf("Error sending logs for %s/%s#%d.%d",
// w.Repo.Owner, w.Repo.Name, w.Build.Number, w.Job.Number)
// }
// stream.Close()
logrus.Infof("Finished build %s/%s#%d.%d",
w.Repo.Owner, w.Repo.Name, w.Build.Number, w.Job.Number)
return nil
}

View file

@ -13,7 +13,6 @@ import (
"github.com/drone/drone/agent"
"github.com/drone/drone/build/docker"
"github.com/drone/drone/model"
"github.com/drone/drone/queue"
"github.com/drone/drone/yaml"
"github.com/codegangsta/cli"
@ -96,7 +95,7 @@ var execCmd = cli.Command{
cli.StringFlag{
EnvVar: "DOCKER_HOST",
Name: "docker-host",
Usage: "docker deamon address",
Usage: "docker daemon address",
Value: "unix:///var/run/docker.sock",
},
cli.BoolFlag{
@ -340,7 +339,7 @@ func exec(c *cli.Context) error {
Pull: c.Bool("pull"),
}
payload := &queue.Work{
payload := &model.Work{
Yaml: string(file),
Verified: c.BoolT("yaml.verified"),
Signed: c.BoolT("yaml.signed"),

11
drone/global.go Normal file
View file

@ -0,0 +1,11 @@
package main
import "github.com/codegangsta/cli"
var globalCmd = cli.Command{
Name: "global",
Usage: "manage global state",
Subcommands: []cli.Command{
globalSecretCmd,
},
}

13
drone/global_secret.go Normal file
View file

@ -0,0 +1,13 @@
package main
import "github.com/codegangsta/cli"
var globalSecretCmd = cli.Command{
Name: "secret",
Usage: "manage secrets",
Subcommands: []cli.Command{
globalSecretAddCmd,
globalSecretRemoveCmd,
globalSecretListCmd,
},
}

View file

@ -0,0 +1,41 @@
package main
import (
"log"
"github.com/codegangsta/cli"
)
var globalSecretAddCmd = cli.Command{
Name: "add",
Usage: "adds a secret",
ArgsUsage: "[key] [value]",
Action: func(c *cli.Context) {
if err := globalSecretAdd(c); err != nil {
log.Fatalln(err)
}
},
Flags: secretAddFlags(),
}
func globalSecretAdd(c *cli.Context) error {
if len(c.Args()) != 2 {
cli.ShowSubcommandHelp(c)
return nil
}
name := c.Args().First()
value := c.Args().Get(1)
secret, err := secretParseCmd(name, value, c)
if err != nil {
return err
}
client, err := newClient(c)
if err != nil {
return err
}
return client.GlobalSecretPost(secret)
}

View file

@ -0,0 +1 @@
package main

View file

@ -0,0 +1,33 @@
package main
import (
"log"
"github.com/codegangsta/cli"
)
var globalSecretListCmd = cli.Command{
Name: "ls",
Usage: "list all secrets",
Action: func(c *cli.Context) {
if err := globalSecretList(c); err != nil {
log.Fatalln(err)
}
},
Flags: secretListFlags(),
}
func globalSecretList(c *cli.Context) error {
client, err := newClient(c)
if err != nil {
return err
}
secrets, err := client.GlobalSecretList()
if err != nil || len(secrets) == 0 {
return err
}
return secretDisplayList(secrets, c)
}

33
drone/global_secret_rm.go Normal file
View file

@ -0,0 +1,33 @@
package main
import (
"log"
"github.com/codegangsta/cli"
)
var globalSecretRemoveCmd = cli.Command{
Name: "rm",
Usage: "remove a secret",
Action: func(c *cli.Context) {
if err := globalSecretRemove(c); err != nil {
log.Fatalln(err)
}
},
}
func globalSecretRemove(c *cli.Context) error {
if len(c.Args()) != 1 {
cli.ShowSubcommandHelp(c)
return nil
}
secret := c.Args().First()
client, err := newClient(c)
if err != nil {
return err
}
return client.GlobalSecretDel(secret)
}

View file

@ -43,6 +43,7 @@ func main() {
repoCmd,
userCmd,
orgCmd,
globalCmd,
}
app.Run(os.Args)

View file

@ -1,13 +1,9 @@
package main
import (
"fmt"
"io/ioutil"
"log"
"strings"
"github.com/codegangsta/cli"
"github.com/drone/drone/model"
)
var orgSecretAddCmd = cli.Command{
@ -19,26 +15,7 @@ var orgSecretAddCmd = cli.Command{
log.Fatalln(err)
}
},
Flags: []cli.Flag{
cli.StringSliceFlag{
Name: "event",
Usage: "inject the secret for these event types",
Value: &cli.StringSlice{
model.EventPush,
model.EventTag,
model.EventDeploy,
},
},
cli.StringSliceFlag{
Name: "image",
Usage: "inject the secret for these image types",
Value: &cli.StringSlice{},
},
cli.StringFlag{
Name: "input",
Usage: "input secret value from a file",
},
},
Flags: secretAddFlags(),
}
func orgSecretAdd(c *cli.Context) error {
@ -51,27 +28,9 @@ func orgSecretAdd(c *cli.Context) error {
name := c.Args().Get(1)
value := c.Args().Get(2)
secret := &model.Secret{}
secret.Name = name
secret.Value = value
secret.Images = c.StringSlice("image")
secret.Events = c.StringSlice("event")
if len(secret.Images) == 0 {
return fmt.Errorf("Please specify the --image parameter")
}
// TODO(bradrydzewski) below we use an @ sybmol to denote that the secret
// value should be loaded from a file (inspired by curl). I'd prefer to use
// a --input flag to explicitly specify a filepath instead.
if strings.HasPrefix(secret.Value, "@") {
path := secret.Value[1:]
out, ferr := ioutil.ReadFile(path)
if ferr != nil {
return ferr
}
secret.Value = string(out)
secret, err := secretParseCmd(name, value, c)
if err != nil {
return err
}
client, err := newClient(c)

View file

@ -2,9 +2,6 @@ package main
import (
"log"
"os"
"strings"
"text/template"
"github.com/codegangsta/cli"
)
@ -17,21 +14,7 @@ var orgSecretListCmd = cli.Command{
log.Fatalln(err)
}
},
Flags: []cli.Flag{
cli.StringFlag{
Name: "format",
Usage: "format output",
Value: tmplOrgSecretList,
},
cli.StringFlag{
Name: "image",
Usage: "filter by image",
},
cli.StringFlag{
Name: "event",
Usage: "filter by event",
},
},
Flags: secretListFlags(),
}
func orgSecretList(c *cli.Context) error {
@ -53,35 +36,5 @@ func orgSecretList(c *cli.Context) error {
return err
}
tmpl, err := template.New("_").Funcs(orgSecretFuncMap).Parse(c.String("format") + "\n")
if err != nil {
return err
}
for _, secret := range secrets {
if c.String("image") != "" && !stringInSlice(c.String("image"), secret.Images) {
continue
}
if c.String("event") != "" && !stringInSlice(c.String("event"), secret.Events) {
continue
}
tmpl.Execute(os.Stdout, secret)
}
return nil
}
// template for secret list items
var tmplOrgSecretList = "\x1b[33m{{ .Name }} \x1b[0m" + `
Images: {{ list .Images }}
Events: {{ list .Events }}
`
var orgSecretFuncMap = template.FuncMap{
"list": func(s []string) string {
return strings.Join(s, ", ")
},
return secretDisplayList(secrets, c)
}

View file

@ -10,5 +10,6 @@ var repoCmd = cli.Command{
repoInfoCmd,
repoAddCmd,
repoRemoveCmd,
repoChownCmd,
},
}

37
drone/repo_chown.go Normal file
View file

@ -0,0 +1,37 @@
package main
import (
"fmt"
"log"
"github.com/codegangsta/cli"
)
var repoChownCmd = cli.Command{
Name: "chown",
Usage: "assume ownership of a repository",
Action: func(c *cli.Context) {
if err := repoChown(c); err != nil {
log.Fatalln(err)
}
},
}
func repoChown(c *cli.Context) error {
repo := c.Args().First()
owner, name, err := parseRepo(repo)
if err != nil {
return err
}
client, err := newClient(c)
if err != nil {
return err
}
if _, err := client.RepoChown(owner, name); err != nil {
return err
}
fmt.Printf("Successfully assumed ownership of repository %s/%s\n", owner, name)
return nil
}

View file

@ -1,6 +1,14 @@
package main
import "github.com/codegangsta/cli"
import (
"io/ioutil"
"os"
"strings"
"text/template"
"github.com/codegangsta/cli"
"github.com/drone/drone/model"
)
var secretCmd = cli.Command{
Name: "secret",
@ -11,3 +19,113 @@ var secretCmd = cli.Command{
secretListCmd,
},
}
func secretAddFlags() []cli.Flag {
return []cli.Flag{
cli.StringSliceFlag{
Name: "event",
Usage: "inject the secret for these event types",
Value: &cli.StringSlice{
model.EventPush,
model.EventTag,
model.EventDeploy,
},
},
cli.StringSliceFlag{
Name: "image",
Usage: "inject the secret for these image types",
Value: &cli.StringSlice{},
},
cli.StringFlag{
Name: "input",
Usage: "input secret value from a file",
},
cli.BoolFlag{
Name: "skip-verify",
Usage: "skip verification for the secret",
},
cli.BoolFlag{
Name: "conceal",
Usage: "conceal secret in build logs",
},
}
}
func secretListFlags() []cli.Flag {
return []cli.Flag{
cli.StringFlag{
Name: "format",
Usage: "format output",
Value: tmplSecretList,
},
cli.StringFlag{
Name: "image",
Usage: "filter by image",
},
cli.StringFlag{
Name: "event",
Usage: "filter by event",
},
}
}
func secretParseCmd(name string, value string, c *cli.Context) (*model.Secret, error) {
secret := &model.Secret{}
secret.Name = name
secret.Value = value
secret.Images = c.StringSlice("image")
secret.Events = c.StringSlice("event")
secret.SkipVerify = c.Bool("skip-verify")
secret.Conceal = c.Bool("conceal")
// TODO(bradrydzewski) below we use an @ sybmol to denote that the secret
// value should be loaded from a file (inspired by curl). I'd prefer to use
// a --input flag to explicitly specify a filepath instead.
if strings.HasPrefix(secret.Value, "@") {
path := secret.Value[1:]
out, ferr := ioutil.ReadFile(path)
if ferr != nil {
return nil, ferr
}
secret.Value = string(out)
}
return secret, nil
}
func secretDisplayList(secrets []*model.Secret, c *cli.Context) error {
tmpl, err := template.New("_").Funcs(secretFuncMap).Parse(c.String("format") + "\n")
if err != nil {
return err
}
for _, secret := range secrets {
if c.String("image") != "" && !stringInSlice(c.String("image"), secret.Images) {
continue
}
if c.String("event") != "" && !stringInSlice(c.String("event"), secret.Events) {
continue
}
tmpl.Execute(os.Stdout, secret)
}
return nil
}
// template for secret list items
var tmplSecretList = "\x1b[33m{{ .Name }} \x1b[0m" + `
Events: {{ list .Events }}
SkipVerify: {{ .SkipVerify }}
Conceal: {{ .Conceal }}
`
var secretFuncMap = template.FuncMap{
"list": func(s []string) string {
return strings.Join(s, ", ")
},
}

View file

@ -1,13 +1,9 @@
package main
import (
"fmt"
"io/ioutil"
"log"
"strings"
"github.com/codegangsta/cli"
"github.com/drone/drone/model"
)
var secretAddCmd = cli.Command{
@ -19,26 +15,7 @@ var secretAddCmd = cli.Command{
log.Fatalln(err)
}
},
Flags: []cli.Flag{
cli.StringSliceFlag{
Name: "event",
Usage: "inject the secret for these event types",
Value: &cli.StringSlice{
model.EventPush,
model.EventTag,
model.EventDeploy,
},
},
cli.StringSliceFlag{
Name: "image",
Usage: "inject the secret for these image types",
Value: &cli.StringSlice{},
},
cli.StringFlag{
Name: "input",
Usage: "input secret value from a file",
},
},
Flags: secretAddFlags(),
}
func secretAdd(c *cli.Context) error {
@ -54,27 +31,9 @@ func secretAdd(c *cli.Context) error {
return nil
}
secret := &model.Secret{}
secret.Name = tail[0]
secret.Value = tail[1]
secret.Images = c.StringSlice("image")
secret.Events = c.StringSlice("event")
if len(secret.Images) == 0 {
return fmt.Errorf("Please specify the --image parameter")
}
// TODO(bradrydzewski) below we use an @ sybmol to denote that the secret
// value should be loaded from a file (inspired by curl). I'd prefer to use
// a --input flag to explicitly specify a filepath instead.
if strings.HasPrefix(secret.Value, "@") {
path := secret.Value[1:]
out, ferr := ioutil.ReadFile(path)
if ferr != nil {
return ferr
}
secret.Value = string(out)
secret, err := secretParseCmd(tail[0], tail[1], c)
if err != nil {
return err
}
client, err := newClient(c)

View file

@ -2,9 +2,6 @@ package main
import (
"log"
"os"
"strings"
"text/template"
"github.com/codegangsta/cli"
)
@ -17,21 +14,7 @@ var secretListCmd = cli.Command{
log.Fatalln(err)
}
},
Flags: []cli.Flag{
cli.StringFlag{
Name: "format",
Usage: "format output",
Value: tmplSecretList,
},
cli.StringFlag{
Name: "image",
Usage: "filter by image",
},
cli.StringFlag{
Name: "event",
Usage: "filter by event",
},
},
Flags: secretListFlags(),
}
func secretList(c *cli.Context) error {
@ -53,35 +36,5 @@ func secretList(c *cli.Context) error {
return err
}
tmpl, err := template.New("_").Funcs(secretFuncMap).Parse(c.String("format") + "\n")
if err != nil {
return err
}
for _, secret := range secrets {
if c.String("image") != "" && !stringInSlice(c.String("image"), secret.Images) {
continue
}
if c.String("event") != "" && !stringInSlice(c.String("event"), secret.Events) {
continue
}
tmpl.Execute(os.Stdout, secret)
}
return nil
}
// template for secret list items
var tmplSecretList = "\x1b[33m{{ .Name }} \x1b[0m" + `
Images: {{ list .Images }}
Events: {{ list .Events }}
`
var secretFuncMap = template.FuncMap{
"list": func(s []string) string {
return strings.Join(s, ", ")
},
return secretDisplayList(secrets, c)
}

View file

@ -6,10 +6,10 @@ import (
"github.com/drone/drone/router"
"github.com/drone/drone/router/middleware"
"github.com/gin-gonic/contrib/ginrus"
"github.com/Sirupsen/logrus"
"github.com/codegangsta/cli"
"github.com/gin-gonic/contrib/ginrus"
)
var serverCmd = cli.Command{
@ -26,6 +26,11 @@ var serverCmd = cli.Command{
Name: "debug",
Usage: "start the server in debug mode",
},
cli.BoolFlag{
EnvVar: "DRONE_BROKER_DEBUG",
Name: "broker-debug",
Usage: "start the broker in debug mode",
},
cli.StringFlag{
EnvVar: "DRONE_SERVER_ADDR",
Name: "server-addr",
@ -64,8 +69,8 @@ var serverCmd = cli.Command{
Value: ".drone.yml",
},
cli.DurationFlag{
EnvVar: "DRONE_CACHE_TTY",
Name: "cache-tty",
EnvVar: "DRONE_CACHE_TTL",
Name: "cache-ttl",
Usage: "cache duration",
Value: time.Minute * 15,
},
@ -288,13 +293,11 @@ func server(c *cli.Context) error {
ginrus.Ginrus(logrus.StandardLogger(), time.RFC3339, true),
middleware.Version,
middleware.Config(c),
middleware.Queue(c),
middleware.Stream(c),
middleware.Bus(c),
middleware.Cache(c),
middleware.Store(c),
middleware.Remote(c),
middleware.Agents(c),
middleware.Broker(c),
)
// start the server with tls enabled

View file

@ -15,7 +15,7 @@ import (
func newClient(c *cli.Context) (client.Client, error) {
var token = c.GlobalString("token")
var server = c.GlobalString("server")
var server = strings.TrimRight(c.GlobalString("server"), "/")
// if no server url is provided we can default
// to the hosted Drone service.
@ -31,7 +31,7 @@ func newClient(c *cli.Context) (client.Client, error) {
tlsConfig := &tls.Config{RootCAs: certs}
// create the drone client with TLS options
return client.NewClientTokenTLS(server, token, tlsConfig), nil
return client.NewClientTokenTLS(server, token, tlsConfig)
}
func parseRepo(str string) (user, repo string, err error) {

View file

@ -5,6 +5,7 @@ type Build struct {
ID int64 `json:"id" meddler:"build_id,pk"`
RepoID int64 `json:"-" meddler:"build_repo_id"`
Number int `json:"number" meddler:"build_number"`
Parent int `json:"parent" meddler:"build_parent"`
Event string `json:"event" meddler:"build_event"`
Status string `json:"status" meddler:"build_status"`
Enqueued int64 `json:"enqueued_at" meddler:"build_enqueued"`
@ -26,6 +27,7 @@ type Build struct {
Link string `json:"link_url" meddler:"build_link"`
Signed bool `json:"signed" meddler:"build_signed"`
Verified bool `json:"verified" meddler:"build_verified"`
Jobs []*Job `json:"jobs,omitempty" meddler:"-"`
}
type BuildGroup struct {

View file

@ -1,6 +1,4 @@
package bus
import "github.com/drone/drone/model"
package model
// EventType defines the possible types of build events.
type EventType string
@ -15,14 +13,14 @@ const (
// Event represents a build event.
type Event struct {
Type EventType `json:"type"`
Repo model.Repo `json:"repo"`
Build model.Build `json:"build"`
Job model.Job `json:"job"`
Repo Repo `json:"repo"`
Build Build `json:"build"`
Job Job `json:"job"`
}
// NewEvent creates a new Event for the build, using copies of
// the build data to avoid possible mutation or race conditions.
func NewEvent(t EventType, r *model.Repo, b *model.Build, j *model.Job) *Event {
func NewEvent(t EventType, r *Repo, b *Build, j *Job) *Event {
return &Event{
Type: t,
Repo: *r,
@ -31,7 +29,7 @@ func NewEvent(t EventType, r *model.Repo, b *model.Build, j *model.Job) *Event {
}
}
func NewBuildEvent(t EventType, r *model.Repo, b *model.Build) *Event {
func NewBuildEvent(t EventType, r *Repo, b *Build) *Event {
return &Event{
Type: t,
Repo: *r,

View file

@ -20,6 +20,12 @@ type RepoSecret struct {
// the secret is restricted to this list of events.
Events []string `json:"event,omitempty" meddler:"secret_events,json"`
// whether the secret requires verification
SkipVerify bool `json:"skip_verify" meddler:"secret_skip_verify"`
// whether the secret should be concealed in the build log
Conceal bool `json:"conceal" meddler:"secret_conceal"`
}
// Secret transforms a repo secret into a simple secret.
@ -29,6 +35,8 @@ func (s *RepoSecret) Secret() *Secret {
Value: s.Value,
Images: s.Images,
Events: s.Events,
SkipVerify: s.SkipVerify,
Conceal: s.Conceal,
}
}
@ -39,6 +47,8 @@ func (s *RepoSecret) Clone() *RepoSecret {
Name: s.Name,
Images: s.Images,
Events: s.Events,
SkipVerify: s.SkipVerify,
Conceal: s.Conceal,
}
}

View file

@ -18,6 +18,12 @@ type Secret struct {
// the secret is restricted to this list of events.
Events []string `json:"event,omitempty"`
// whether the secret requires verification
SkipVerify bool `json:"skip_verify"`
// whether the secret should be concealed in the build log
Conceal bool `json:"conceal"`
}
// Match returns true if an image and event match the restricted list.

View file

@ -48,6 +48,11 @@ func TestSecret(t *testing.T) {
// image is only authorized for golang, not golang:1.4.2
g.Assert(secret.MatchImage("golang:1.4.2")).IsFalse()
})
g.It("should not match empty image", func() {
secret := Secret{}
secret.Images = []string{}
g.Assert(secret.MatchImage("node")).IsFalse()
})
g.It("should not match event", func() {
secret := Secret{}
secret.Events = []string{"pull_request"}

View file

@ -20,6 +20,12 @@ type TeamSecret struct {
// the secret is restricted to this list of events.
Events []string `json:"event,omitempty" meddler:"team_secret_events,json"`
// whether the secret requires verification
SkipVerify bool `json:"skip_verify" meddler:"team_secret_skip_verify"`
// whether the secret should be concealed in the build log
Conceal bool `json:"conceal" meddler:"team_secret_conceal"`
}
// Secret transforms a repo secret into a simple secret.
@ -29,6 +35,8 @@ func (s *TeamSecret) Secret() *Secret {
Value: s.Value,
Images: s.Images,
Events: s.Events,
SkipVerify: s.SkipVerify,
Conceal: s.Conceal,
}
}
@ -39,6 +47,8 @@ func (s *TeamSecret) Clone() *TeamSecret {
Name: s.Name,
Images: s.Images,
Events: s.Events,
SkipVerify: s.SkipVerify,
Conceal: s.Conceal,
}
}

19
model/work.go Normal file
View file

@ -0,0 +1,19 @@
package model
// Work represents an item for work to be
// processed by a worker.
type Work struct {
Signed bool `json:"signed"`
Verified bool `json:"verified"`
Yaml string `json:"config"`
YamlEnc string `json:"secret"`
Repo *Repo `json:"repo"`
Build *Build `json:"build"`
BuildLast *Build `json:"build_last"`
Job *Job `json:"job"`
Netrc *Netrc `json:"netrc"`
Keys *Key `json:"keys"`
System *System `json:"system"`
Secrets []*Secret `json:"secrets"`
User *User `json:"user"`
}

View file

@ -1,23 +0,0 @@
package queue
import (
"golang.org/x/net/context"
)
const key = "queue"
// Setter defines a context that enables setting values.
type Setter interface {
Set(string, interface{})
}
// FromContext returns the Queue associated with this context.
func FromContext(c context.Context) Queue {
return c.Value(key).(Queue)
}
// ToContext adds the Queue to this context if it supports
// the Setter interface.
func ToContext(c Setter, q Queue) {
c.Set(key, q)
}

View file

@ -1,67 +0,0 @@
package queue
//go:generate mockery -name Queue -output mock -case=underscore
import (
"errors"
"golang.org/x/net/context"
)
// ErrNotFound indicates the requested work item does not
// exist in the queue.
var ErrNotFound = errors.New("queue item not found")
type Queue interface {
// Publish inserts work at the tail of this queue, waiting for
// space to become available if the queue is full.
Publish(*Work) error
// Remove removes the specified work item from this queue,
// if it is present.
Remove(*Work) error
// PullClose retrieves and removes the head of this queue,
// waiting if necessary until work becomes available.
Pull() *Work
// PullClose retrieves and removes the head of this queue,
// waiting if necessary until work becomes available. The
// CloseNotifier should be provided to clone the channel
// if the subscribing client terminates its connection.
PullClose(CloseNotifier) *Work
}
// Publish inserts work at the tail of this queue, waiting for
// space to become available if the queue is full.
func Publish(c context.Context, w *Work) error {
return FromContext(c).Publish(w)
}
// Remove removes the specified work item from this queue,
// if it is present.
func Remove(c context.Context, w *Work) error {
return FromContext(c).Remove(w)
}
// Pull retrieves and removes the head of this queue,
// waiting if necessary until work becomes available.
func Pull(c context.Context) *Work {
return FromContext(c).Pull()
}
// PullClose retrieves and removes the head of this queue,
// waiting if necessary until work becomes available. The
// CloseNotifier should be provided to clone the channel
// if the subscribing client terminates its connection.
func PullClose(c context.Context, cn CloseNotifier) *Work {
return FromContext(c).PullClose(cn)
}
// CloseNotifier defines a datastructure that is capable of notifying
// a subscriber when its connection is closed.
type CloseNotifier interface {
// CloseNotify returns a channel that receives a single value
// when the client connection has gone away.
CloseNotify() <-chan bool
}

View file

@ -1,85 +0,0 @@
package queue
import "sync"
type queue struct {
sync.Mutex
items map[*Work]struct{}
itemc chan *Work
}
func New() Queue {
return newQueue()
}
func newQueue() *queue {
return &queue{
items: make(map[*Work]struct{}),
itemc: make(chan *Work, 999),
}
}
func (q *queue) Publish(work *Work) error {
q.Lock()
q.items[work] = struct{}{}
q.Unlock()
q.itemc <- work
return nil
}
func (q *queue) Remove(work *Work) error {
q.Lock()
defer q.Unlock()
_, ok := q.items[work]
if !ok {
return ErrNotFound
}
var items []*Work
// loop through and drain all items
// from the
drain:
for {
select {
case item := <-q.itemc:
items = append(items, item)
default:
break drain
}
}
// re-add all items to the queue except
// the item we're trying to remove
for _, item := range items {
if item == work {
delete(q.items, work)
continue
}
q.itemc <- item
}
return nil
}
func (q *queue) Pull() *Work {
work := <-q.itemc
q.Lock()
delete(q.items, work)
q.Unlock()
return work
}
func (q *queue) PullClose(cn CloseNotifier) *Work {
for {
select {
case <-cn.CloseNotify():
return nil
case work := <-q.itemc:
q.Lock()
delete(q.items, work)
q.Unlock()
return work
}
}
}

View file

@ -1,93 +0,0 @@
package queue
import (
"sync"
"testing"
. "github.com/franela/goblin"
"github.com/gin-gonic/gin"
)
func TestBuild(t *testing.T) {
g := Goblin(t)
g.Describe("Queue", func() {
g.It("Should publish item", func() {
c := new(gin.Context)
q := newQueue()
ToContext(c, q)
w1 := &Work{}
w2 := &Work{}
Publish(c, w1)
Publish(c, w2)
g.Assert(len(q.items)).Equal(2)
g.Assert(len(q.itemc)).Equal(2)
})
g.It("Should remove item", func() {
c := new(gin.Context)
q := newQueue()
ToContext(c, q)
w1 := &Work{}
w2 := &Work{}
w3 := &Work{}
Publish(c, w1)
Publish(c, w2)
Publish(c, w3)
Remove(c, w2)
g.Assert(len(q.items)).Equal(2)
g.Assert(len(q.itemc)).Equal(2)
g.Assert(Pull(c)).Equal(w1)
g.Assert(Pull(c)).Equal(w3)
g.Assert(Remove(c, w2)).Equal(ErrNotFound)
})
g.It("Should pull item", func() {
c := new(gin.Context)
q := New()
ToContext(c, q)
cn := new(closeNotifier)
cn.closec = make(chan bool, 1)
w1 := &Work{}
w2 := &Work{}
Publish(c, w1)
g.Assert(Pull(c)).Equal(w1)
Publish(c, w2)
g.Assert(PullClose(c, cn)).Equal(w2)
})
g.It("Should cancel pulling item", func() {
c := new(gin.Context)
q := New()
ToContext(c, q)
cn := new(closeNotifier)
cn.closec = make(chan bool, 1)
var wg sync.WaitGroup
go func() {
wg.Add(1)
g.Assert(PullClose(c, cn) == nil).IsTrue()
wg.Done()
}()
go func() {
cn.closec <- true
}()
wg.Wait()
})
})
}
type closeNotifier struct {
closec chan bool
}
func (c *closeNotifier) CloseNotify() <-chan bool {
return c.closec
}

View file

@ -1,21 +0,0 @@
package queue
import "github.com/drone/drone/model"
// Work represents an item for work to be
// processed by a worker.
type Work struct {
Signed bool `json:"signed"`
Verified bool `json:"verified"`
Yaml string `json:"config"`
YamlEnc string `json:"secret"`
Repo *model.Repo `json:"repo"`
Build *model.Build `json:"build"`
BuildLast *model.Build `json:"build_last"`
Job *model.Job `json:"job"`
Netrc *model.Netrc `json:"netrc"`
Keys *model.Key `json:"keys"`
System *model.System `json:"system"`
Secrets []*model.Secret `json:"secrets"`
User *model.User `json:"user"`
}

View file

@ -39,13 +39,23 @@ func New(client, secret string) remote.Remote {
// Login authenticates an account with Bitbucket using the oauth2 protocol. The
// Bitbucket account details are returned when the user is successfully authenticated.
func (c *config) Login(w http.ResponseWriter, r *http.Request) (*model.User, error) {
redirect := httputil.GetURL(r)
func (c *config) Login(w http.ResponseWriter, req *http.Request) (*model.User, error) {
redirect := httputil.GetURL(req)
config := c.newConfig(redirect)
code := r.FormValue("code")
// get the OAuth errors
if err := req.FormValue("error"); err != "" {
return nil, &remote.AuthError{
Err: err,
Description: req.FormValue("error_description"),
URI: req.FormValue("error_uri"),
}
}
// get the OAuth code
code := req.FormValue("code")
if len(code) == 0 {
http.Redirect(w, r, config.AuthCodeURL("drone"), http.StatusSeeOther)
http.Redirect(w, req, config.AuthCodeURL("drone"), http.StatusSeeOther)
return nil, nil
}
@ -104,6 +114,11 @@ func (c *config) Teams(u *model.User) ([]*model.Team, error) {
return convertTeamList(resp.Values), nil
}
// TeamPerm is not supported by the Bitbucket driver.
func (c *config) TeamPerm(u *model.User, org string) (*model.Perm, error) {
return nil, nil
}
// Repo returns the named Bitbucket repository.
func (c *config) Repo(u *model.User, owner, name string) (*model.Repo, error) {
repo, err := c.newClient(u).FindRepo(owner, name)
@ -232,8 +247,8 @@ func (c *config) Netrc(u *model.User, r *model.Repo) (*model.Netrc, error) {
// Hook parses the incoming Bitbucket hook and returns the Repository and
// Build details. If the hook is unsupported nil values are returned.
func (c *config) Hook(r *http.Request) (*model.Repo, *model.Build, error) {
return parseHook(r)
func (c *config) Hook(req *http.Request) (*model.Repo, *model.Build, error) {
return parseHook(req)
}
// helper function to return the bitbucket oauth2 client

View file

@ -70,6 +70,11 @@ func Test_bitbucket(t *testing.T) {
_, err := c.Login(nil, r)
g.Assert(err != nil).IsTrue()
})
g.It("Should handle authentication errors", func() {
r, _ := http.NewRequest("GET", "?error=invalid_scope", nil)
_, err := c.Login(nil, r)
g.Assert(err != nil).IsTrue()
})
})
g.Describe("Given an access token", func() {

View file

@ -2,6 +2,7 @@ package bitbucket
import (
"fmt"
"regexp"
"net/url"
"strings"
@ -152,7 +153,11 @@ func convertPullHook(from *internal.PullRequestHook) *model.Build {
Event: model.EventPull,
Commit: from.PullRequest.Dest.Commit.Hash,
Ref: fmt.Sprintf("refs/heads/%s", from.PullRequest.Dest.Branch.Name),
Remote: cloneLink(&from.PullRequest.Dest.Repo),
Refspec: fmt.Sprintf("%s:%s",
from.PullRequest.Source.Branch.Name,
from.PullRequest.Dest.Branch.Name,
),
Remote: fmt.Sprintf("https://bitbucket.org/%s", from.PullRequest.Source.Repo.FullName),
Link: from.PullRequest.Links.Html.Href,
Branch: from.PullRequest.Dest.Branch.Name,
Message: from.PullRequest.Desc,
@ -182,5 +187,20 @@ func convertPushHook(hook *internal.PushHook, change *internal.Change) *model.Bu
build.Event = model.EventPush
build.Ref = fmt.Sprintf("refs/heads/%s", change.New.Name)
}
if len(change.New.Target.Author.Raw) != 0 {
build.Email = extractEmail(change.New.Target.Author.Raw)
}
return build
}
// regex for git author fields ("name <name@mail.tld>")
var reGitMail = regexp.MustCompile("<(.*)>")
// extracts the email from a git commit author string
func extractEmail(gitauthor string) (author string) {
matches := reGitMail.FindAllStringSubmatch(gitauthor,-1)
if len(matches) == 1 {
author = matches[0][1]
}
return
}

View file

@ -139,6 +139,8 @@ func Test_helper(t *testing.T) {
hook.PullRequest.Dest.Commit.Hash = "73f9c44d"
hook.PullRequest.Dest.Branch.Name = "master"
hook.PullRequest.Dest.Repo.Links.Html.Href = "https://bitbucket.org/foo/bar"
hook.PullRequest.Source.Branch.Name = "change"
hook.PullRequest.Source.Repo.FullName = "baz/bar"
hook.PullRequest.Links.Html.Href = "https://bitbucket.org/foo/bar/pulls/5"
hook.PullRequest.Desc = "updated README"
hook.PullRequest.Updated = time.Now()
@ -151,6 +153,8 @@ func Test_helper(t *testing.T) {
g.Assert(build.Branch).Equal(hook.PullRequest.Dest.Branch.Name)
g.Assert(build.Link).Equal(hook.PullRequest.Links.Html.Href)
g.Assert(build.Ref).Equal("refs/heads/master")
g.Assert(build.Refspec).Equal("change:master")
g.Assert(build.Remote).Equal("https://bitbucket.org/baz/bar")
g.Assert(build.Message).Equal(hook.PullRequest.Desc)
g.Assert(build.Timestamp).Equal(hook.PullRequest.Updated.Unix())
})
@ -162,6 +166,7 @@ func Test_helper(t *testing.T) {
change.New.Target.Links.Html.Href = "https://bitbucket.org/foo/bar/commits/73f9c44d"
change.New.Target.Message = "updated README"
change.New.Target.Date = time.Now()
change.New.Target.Author.Raw = "Test <test@domain.tld>"
hook := internal.PushHook{}
hook.Actor.Login = "octocat"
@ -169,6 +174,7 @@ func Test_helper(t *testing.T) {
build := convertPushHook(&hook, &change)
g.Assert(build.Event).Equal(model.EventPush)
g.Assert(build.Email).Equal("test@domain.tld")
g.Assert(build.Author).Equal(hook.Actor.Login)
g.Assert(build.Avatar).Equal(hook.Actor.Links.Avatar.Href)
g.Assert(build.Commit).Equal(change.New.Target.Hash)

View file

@ -27,6 +27,11 @@ func Handler() http.Handler {
}
func getOauth(c *gin.Context) {
switch c.PostForm("error") {
case "invalid_scope":
c.String(500, "")
}
switch c.PostForm("code") {
case "code_bad_request":
c.String(500, "")

View file

@ -33,6 +33,7 @@ const HookPush = `
"type": "commit",
"hash": "709d658dc5b6d6afcd46049c2f332ee3f515a67d",
"author": {
"raw": "emmap1 <email@domain.tld>",
"username": "emmap1",
"links": {
"avatar": {

View file

@ -115,6 +115,11 @@ func (*Config) Teams(u *model.User) ([]*model.Team, error) {
return teams, nil
}
// TeamPerm is not supported by the Stash driver.
func (*Config) TeamPerm(u *model.User, org string) (*model.Perm, error) {
return nil, nil
}
func (c *Config) Repo(u *model.User, owner, name string) (*model.Repo, error) {
repo, err := internal.NewClientWithToken(c.URL, c.Consumer, u.Token).FindRepo(owner, name)
if err != nil {

23
remote/errors.go Normal file
View file

@ -0,0 +1,23 @@
package remote
// AuthError represents remote authentication error.
type AuthError struct {
Err string
Description string
URI string
}
// Error implements error interface.
func (ae *AuthError) Error() string {
err := ae.Err
if ae.Description != "" {
err += " " + ae.Description
}
if ae.URI != "" {
err += " " + ae.URI
}
return err
}
// check interface
var _ error = new(AuthError)

View file

@ -69,6 +69,11 @@ func (c *client) Teams(u *model.User) ([]*model.Team, error) {
return empty, nil
}
// TeamPerm is not supported by the Gerrit driver.
func (c *client) TeamPerm(u *model.User, org string) (*model.Perm, error) {
return nil, nil
}
// Repo is not supported by the Gerrit driver.
func (c *client) Repo(u *model.User, owner, name string) (*model.Repo, error) {
return nil, nil

View file

@ -28,6 +28,7 @@ const (
const (
headRefs = "refs/pull/%d/head" // pull request unmerged
mergeRefs = "refs/pull/%d/merge" // pull request merged with base
refspec = "%s:%s"
)
// convertStatus is a helper function used to convert a Drone status to a
@ -93,6 +94,18 @@ func convertPerm(from *github.Repository) *model.Perm {
}
}
// convertTeamPerm is a helper function used to convert a GitHub organization
// permissions to the common Drone permissions structure.
func convertTeamPerm(from *github.Membership) *model.Perm {
admin := false
if *from.Role == "admin" {
admin = true
}
return &model.Perm{
Admin: admin,
}
}
// convertRepoList is a helper function used to convert a GitHub repository
// list to the common Drone repository structure.
func convertRepoList(from []github.Repository) []*model.RepoLite {
@ -224,11 +237,16 @@ func convertPullHook(from *webhook, merge bool) *model.Build {
Commit: from.PullRequest.Head.SHA,
Link: from.PullRequest.HTMLURL,
Ref: fmt.Sprintf(headRefs, from.PullRequest.Number),
Branch: from.PullRequest.Head.Ref,
Branch: from.PullRequest.Base.Ref,
Message: from.PullRequest.Title,
Author: from.PullRequest.User.Login,
Avatar: from.PullRequest.User.Avatar,
Title: from.PullRequest.Title,
Remote: from.PullRequest.Head.Repo.CloneURL,
Refspec: fmt.Sprintf(refspec,
from.PullRequest.Head.Ref,
from.PullRequest.Base.Ref,
),
}
if merge {
build.Ref = fmt.Sprintf(mergeRefs, from.PullRequest.Number)

View file

@ -172,8 +172,10 @@ func Test_helper(t *testing.T) {
g.It("should convert a pull request from webhook", func() {
from := &webhook{}
from.PullRequest.Head.Ref = "master"
from.PullRequest.Base.Ref = "master"
from.PullRequest.Head.Ref = "changes"
from.PullRequest.Head.SHA = "f72fc19"
from.PullRequest.Head.Repo.CloneURL = "https://github.com/octocat/hello-world-fork"
from.PullRequest.HTMLURL = "https://github.com/octocat/hello-world/pulls/42"
from.PullRequest.Number = 42
from.PullRequest.Title = "Updated README.md"
@ -182,8 +184,10 @@ func Test_helper(t *testing.T) {
build := convertPullHook(from, true)
g.Assert(build.Event).Equal(model.EventPull)
g.Assert(build.Branch).Equal(from.PullRequest.Head.Ref)
g.Assert(build.Branch).Equal(from.PullRequest.Base.Ref)
g.Assert(build.Ref).Equal("refs/pull/42/merge")
g.Assert(build.Refspec).Equal("changes:master")
g.Assert(build.Remote).Equal("https://github.com/octocat/hello-world-fork")
g.Assert(build.Commit).Equal(from.PullRequest.Head.SHA)
g.Assert(build.Message).Equal(from.PullRequest.Title)
g.Assert(build.Title).Equal(from.PullRequest.Title)

View file

@ -13,6 +13,8 @@ func Handler() http.Handler {
e := gin.New()
e.GET("/api/v3/repos/:owner/:name", getRepo)
e.GET("/api/v3/orgs/:org/memberships/:user", getMembership)
e.GET("/api/v3/user/memberships/orgs/:org", getMembership)
return e
}
@ -26,6 +28,17 @@ func getRepo(c *gin.Context) {
}
}
func getMembership(c *gin.Context) {
switch c.Param("org") {
case "org_not_found":
c.String(404, "")
case "github":
c.String(200, membershipIsMemberPayload)
default:
c.String(200, membershipIsOwnerPayload)
}
}
var repoPayload = `
{
"owner": {
@ -45,3 +58,85 @@ var repoPayload = `
}
}
`
var membershipIsOwnerPayload = `
{
"url": "https://api.github.com/orgs/octocat/memberships/octocat",
"state": "active",
"role": "admin",
"organization_url": "https://api.github.com/orgs/octocat",
"user": {
"login": "octocat",
"id": 5555555,
"avatar_url": "https://github.com/images/error/octocat_happy.gif",
"gravatar_id": "",
"url": "https://api.github.com/users/octocat",
"html_url": "https://github.com/octocat",
"followers_url": "https://api.github.com/users/octocat/followers",
"following_url": "https://api.github.com/users/octocat/following{/other_user}",
"gists_url": "https://api.github.com/users/octocat/gists{/gist_id}",
"starred_url": "https://api.github.com/users/octocat/starred{/owner}{/repo}",
"subscriptions_url": "https://api.github.com/users/octocat/subscriptions",
"organizations_url": "https://api.github.com/users/octocat/orgs",
"repos_url": "https://api.github.com/users/octocat/repos",
"events_url": "https://api.github.com/users/octocat/events{/privacy}",
"received_events_url": "https://api.github.com/users/octocat/received_events",
"type": "User",
"site_admin": false
},
"organization": {
"login": "octocat",
"id": 5555556,
"url": "https://api.github.com/orgs/octocat",
"repos_url": "https://api.github.com/orgs/octocat/repos",
"events_url": "https://api.github.com/orgs/octocat/events",
"hooks_url": "https://api.github.com/orgs/octocat/hooks",
"issues_url": "https://api.github.com/orgs/octocat/issues",
"members_url": "https://api.github.com/orgs/octocat/members{/member}",
"public_members_url": "https://api.github.com/orgs/octocat/public_members{/member}",
"avatar_url": "https://github.com/images/error/octocat_happy.gif",
"description": ""
}
}
`
var membershipIsMemberPayload = `
{
"url": "https://api.github.com/orgs/github/memberships/octocat",
"state": "active",
"role": "member",
"organization_url": "https://api.github.com/orgs/github",
"user": {
"login": "octocat",
"id": 5555555,
"avatar_url": "https://github.com/images/error/octocat_happy.gif",
"gravatar_id": "",
"url": "https://api.github.com/users/octocat",
"html_url": "https://github.com/octocat",
"followers_url": "https://api.github.com/users/octocat/followers",
"following_url": "https://api.github.com/users/octocat/following{/other_user}",
"gists_url": "https://api.github.com/users/octocat/gists{/gist_id}",
"starred_url": "https://api.github.com/users/octocat/starred{/owner}{/repo}",
"subscriptions_url": "https://api.github.com/users/octocat/subscriptions",
"organizations_url": "https://api.github.com/users/octocat/orgs",
"repos_url": "https://api.github.com/users/octocat/repos",
"events_url": "https://api.github.com/users/octocat/events{/privacy}",
"received_events_url": "https://api.github.com/users/octocat/received_events",
"type": "User",
"site_admin": false
},
"organization": {
"login": "octocat",
"id": 5555557,
"url": "https://api.github.com/orgs/github",
"repos_url": "https://api.github.com/orgs/github/repos",
"events_url": "https://api.github.com/orgs/github/events",
"hooks_url": "https://api.github.com/orgs/github/hooks",
"issues_url": "https://api.github.com/orgs/github/issues",
"members_url": "https://api.github.com/orgs/github/members{/member}",
"public_members_url": "https://api.github.com/orgs/github/public_members{/member}",
"avatar_url": "https://github.com/images/error/octocat_happy.gif",
"description": ""
}
}
`

View file

@ -70,6 +70,11 @@ const HookPullRequest = `
"login": "baxterthehacker",
"avatar_url": "https://avatars.githubusercontent.com/u/6752317?v=3"
},
"base": {
"label": "baxterthehacker:master",
"ref": "master",
"sha": "9353195a19e45482665306e466c832c46560532d"
},
"head": {
"label": "baxterthehacker:changes",
"ref": "changes",

View file

@ -92,6 +92,16 @@ type client struct {
func (c *client) Login(res http.ResponseWriter, req *http.Request) (*model.User, error) {
config := c.newConfig(httputil.GetURL(req))
// get the OAuth errors
if err := req.FormValue("error"); err != "" {
return nil, &remote.AuthError{
Err: err,
Description: req.FormValue("error_description"),
URI: req.FormValue("error_uri"),
}
}
// get the OAuth code
code := req.FormValue("code")
if len(code) == 0 {
// TODO(bradrydzewski) we really should be using a random value here and
@ -158,6 +168,16 @@ func (c *client) Teams(u *model.User) ([]*model.Team, error) {
return teams, nil
}
// TeamPerm returns the user permissions for the named GitHub organization.
func (c *client) TeamPerm(u *model.User, org string) (*model.Perm, error) {
client := c.newClientToken(u.Token)
membership, _, err := client.Organizations.GetOrgMembership(u.Login, org)
if err != nil {
return nil, err
}
return convertTeamPerm(membership), nil
}
// Repo returns the named GitHub repository.
func (c *client) Repo(u *model.User, owner, name string) (*model.Repo, error) {
client := c.newClientToken(u.Token)

View file

@ -110,6 +110,23 @@ func Test_github(t *testing.T) {
})
})
g.Describe("Requesting organization permissions", func() {
g.It("Should return the permission details of an admin", func() {
perm, err := c.TeamPerm(fakeUser, "octocat")
g.Assert(err == nil).IsTrue()
g.Assert(perm.Admin).IsTrue()
})
g.It("Should return the permission details of a member", func() {
perm, err := c.TeamPerm(fakeUser, "github")
g.Assert(err == nil).IsTrue()
g.Assert(perm.Admin).IsFalse()
})
g.It("Should handle a not found error", func() {
_, err := c.TeamPerm(fakeUser, "org_not_found")
g.Assert(err != nil).IsTrue()
})
})
g.It("Should return a user repository list")
g.It("Should return a user team list")
@ -123,6 +140,7 @@ func Test_github(t *testing.T) {
g.It("Should create an access token")
g.It("Should handle an access token error")
g.It("Should return the authenticated user")
g.It("Should handle authentication errors")
})
})
}

View file

@ -68,9 +68,16 @@ type webhook struct {
Avatar string `json:"avatar_url"`
} `json:"user"`
Base struct {
Ref string `json:"ref"`
} `json:"base"`
Head struct {
SHA string
Ref string
SHA string `json:"sha"`
Ref string `json:"ref"`
Repo struct {
CloneURL string `json:"clone_url"`
} `json:"repo"`
} `json:"head"`
} `json:"pull_request"`
}

View file

@ -115,6 +115,15 @@ func (g *Gitlab) Login(res http.ResponseWriter, req *http.Request) (*model.User,
TLSClientConfig: &tls.Config{InsecureSkipVerify: g.SkipVerify},
}
// get the OAuth errors
if err := req.FormValue("error"); err != "" {
return nil, &remote.AuthError{
Err: err,
Description: req.FormValue("error_description"),
URI: req.FormValue("error_uri"),
}
}
// get the OAuth code
var code = req.FormValue("code")
if len(code) == 0 {
@ -194,6 +203,11 @@ func (g *Gitlab) Teams(u *model.User) ([]*model.Team, error) {
return teams, nil
}
// TeamPerm is not supported by the Gitlab driver.
func (g *Gitlab) TeamPerm(u *model.User, org string) (*model.Perm, error) {
return nil, nil
}
// Repo fetches the named repository from the remote system.
func (g *Gitlab) Repo(u *model.User, owner, name string) (*model.Repo, error) {
client := NewClient(g.URL, u.Token, g.SkipVerify)

View file

@ -30,12 +30,13 @@ func getRepo(c *gin.Context) {
}
func getRepoFile(c *gin.Context) {
switch c.Param("file") {
case "file_not_found":
if c.Param("file") == "file_not_found" {
c.String(404, "")
default:
}
if c.Param("commit") == "v1.0.0" || c.Param("commit") == "9ecad50" {
c.String(200, repoFilePayload)
}
c.String(404, "")
}
func createRepoHook(c *gin.Context) {

View file

@ -1,6 +1,6 @@
package fixtures
// old version ?
// Sample Gogs push hook
var HookPush = `
{
"ref": "refs/heads/master",
@ -22,7 +22,10 @@ var HookPush = `
"repository": {
"id": 1,
"name": "hello-world",
"url": "http://gogs.golang.org/gordon/hello-world",
"full_name": "gordon/hello-world",
"html_url": "http://gogs.golang.org/gordon/hello-world",
"ssh_url": "git@gogs.golang.org:gordon/hello-world.git",
"clone_url": "http://gogs.golang.org/gordon/hello-world.git",
"description": "",
"website": "",
"watchers": 1,
@ -46,222 +49,89 @@ var HookPush = `
}
`
// Sampled from Gogs version 0.9.97
// X-Gogs-Event: push
var HookPushNew = `
{
"secret": "a_secret",
"ref": "refs/heads/master",
"before": "117b1990205dbc6395656ef1ed2125719aa7f4d3",
"after": "7d7605add378b55e6154d96b3e0957d392e2cc14",
"compare_url": "http://cdb:3000/org1/test3/compare/117b1990205dbc6395656ef1ed2125719aa7f4d3...7d7605add378b55e6154d96b3e0957d392e2cc14",
"commits": [
{
"id": "7d7605add378b55e6154d96b3e0957d392e2cc14",
"message": "Capitalize\n",
"url": "http://cdb:3000/org1/test3/commit/7d7605add378b55e6154d96b3e0957d392e2cc14",
"author": {
"name": "Sandro Santilli",
"email": "strk@kbt.io",
"username": "strk"
},
"committer": {
"name": "Sandro Santilli",
"email": "strk@kbt.io",
"username": "strk"
},
"timestamp": "2016-08-31T22:51:59+02:00"
},
{
"id": "85800d8ecf8107626dc43a0cbdf218c31cd04779",
"message": "dot\n",
"url": "http://cdb:3000/org1/test3/commit/85800d8ecf8107626dc43a0cbdf218c31cd04779",
"author": {
"name": "Sandro Santilli",
"email": "strk@kbt.io",
"username": "strk"
},
"committer": {
"name": "Sandro Santilli",
"email": "strk@kbt.io",
"username": "strk"
},
"timestamp": "2016-08-31T22:46:53+02:00"
}
],
// Sample Gogs tag hook
var HookPushTag = `{
"secret": "l26Un7G7HXogLAvsyf2hOA4EMARSTsR3",
"ref": "v1.0.0",
"ref_type": "tag",
"repository": {
"id": 5,
"owner": {
"id": 5,
"username": "org1",
"full_name": "org1",
"email": "",
"avatar_url": "http://cdb:3000/avatars/5"
},
"name": "test3",
"full_name": "org1/test3",
"description": "just a test",
"private": false,
"fork": false,
"html_url": "http://cdb:3000/org1/test3",
"ssh_url": "strk@git.osgeo.org:org1/test3.git",
"clone_url": "http://cdb:3000/org1/test3.git",
"website": "",
"stars_count": 0,
"forks_count": 1,
"watchers_count": 2,
"open_issues_count": 0,
"default_branch": "master",
"created_at": "2016-08-31T22:45:16+02:00",
"updated_at": "2016-08-31T22:45:31+02:00"
},
"pusher": {
"id": 1,
"username": "strk",
"full_name": "",
"email": "strk@kbt.io",
"avatar_url": "https://avatars.kbt.io/avatar/fe2a9e759730ee64c44bf8901bf4ccc3"
"owner": {
"id": 1,
"username": "gordon",
"full_name": "Gordon the Gopher",
"email": "gordon@golang.org",
"avatar_url": "https://secure.gravatar.com/avatar/8c58a0be77ee441bb8f8595b7f1b4e87"
},
"name": "hello-world",
"full_name": "gordon/hello-world",
"description": "",
"private": true,
"fork": false,
"html_url": "http://gogs.golang.org/gordon/hello-world",
"ssh_url": "git@gogs.golang.org:gordon/hello-world.git",
"clone_url": "http://gogs.golang.org/gordon/hello-world.git",
"default_branch": "master",
"created_at": "2015-10-22T19:32:44Z",
"updated_at": "2016-11-24T13:37:16Z"
},
"sender": {
"id": 1,
"username": "strk",
"full_name": "",
"email": "strk@kbt.io",
"avatar_url": "https://avatars.kbt.io/avatar/fe2a9e759730ee64c44bf8901bf4ccc3"
"username": "gordon",
"full_name": "Gordon the Gopher",
"email": "gordon@golang.org",
"avatar_url": "https://secure.gravatar.com/avatar/8c58a0be77ee441bb8f8595b7f1b4e87"
}
}
`
}`
// Sampled from Gogs version 0.9.97
// X-Gogs-Event: pull_request
var HookPullRequestOpenNew = `
{
"secret": "a_secret",
// HookPullRequest is a sample pull_request webhook payload
var HookPullRequest = `{
"action": "opened",
"number": 1,
"pull_request": {
"id": 2,
"number": 1,
"html_url": "http://gogs.golang.org/gordon/hello-world/pull/1",
"state": "open",
"title": "Update the README with new information",
"body": "please merge",
"user": {
"id": 1,
"username": "strk",
"full_name": "",
"email": "strk@kbt.io",
"avatar_url": "https://avatars.kbt.io/avatar/fe2a9e759730ee64c44bf8901bf4ccc3"
"username": "gordon",
"full_name": "Gordon the Gopher",
"email": "gordon@golang.org",
"avatar_url": "http://gogs.golang.org///1.gravatar.com/avatar/8c58a0be77ee441bb8f8595b7f1b4e87"
},
"title": "dot",
"body": "could you figure",
"labels": [],
"milestone": null,
"assignee": null,
"state": "open",
"comments": 0,
"html_url": "http://cdb:3000/org1/test3/pulls/1",
"mergeable": true,
"merged": false,
"merged_at": null,
"merge_commit_sha": null,
"merged_by": null
"base": {
"label": "master",
"ref": "master",
"sha": "9353195a19e45482665306e466c832c46560532d"
},
"head": {
"label": "feature/changes",
"ref": "feature/changes",
"sha": "0d1a26e67d8f5eaf1f6ba5c57fc3c7d91ac0fd1c"
}
},
"repository": {
"id": 5,
"id": 35129377,
"name": "hello-world",
"full_name": "gordon/hello-world",
"owner": {
"id": 5,
"username": "org1",
"full_name": "org1",
"email": "",
"avatar_url": "http://cdb:3000/avatars/5"
"id": 1,
"username": "gordon",
"full_name": "Gordon the Gopher",
"email": "gordon@golang.org",
"avatar_url": "https://secure.gravatar.com/avatar/8c58a0be77ee441bb8f8595b7f1b4e87"
},
"name": "test3",
"full_name": "org1/test3",
"description": "just a test",
"private": false,
"fork": false,
"html_url": "http://cdb:3000/org1/test3",
"ssh_url": "strk@git.osgeo.org:org1/test3.git",
"clone_url": "http://cdb:3000/org1/test3.git",
"website": "",
"stars_count": 0,
"forks_count": 1,
"watchers_count": 2,
"open_issues_count": 0,
"default_branch": "master",
"created_at": "2016-08-31T22:45:16+02:00",
"updated_at": "2016-08-31T22:45:31+02:00"
"private": true,
"html_url": "http://gogs.golang.org/gordon/hello-world",
"clone_url": "https://gogs.golang.org/gordon/hello-world.git",
"default_branch": "master"
},
"sender": {
"id": 1,
"username": "strk",
"full_name": "",
"email": "strk@kbt.io",
"avatar_url": "https://avatars.kbt.io/avatar/fe2a9e759730ee64c44bf8901bf4ccc3"
"username": "gordon",
"full_name": "Gordon the Gopher",
"email": "gordon@golang.org",
"avatar_url": "https://secure.gravatar.com/avatar/8c58a0be77ee441bb8f8595b7f1b4e87"
}
}
`
// Sampled from Gogs version 0.9.97
// X-Gogs-Event: pull_request
var HookPullRequestSynchronize = `
{
"secret": "a_secret",
"action": "synchronized",
"number": 1,
"pull_request": {
"id": 2,
"number": 1,
"user": {
"id": 1,
"username": "strk",
"full_name": "",
"email": "strk@kbt.io",
"avatar_url": "https://avatars.kbt.io/avatar/fe2a9e759730ee64c44bf8901bf4ccc3"
},
"title": "dot",
"body": "could you figure",
"labels": [],
"milestone": null,
"assignee": null,
"state": "open",
"comments": 0,
"html_url": "http://cdb:3000/org1/test3/pulls/1",
"mergeable": true,
"merged": false,
"merged_at": null,
"merge_commit_sha": null,
"merged_by": null
},
"repository": {
"id": 5,
"owner": {
"id": 5,
"username": "org1",
"full_name": "org1",
"email": "",
"avatar_url": "http://cdb:3000/avatars/5"
},
"name": "test3",
"full_name": "org1/test3",
"description": "just a test",
"private": false,
"fork": false,
"html_url": "http://cdb:3000/org1/test3",
"ssh_url": "strk@git.osgeo.org:org1/test3.git",
"clone_url": "http://cdb:3000/org1/test3.git",
"website": "",
"stars_count": 0,
"forks_count": 1,
"watchers_count": 2,
"open_issues_count": 0,
"default_branch": "master",
"created_at": "2016-08-31T22:45:16+02:00",
"updated_at": "2016-08-31T22:45:31+02:00"
},
"sender": {
"id": 1,
"username": "strk",
"full_name": "",
"email": "strk@kbt.io",
"avatar_url": "https://avatars.kbt.io/avatar/fe2a9e759730ee64c44bf8901bf4ccc3"
}
}
`
}`

View file

@ -6,6 +6,7 @@ import (
"net"
"net/http"
"net/url"
"strings"
"github.com/drone/drone/model"
"github.com/drone/drone/remote"
@ -126,6 +127,11 @@ func (c *client) Teams(u *model.User) ([]*model.Team, error) {
return teams, nil
}
// TeamPerm is not supported by the Gogs driver.
func (c *client) TeamPerm(u *model.User, org string) (*model.Perm, error) {
return nil, nil
}
// Repo returns the named Gogs repository.
func (c *client) Repo(u *model.User, owner, name string) (*model.Repo, error) {
client := c.newClientToken(u.Token)
@ -133,6 +139,9 @@ func (c *client) Repo(u *model.User, owner, name string) (*model.Repo, error) {
if err != nil {
return nil, err
}
if c.PrivateMode {
repo.Private = true
}
return toRepo(repo), nil
}
@ -166,7 +175,18 @@ func (c *client) Perm(u *model.User, owner, name string) (*model.Perm, error) {
// File fetches the file from the Gogs repository and returns its contents.
func (c *client) File(u *model.User, r *model.Repo, b *model.Build, f string) ([]byte, error) {
client := c.newClientToken(u.Token)
cfg, err := client.GetFile(r.Owner, r.Name, b.Commit, f)
buildRef := b.Commit
if buildRef == "" {
// Remove refs/tags or refs/heads, Gogs needs a short ref
buildRef = strings.TrimPrefix(
strings.TrimPrefix(
b.Ref,
"refs/heads/",
),
"refs/tags/",
)
}
cfg, err := client.GetFile(r.Owner, r.Name, buildRef, f)
return cfg, err
}
@ -204,6 +224,7 @@ func (c *client) Activate(u *model.User, r *model.Repo, link string) error {
hook := gogs.CreateHookOption{
Type: "gogs",
Config: config,
Events: []string{"push", "create", "pull_request"},
Active: true,
}
@ -220,22 +241,7 @@ func (c *client) Deactivate(u *model.User, r *model.Repo, link string) error {
// Hook parses the incoming Gogs hook and returns the Repository and Build
// details. If the hook is unsupported nil values are returned.
func (c *client) Hook(r *http.Request) (*model.Repo, *model.Build, error) {
var (
err error
repo *model.Repo
build *model.Build
)
switch r.Header.Get("X-Gogs-Event") {
case "push":
var push *pushHook
push, err = parsePush(r.Body)
if err == nil {
repo = repoFromPush(push)
build = buildFromPush(push)
}
}
return repo, build, err
return parseHook(r)
}
// helper function to return the Gogs client

View file

@ -128,6 +128,12 @@ func Test_gogs(t *testing.T) {
g.Assert(string(raw)).Equal("{ platform: linux/amd64 }")
})
g.It("Should return a repository file from a ref", func() {
raw, err := c.File(fakeUser, fakeRepo, fakeBuildWithRef, ".drone.yml")
g.Assert(err == nil).IsTrue()
g.Assert(string(raw)).Equal("{ platform: linux/amd64 }")
})
g.Describe("Given an authentication request", func() {
g.It("Should redirect to login form")
g.It("Should create an access token")
@ -178,4 +184,8 @@ var (
fakeBuild = &model.Build{
Commit: "9ecad50",
}
fakeBuildWithRef = &model.Build{
Ref: "refs/tags/v1.0.0",
}
)

View file

@ -70,6 +70,11 @@ func buildFromPush(hook *pushHook) *model.Build {
hook.Repo.URL,
fixMalformedAvatar(hook.Sender.Avatar),
)
author := hook.Sender.Login
if author == "" {
author = hook.Sender.Username
}
return &model.Build{
Event: model.EventPush,
Commit: hook.After,
@ -78,22 +83,75 @@ func buildFromPush(hook *pushHook) *model.Build {
Branch: strings.TrimPrefix(hook.Ref, "refs/heads/"),
Message: hook.Commits[0].Message,
Avatar: avatar,
Author: hook.Sender.Login,
Author: author,
Timestamp: time.Now().UTC().Unix(),
}
}
// helper function that extracts the Build data from a Gogs tag hook
func buildFromTag(hook *pushHook) *model.Build {
avatar := expandAvatar(
hook.Repo.URL,
fixMalformedAvatar(hook.Sender.Avatar),
)
author := hook.Sender.Login
if author == "" {
author = hook.Sender.Username
}
return &model.Build{
Event: model.EventTag,
Commit: hook.After,
Ref: fmt.Sprintf("refs/tags/%s", hook.Ref),
Link: fmt.Sprintf("%s/src/%s", hook.Repo.URL, hook.Ref),
Branch: fmt.Sprintf("refs/tags/%s", hook.Ref),
Message: fmt.Sprintf("created tag %s", hook.Ref),
Avatar: avatar,
Author: author,
Timestamp: time.Now().UTC().Unix(),
}
}
// helper function that extracts the Build data from a Gogs pull_request hook
func buildFromPullRequest(hook *pullRequestHook) *model.Build {
avatar := expandAvatar(
hook.Repo.URL,
fixMalformedAvatar(hook.PullRequest.User.Avatar),
)
build := &model.Build{
Event: model.EventPull,
Commit: hook.PullRequest.Head.Sha,
Link: hook.PullRequest.URL,
Ref: fmt.Sprintf("refs/pull/%d/head", hook.Number),
Branch: hook.PullRequest.Base.Ref,
Message: hook.PullRequest.Title,
Author: hook.PullRequest.User.Username,
Avatar: avatar,
Title: hook.PullRequest.Title,
Refspec: fmt.Sprintf("%s:%s",
hook.PullRequest.Head.Ref,
hook.PullRequest.Base.Ref,
),
}
return build
}
// helper function that extracts the Repository data from a Gogs push hook
func repoFromPush(hook *pushHook) *model.Repo {
fullName := fmt.Sprintf(
"%s/%s",
hook.Repo.Owner.Username,
hook.Repo.Name,
)
return &model.Repo{
Name: hook.Repo.Name,
Owner: hook.Repo.Owner.Username,
FullName: fullName,
FullName: hook.Repo.FullName,
Link: hook.Repo.URL,
}
}
// helper function that extracts the Repository data from a Gogs pull_request hook
func repoFromPullRequest(hook *pullRequestHook) *model.Repo {
return &model.Repo{
Name: hook.Repo.Name,
Owner: hook.Repo.Owner.Username,
FullName: hook.Repo.FullName,
Link: hook.Repo.URL,
}
}
@ -105,6 +163,12 @@ func parsePush(r io.Reader) (*pushHook, error) {
return push, err
}
func parsePullRequest(r io.Reader) (*pullRequestHook, error) {
pr := new(pullRequestHook)
err := json.NewDecoder(r).Decode(pr)
return pr, err
}
// fixMalformedAvatar is a helper function that fixes an avatar url if malformed
// (currently a known bug with gogs)
func fixMalformedAvatar(url string) string {

View file

@ -27,6 +27,7 @@ func Test_parse(t *testing.T) {
g.Assert(hook.Repo.Name).Equal("hello-world")
g.Assert(hook.Repo.URL).Equal("http://gogs.golang.org/gordon/hello-world")
g.Assert(hook.Repo.Owner.Name).Equal("gordon")
g.Assert(hook.Repo.FullName).Equal("gordon/hello-world")
g.Assert(hook.Repo.Owner.Email).Equal("gordon@golang.org")
g.Assert(hook.Repo.Owner.Username).Equal("gordon")
g.Assert(hook.Repo.Private).Equal(true)
@ -37,6 +38,47 @@ func Test_parse(t *testing.T) {
g.Assert(hook.Sender.Avatar).Equal("http://gogs.golang.org///1.gravatar.com/avatar/8c58a0be77ee441bb8f8595b7f1b4e87")
})
g.It("Should parse tag hook payload", func() {
buf := bytes.NewBufferString(fixtures.HookPushTag)
hook, err := parsePush(buf)
g.Assert(err == nil).IsTrue()
g.Assert(hook.Ref).Equal("v1.0.0")
g.Assert(hook.Repo.Name).Equal("hello-world")
g.Assert(hook.Repo.URL).Equal("http://gogs.golang.org/gordon/hello-world")
g.Assert(hook.Repo.FullName).Equal("gordon/hello-world")
g.Assert(hook.Repo.Owner.Email).Equal("gordon@golang.org")
g.Assert(hook.Repo.Owner.Username).Equal("gordon")
g.Assert(hook.Repo.Private).Equal(true)
g.Assert(hook.Sender.Username).Equal("gordon")
g.Assert(hook.Sender.Avatar).Equal("https://secure.gravatar.com/avatar/8c58a0be77ee441bb8f8595b7f1b4e87")
})
g.It("Should parse pull_request hook payload", func() {
buf := bytes.NewBufferString(fixtures.HookPullRequest)
hook, err := parsePullRequest(buf)
g.Assert(err == nil).IsTrue()
g.Assert(hook.Action).Equal("opened")
g.Assert(hook.Number).Equal(int64(1))
g.Assert(hook.Repo.Name).Equal("hello-world")
g.Assert(hook.Repo.URL).Equal("http://gogs.golang.org/gordon/hello-world")
g.Assert(hook.Repo.FullName).Equal("gordon/hello-world")
g.Assert(hook.Repo.Owner.Email).Equal("gordon@golang.org")
g.Assert(hook.Repo.Owner.Username).Equal("gordon")
g.Assert(hook.Repo.Private).Equal(true)
g.Assert(hook.Sender.Username).Equal("gordon")
g.Assert(hook.Sender.Avatar).Equal("https://secure.gravatar.com/avatar/8c58a0be77ee441bb8f8595b7f1b4e87")
g.Assert(hook.PullRequest.Title).Equal("Update the README with new information")
g.Assert(hook.PullRequest.Body).Equal("please merge")
g.Assert(hook.PullRequest.State).Equal("open")
g.Assert(hook.PullRequest.User.Username).Equal("gordon")
g.Assert(hook.PullRequest.Base.Label).Equal("master")
g.Assert(hook.PullRequest.Base.Ref).Equal("master")
g.Assert(hook.PullRequest.Head.Label).Equal("feature/changes")
g.Assert(hook.PullRequest.Head.Ref).Equal("feature/changes")
})
g.It("Should return a Build struct from a push hook", func() {
buf := bytes.NewBufferString(fixtures.HookPush)
hook, _ := parsePush(buf)
@ -62,6 +104,31 @@ func Test_parse(t *testing.T) {
g.Assert(repo.Link).Equal(hook.Repo.URL)
})
g.It("Should return a Build struct from a pull_request hook", func() {
buf := bytes.NewBufferString(fixtures.HookPullRequest)
hook, _ := parsePullRequest(buf)
build := buildFromPullRequest(hook)
g.Assert(build.Event).Equal(model.EventPull)
g.Assert(build.Commit).Equal(hook.PullRequest.Head.Sha)
g.Assert(build.Ref).Equal("refs/pull/1/head")
g.Assert(build.Link).Equal(hook.PullRequest.URL)
g.Assert(build.Branch).Equal("master")
g.Assert(build.Message).Equal(hook.PullRequest.Title)
g.Assert(build.Avatar).Equal("http://1.gravatar.com/avatar/8c58a0be77ee441bb8f8595b7f1b4e87")
g.Assert(build.Author).Equal(hook.PullRequest.User.Username)
})
g.It("Should return a Repo struct from a pull_request hook", func() {
buf := bytes.NewBufferString(fixtures.HookPullRequest)
hook, _ := parsePullRequest(buf)
repo := repoFromPullRequest(hook)
g.Assert(repo.Name).Equal(hook.Repo.Name)
g.Assert(repo.Owner).Equal(hook.Repo.Owner.Username)
g.Assert(repo.FullName).Equal("gordon/hello-world")
g.Assert(repo.Link).Equal(hook.Repo.URL)
})
g.It("Should return a Perm struct from a Gogs Perm", func() {
perms := []gogs.Permission{
{true, true, true},

107
remote/gogs/parse.go Normal file
View file

@ -0,0 +1,107 @@
package gogs
import (
"io"
"net/http"
"github.com/drone/drone/model"
)
const (
hookEvent = "X-Gogs-Event"
hookPush = "push"
hookCreated = "create"
hookPullRequest = "pull_request"
actionOpen = "opened"
actionSync = "synchronize"
stateOpen = "open"
refBranch = "branch"
refTag = "tag"
)
// parseHook parses a Bitbucket hook from an http.Request request and returns
// Repo and Build detail. If a hook type is unsupported nil values are returned.
func parseHook(r *http.Request) (*model.Repo, *model.Build, error) {
switch r.Header.Get(hookEvent) {
case hookPush:
return parsePushHook(r.Body)
case hookCreated:
return parseCreatedHook(r.Body)
case hookPullRequest:
return parsePullRequestHook(r.Body)
}
return nil, nil, nil
}
// parsePushHook parses a push hook and returns the Repo and Build details.
// If the commit type is unsupported nil values are returned.
func parsePushHook(payload io.Reader) (*model.Repo, *model.Build, error) {
var (
repo *model.Repo
build *model.Build
)
push, err := parsePush(payload)
if err != nil {
return nil, nil, err
}
// is this even needed?
if push.RefType == refBranch {
return nil, nil, nil
}
repo = repoFromPush(push)
build = buildFromPush(push)
return repo, build, err
}
// parseCreatedHook parses a push hook and returns the Repo and Build details.
// If the commit type is unsupported nil values are returned.
func parseCreatedHook(payload io.Reader) (*model.Repo, *model.Build, error) {
var (
repo *model.Repo
build *model.Build
)
push, err := parsePush(payload)
if err != nil {
return nil, nil, err
}
if push.RefType != refTag {
return nil, nil, nil
}
repo = repoFromPush(push)
build = buildFromTag(push)
return repo, build, err
}
// parsePullRequestHook parses a pull_request hook and returns the Repo and Build details.
func parsePullRequestHook(payload io.Reader) (*model.Repo, *model.Build, error) {
var (
repo *model.Repo
build *model.Build
)
pr, err := parsePullRequest(payload)
if err != nil {
return nil, nil, err
}
// Don't trigger builds for non-code changes, or if PR is not open
if pr.Action != actionOpen && pr.Action != actionSync {
return nil, nil, nil
}
if pr.PullRequest.State != stateOpen {
return nil, nil, nil
}
repo = repoFromPullRequest(pr)
build = buildFromPullRequest(pr)
return repo, build, err
}

View file

@ -0,0 +1 @@
package gogs

View file

@ -5,17 +5,20 @@ type pushHook struct {
Before string `json:"before"`
After string `json:"after"`
Compare string `json:"compare_url"`
RefType string `json:"ref_type"`
Pusher struct {
Name string `json:"name"`
Email string `json:"email"`
Login string `json:"login"`
Username string `json:"username"`
} `json:"pusher"`
Repo struct {
ID int64 `json:"id"`
Name string `json:"name"`
URL string `json:"url"`
FullName string `json:"full_name"`
URL string `json:"html_url"`
Private bool `json:"private"`
Owner struct {
Name string `json:"name"`
@ -33,6 +36,89 @@ type pushHook struct {
Sender struct {
ID int64 `json:"id"`
Login string `json:"login"`
Username string `json:"username"`
Avatar string `json:"avatar_url"`
} `json:"sender"`
}
type pullRequestHook struct {
Action string `json:"action"`
Number int64 `json:"number"`
PullRequest struct {
ID int64 `json:"id"`
User struct {
ID int64 `json:"id"`
Username string `json:"username"`
Name string `json:"full_name"`
Email string `json:"email"`
Avatar string `json:"avatar_url"`
} `json:"user"`
Title string `json:"title"`
Body string `json:"body"`
Labels []string `json:"labels"`
State string `json:"state"`
URL string `json:"html_url"`
Mergeable bool `json:"mergeable"`
Merged bool `json:"merged"`
MergeBase string `json:"merge_base"`
Base struct {
Label string `json:"label"`
Ref string `json:"ref"`
Sha string `json:"sha"`
Repo struct {
ID int64 `json:"id"`
Name string `json:"name"`
FullName string `json:"full_name"`
URL string `json:"html_url"`
Private bool `json:"private"`
Owner struct {
ID int64 `json:"id"`
Username string `json:"username"`
Name string `json:"full_name"`
Email string `json:"email"`
Avatar string `json:"avatar_url"`
} `json:"owner"`
} `json:"repo"`
} `json:"base"`
Head struct {
Label string `json:"label"`
Ref string `json:"ref"`
Sha string `json:"sha"`
Repo struct {
ID int64 `json:"id"`
Name string `json:"name"`
FullName string `json:"full_name"`
URL string `json:"html_url"`
Private bool `json:"private"`
Owner struct {
ID int64 `json:"id"`
Username string `json:"username"`
Name string `json:"full_name"`
Email string `json:"email"`
Avatar string `json:"avatar_url"`
} `json:"owner"`
} `json:"repo"`
} `json:"head"`
} `json:"pull_request"`
Repo struct {
ID int64 `json:"id"`
Name string `json:"name"`
FullName string `json:"full_name"`
URL string `json:"html_url"`
Private bool `json:"private"`
Owner struct {
ID int64 `json:"id"`
Username string `json:"username"`
Name string `json:"full_name"`
Email string `json:"email"`
Avatar string `json:"avatar_url"`
} `json:"owner"`
} `json:"repository"`
Sender struct {
ID int64 `json:"id"`
Username string `json:"username"`
Name string `json:"full_name"`
Email string `json:"email"`
Avatar string `json:"avatar_url"`
} `json:"sender"`
}

View file

@ -267,3 +267,26 @@ func (_m *Remote) Teams(u *model.User) ([]*model.Team, error) {
return r0, r1
}
// TeamPerm provides a mock function with given fields: u, org
func (_m *Remote) TeamPerm(u *model.User, org string) (*model.Perm, error) {
ret := _m.Called(u, org)
var r0 *model.Perm
if rf, ok := ret.Get(0).(func(*model.User, string) *model.Perm); ok {
r0 = rf(u, org)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(*model.Perm)
}
}
var r1 error
if rf, ok := ret.Get(1).(func(*model.User, string) error); ok {
r1 = rf(u, org)
} else {
r1 = ret.Error(1)
}
return r0, r1
}

View file

@ -4,6 +4,7 @@ package remote
import (
"net/http"
"time"
"github.com/drone/drone/model"
@ -22,6 +23,10 @@ type Remote interface {
// Teams fetches a list of team memberships from the remote system.
Teams(u *model.User) ([]*model.Team, error)
// TeamPerm fetches the named organization permissions from
// the remote system for the specified user.
TeamPerm(u *model.User, org string) (*model.Perm, error)
// Repo fetches the named repository from the remote system.
Repo(u *model.User, owner, repo string) (*model.Repo, error)
@ -80,6 +85,12 @@ func Teams(c context.Context, u *model.User) ([]*model.Team, error) {
return FromContext(c).Teams(u)
}
// TeamPerm fetches the named organization permissions from
// the remote system for the specified user.
func TeamPerm(c context.Context, u *model.User, org string) (*model.Perm, error) {
return FromContext(c).TeamPerm(u, org)
}
// Repo fetches the named repository from the remote system.
func Repo(c context.Context, u *model.User, owner, repo string) (*model.Repo, error) {
return FromContext(c).Repo(u, owner, repo)
@ -97,8 +108,15 @@ func Perm(c context.Context, u *model.User, owner, repo string) (*model.Perm, er
}
// File fetches a file from the remote repository and returns in string format.
func File(c context.Context, u *model.User, r *model.Repo, b *model.Build, f string) ([]byte, error) {
return FromContext(c).File(u, r, b, f)
func File(c context.Context, u *model.User, r *model.Repo, b *model.Build, f string) (out []byte, err error) {
for i:=0;i<5;i++ {
out, err = FromContext(c).File(u, r, b, f)
if err == nil {
return
}
time.Sleep(1*time.Second)
}
return
}
// Status sends the commit status to the remote system.

View file

@ -0,0 +1,63 @@
package middleware
import (
"os"
"sync"
handlers "github.com/drone/drone/server"
"github.com/codegangsta/cli"
"github.com/drone/mq/logger"
"github.com/drone/mq/server"
"github.com/drone/mq/stomp"
"github.com/Sirupsen/logrus"
"github.com/gin-gonic/gin"
"github.com/tidwall/redlog"
)
const (
serverKey = "broker"
clientKey = "stomp.client" // mirrored from stomp/context
)
// Broker is a middleware function that initializes the broker
// and adds the broker client to the request context.
func Broker(cli *cli.Context) gin.HandlerFunc {
secret := cli.String("agent-secret")
if secret == "" {
logrus.Fatalf("fatal error. please provide the DRONE_SECRET")
}
// setup broker logging.
log := redlog.New(os.Stderr)
log.SetLevel(2)
logger.SetLogger(log)
if cli.Bool("broker-debug") {
log.SetLevel(1)
}
broker := server.NewServer(
server.WithCredentials("x-token", secret),
)
client := broker.Client()
var once sync.Once
return func(c *gin.Context) {
c.Set(serverKey, broker)
c.Set(clientKey, client)
once.Do(func() {
// this is some really hacky stuff
// turns out I need to do some refactoring
// don't judge!
// will fix in 0.6 release
ctx := c.Copy()
client.Connect(
stomp.WithCredentials("x-token", secret),
)
client.Subscribe("/queue/updates", stomp.HandlerFunc(func(m *stomp.Message) {
go handlers.HandleUpdate(ctx, m.Copy())
}))
})
}
}

View file

@ -1,17 +0,0 @@
package middleware
import (
"github.com/drone/drone/bus"
"github.com/codegangsta/cli"
"github.com/gin-gonic/gin"
)
// Bus is a middleware function that initializes the Event Bus and attaches to
// the context of every http.Request.
func Bus(cli *cli.Context) gin.HandlerFunc {
v := bus.New()
return func(c *gin.Context) {
bus.ToContext(c, v)
}
}

View file

@ -1,17 +0,0 @@
package middleware
import (
"github.com/drone/drone/queue"
"github.com/codegangsta/cli"
"github.com/gin-gonic/gin"
)
// Queue is a middleware function that initializes the Queue and attaches to
// the context of every http.Request.
func Queue(cli *cli.Context) gin.HandlerFunc {
v := queue.New()
return func(c *gin.Context) {
queue.ToContext(c, v)
}
}

View file

@ -1,21 +1,73 @@
package session
import (
"github.com/drone/drone/cache"
"github.com/drone/drone/model"
log "github.com/Sirupsen/logrus"
"github.com/gin-gonic/gin"
)
func TeamPerm(c *gin.Context) *model.Perm {
user := User(c)
team := c.Param("team")
perm := &model.Perm{}
switch {
// if the user is not authenticated
case user == nil:
perm.Admin = false
perm.Pull = false
perm.Push = false
// if the user is a DRONE_ADMIN
case user.Admin:
perm.Admin = true
perm.Pull = true
perm.Push = true
// otherwise if the user is authenticated we should
// check the remote system to get the users permissiosn.
default:
log.Debugf("Fetching team permission for %s %s",
user.Login, team)
var err error
perm, err = cache.GetTeamPerms(c, user, team)
if err != nil {
// debug
log.Errorf("Error fetching team permission for %s %s",
user.Login, team)
perm.Admin = false
perm.Pull = false
perm.Push = false
}
}
if user != nil {
log.Debugf("%s granted %+v team permission to %s",
user.Login, perm, team)
} else {
log.Debugf("Guest granted %+v to %s", perm, team)
perm.Admin = false
perm.Pull = false
perm.Push = false
}
return perm
}
func MustTeamAdmin() gin.HandlerFunc {
return func(c *gin.Context) {
user := User(c)
switch {
case user == nil:
perm := TeamPerm(c)
if perm.Admin {
c.Next()
} else {
c.String(401, "User not authorized")
c.Abort()
case user.Admin == false:
c.String(413, "User not authorized")
c.Abort()
default:
c.Next()
}
}
}

View file

@ -0,0 +1,95 @@
package session
import (
"testing"
"github.com/drone/drone/cache"
"github.com/drone/drone/model"
"github.com/franela/goblin"
"github.com/gin-gonic/gin"
)
func TestTeamPerm(t *testing.T) {
g := goblin.Goblin(t)
g.Describe("TeamPerm", func() {
var c *gin.Context
g.BeforeEach(func() {
c = new(gin.Context)
cache.ToContext(c, cache.Default())
})
g.It("Should set admin to false (user not logged in)", func() {
p := TeamPerm(c)
g.Assert(p.Admin).IsFalse("admin should be false")
})
g.It("Should set admin to true (user is DRONE_ADMIN)", func() {
// Set DRONE_ADMIN user
c.Set("user", fakeUserAdmin)
p := TeamPerm(c)
g.Assert(p.Admin).IsTrue("admin should be false")
})
g.It("Should set admin to false (user logged in, not owner of org)", func() {
// Set fake org
params := gin.Params{
gin.Param{
Key: "team",
Value: "test_org",
},
}
c.Params = params
// Set cache to show user does not Owner/Admin
cache.Set(c, "perms:octocat:test_org", fakeTeamPerm)
// Set User
c.Set("user", fakeUser)
p := TeamPerm(c)
g.Assert(p.Admin).IsFalse("admin should be false")
})
g.It("Should set admin to true (user logged in, owner of org)", func() {
// Set fake org
params := gin.Params{
gin.Param{
Key: "team",
Value: "test_org",
},
}
c.Params = params
// Set cache to show user is Owner/Admin
cache.Set(c, "perms:octocat:test_org", fakeTeamPermAdmin)
// Set User
c.Set("user", fakeUser)
p := TeamPerm(c)
g.Assert(p.Admin).IsTrue("admin should be true")
})
})
}
var (
fakeUserAdmin = &model.User{
Login: "octocatAdmin",
Token: "cfcd2084",
Admin: true,
}
fakeUser = &model.User{
Login: "octocat",
Token: "cfcd2084",
Admin: false,
}
fakeTeamPermAdmin = &model.Perm{
Admin: true,
}
fakeTeamPerm = &model.Perm{
Admin: false,
}
)

View file

@ -1,17 +0,0 @@
package middleware
import (
"github.com/drone/drone/stream"
"github.com/codegangsta/cli"
"github.com/gin-gonic/gin"
)
// Stream is a middleware function that initializes the Stream and attaches to
// the context of every http.Request.
func Stream(cli *cli.Context) gin.HandlerFunc {
v := stream.New()
return func(c *gin.Context) {
stream.ToContext(c, v)
}
}

View file

@ -64,7 +64,7 @@ func Load(middleware ...gin.HandlerFunc) http.Handler {
teams := e.Group("/api/teams")
{
user.Use(session.MustTeamAdmin())
teams.Use(session.MustTeamAdmin())
team := teams.Group("/:team")
{
@ -74,6 +74,15 @@ func Load(middleware ...gin.HandlerFunc) http.Handler {
}
}
global := e.Group("/api/global")
{
global.Use(session.MustAdmin())
global.GET("/secrets", server.GetGlobalSecrets)
global.POST("/secrets", server.PostGlobalSecret)
global.DELETE("/secrets/:secret", server.DeleteGlobalSecret)
}
repos := e.Group("/api/repos/:owner/:name")
{
repos.POST("", server.PostRepo)
@ -113,17 +122,9 @@ func Load(middleware ...gin.HandlerFunc) http.Handler {
e.POST("/hook", server.PostHook)
e.POST("/api/hook", server.PostHook)
stream := e.Group("/api/stream")
{
stream.Use(session.SetRepo())
stream.Use(session.SetPerm())
stream.Use(session.MustPull)
stream.GET("/:owner/:name", server.GetRepoEvents)
stream.GET("/:owner/:name/:build/:number", server.GetStream)
}
ws := e.Group("/ws")
{
ws.GET("/broker", server.Broker)
ws.GET("/feed", server.EventStream)
ws.GET("/logs/:owner/:name/:build/:number",
session.SetRepo(),
@ -152,18 +153,19 @@ func Load(middleware ...gin.HandlerFunc) http.Handler {
agents.GET("", server.GetAgents)
}
queue := e.Group("/api/queue")
debug := e.Group("/api/debug")
{
queue.Use(session.AuthorizeAgent)
queue.POST("/pull", server.Pull)
queue.POST("/pull/:os/:arch", server.Pull)
queue.POST("/wait/:id", server.Wait)
queue.POST("/stream/:id", server.Stream)
queue.POST("/status/:id", server.Update)
queue.POST("/ping", server.Ping)
queue.POST("/logs/:id", server.PostLogs)
queue.GET("/logs/:id", server.WriteLogs)
debug.Use(session.MustAdmin())
debug.GET("/pprof/", server.IndexHandler())
debug.GET("/pprof/heap", server.HeapHandler())
debug.GET("/pprof/goroutine", server.GoroutineHandler())
debug.GET("/pprof/block", server.BlockHandler())
debug.GET("/pprof/threadcreate", server.ThreadCreateHandler())
debug.GET("/pprof/cmdline", server.CmdlineHandler())
debug.GET("/pprof/profile", server.ProfileHandler())
debug.GET("/pprof/symbol", server.SymbolHandler())
debug.POST("/pprof/symbol", server.SymbolHandler())
debug.GET("/pprof/trace", server.TraceHandler())
}
// DELETE THESE

13
server/broker.go Normal file
View file

@ -0,0 +1,13 @@
package server
import (
"net/http"
"github.com/gin-gonic/gin"
)
// Broker handles connections to the embedded message broker.
func Broker(c *gin.Context) {
broker := c.MustGet("broker").(http.Handler)
broker.ServeHTTP(c.Writer, c.Request)
}

View file

@ -1,22 +1,23 @@
package server
import (
"bufio"
"io"
"net/http"
"strconv"
"time"
log "github.com/Sirupsen/logrus"
"github.com/drone/drone/bus"
"github.com/drone/drone/queue"
"github.com/drone/drone/remote"
"github.com/drone/drone/shared/httputil"
"github.com/drone/drone/store"
"github.com/drone/drone/stream"
"github.com/drone/drone/yaml"
"github.com/gin-gonic/gin"
"github.com/square/go-jose"
"github.com/drone/drone/model"
"github.com/drone/drone/router/middleware/session"
"github.com/drone/mq/stomp"
)
func GetBuilds(c *gin.Context) {
@ -112,7 +113,7 @@ func GetBuildLogs(c *gin.Context) {
}
c.Header("Content-Type", "application/json")
stream.Copy(c.Writer, r)
copyLogs(c.Writer, r)
}
func DeleteBuild(c *gin.Context) {
@ -148,7 +149,14 @@ func DeleteBuild(c *gin.Context) {
job.ExitCode = 137
store.UpdateBuildJob(c, build, job)
bus.Publish(c, bus.NewEvent(bus.Cancelled, repo, build, job))
client := stomp.MustFromContext(c)
client.SendJSON("/topic/cancel", model.Event{
Type: model.Cancelled,
Repo: *repo,
Build: *build,
Job: *job,
}, stomp.WithHeader("job-id", strconv.FormatInt(job.ID, 10)))
c.String(204, "")
}
@ -228,6 +236,7 @@ func PostBuild(c *gin.Context) {
if forkit, _ := strconv.ParseBool(fork); forkit {
build.ID = 0
build.Number = 0
build.Parent = num
for _, job := range jobs {
job.ID = 0
job.NodeID = 0
@ -293,7 +302,7 @@ func PostBuild(c *gin.Context) {
last, _ := store.GetBuildLastBefore(c, repo, build.Branch, build.ID)
secs, err := store.GetMergedSecretList(c, repo)
if err != nil {
log.Errorf("Error getting secrets for %s#%d. %s", repo.FullName, build.Number, err)
log.Debugf("Error getting secrets for %s#%d. %s", repo.FullName, build.Number, err)
}
var signed bool
@ -318,9 +327,19 @@ func PostBuild(c *gin.Context) {
log.Debugf(".drone.yml is signed=%v and verified=%v", signed, verified)
bus.Publish(c, bus.NewBuildEvent(bus.Enqueued, repo, build))
client := stomp.MustFromContext(c)
client.SendJSON("/topic/events", model.Event{
Type: model.Enqueued,
Repo: *repo,
Build: *build,
},
stomp.WithHeader("repo", repo.FullName),
stomp.WithHeader("private", strconv.FormatBool(repo.IsPrivate)),
)
for _, job := range jobs {
queue.Publish(c, &queue.Work{
broker, _ := stomp.FromContext(c)
broker.SendJSON("/queue/pending", &model.Work{
Signed: signed,
Verified: verified,
User: user,
@ -332,7 +351,15 @@ func PostBuild(c *gin.Context) {
Yaml: string(raw),
Secrets: secs,
System: &model.System{Link: httputil.GetURL(c.Request)},
})
},
stomp.WithHeader(
"platform",
yaml.ParsePlatformDefault(raw, "linux/amd64"),
),
stomp.WithHeaders(
yaml.ParseLabel(raw),
),
)
}
}
@ -344,3 +371,20 @@ func GetBuildQueue(c *gin.Context) {
}
c.JSON(200, out)
}
// copyLogs copies the stream from the source to the destination in valid JSON
// format. This converts the logs, which are per-line JSON objects, to a
// proper JSON array.
func copyLogs(dest io.Writer, src io.Reader) error {
io.WriteString(dest, "[")
scanner := bufio.NewScanner(src)
for scanner.Scan() {
io.WriteString(dest, scanner.Text())
io.WriteString(dest, ",\n")
}
io.WriteString(dest, "{}]")
return nil
}

70
server/debug.go Normal file
View file

@ -0,0 +1,70 @@
package server
import (
"net/http/pprof"
"github.com/gin-gonic/gin"
)
// IndexHandler will pass the call from /debug/pprof to pprof
func IndexHandler() gin.HandlerFunc {
return func(c *gin.Context) {
pprof.Index(c.Writer, c.Request)
}
}
// HeapHandler will pass the call from /debug/pprof/heap to pprof
func HeapHandler() gin.HandlerFunc {
return func(c *gin.Context) {
pprof.Handler("heap").ServeHTTP(c.Writer, c.Request)
}
}
// GoroutineHandler will pass the call from /debug/pprof/goroutine to pprof
func GoroutineHandler() gin.HandlerFunc {
return func(c *gin.Context) {
pprof.Handler("goroutine").ServeHTTP(c.Writer, c.Request)
}
}
// BlockHandler will pass the call from /debug/pprof/block to pprof
func BlockHandler() gin.HandlerFunc {
return func(c *gin.Context) {
pprof.Handler("block").ServeHTTP(c.Writer, c.Request)
}
}
// ThreadCreateHandler will pass the call from /debug/pprof/threadcreate to pprof
func ThreadCreateHandler() gin.HandlerFunc {
return func(c *gin.Context) {
pprof.Handler("threadcreate").ServeHTTP(c.Writer, c.Request)
}
}
// CmdlineHandler will pass the call from /debug/pprof/cmdline to pprof
func CmdlineHandler() gin.HandlerFunc {
return func(c *gin.Context) {
pprof.Cmdline(c.Writer, c.Request)
}
}
// ProfileHandler will pass the call from /debug/pprof/profile to pprof
func ProfileHandler() gin.HandlerFunc {
return func(c *gin.Context) {
pprof.Profile(c.Writer, c.Request)
}
}
// SymbolHandler will pass the call from /debug/pprof/symbol to pprof
func SymbolHandler() gin.HandlerFunc {
return func(c *gin.Context) {
pprof.Symbol(c.Writer, c.Request)
}
}
// TraceHandler will pass the call from /debug/pprof/trace to pprof
func TraceHandler() gin.HandlerFunc {
return func(c *gin.Context) {
pprof.Trace(c.Writer, c.Request)
}
}

62
server/global_secret.go Normal file
View file

@ -0,0 +1,62 @@
package server
import (
"net/http"
"github.com/drone/drone/model"
"github.com/drone/drone/store"
"github.com/gin-gonic/gin"
)
func GetGlobalSecrets(c *gin.Context) {
secrets, err := store.GetGlobalSecretList(c)
if err != nil {
c.AbortWithStatus(http.StatusInternalServerError)
return
}
var list []*model.TeamSecret
for _, s := range secrets {
list = append(list, s.Clone())
}
c.JSON(http.StatusOK, list)
}
func PostGlobalSecret(c *gin.Context) {
in := &model.TeamSecret{}
err := c.Bind(in)
if err != nil {
c.String(http.StatusBadRequest, "Invalid JSON input. %s", err.Error())
return
}
in.ID = 0
err = store.SetGlobalSecret(c, in)
if err != nil {
c.String(http.StatusInternalServerError, "Unable to persist global secret. %s", err.Error())
return
}
c.String(http.StatusOK, "")
}
func DeleteGlobalSecret(c *gin.Context) {
name := c.Param("secret")
secret, err := store.GetGlobalSecret(c, name)
if err != nil {
c.String(http.StatusNotFound, "Cannot find secret %s.", name)
return
}
err = store.DeleteGlobalSecret(c, secret)
if err != nil {
c.String(http.StatusInternalServerError, "Unable to delete global secret. %s", err.Error())
return
}
c.String(http.StatusOK, "")
}

View file

@ -3,19 +3,19 @@ package server
import (
"fmt"
"regexp"
"strconv"
"github.com/gin-gonic/gin"
"github.com/square/go-jose"
log "github.com/Sirupsen/logrus"
"github.com/drone/drone/bus"
"github.com/drone/drone/model"
"github.com/drone/drone/queue"
"github.com/drone/drone/remote"
"github.com/drone/drone/shared/httputil"
"github.com/drone/drone/shared/token"
"github.com/drone/drone/store"
"github.com/drone/drone/yaml"
"github.com/drone/mq/stomp"
)
var skipRe = regexp.MustCompile(`\[(?i:ci *skip|skip *ci)\]`)
@ -208,12 +208,22 @@ func PostHook(c *gin.Context) {
last, _ := store.GetBuildLastBefore(c, repo, build.Branch, build.ID)
secs, err := store.GetMergedSecretList(c, repo)
if err != nil {
log.Errorf("Error getting secrets for %s#%d. %s", repo.FullName, build.Number, err)
log.Debugf("Error getting secrets for %s#%d. %s", repo.FullName, build.Number, err)
}
bus.Publish(c, bus.NewBuildEvent(bus.Enqueued, repo, build))
client := stomp.MustFromContext(c)
client.SendJSON("/topic/events", model.Event{
Type: model.Enqueued,
Repo: *repo,
Build: *build,
},
stomp.WithHeader("repo", repo.FullName),
stomp.WithHeader("private", strconv.FormatBool(repo.IsPrivate)),
)
for _, job := range jobs {
queue.Publish(c, &queue.Work{
broker, _ := stomp.FromContext(c)
broker.SendJSON("/queue/pending", &model.Work{
Signed: build.Signed,
Verified: build.Verified,
User: user,
@ -225,7 +235,15 @@ func PostHook(c *gin.Context) {
Yaml: string(raw),
Secrets: secs,
System: &model.System{Link: httputil.GetURL(c.Request)},
})
},
stomp.WithHeader(
"platform",
yaml.ParsePlatformDefault(raw, "linux/amd64"),
),
stomp.WithHeaders(
yaml.ParseLabel(raw),
),
)
}
}

View file

@ -117,7 +117,7 @@ func GetLogin(c *gin.Context) {
func GetLogout(c *gin.Context) {
httputil.DelCookie(c.Writer, c.Request, "user_sess")
httputil.DelCookie(c.Writer, c.Request, "user_last")
c.Redirect(303, "/login")
c.Redirect(303, "/")
}
func GetLoginToken(c *gin.Context) {

View file

@ -1,80 +1,46 @@
package server
import (
"bytes"
"fmt"
"io"
"net/http"
"strconv"
"sync"
"time"
"golang.org/x/net/context"
"github.com/Sirupsen/logrus"
"github.com/drone/drone/bus"
"github.com/drone/drone/model"
"github.com/drone/drone/queue"
"github.com/drone/drone/remote"
"github.com/drone/drone/store"
"github.com/drone/drone/stream"
"github.com/gin-gonic/gin"
"github.com/drone/mq/stomp"
"github.com/gorilla/websocket"
)
// Pull is a long request that polls and attemts to pull work off the queue stack.
func Pull(c *gin.Context) {
logrus.Debugf("Agent %s connected.", c.ClientIP())
// newline defines a newline constant to separate lines in the build output
var newline = []byte{'\n'}
w := queue.PullClose(c, c.Writer)
if w == nil {
logrus.Debugf("Agent %s could not pull work.", c.ClientIP())
} else {
// setup the channel to stream logs
if err := stream.Create(c, stream.ToKey(w.Job.ID)); err != nil {
logrus.Errorf("Unable to create stream. %s", err)
}
c.JSON(202, w)
logrus.Debugf("Agent %s assigned work. %s/%s#%d.%d",
c.ClientIP(),
w.Repo.Owner,
w.Repo.Name,
w.Build.Number,
w.Job.Number,
)
}
// upgrader defines the default behavior for upgrading the websocket.
var upgrader = websocket.Upgrader{
ReadBufferSize: 1024,
WriteBufferSize: 1024,
CheckOrigin: func(r *http.Request) bool {
return true
},
}
// Wait is a long request that polls and waits for cancelled build requests.
func Wait(c *gin.Context) {
id, err := strconv.ParseInt(c.Param("id"), 10, 64)
if err != nil {
c.String(500, "Invalid input. %s", err)
return
// HandleUpdate handles build updates from the agent and persists to the database.
func HandleUpdate(c context.Context, message *stomp.Message) {
defer func() {
message.Release()
if r := recover(); r != nil {
err := r.(error)
logrus.Errorf("Panic recover: broker update handler: %s", err)
}
}()
eventc := make(chan *bus.Event, 1)
bus.Subscribe(c, eventc)
defer bus.Unsubscribe(c, eventc)
for {
select {
case event := <-eventc:
if event.Job.ID == id && event.Type == bus.Cancelled {
c.JSON(200, event.Job)
return
}
case <-c.Writer.CloseNotify():
return
}
}
}
// Update handles build updates from the agent and persists to the database.
func Update(c *gin.Context) {
work := &queue.Work{}
if err := c.BindJSON(work); err != nil {
work := new(model.Work)
if err := message.Unmarshal(work); err != nil {
logrus.Errorf("Invalid input. %s", err)
return
}
@ -85,12 +51,12 @@ func Update(c *gin.Context) {
// empty values if we just saved what was coming in the http.Request body.
build, err := store.GetBuild(c, work.Build.ID)
if err != nil {
c.String(404, "Unable to find build. %s", err)
logrus.Errorf("Unable to find build. %s", err)
return
}
job, err := store.GetJob(c, work.Job.ID)
if err != nil {
c.String(404, "Unable to find job. %s", err)
logrus.Errorf("Unable to find job. %s", err)
return
}
build.Started = work.Build.Started
@ -117,189 +83,81 @@ func Update(c *gin.Context) {
ok, err := store.UpdateBuildJob(c, build, job)
if err != nil {
c.String(500, "Unable to update job. %s", err)
logrus.Errorf("Unable to update job. %s", err)
return
}
if ok && build.Status != model.StatusRunning {
if ok {
// get the user because we transfer the user form the server to agent
// and back we lose the token which does not get serialized to json.
user, err := store.GetUser(c, work.User.ID)
if err != nil {
c.String(500, "Unable to find user. %s", err)
user, uerr := store.GetUser(c, work.User.ID)
if uerr != nil {
logrus.Errorf("Unable to find user. %s", err)
return
}
remote.Status(c, user, work.Repo, build,
fmt.Sprintf("%s/%s/%d", work.System.Link, work.Repo.FullName, work.Build.Number))
}
if build.Status == model.StatusRunning {
bus.Publish(c, bus.NewEvent(bus.Started, work.Repo, build, job))
} else {
bus.Publish(c, bus.NewEvent(bus.Finished, work.Repo, build, job))
client := stomp.MustFromContext(c)
err = client.SendJSON("/topic/events", model.Event{
Type: func() model.EventType {
// HACK we don't even really care about the event type.
// so we should just simplify how events are triggered.
if job.Status == model.StatusRunning {
return model.Started
}
c.JSON(200, work)
}
// Stream streams the logs to disk or memory for broadcasing to listeners. Once
// the stream is closed it is moved to permanent storage in the database.
func Stream(c *gin.Context) {
id, err := strconv.ParseInt(c.Param("id"), 10, 64)
if err != nil {
c.String(500, "Invalid input. %s", err)
return
}
key := c.Param("id")
logrus.Infof("Agent %s creating stream %s.", c.ClientIP(), key)
wc, err := stream.Writer(c, key)
if err != nil {
c.String(500, "Failed to create stream writer. %s", err)
return
}
defer func() {
wc.Close()
stream.Delete(c, key)
}()
io.Copy(wc, c.Request.Body)
rc, err := stream.Reader(c, key)
if err != nil {
c.String(500, "Failed to create stream reader. %s", err)
return
}
wg := sync.WaitGroup{}
wg.Add(1)
go func() {
defer recover()
store.WriteLog(c, &model.Job{ID: id}, rc)
wg.Done()
}()
wc.Close()
wg.Wait()
c.String(200, "")
logrus.Debugf("Agent %s wrote stream to database", c.ClientIP())
}
func Ping(c *gin.Context) {
agent, err := store.GetAgentAddr(c, c.ClientIP())
if err == nil {
agent.Updated = time.Now().Unix()
err = store.UpdateAgent(c, agent)
} else {
err = store.CreateAgent(c, &model.Agent{
Address: c.ClientIP(),
Platform: "linux/amd64",
Capacity: 2,
Created: time.Now().Unix(),
Updated: time.Now().Unix(),
})
}
if err != nil {
logrus.Errorf("Unable to register agent. %s", err.Error())
}
c.String(200, "PONG")
}
//
//
// Below are alternate implementations for the Queue that use websockets.
//
//
// PostLogs handles an http request from the agent to post build logs. These
// logs are posted at the end of the build process.
func PostLogs(c *gin.Context) {
id, _ := strconv.ParseInt(c.Param("id"), 10, 64)
job, err := store.GetJob(c, id)
if err != nil {
c.String(404, "Cannot upload logs. %s", err)
return
}
if err := store.WriteLog(c, job, c.Request.Body); err != nil {
c.String(500, "Cannot persist logs", err)
return
}
c.String(200, "")
}
// WriteLogs handles an http request from the agent to stream build logs from
// the agent to the server to enable real time streamings to the client.
func WriteLogs(c *gin.Context) {
id, err := strconv.ParseInt(c.Param("id"), 10, 64)
if err != nil {
c.String(500, "Invalid input. %s", err)
return
}
conn, err := upgrader.Upgrade(c.Writer, c.Request, nil)
if err != nil {
c.String(500, "Cannot upgrade to websocket. %s", err)
return
}
defer conn.Close()
wc, err := stream.Writer(c, stream.ToKey(id))
if err != nil {
c.String(500, "Cannot create stream writer. %s", err)
return
}
defer func() {
wc.Close()
stream.Delete(c, stream.ToKey(id))
}()
var msg []byte
for {
_, msg, err = conn.ReadMessage()
if err != nil {
break
}
wc.Write(msg)
wc.Write(newline)
}
if err != nil && err != io.EOF {
c.String(500, "Error reading logs. %s", err)
return
}
//
// rc, err := stream.Reader(c, stream.ToKey(id))
// if err != nil {
// c.String(500, "Failed to create stream reader. %s", err)
// return
// }
//
// wg := sync.WaitGroup{}
// wg.Add(1)
//
// go func() {
// defer recover()
// store.WriteLog(c, &model.Job{ID: id}, rc)
// wg.Done()
// }()
//
// wc.Close()
// wg.Wait()
}
// newline defines a newline constant to separate lines in the build output
var newline = []byte{'\n'}
// upgrader defines the default behavior for upgrading the websocket.
var upgrader = websocket.Upgrader{
ReadBufferSize: 1024,
WriteBufferSize: 1024,
CheckOrigin: func(r *http.Request) bool {
return true
return model.Finished
}(),
Repo: *work.Repo,
Build: *build,
Job: *job,
},
stomp.WithHeader("repo", work.Repo.FullName),
stomp.WithHeader("private", strconv.FormatBool(work.Repo.IsPrivate)),
)
if err != nil {
logrus.Errorf("Unable to publish to /topic/events. %s", err)
}
if job.Status == model.StatusRunning {
return
}
var buf bytes.Buffer
var sub []byte
done := make(chan bool)
dest := fmt.Sprintf("/topic/logs.%d", job.ID)
sub, err = client.Subscribe(dest, stomp.HandlerFunc(func(m *stomp.Message) {
defer m.Release()
if m.Header.GetBool("eof") {
done <- true
return
}
buf.Write(m.Body)
buf.WriteByte('\n')
}))
if err != nil {
logrus.Errorf("Unable to read logs from broker. %s", err)
return
}
defer func() {
client.Send(dest, []byte{}, stomp.WithRetain("remove"))
client.Unsubscribe(sub)
}()
select {
case <-done:
case <-time.After(30 * time.Second):
logrus.Errorf("Unable to read logs from broker. Timeout. %s", err)
return
}
if err := store.WriteLog(c, job, &buf); err != nil {
logrus.Errorf("Unable to write logs to store. %s", err)
return
}
}

View file

@ -1,121 +1,21 @@
package server
import (
"bufio"
"encoding/json"
"io"
"fmt"
"strconv"
"time"
"github.com/drone/drone/bus"
"github.com/drone/drone/cache"
"github.com/drone/drone/model"
"github.com/drone/drone/router/middleware/session"
"github.com/drone/drone/store"
"github.com/drone/drone/stream"
"github.com/drone/mq/stomp"
"github.com/Sirupsen/logrus"
"github.com/gin-gonic/gin"
"github.com/gorilla/websocket"
"github.com/manucorporat/sse"
)
// GetRepoEvents will upgrade the connection to a Websocket and will stream
// event updates to the browser.
func GetRepoEvents(c *gin.Context) {
repo := session.Repo(c)
c.Writer.Header().Set("Content-Type", "text/event-stream")
eventc := make(chan *bus.Event, 1)
bus.Subscribe(c, eventc)
defer func() {
bus.Unsubscribe(c, eventc)
close(eventc)
logrus.Infof("closed event stream")
}()
c.Stream(func(w io.Writer) bool {
select {
case event := <-eventc:
if event == nil {
logrus.Infof("nil event received")
return false
}
// TODO(bradrydzewski) This is a super hacky workaround until we improve
// the actual bus. Having a per-call database event is just plain stupid.
if event.Repo.FullName == repo.FullName {
var payload = struct {
model.Build
Jobs []*model.Job `json:"jobs"`
}{}
payload.Build = event.Build
payload.Jobs, _ = store.GetJobList(c, &event.Build)
data, _ := json.Marshal(&payload)
sse.Encode(w, sse.Event{
Event: "message",
Data: string(data),
})
}
case <-c.Writer.CloseNotify():
return false
}
return true
})
}
func GetStream(c *gin.Context) {
repo := session.Repo(c)
buildn, _ := strconv.Atoi(c.Param("build"))
jobn, _ := strconv.Atoi(c.Param("number"))
c.Writer.Header().Set("Content-Type", "text/event-stream")
build, err := store.GetBuildNumber(c, repo, buildn)
if err != nil {
logrus.Debugln("stream cannot get build number.", err)
c.AbortWithError(404, err)
return
}
job, err := store.GetJobNumber(c, build, jobn)
if err != nil {
logrus.Debugln("stream cannot get job number.", err)
c.AbortWithError(404, err)
return
}
rc, err := stream.Reader(c, stream.ToKey(job.ID))
if err != nil {
c.AbortWithError(404, err)
return
}
go func() {
<-c.Writer.CloseNotify()
rc.Close()
}()
var line int
var scanner = bufio.NewScanner(rc)
for scanner.Scan() {
line++
var err = sse.Encode(c.Writer, sse.Event{
Id: strconv.Itoa(line),
Event: "message",
Data: scanner.Text(),
})
if err != nil {
break
}
c.Writer.Flush()
}
logrus.Debugf("Closed stream %s#%d", repo.FullName, build.Number)
}
var (
// Time allowed to write the file to the client.
writeWait = 5 * time.Second
@ -165,28 +65,34 @@ func LogStream(c *gin.Context) {
ticker := time.NewTicker(pingPeriod)
defer ticker.Stop()
rc, err := stream.Reader(c, stream.ToKey(job.ID))
logs := make(chan []byte)
done := make(chan bool)
dest := fmt.Sprintf("/topic/logs.%d", job.ID)
client, _ := stomp.FromContext(c)
sub, err := client.Subscribe(dest, stomp.HandlerFunc(func(m *stomp.Message) {
if m.Header.GetBool("eof") {
done <- true
} else {
logs <- m.Body
}
m.Release()
}))
if err != nil {
c.AbortWithError(404, err)
logrus.Errorf("Unable to read logs from broker. %s", err)
return
}
quitc := make(chan bool)
defer func() {
quitc <- true
close(quitc)
rc.Close()
ws.Close()
logrus.Debug("Successfully closed websocket")
client.Unsubscribe(sub)
close(done)
close(logs)
}()
go func() {
defer func() {
recover()
}()
for {
select {
case <-quitc:
case buf := <-logs:
ws.SetWriteDeadline(time.Now().Add(writeWait))
ws.WriteMessage(websocket.TextMessage, buf)
case <-done:
return
case <-ticker.C:
err := ws.WriteControl(websocket.PingMessage, []byte{}, time.Now().Add(writeWait))
@ -195,18 +101,6 @@ func LogStream(c *gin.Context) {
}
}
}
}()
var scanner = bufio.NewScanner(rc)
var b []byte
for scanner.Scan() {
b = scanner.Bytes()
if len(b) == 0 {
continue
}
ws.SetWriteDeadline(time.Now().Add(writeWait))
ws.WriteMessage(websocket.TextMessage, b)
}
}
// EventStream produces the User event stream, sending all repository, build
@ -227,20 +121,34 @@ func EventStream(c *gin.Context) {
repo, _ = cache.GetRepoMap(c, user)
}
ticker := time.NewTicker(pingPeriod)
eventc := make(chan []byte, 10)
quitc := make(chan bool)
eventc := make(chan *bus.Event, 10)
bus.Subscribe(c, eventc)
tick := time.NewTicker(pingPeriod)
defer func() {
ticker.Stop()
bus.Unsubscribe(c, eventc)
quitc <- true
close(quitc)
close(eventc)
tick.Stop()
ws.Close()
logrus.Debug("Successfully closed websocket")
}()
client := stomp.MustFromContext(c)
sub, err := client.Subscribe("/topic/events", stomp.HandlerFunc(func(m *stomp.Message) {
name := m.Header.GetString("repo")
priv := m.Header.GetBool("private")
if repo[name] || !priv {
eventc <- m.Body
}
m.Release()
}))
if err != nil {
logrus.Errorf("Unable to read logs from broker. %s", err)
return
}
defer func() {
client.Unsubscribe(sub)
close(quitc)
close(eventc)
}()
go func() {
defer func() {
recover()
@ -249,15 +157,13 @@ func EventStream(c *gin.Context) {
select {
case <-quitc:
return
case event := <-eventc:
if event == nil {
case event, ok := <-eventc:
if !ok {
return
}
if repo[event.Repo.FullName] || !event.Repo.IsPrivate {
ws.SetWriteDeadline(time.Now().Add(writeWait))
ws.WriteJSON(event)
}
case <-ticker.C:
ws.WriteMessage(websocket.TextMessage, event)
case <-tick.C:
err := ws.WriteControl(websocket.PingMessage, []byte{}, time.Now().Add(writeWait))
if err != nil {
return

View file

@ -218,6 +218,9 @@ func (c *Config) AuthCodeURL(state string) string {
if err != nil {
panic("AuthURL malformed: " + err.Error())
}
if err := url_.Query().Get("error"); err != "" {
panic("AuthURL contains error: " + err)
}
q := url.Values{
"response_type": {"code"},
"client_id": {c.ClientId},

View file

@ -0,0 +1,12 @@
-- +migrate Up
ALTER TABLE secrets ADD COLUMN secret_conceal BOOLEAN;
ALTER TABLE team_secrets ADD COLUMN team_secret_conceal BOOLEAN;
UPDATE secrets SET secret_conceal = false;
UPDATE team_secrets SET team_secret_conceal = false;
-- +migrate Down
ALTER TABLE secrets DROP COLUMN secret_conceal;
ALTER TABLE team_secrets DROP COLUMN team_secret_conceal;

View file

@ -2,7 +2,7 @@
CREATE TABLE agents (
agent_id INTEGER PRIMARY KEY AUTO_INCREMENT
,agent_addr VARCHAR(500)
,agent_addr VARCHAR(255)
,agent_platform VARCHAR(500)
,agent_capacity INTEGER
,agent_created INTEGER

View file

@ -0,0 +1,7 @@
-- +migrate Up
ALTER TABLE builds ADD COLUMN build_parent INTEGER DEFAULT 0;
-- +migrate Down
ALTER TABLE builds DROP COLUMN build_parent;

View file

@ -0,0 +1,12 @@
-- +migrate Up
ALTER TABLE secrets ADD COLUMN secret_skip_verify BOOLEAN;
ALTER TABLE team_secrets ADD COLUMN team_secret_skip_verify BOOLEAN;
UPDATE secrets SET secret_skip_verify = false;
UPDATE team_secrets SET team_secret_skip_verify = false;
-- +migrate Down
ALTER TABLE secrets DROP COLUMN secret_skip_verify;
ALTER TABLE team_secrets DROP COLUMN team_secret_skip_verify;

View file

@ -0,0 +1,12 @@
-- +migrate Up
ALTER TABLE secrets ADD COLUMN secret_conceal BOOLEAN;
ALTER TABLE team_secrets ADD COLUMN team_secret_conceal BOOLEAN;
UPDATE secrets SET secret_conceal = false;
UPDATE team_secrets SET team_secret_conceal = false;
-- +migrate Down
ALTER TABLE secrets DROP COLUMN secret_conceal;
ALTER TABLE team_secrets DROP COLUMN team_secret_conceal;

View file

@ -0,0 +1,7 @@
-- +migrate Up
ALTER TABLE builds ADD COLUMN build_parent INTEGER DEFAULT 0;
-- +migrate Down
ALTER TABLE builds DROP COLUMN build_parent;

Some files were not shown because too many files have changed in this diff Show more