diff --git a/.drone.yml b/.drone.yml index 9b79c0451..959e9585d 100644 --- a/.drone.yml +++ b/.drone.yml @@ -22,17 +22,16 @@ pipeline: when: event: push - - # archive: - # image: plugins/s3 - # acl: public-read - # bucket: downloads.drone.io - # source: release/**/*.* - # access_key: ${AWS_ACCESS_KEY_ID} - # secret_key: ${AWS_SECRET_ACCESS_KEY} - # when: - # event: push - # branch: master + archive: + image: plugins/s3 + acl: public-read + bucket: downloads.drone.io + source: release/**/*.* + access_key: ${AWS_ACCESS_KEY_ID} + secret_key: ${AWS_SECRET_ACCESS_KEY} + when: + event: push + branch: master publish: image: plugins/docker diff --git a/agent/agent.go b/agent/agent.go deleted file mode 100644 index 93fc0e552..000000000 --- a/agent/agent.go +++ /dev/null @@ -1,329 +0,0 @@ -package agent - -import ( - "fmt" - "net" - "net/url" - "path/filepath" - "regexp" - "strings" - "time" - - "github.com/drone/drone/build" - "github.com/drone/drone/model" - "github.com/drone/drone/version" - "github.com/drone/drone/yaml" - "github.com/drone/drone/yaml/transform" - "github.com/drone/envsubst" -) - -type Logger interface { - Write(*build.Line) -} - -type Agent struct { - Update UpdateFunc - Logger LoggerFunc - Engine build.Engine - Timeout time.Duration - Platform string - Namespace string - Extension []string - Escalate []string - Netrc []string - Local string - Pull bool -} - -func (a *Agent) Poll() error { - - // logrus.Infof("Starting build %s/%s#%d.%d", - // payload.Repo.Owner, payload.Repo.Name, payload.Build.Number, payload.Job.Number) - // - // - // logrus.Infof("Finished build %s/%s#%d.%d", - // payload.Repo.Owner, payload.Repo.Name, payload.Build.Number, payload.Job.Number) - - return nil -} - -func (a *Agent) Run(payload *model.Work, cancel <-chan bool) error { - - payload.Job.Status = model.StatusRunning - payload.Job.Started = time.Now().Unix() - - spec, err := a.prep(payload) - if err != nil { - payload.Job.Error = err.Error() - payload.Job.ExitCode = 255 - payload.Job.Finished = payload.Job.Started - payload.Job.Status = model.StatusError - a.Update(payload) - return err - } - a.Update(payload) - err = a.exec(spec, payload, cancel) - - if err != nil { - payload.Job.ExitCode = 255 - payload.Job.Error = err.Error() - } - if exitErr, ok := err.(*build.ExitError); ok { - payload.Job.ExitCode = exitErr.Code - payload.Job.Error = "" // exit errors are already written to the log - } - - payload.Job.Finished = time.Now().Unix() - - switch payload.Job.ExitCode { - case 128, 130, 137: - payload.Job.Status = model.StatusKilled - case 0: - payload.Job.Status = model.StatusSuccess - default: - payload.Job.Status = model.StatusFailure - } - - a.Update(payload) - - return err -} - -func (a *Agent) prep(w *model.Work) (*yaml.Config, error) { - - envs := toEnv(w) - envSecrets := map[string]string{} - - // list of secrets to interpolate in the yaml - for _, secret := range w.Secrets { - if (w.Verified || secret.SkipVerify) && secret.MatchEvent(w.Build.Event) { - envSecrets[secret.Name] = secret.Value - } - } - - var err error - w.Yaml, err = envsubst.Eval(w.Yaml, func(s string) string { - env, ok := envSecrets[s] - if !ok { - env, _ = envs[s] - } - if strings.Contains(env, "\n") { - env = fmt.Sprintf("%q", env) - } - return env - }) - if err != nil { - return nil, err - } - - // append secrets when verified or when a secret does not require - // verification - var secrets []*model.Secret - for _, secret := range w.Secrets { - if w.Verified || secret.SkipVerify { - secrets = append(secrets, secret) - } - } - - // inject the netrc file into the clone plugin if the repository is - // private and requires authentication. - if w.Repo.IsPrivate { - secrets = append(secrets, &model.Secret{ - Name: "DRONE_NETRC_USERNAME", - Value: w.Netrc.Login, - Images: []string{"*"}, - Events: []string{"*"}, - }) - secrets = append(secrets, &model.Secret{ - Name: "DRONE_NETRC_PASSWORD", - Value: w.Netrc.Password, - Images: []string{"*"}, - Events: []string{"*"}, - }) - secrets = append(secrets, &model.Secret{ - Name: "DRONE_NETRC_MACHINE", - Value: w.Netrc.Machine, - Images: []string{"*"}, - Events: []string{"*"}, - }) - } - - conf, err := yaml.ParseString(w.Yaml) - if err != nil { - return nil, err - } - - src := "src" - if url, _ := url.Parse(w.Repo.Link); url != nil { - host, _, err := net.SplitHostPort(url.Host) - if err == nil { - url.Host = host - } - src = filepath.Join(src, url.Host, url.Path) - } - - transform.Clone(conf, w.Repo.Kind) - transform.Environ(conf, envs) - transform.DefaultFilter(conf) - if w.BuildLast != nil { - transform.ChangeFilter(conf, w.BuildLast.Status) - } - - transform.ImageSecrets(conf, secrets, w.Build.Event) - transform.Identifier(conf) - transform.WorkspaceTransform(conf, "/drone", src) - - if err := transform.Check(conf, w.Repo.IsTrusted); err != nil { - return nil, err - } - - transform.CommandTransform(conf) - transform.ImagePull(conf, a.Pull) - transform.ImageTag(conf) - if err := transform.ImageEscalate(conf, a.Escalate); err != nil { - return nil, err - } - transform.PluginParams(conf) - - if a.Local != "" { - transform.PluginDisable(conf, true) - transform.ImageVolume(conf, []string{a.Local + ":" + conf.Workspace.Path}) - } - - transform.Pod(conf, a.Platform) - if err := transform.RemoteTransform(conf, a.Extension); err != nil { - return nil, err - } - - return conf, nil -} - -func (a *Agent) exec(spec *yaml.Config, payload *model.Work, cancel <-chan bool) error { - - conf := build.Config{ - Engine: a.Engine, - Buffer: 500, - } - - pipeline := conf.Pipeline(spec) - defer pipeline.Teardown() - - // setup the build environment - if err := pipeline.Setup(); err != nil { - return err - } - - replacer := NewSecretReplacer(payload.Secrets) - timeout := time.After(time.Duration(payload.Repo.Timeout) * time.Minute) - - for { - select { - case <-pipeline.Done(): - return pipeline.Err() - case <-cancel: - pipeline.Stop() - return fmt.Errorf("termination request received, build cancelled") - case <-timeout: - pipeline.Stop() - return fmt.Errorf("maximum time limit exceeded, build cancelled") - case <-time.After(a.Timeout): - pipeline.Stop() - return fmt.Errorf("terminal inactive for %v, build cancelled", a.Timeout) - case <-pipeline.Next(): - - // TODO(bradrydzewski) this entire block of code should probably get - // encapsulated in the pipeline. - status := model.StatusSuccess - if pipeline.Err() != nil { - status = model.StatusFailure - } - // updates the build status passed into each container. I realize this is - // a bit out of place and will work to resolve. - pipeline.Head().Environment["DRONE_BUILD_STATUS"] = status - - if !pipeline.Head().Constraints.Match( - a.Platform, - payload.Build.Deploy, - payload.Build.Event, - payload.Build.Branch, - status, payload.Job.Environment) { // TODO: fix this whole section - - pipeline.Skip() - } else { - pipeline.Exec() - } - case line := <-pipeline.Pipe(): - line.Out = replacer.Replace(line.Out) - a.Logger(line) - } - } -} - -func toEnv(w *model.Work) map[string]string { - envs := map[string]string{ - "CI": "drone", - "DRONE": "true", - "DRONE_ARCH": "linux/amd64", - "DRONE_REPO": w.Repo.FullName, - "DRONE_REPO_SCM": w.Repo.Kind, - "DRONE_REPO_OWNER": w.Repo.Owner, - "DRONE_REPO_NAME": w.Repo.Name, - "DRONE_REPO_LINK": w.Repo.Link, - "DRONE_REPO_AVATAR": w.Repo.Avatar, - "DRONE_REPO_BRANCH": w.Repo.Branch, - "DRONE_REPO_PRIVATE": fmt.Sprintf("%v", w.Repo.IsPrivate), - "DRONE_REPO_TRUSTED": fmt.Sprintf("%v", w.Repo.IsTrusted), - "DRONE_REMOTE_URL": w.Repo.Clone, - "DRONE_COMMIT_SHA": w.Build.Commit, - "DRONE_COMMIT_REF": w.Build.Ref, - "DRONE_COMMIT_REFSPEC": w.Build.Refspec, - "DRONE_COMMIT_BRANCH": w.Build.Branch, - "DRONE_COMMIT_LINK": w.Build.Link, - "DRONE_COMMIT_MESSAGE": w.Build.Message, - "DRONE_COMMIT_AUTHOR": w.Build.Author, - "DRONE_COMMIT_AUTHOR_EMAIL": w.Build.Email, - "DRONE_COMMIT_AUTHOR_AVATAR": w.Build.Avatar, - "DRONE_BUILD_NUMBER": fmt.Sprintf("%d", w.Build.Number), - "DRONE_BUILD_EVENT": w.Build.Event, - "DRONE_BUILD_STATUS": w.Build.Status, - "DRONE_BUILD_LINK": fmt.Sprintf("%s/%s/%d", w.System.Link, w.Repo.FullName, w.Build.Number), - "DRONE_BUILD_CREATED": fmt.Sprintf("%d", w.Build.Created), - "DRONE_BUILD_STARTED": fmt.Sprintf("%d", w.Build.Started), - "DRONE_BUILD_FINISHED": fmt.Sprintf("%d", w.Build.Finished), - "DRONE_JOB_NUMBER": fmt.Sprintf("%d", w.Job.Number), - "DRONE_JOB_STATUS": w.Job.Status, - "DRONE_JOB_ERROR": w.Job.Error, - "DRONE_JOB_EXIT_CODE": fmt.Sprintf("%d", w.Job.ExitCode), - "DRONE_JOB_STARTED": fmt.Sprintf("%d", w.Job.Started), - "DRONE_JOB_FINISHED": fmt.Sprintf("%d", w.Job.Finished), - "DRONE_YAML_VERIFIED": fmt.Sprintf("%v", w.Verified), - "DRONE_YAML_SIGNED": fmt.Sprintf("%v", w.Signed), - "DRONE_BRANCH": w.Build.Branch, - "DRONE_COMMIT": w.Build.Commit, - "DRONE_VERSION": version.Version, - } - - if w.Build.Event == model.EventTag { - envs["DRONE_TAG"] = strings.TrimPrefix(w.Build.Ref, "refs/tags/") - } - if w.Build.Event == model.EventPull { - envs["DRONE_PULL_REQUEST"] = pullRegexp.FindString(w.Build.Ref) - } - if w.Build.Event == model.EventDeploy { - envs["DRONE_DEPLOY_TO"] = w.Build.Deploy - } - - if w.BuildLast != nil { - envs["DRONE_PREV_BUILD_STATUS"] = w.BuildLast.Status - envs["DRONE_PREV_BUILD_NUMBER"] = fmt.Sprintf("%v", w.BuildLast.Number) - envs["DRONE_PREV_COMMIT_SHA"] = w.BuildLast.Commit - } - - // inject matrix values as environment variables - for key, val := range w.Job.Environment { - envs[key] = val - } - return envs -} - -var pullRegexp = regexp.MustCompile("\\d+") diff --git a/agent/secret.go b/agent/secret.go deleted file mode 100644 index 531963bd2..000000000 --- a/agent/secret.go +++ /dev/null @@ -1,46 +0,0 @@ -package agent - -import ( - "strings" - - "github.com/drone/drone/model" -) - -// SecretReplacer hides secrets from being exposed by the build output. -type SecretReplacer interface { - // Replace conceals instances of secrets found in s. - Replace(s string) string -} - -// NewSecretReplacer creates a SecretReplacer based on whether any value in -// secrets requests it be hidden. -func NewSecretReplacer(secrets []*model.Secret) SecretReplacer { - var r []string - for _, s := range secrets { - if s.Conceal { - r = append(r, s.Value, "*****") - } - } - - if len(r) == 0 { - return &noopReplacer{} - } - - return &secretReplacer{ - replacer: strings.NewReplacer(r...), - } -} - -type noopReplacer struct{} - -func (*noopReplacer) Replace(s string) string { - return s -} - -type secretReplacer struct { - replacer *strings.Replacer -} - -func (r *secretReplacer) Replace(s string) string { - return r.replacer.Replace(s) -} diff --git a/agent/secret_test.go b/agent/secret_test.go deleted file mode 100644 index a88b523c0..000000000 --- a/agent/secret_test.go +++ /dev/null @@ -1,39 +0,0 @@ -package agent - -import ( - "testing" - - "github.com/drone/drone/model" - "github.com/franela/goblin" -) - -const testString = "This is SECRET: secret_value" - -func TestSecret(t *testing.T) { - g := goblin.Goblin(t) - g.Describe("SecretReplacer", func() { - g.It("Should conceal secret", func() { - secrets := []*model.Secret{ - { - Name: "SECRET", - Value: "secret_value", - Conceal: true, - }, - } - r := NewSecretReplacer(secrets) - g.Assert(r.Replace(testString)).Equal("This is SECRET: *****") - }) - - g.It("Should not conceal secret", func() { - secrets := []*model.Secret{ - { - Name: "SECRET", - Value: "secret_value", - Conceal: false, - }, - } - r := NewSecretReplacer(secrets) - g.Assert(r.Replace(testString)).Equal(testString) - }) - }) -} diff --git a/agent/updater.go b/agent/updater.go deleted file mode 100644 index 90fc999f7..000000000 --- a/agent/updater.go +++ /dev/null @@ -1,67 +0,0 @@ -package agent - -import ( - "fmt" - - "github.com/Sirupsen/logrus" - "github.com/drone/drone/build" - "github.com/drone/drone/model" - "github.com/drone/mq/logger" - "github.com/drone/mq/stomp" -) - -// UpdateFunc handles buid pipeline status updates. -type UpdateFunc func(*model.Work) - -// LoggerFunc handles buid pipeline logging updates. -type LoggerFunc func(*build.Line) - -var NoopUpdateFunc = func(*model.Work) {} - -var TermLoggerFunc = func(line *build.Line) { - fmt.Println(line) -} - -// NewClientUpdater returns an updater that sends updated build details -// to the drone server. -func NewClientUpdater(client *stomp.Client) UpdateFunc { - return func(w *model.Work) { - err := client.SendJSON("/queue/updates", w) - if err != nil { - logger.Warningf("Error updating %s/%s#%d.%d. %s", - w.Repo.Owner, w.Repo.Name, w.Build.Number, w.Job.Number, err) - } - if w.Job.Status != model.StatusRunning { - var dest = fmt.Sprintf("/topic/logs.%d", w.Job.ID) - var opts = []stomp.MessageOption{ - stomp.WithHeader("eof", "true"), - stomp.WithRetain("all"), - } - - if err := client.Send(dest, []byte("eof"), opts...); err != nil { - logger.Warningf("Error sending eof %s/%s#%d.%d. %s", - w.Repo.Owner, w.Repo.Name, w.Build.Number, w.Job.Number, err) - } - } - } -} - -func NewClientLogger(client *stomp.Client, id int64, limit int64) LoggerFunc { - - var size int64 - var dest = fmt.Sprintf("/topic/logs.%d", id) - var opts = []stomp.MessageOption{ - stomp.WithRetain("all"), - } - - return func(line *build.Line) { - if size > limit { - return - } - if err := client.SendJSON(dest, line, opts...); err != nil { - logrus.Errorf("Error streaming build logs. %s", err) - } - - size += int64(len(line.Out)) - } -} diff --git a/build/config.go b/build/config.go deleted file mode 100644 index d8d3232eb..000000000 --- a/build/config.go +++ /dev/null @@ -1,48 +0,0 @@ -package build - -import "github.com/drone/drone/yaml" - -// Config defines the configuration for creating the Pipeline. -type Config struct { - Engine Engine - - // Buffer defines the size of the buffer for the channel to which the - // console output is streamed. - Buffer uint -} - -// Pipeline creates a build Pipeline using the specific configuration for -// the given Yaml specification. -func (c *Config) Pipeline(spec *yaml.Config) *Pipeline { - - pipeline := Pipeline{ - engine: c.Engine, - pipe: make(chan *Line, c.Buffer), - next: make(chan error), - done: make(chan error), - } - - var containers []*yaml.Container - containers = append(containers, spec.Services...) - containers = append(containers, spec.Pipeline...) - - for _, c := range containers { - if c.Disabled { - continue - } - next := &element{Container: c} - if pipeline.head == nil { - pipeline.head = next - pipeline.tail = next - } else { - pipeline.tail.next = next - pipeline.tail = next - } - } - - go func() { - pipeline.next <- nil - }() - - return &pipeline -} diff --git a/build/docker/docker.go b/build/docker/docker.go deleted file mode 100644 index 4f8416ee7..000000000 --- a/build/docker/docker.go +++ /dev/null @@ -1,112 +0,0 @@ -package docker - -import ( - "io" - - "github.com/drone/drone/build" - "github.com/drone/drone/build/docker/internal" - "github.com/drone/drone/yaml" - - "github.com/samalba/dockerclient" -) - -type dockerEngine struct { - client dockerclient.Client -} - -func (e *dockerEngine) ContainerStart(container *yaml.Container) (string, error) { - conf := toContainerConfig(container) - auth := toAuthConfig(container) - - // pull the image if it does not exists or if the Container - // is configured to always pull a new image. - _, err := e.client.InspectImage(container.Image) - if err != nil || container.Pull { - e.client.PullImage(container.Image, auth) - } - - // create and start the container and return the Container ID. - id, err := e.client.CreateContainer(conf, container.ID, auth) - if err != nil { - return id, err - } - err = e.client.StartContainer(id, &conf.HostConfig) - if err != nil { - - // remove the container if it cannot be started - e.client.RemoveContainer(id, true, true) - return id, err - } - return id, nil -} - -func (e *dockerEngine) ContainerStop(id string) error { - e.client.StopContainer(id, 1) - e.client.KillContainer(id, "9") - return nil -} - -func (e *dockerEngine) ContainerRemove(id string) error { - e.client.StopContainer(id, 1) - e.client.KillContainer(id, "9") - e.client.RemoveContainer(id, true, true) - return nil -} - -func (e *dockerEngine) ContainerWait(id string) (*build.State, error) { - // wait for the container to exit - // - // TODO(bradrydzewski) we should have a for loop here - // to re-connect and wait if this channel returns a - // result even though the container is still running. - // - <-e.client.Wait(id) - v, err := e.client.InspectContainer(id) - if err != nil { - return nil, err - } - return &build.State{ - ExitCode: v.State.ExitCode, - OOMKilled: v.State.OOMKilled, - }, nil -} - -func (e *dockerEngine) ContainerLogs(id string) (io.ReadCloser, error) { - opts := &dockerclient.LogOptions{ - Follow: true, - Stdout: true, - Stderr: true, - } - - piper, pipew := io.Pipe() - go func() { - defer pipew.Close() - - // sometimes the docker logs fails due to parsing errors. this - // routine will check for such a failure and attempt to resume - // if necessary. - for i := 0; i < 5; i++ { - if i > 0 { - opts.Tail = 1 - } - - rc, err := e.client.ContainerLogs(id, opts) - if err != nil { - return - } - defer rc.Close() - - // use Docker StdCopy - internal.StdCopy(pipew, pipew, rc) - - // check to see if the container is still running. If not, - // we can safely exit and assume there are no more logs left - // to stream. - v, err := e.client.InspectContainer(id) - if err != nil || !v.State.Running { - return - } - } - }() - return piper, nil -} diff --git a/build/docker/docker_test.go b/build/docker/docker_test.go deleted file mode 100644 index 1cdc3ff91..000000000 --- a/build/docker/docker_test.go +++ /dev/null @@ -1 +0,0 @@ -package docker diff --git a/build/docker/helper.go b/build/docker/helper.go deleted file mode 100644 index 46bcc3cdb..000000000 --- a/build/docker/helper.go +++ /dev/null @@ -1,25 +0,0 @@ -package docker - -import ( - "github.com/drone/drone/build" - "github.com/samalba/dockerclient" -) - -// NewClient returns a new Docker engine using the provided Docker client. -func NewClient(client dockerclient.Client) build.Engine { - return &dockerEngine{client} -} - -// New returns a new Docker engine from the provided DOCKER_HOST and -// DOCKER_CERT_PATH environment variables. -func New(host, cert string, tls bool) (build.Engine, error) { - config, err := dockerclient.TLSConfigFromCertPath(cert) - if err == nil && tls { - config.InsecureSkipVerify = true - } - client, err := dockerclient.NewDockerClient(host, config) - if err != nil { - return nil, err - } - return NewClient(client), nil -} diff --git a/build/docker/helper_test.go b/build/docker/helper_test.go deleted file mode 100644 index 1cdc3ff91..000000000 --- a/build/docker/helper_test.go +++ /dev/null @@ -1 +0,0 @@ -package docker diff --git a/build/docker/internal/README b/build/docker/internal/README deleted file mode 100644 index 2bd3e9830..000000000 --- a/build/docker/internal/README +++ /dev/null @@ -1 +0,0 @@ -This is an internal copy of the Docker stdcopy package that removes the logrus debug logging. The original package is found at https://github.com/docker/docker/tree/master/pkg/stdcopy \ No newline at end of file diff --git a/build/docker/internal/stdcopy.go b/build/docker/internal/stdcopy.go deleted file mode 100644 index db61b0c88..000000000 --- a/build/docker/internal/stdcopy.go +++ /dev/null @@ -1,167 +0,0 @@ -package internal - -import ( - "encoding/binary" - "errors" - "fmt" - "io" -) - -// StdType is the type of standard stream -// a writer can multiplex to. -type StdType byte - -const ( - // Stdin represents standard input stream type. - Stdin StdType = iota - // Stdout represents standard output stream type. - Stdout - // Stderr represents standard error steam type. - Stderr - - stdWriterPrefixLen = 8 - stdWriterFdIndex = 0 - stdWriterSizeIndex = 4 - - startingBufLen = 32*1024 + stdWriterPrefixLen + 1 -) - -// stdWriter is wrapper of io.Writer with extra customized info. -type stdWriter struct { - io.Writer - prefix byte -} - -// Write sends the buffer to the underneath writer. -// It insert the prefix header before the buffer, -// so stdcopy.StdCopy knows where to multiplex the output. -// It makes stdWriter to implement io.Writer. -func (w *stdWriter) Write(buf []byte) (n int, err error) { - if w == nil || w.Writer == nil { - return 0, errors.New("Writer not instantiated") - } - if buf == nil { - return 0, nil - } - - header := [stdWriterPrefixLen]byte{stdWriterFdIndex: w.prefix} - binary.BigEndian.PutUint32(header[stdWriterSizeIndex:], uint32(len(buf))) - - line := append(header[:], buf...) - - n, err = w.Writer.Write(line) - n -= stdWriterPrefixLen - - if n < 0 { - n = 0 - } - return -} - -// NewStdWriter instantiates a new Writer. -// Everything written to it will be encapsulated using a custom format, -// and written to the underlying `w` stream. -// This allows multiple write streams (e.g. stdout and stderr) to be muxed into a single connection. -// `t` indicates the id of the stream to encapsulate. -// It can be stdcopy.Stdin, stdcopy.Stdout, stdcopy.Stderr. -func NewStdWriter(w io.Writer, t StdType) io.Writer { - return &stdWriter{ - Writer: w, - prefix: byte(t), - } -} - -// StdCopy is a modified version of io.Copy. -// -// StdCopy will demultiplex `src`, assuming that it contains two streams, -// previously multiplexed together using a StdWriter instance. -// As it reads from `src`, StdCopy will write to `dstout` and `dsterr`. -// -// StdCopy will read until it hits EOF on `src`. It will then return a nil error. -// In other words: if `err` is non nil, it indicates a real underlying error. -// -// `written` will hold the total number of bytes written to `dstout` and `dsterr`. -func StdCopy(dstout, dsterr io.Writer, src io.Reader) (written int64, err error) { - var ( - buf = make([]byte, startingBufLen) - bufLen = len(buf) - nr, nw int - er, ew error - out io.Writer - frameSize int - ) - - for { - // Make sure we have at least a full header - for nr < stdWriterPrefixLen { - var nr2 int - nr2, er = src.Read(buf[nr:]) - nr += nr2 - if er == io.EOF { - if nr < stdWriterPrefixLen { - return written, nil - } - break - } - if er != nil { - return 0, er - } - } - - // Check the first byte to know where to write - switch StdType(buf[stdWriterFdIndex]) { - case Stdin: - fallthrough - case Stdout: - // Write on stdout - out = dstout - case Stderr: - // Write on stderr - out = dsterr - default: - return 0, fmt.Errorf("Unrecognized input header: %d", buf[stdWriterFdIndex]) - } - - // Retrieve the size of the frame - frameSize = int(binary.BigEndian.Uint32(buf[stdWriterSizeIndex : stdWriterSizeIndex+4])) - - // Check if the buffer is big enough to read the frame. - // Extend it if necessary. - if frameSize+stdWriterPrefixLen > bufLen { - buf = append(buf, make([]byte, frameSize+stdWriterPrefixLen-bufLen+1)...) - bufLen = len(buf) - } - - // While the amount of bytes read is less than the size of the frame + header, we keep reading - for nr < frameSize+stdWriterPrefixLen { - var nr2 int - nr2, er = src.Read(buf[nr:]) - nr += nr2 - if er == io.EOF { - if nr < frameSize+stdWriterPrefixLen { - return written, nil - } - break - } - if er != nil { - return 0, er - } - } - - // Write the retrieved frame (without header) - nw, ew = out.Write(buf[stdWriterPrefixLen : frameSize+stdWriterPrefixLen]) - if ew != nil { - return 0, ew - } - // If the frame has not been fully written: error - if nw != frameSize { - return 0, io.ErrShortWrite - } - written += int64(nw) - - // Move the rest of the buffer to the beginning - copy(buf, buf[frameSize+stdWriterPrefixLen:]) - // Move the index - nr -= frameSize + stdWriterPrefixLen - } -} diff --git a/build/docker/internal/stdcopy_test.go b/build/docker/internal/stdcopy_test.go deleted file mode 100644 index 7a443bb8b..000000000 --- a/build/docker/internal/stdcopy_test.go +++ /dev/null @@ -1,260 +0,0 @@ -package internal - -import ( - "bytes" - "errors" - "io" - "io/ioutil" - "strings" - "testing" -) - -func TestNewStdWriter(t *testing.T) { - writer := NewStdWriter(ioutil.Discard, Stdout) - if writer == nil { - t.Fatalf("NewStdWriter with an invalid StdType should not return nil.") - } -} - -func TestWriteWithUnitializedStdWriter(t *testing.T) { - writer := stdWriter{ - Writer: nil, - prefix: byte(Stdout), - } - n, err := writer.Write([]byte("Something here")) - if n != 0 || err == nil { - t.Fatalf("Should fail when given an uncomplete or uninitialized StdWriter") - } -} - -func TestWriteWithNilBytes(t *testing.T) { - writer := NewStdWriter(ioutil.Discard, Stdout) - n, err := writer.Write(nil) - if err != nil { - t.Fatalf("Shouldn't have fail when given no data") - } - if n > 0 { - t.Fatalf("Write should have written 0 byte, but has written %d", n) - } -} - -func TestWrite(t *testing.T) { - writer := NewStdWriter(ioutil.Discard, Stdout) - data := []byte("Test StdWrite.Write") - n, err := writer.Write(data) - if err != nil { - t.Fatalf("Error while writing with StdWrite") - } - if n != len(data) { - t.Fatalf("Write should have written %d byte but wrote %d.", len(data), n) - } -} - -type errWriter struct { - n int - err error -} - -func (f *errWriter) Write(buf []byte) (int, error) { - return f.n, f.err -} - -func TestWriteWithWriterError(t *testing.T) { - expectedError := errors.New("expected") - expectedReturnedBytes := 10 - writer := NewStdWriter(&errWriter{ - n: stdWriterPrefixLen + expectedReturnedBytes, - err: expectedError}, Stdout) - data := []byte("This won't get written, sigh") - n, err := writer.Write(data) - if err != expectedError { - t.Fatalf("Didn't get expected error.") - } - if n != expectedReturnedBytes { - t.Fatalf("Didn't get expected written bytes %d, got %d.", - expectedReturnedBytes, n) - } -} - -func TestWriteDoesNotReturnNegativeWrittenBytes(t *testing.T) { - writer := NewStdWriter(&errWriter{n: -1}, Stdout) - data := []byte("This won't get written, sigh") - actual, _ := writer.Write(data) - if actual != 0 { - t.Fatalf("Expected returned written bytes equal to 0, got %d", actual) - } -} - -func getSrcBuffer(stdOutBytes, stdErrBytes []byte) (buffer *bytes.Buffer, err error) { - buffer = new(bytes.Buffer) - dstOut := NewStdWriter(buffer, Stdout) - _, err = dstOut.Write(stdOutBytes) - if err != nil { - return - } - dstErr := NewStdWriter(buffer, Stderr) - _, err = dstErr.Write(stdErrBytes) - return -} - -func TestStdCopyWriteAndRead(t *testing.T) { - stdOutBytes := []byte(strings.Repeat("o", startingBufLen)) - stdErrBytes := []byte(strings.Repeat("e", startingBufLen)) - buffer, err := getSrcBuffer(stdOutBytes, stdErrBytes) - if err != nil { - t.Fatal(err) - } - written, err := StdCopy(ioutil.Discard, ioutil.Discard, buffer) - if err != nil { - t.Fatal(err) - } - expectedTotalWritten := len(stdOutBytes) + len(stdErrBytes) - if written != int64(expectedTotalWritten) { - t.Fatalf("Expected to have total of %d bytes written, got %d", expectedTotalWritten, written) - } -} - -type customReader struct { - n int - err error - totalCalls int - correctCalls int - src *bytes.Buffer -} - -func (f *customReader) Read(buf []byte) (int, error) { - f.totalCalls++ - if f.totalCalls <= f.correctCalls { - return f.src.Read(buf) - } - return f.n, f.err -} - -func TestStdCopyReturnsErrorReadingHeader(t *testing.T) { - expectedError := errors.New("error") - reader := &customReader{ - err: expectedError} - written, err := StdCopy(ioutil.Discard, ioutil.Discard, reader) - if written != 0 { - t.Fatalf("Expected 0 bytes read, got %d", written) - } - if err != expectedError { - t.Fatalf("Didn't get expected error") - } -} - -func TestStdCopyReturnsErrorReadingFrame(t *testing.T) { - expectedError := errors.New("error") - stdOutBytes := []byte(strings.Repeat("o", startingBufLen)) - stdErrBytes := []byte(strings.Repeat("e", startingBufLen)) - buffer, err := getSrcBuffer(stdOutBytes, stdErrBytes) - if err != nil { - t.Fatal(err) - } - reader := &customReader{ - correctCalls: 1, - n: stdWriterPrefixLen + 1, - err: expectedError, - src: buffer} - written, err := StdCopy(ioutil.Discard, ioutil.Discard, reader) - if written != 0 { - t.Fatalf("Expected 0 bytes read, got %d", written) - } - if err != expectedError { - t.Fatalf("Didn't get expected error") - } -} - -func TestStdCopyDetectsCorruptedFrame(t *testing.T) { - stdOutBytes := []byte(strings.Repeat("o", startingBufLen)) - stdErrBytes := []byte(strings.Repeat("e", startingBufLen)) - buffer, err := getSrcBuffer(stdOutBytes, stdErrBytes) - if err != nil { - t.Fatal(err) - } - reader := &customReader{ - correctCalls: 1, - n: stdWriterPrefixLen + 1, - err: io.EOF, - src: buffer} - written, err := StdCopy(ioutil.Discard, ioutil.Discard, reader) - if written != startingBufLen { - t.Fatalf("Expected %d bytes read, got %d", startingBufLen, written) - } - if err != nil { - t.Fatal("Didn't get nil error") - } -} - -func TestStdCopyWithInvalidInputHeader(t *testing.T) { - dstOut := NewStdWriter(ioutil.Discard, Stdout) - dstErr := NewStdWriter(ioutil.Discard, Stderr) - src := strings.NewReader("Invalid input") - _, err := StdCopy(dstOut, dstErr, src) - if err == nil { - t.Fatal("StdCopy with invalid input header should fail.") - } -} - -func TestStdCopyWithCorruptedPrefix(t *testing.T) { - data := []byte{0x01, 0x02, 0x03} - src := bytes.NewReader(data) - written, err := StdCopy(nil, nil, src) - if err != nil { - t.Fatalf("StdCopy should not return an error with corrupted prefix.") - } - if written != 0 { - t.Fatalf("StdCopy should have written 0, but has written %d", written) - } -} - -func TestStdCopyReturnsWriteErrors(t *testing.T) { - stdOutBytes := []byte(strings.Repeat("o", startingBufLen)) - stdErrBytes := []byte(strings.Repeat("e", startingBufLen)) - buffer, err := getSrcBuffer(stdOutBytes, stdErrBytes) - if err != nil { - t.Fatal(err) - } - expectedError := errors.New("expected") - - dstOut := &errWriter{err: expectedError} - - written, err := StdCopy(dstOut, ioutil.Discard, buffer) - if written != 0 { - t.Fatalf("StdCopy should have written 0, but has written %d", written) - } - if err != expectedError { - t.Fatalf("Didn't get expected error, got %v", err) - } -} - -func TestStdCopyDetectsNotFullyWrittenFrames(t *testing.T) { - stdOutBytes := []byte(strings.Repeat("o", startingBufLen)) - stdErrBytes := []byte(strings.Repeat("e", startingBufLen)) - buffer, err := getSrcBuffer(stdOutBytes, stdErrBytes) - if err != nil { - t.Fatal(err) - } - dstOut := &errWriter{n: startingBufLen - 10} - - written, err := StdCopy(dstOut, ioutil.Discard, buffer) - if written != 0 { - t.Fatalf("StdCopy should have return 0 written bytes, but returned %d", written) - } - if err != io.ErrShortWrite { - t.Fatalf("Didn't get expected io.ErrShortWrite error") - } -} - -func BenchmarkWrite(b *testing.B) { - w := NewStdWriter(ioutil.Discard, Stdout) - data := []byte("Test line for testing stdwriter performance\n") - data = bytes.Repeat(data, 100) - b.SetBytes(int64(len(data))) - b.ResetTimer() - for i := 0; i < b.N; i++ { - if _, err := w.Write(data); err != nil { - b.Fatal(err) - } - } -} diff --git a/build/docker/util.go b/build/docker/util.go deleted file mode 100644 index 3d57228d7..000000000 --- a/build/docker/util.go +++ /dev/null @@ -1,102 +0,0 @@ -package docker - -import ( - "fmt" - "strings" - - "github.com/drone/drone/yaml" - "github.com/samalba/dockerclient" -) - -// helper function that converts the Continer data structure to the exepcted -// dockerclient.ContainerConfig. -func toContainerConfig(c *yaml.Container) *dockerclient.ContainerConfig { - config := &dockerclient.ContainerConfig{ - Image: c.Image, - Env: toEnvironmentSlice(c.Environment), - Labels: c.Labels, - Cmd: c.Command, - Entrypoint: c.Entrypoint, - WorkingDir: c.WorkingDir, - HostConfig: dockerclient.HostConfig{ - Privileged: c.Privileged, - NetworkMode: c.Network, - Memory: c.MemLimit, - ShmSize: c.ShmSize, - CpuShares: c.CPUShares, - CpuQuota: c.CPUQuota, - CpusetCpus: c.CPUSet, - MemorySwappiness: -1, - OomKillDisable: c.OomKillDisable, - }, - } - - if len(config.Entrypoint) == 0 { - config.Entrypoint = nil - } - if len(config.Cmd) == 0 { - config.Cmd = nil - } - if len(c.ExtraHosts) > 0 { - config.HostConfig.ExtraHosts = c.ExtraHosts - } - if len(c.DNS) != 0 { - config.HostConfig.Dns = c.DNS - } - if len(c.DNSSearch) != 0 { - config.HostConfig.DnsSearch = c.DNSSearch - } - if len(c.VolumesFrom) != 0 { - config.HostConfig.VolumesFrom = c.VolumesFrom - } - - config.Volumes = map[string]struct{}{} - for _, path := range c.Volumes { - if strings.Index(path, ":") == -1 { - config.Volumes[path] = struct{}{} - continue - } - parts := strings.Split(path, ":") - config.Volumes[parts[1]] = struct{}{} - config.HostConfig.Binds = append(config.HostConfig.Binds, path) - } - - for _, path := range c.Devices { - if strings.Index(path, ":") == -1 { - continue - } - parts := strings.Split(path, ":") - device := dockerclient.DeviceMapping{ - PathOnHost: parts[0], - PathInContainer: parts[1], - CgroupPermissions: "rwm", - } - config.HostConfig.Devices = append(config.HostConfig.Devices, device) - } - - return config -} - -// helper function that converts the AuthConfig data structure to the exepcted -// dockerclient.AuthConfig. -func toAuthConfig(container *yaml.Container) *dockerclient.AuthConfig { - if container.AuthConfig.Username == "" && - container.AuthConfig.Password == "" { - return nil - } - return &dockerclient.AuthConfig{ - Email: container.AuthConfig.Email, - Username: container.AuthConfig.Username, - Password: container.AuthConfig.Password, - } -} - -// helper function that converts a key value map of environment variables to a -// string slice in key=value format. -func toEnvironmentSlice(env map[string]string) []string { - var envs []string - for k, v := range env { - envs = append(envs, fmt.Sprintf("%s=%s", k, v)) - } - return envs -} diff --git a/build/docker/util_test.go b/build/docker/util_test.go deleted file mode 100644 index 1a4a8ce3c..000000000 --- a/build/docker/util_test.go +++ /dev/null @@ -1,24 +0,0 @@ -package docker - -import ( - "testing" -) - -func Test_toContainerConfig(t *testing.T) { - t.Skip() -} - -func Test_toAuthConfig(t *testing.T) { - t.Skip() -} - -func Test_toEnvironmentSlice(t *testing.T) { - env := map[string]string{ - "HOME": "/root", - } - envs := toEnvironmentSlice(env) - want, got := "HOME=/root", envs[0] - if want != got { - t.Errorf("Wanted envar %s got %s", want, got) - } -} diff --git a/build/engine.go b/build/engine.go deleted file mode 100644 index b93065680..000000000 --- a/build/engine.go +++ /dev/null @@ -1,16 +0,0 @@ -package build - -import ( - "io" - - "github.com/drone/drone/yaml" -) - -// Engine defines the container runtime engine. -type Engine interface { - ContainerStart(*yaml.Container) (string, error) - ContainerStop(string) error - ContainerRemove(string) error - ContainerWait(string) (*State, error) - ContainerLogs(string) (io.ReadCloser, error) -} diff --git a/build/error.go b/build/error.go deleted file mode 100644 index 9190de8a8..000000000 --- a/build/error.go +++ /dev/null @@ -1,37 +0,0 @@ -package build - -import ( - "errors" - "fmt" -) - -var ( - // ErrSkip is used as a return value when container execution should be - // skipped at runtime. It is not returned as an error by any function. - ErrSkip = errors.New("Skip") - - // ErrTerm is used as a return value when the runner should terminate - // execution and exit. It is not returned as an error by any function. - ErrTerm = errors.New("Terminate") -) - -// An ExitError reports an unsuccessful exit. -type ExitError struct { - Name string - Code int -} - -// Error returns the error message in string format. -func (e *ExitError) Error() string { - return fmt.Sprintf("%s : exit code %d", e.Name, e.Code) -} - -// An OomError reports the process received an OOMKill from the kernel. -type OomError struct { - Name string -} - -// Error returns the error message in string format. -func (e *OomError) Error() string { - return fmt.Sprintf("%s : received oom kill", e.Name) -} diff --git a/build/error_test.go b/build/error_test.go deleted file mode 100644 index 0e99a2119..000000000 --- a/build/error_test.go +++ /dev/null @@ -1,26 +0,0 @@ -package build - -import ( - "testing" - - "github.com/franela/goblin" -) - -func TestErrors(t *testing.T) { - g := goblin.Goblin(t) - - g.Describe("Error messages", func() { - - g.It("should include OOM details", func() { - err := OomError{Name: "golang"} - got, want := err.Error(), "golang : received oom kill" - g.Assert(got).Equal(want) - }) - - g.It("should include Exit code", func() { - err := ExitError{Name: "golang", Code: 255} - got, want := err.Error(), "golang : exit code 255" - g.Assert(got).Equal(want) - }) - }) -} diff --git a/build/pipeline.go b/build/pipeline.go deleted file mode 100644 index 7b83d1f46..000000000 --- a/build/pipeline.go +++ /dev/null @@ -1,241 +0,0 @@ -package build - -import ( - "bufio" - "strconv" - "sync" - "time" - - "github.com/Sirupsen/logrus" - "github.com/drone/drone/yaml" -) - -// element represents a link in the linked list. -type element struct { - *yaml.Container - next *element -} - -// Pipeline represents a build pipeline. -type Pipeline struct { - conf *yaml.Config - head *element - tail *element - wait sync.WaitGroup - pipe chan (*Line) - next chan (error) - done chan (error) - err error - - containers []string - volumes []string - networks []string - - engine Engine -} - -// Done returns when the process is done executing. -func (p *Pipeline) Done() <-chan error { - return p.done -} - -// Err returns the error for the current process. -func (p *Pipeline) Err() error { - return p.err -} - -// Next returns the next step in the process. -func (p *Pipeline) Next() <-chan error { - return p.next -} - -// Exec executes the current step. -func (p *Pipeline) Exec() { - go func() { - defer func() { - if r := recover(); r != nil { - logrus.Errorln("recover executing build step", r) - } - }() - - err := p.exec(p.head.Container) - if err != nil { - p.err = err - } - p.step() - }() -} - -// Skip skips the current step. -func (p *Pipeline) Skip() { - p.step() -} - -// Pipe returns the build output pipe. -func (p *Pipeline) Pipe() <-chan *Line { - return p.pipe -} - -// Head returns the head item in the list. -func (p *Pipeline) Head() *yaml.Container { - return p.head.Container -} - -// Tail returns the tail item in the list. -func (p *Pipeline) Tail() *yaml.Container { - return p.tail.Container -} - -// Stop stops the pipeline. -func (p *Pipeline) Stop() { - go func() { - defer func() { - if r := recover(); r != nil { - logrus.Errorln("recover stopping the pipeline", r) - } - }() - p.done <- ErrTerm - }() -} - -// Setup prepares the build pipeline environment. -func (p *Pipeline) Setup() error { - return nil -} - -// Teardown removes the pipeline environment. -func (p *Pipeline) Teardown() { - - for _, id := range p.containers { - p.engine.ContainerRemove(id) - } - - close(p.next) - close(p.done) - - // TODO we have a race condition here where the program can try to async - // write to a closed pipe channel. This package, in general, needs to be - // tested for race conditions. - // close(p.pipe) -} - -// step steps through the pipeline to head.next -func (p *Pipeline) step() { - if p.head == p.tail { - go func() { - defer func() { - if r := recover(); r != nil { - logrus.Errorln("recover executing step function", r) - } - }() - - // stop all containers - for _, id := range p.containers { - p.engine.ContainerStop(id) - } - - // wait for all logs to terminate - // p.wait.Done() // this is for the ambassador - p.wait.Wait() - - // signal completion - p.done <- nil - }() - } else { - go func() { - defer func() { - if r := recover(); r != nil { - logrus.Errorln("recover executing step to head function", r) - } - }() - - p.head = p.head.next - p.next <- nil - }() - } -} - -// close closes open channels and signals the pipeline is done. -func (p *Pipeline) close(err error) { - go func() { - defer func() { - if r := recover(); r != nil { - logrus.Errorln("recover closing the pipeline", r) - } - }() - p.done <- err - }() -} - -func (p *Pipeline) exec(c *yaml.Container) error { - - name, err := p.engine.ContainerStart(c) - if err != nil { - return err - } - p.containers = append(p.containers, name) - - p.wait.Add(1) - go func() { - defer func() { - if r := recover(); r != nil { - logrus.Errorln("recover writing build output", r) - } - - p.wait.Done() - }() - - rc, rerr := p.engine.ContainerLogs(name) - if rerr != nil { - return - } - defer rc.Close() - - num := 0 - now := time.Now().UTC() - scanner := bufio.NewScanner(rc) - for scanner.Scan() { - p.pipe <- &Line{ - Proc: c.Name, - Time: int64(time.Since(now).Seconds()), - Pos: num, - Out: scanner.Text(), - } - num++ - } - }() - - // exit when running container in detached mode in background - if c.Detached { - return nil - } - - state, err := p.engine.ContainerWait(name) - if err != nil { - return err - } - - p.wait.Add(1) - go func() { - defer func() { - if r := recover(); r != nil { - logrus.Errorln("recover writing exit code to output", r) - } - p.wait.Done() - }() - - p.pipe <- &Line{ - Proc: c.Name, - Type: ExitCodeLine, - Out: strconv.Itoa(state.ExitCode), - } - }() - - if state.OOMKilled { - return &OomError{c.Name} - } else if state.ExitCode != 0 { - return &ExitError{c.Name, state.ExitCode} - } - - return nil -} diff --git a/build/pipeline_test.go b/build/pipeline_test.go deleted file mode 100644 index 639d146f5..000000000 --- a/build/pipeline_test.go +++ /dev/null @@ -1,42 +0,0 @@ -package build - -var sampleYaml = ` -image: hello-world -build: - context: . - dockerfile: Dockerfile - -workspace: - path: src/github.com/octocat/hello-world - base: /go - -pipeline: - test: - image: golang - commands: - - go install - - go test - build: - image: golang - commands: - - go build - when: - event: push - notify: - image: slack - channel: dev - when: - event: failure - -services: - database: - image: mysql - -networks: - custom: - driver: overlay - -volumes: - custom: - driver: blockbridge -` diff --git a/build/types.go b/build/types.go deleted file mode 100644 index 1a79e98ab..000000000 --- a/build/types.go +++ /dev/null @@ -1,35 +0,0 @@ -package build - -import "fmt" - -const ( - StdoutLine int = iota - StderrLine - ExitCodeLine - MetadataLine - ProgressLine -) - -// Line is a line of console output. -type Line struct { - Proc string `json:"proc,omitempty"` - Time int64 `json:"time,omitempty"` - Type int `json:"type,omitempty"` - Pos int `json:"pos,omityempty"` - Out string `json:"out,omitempty"` -} - -func (l *Line) String() string { - switch l.Type { - case ExitCodeLine: - return fmt.Sprintf("[%s] exit code %s", l.Proc, l.Out) - default: - return fmt.Sprintf("[%s:L%v:%vs] %s", l.Proc, l.Pos, l.Time, l.Out) - } -} - -// State defines the state of the container. -type State struct { - ExitCode int // container exit code - OOMKilled bool // container exited due to oom error -} diff --git a/build/types_test.go b/build/types_test.go deleted file mode 100644 index c0fc0abac..000000000 --- a/build/types_test.go +++ /dev/null @@ -1,23 +0,0 @@ -package build - -import ( - "testing" - - "github.com/franela/goblin" -) - -func TestLine(t *testing.T) { - g := goblin.Goblin(t) - - g.Describe("Line output", func() { - g.It("should prefix string() with metadata", func() { - line := Line{ - Proc: "redis", - Time: 60, - Pos: 1, - Out: "starting redis server", - } - g.Assert(line.String()).Equal("[redis:L1:60s] starting redis server") - }) - }) -} diff --git a/drone/agent/agent.go b/drone/agent/agent.go index 3ea456b77..46a516f4d 100644 --- a/drone/agent/agent.go +++ b/drone/agent/agent.go @@ -1,77 +1,37 @@ package agent import ( - "fmt" + "context" + "io" + "log" "math" - "os" - "os/signal" - "strings" + "net/url" "sync" - "syscall" "time" - "github.com/drone/drone/model" - "github.com/drone/mq/logger" - "github.com/drone/mq/stomp" - "github.com/tidwall/redlog" + "github.com/cncd/pipeline/pipeline" + "github.com/cncd/pipeline/pipeline/backend" + "github.com/cncd/pipeline/pipeline/backend/docker" + "github.com/cncd/pipeline/pipeline/interrupt" + "github.com/cncd/pipeline/pipeline/multipart" + "github.com/cncd/pipeline/pipeline/rpc" - "github.com/Sirupsen/logrus" - "github.com/codegangsta/cli" - "github.com/samalba/dockerclient" + "github.com/tevino/abool" + "github.com/urfave/cli" ) // AgentCmd is the exported command for starting the drone agent. var AgentCmd = cli.Command{ Name: "agent", Usage: "starts the drone agent", - Action: start, + Action: loop, Flags: []cli.Flag{ - cli.StringFlag{ - EnvVar: "DOCKER_HOST", - Name: "docker-host", - Usage: "docker daemon address", - Value: "unix:///var/run/docker.sock", - }, - cli.BoolFlag{ - EnvVar: "DOCKER_TLS_VERIFY", - Name: "docker-tls-verify", - Usage: "docker daemon supports tlsverify", - }, - cli.StringFlag{ - EnvVar: "DOCKER_CERT_PATH", - Name: "docker-cert-path", - Usage: "docker certificate directory", - Value: "", - }, - cli.IntFlag{ - EnvVar: "DOCKER_MAX_PROCS", - Name: "docker-max-procs", - Usage: "limit number of running docker processes", - Value: 2, - }, - cli.StringFlag{ - EnvVar: "DOCKER_OS", - Name: "docker-os", - Usage: "docker operating system", - Value: "linux", - }, - cli.StringFlag{ - EnvVar: "DOCKER_ARCH", - Name: "docker-arch", - Usage: "docker architecture system", - Value: "amd64", - }, cli.StringFlag{ EnvVar: "DRONE_SERVER,DRONE_ENDPOINT", Name: "drone-server", Usage: "drone server address", Value: "ws://localhost:8000/ws/broker", }, - cli.StringFlag{ - EnvVar: "DRONE_TOKEN", - Name: "drone-token", - Usage: "drone authorization token", - }, cli.StringFlag{ EnvVar: "DRONE_SECRET,DRONE_AGENT_SECRET", Name: "drone-secret", @@ -83,89 +43,21 @@ var AgentCmd = cli.Command{ Usage: "drone server backoff interval", Value: time.Second * 15, }, - cli.DurationFlag{ - EnvVar: "DRONE_PING", - Name: "ping", - Usage: "drone server ping frequency", - Value: time.Minute * 5, + cli.IntFlag{ + Name: "retry-limit", + EnvVar: "DRONE_RETRY_LIMIT", + Value: math.MaxInt32, }, cli.BoolFlag{ EnvVar: "DRONE_DEBUG", Name: "debug", Usage: "start the agent in debug mode", }, - cli.DurationFlag{ - EnvVar: "DRONE_TIMEOUT", - Name: "timeout", - Usage: "drone timeout due to log inactivity", - Value: time.Minute * 15, - }, cli.StringFlag{ EnvVar: "DRONE_FILTER", Name: "filter", Usage: "filter jobs processed by this agent", }, - cli.IntFlag{ - EnvVar: "DRONE_MAX_LOGS", - Name: "max-log-size", - Usage: "drone maximum log size in megabytes", - Value: 5, - }, - cli.StringSliceFlag{ - EnvVar: "DRONE_PLUGIN_PRIVILEGED", - Name: "privileged", - Usage: "plugins that require privileged mode", - Value: &cli.StringSlice{ - "plugins/docker", - "plugins/docker:*", - "plugins/gcr", - "plugins/gcr:*", - "plugins/ecr", - "plugins/ecr:*", - }, - }, - cli.StringFlag{ - EnvVar: "DRONE_PLUGIN_NAMESPACE", - Name: "namespace", - Value: "plugins", - Usage: "default plugin image namespace", - }, - cli.BoolTFlag{ - EnvVar: "DRONE_PLUGIN_PULL", - Name: "pull", - Usage: "always pull latest plugin images", - }, - cli.StringSliceFlag{ - EnvVar: "DRONE_YAML_EXTENSION", - Name: "extension", - Usage: "custom plugin extension endpoint", - }, - - // - // - // - - cli.BoolFlag{ - EnvVar: "DRONE_CANARY", - Name: "canary", - Usage: "enable experimental features at your own risk", - }, - - // cli.StringFlag{ - // Name: "endpoint", - // EnvVar: "DRONE_ENDPOINT,DRONE_SERVER", - // Value: "ws://localhost:9999/ws/rpc", - // }, - // cli.DurationFlag{ - // Name: "backoff", - // EnvVar: "DRONE_BACKOFF", - // Value: time.Second * 15, - // }, - cli.IntFlag{ - Name: "retry-limit", - EnvVar: "DRONE_RETRY_LIMIT", - Value: math.MaxInt32, - }, cli.IntFlag{ Name: "max-procs", EnvVar: "DRONE_MAX_PROCS", @@ -179,139 +71,188 @@ var AgentCmd = cli.Command{ }, } -func start(c *cli.Context) { - - if c.Bool("canary") { - if err := loop(c); err != nil { - fmt.Println(err) - os.Exit(1) - } - return - } - - log := redlog.New(os.Stderr) - log.SetLevel(0) - logger.SetLogger(log) - - // debug level if requested by user - if c.Bool("debug") { - logrus.SetLevel(logrus.DebugLevel) - - log.SetLevel(1) - } else { - logrus.SetLevel(logrus.WarnLevel) - } - - var accessToken string - if c.String("drone-secret") != "" { - // secretToken := c.String("drone-secret") - accessToken = c.String("drone-secret") - // accessToken, _ = token.New(token.AgentToken, "").Sign(secretToken) - } else { - accessToken = c.String("drone-token") - } - - logger.Noticef("connecting to server %s", c.String("drone-server")) - - server := strings.TrimRight(c.String("drone-server"), "/") - - tls, err := dockerclient.TLSConfigFromCertPath(c.String("docker-cert-path")) - if err == nil { - tls.InsecureSkipVerify = c.Bool("docker-tls-verify") - } - docker, err := dockerclient.NewDockerClient(c.String("docker-host"), tls) +func loop(c *cli.Context) error { + endpoint, err := url.Parse( + c.String("drone-server"), + ) if err != nil { - logrus.Fatal(err) + return err + } + filter := rpc.Filter{ + Labels: map[string]string{ + "platform": c.String("platform"), + }, } - var client *stomp.Client + client, err := rpc.NewClient( + endpoint.String(), + rpc.WithRetryLimit( + c.Int("retry-limit"), + ), + rpc.WithBackoff( + c.Duration("backoff"), + ), + rpc.WithToken( + c.String("drone-secret"), + ), + ) + if err != nil { + return err + } + defer client.Close() - handler := func(m *stomp.Message) { - running.Add(1) - defer func() { - running.Done() - client.Ack(m.Ack) + sigterm := abool.New() + ctx := context.Background() + ctx = interrupt.WithContextFunc(ctx, func() { + println("ctrl+c received, terminating process") + sigterm.Set() + }) + + var wg sync.WaitGroup + parallel := c.Int("max-procs") + wg.Add(parallel) + + for i := 0; i < parallel; i++ { + go func() { + defer wg.Done() + for { + if sigterm.IsSet() { + return + } + if err := run(ctx, client, filter); err != nil { + log.Printf("build runner encountered error: exiting: %s", err) + return + } + } }() - - r := pipelinet{ - drone: client, - docker: docker, - config: config{ - platform: c.String("docker-os") + "/" + c.String("docker-arch"), - timeout: c.Duration("timeout"), - namespace: c.String("namespace"), - privileged: c.StringSlice("privileged"), - pull: c.BoolT("pull"), - logs: int64(c.Int("max-log-size")) * 1000000, - extension: c.StringSlice("extension"), - }, - } - - work := new(model.Work) - m.Unmarshal(work) - r.run(work) } - handleSignals() - - backoff := c.Duration("backoff") - - for { - // dial the drone server to establish a TCP connection. - client, err = stomp.Dial(server) - if err != nil { - logger.Warningf("connection failed, retry in %v. %s", backoff, err) - <-time.After(backoff) - continue - } - opts := []stomp.MessageOption{ - stomp.WithCredentials("x-token", accessToken), - } - - // initialize the stomp session and authenticate. - if err = client.Connect(opts...); err != nil { - logger.Warningf("session failed, retry in %v. %s", backoff, err) - <-time.After(backoff) - continue - } - - opts = []stomp.MessageOption{ - stomp.WithAck("client"), - stomp.WithPrefetch( - c.Int("docker-max-procs"), - ), - } - if filter := c.String("filter"); filter != "" { - opts = append(opts, stomp.WithSelector(filter)) - } - - // subscribe to the pending build queue. - client.Subscribe("/queue/pending", stomp.HandlerFunc(func(m *stomp.Message) { - go handler(m) // HACK until we a channel based Subscribe implementation - }), opts...) - - logger.Noticef("connection established, ready to process builds.") - <-client.Done() - - logger.Warningf("connection interrupted, attempting to reconnect.") - } + wg.Wait() + return nil } -// tracks running builds -var running sync.WaitGroup +const ( + maxFileUpload = 5000000 + maxLogsUpload = 5000000 +) -func handleSignals() { - // Graceful shut-down on SIGINT/SIGTERM - c := make(chan os.Signal, 1) - signal.Notify(c, os.Interrupt) - signal.Notify(c, syscall.SIGTERM) +func run(ctx context.Context, client rpc.Peer, filter rpc.Filter) error { + log.Println("pipeline: request next execution") + + // get the next job from the queue + work, err := client.Next(ctx, filter) + if err != nil { + return err + } + if work == nil { + return nil + } + log.Printf("pipeline: received next execution: %s", work.ID) + + // new docker engine + engine, err := docker.NewEnv() + if err != nil { + return err + } + + timeout := time.Hour + if minutes := work.Timeout; minutes != 0 { + timeout = time.Duration(minutes) * time.Minute + } + + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + cancelled := abool.New() + go func() { + if werr := client.Wait(ctx, work.ID); werr != nil { + cancelled.SetTo(true) + log.Printf("pipeline: cancel signal received: %s: %s", work.ID, werr) + cancel() + } else { + log.Printf("pipeline: cancel channel closed: %s", work.ID) + } + }() go func() { - <-c - logger.Warningf("SIGTERM received.") - logger.Warningf("wait for running builds to finish.") - running.Wait() - logger.Warningf("done.") - os.Exit(0) + for { + select { + case <-ctx.Done(): + log.Printf("pipeline: cancel ping loop: %s", work.ID) + return + case <-time.After(time.Minute): + log.Printf("pipeline: ping queue: %s", work.ID) + client.Extend(ctx, work.ID) + } + } }() + + state := rpc.State{} + state.Started = time.Now().Unix() + err = client.Update(context.Background(), work.ID, state) + if err != nil { + log.Printf("pipeline: error updating pipeline status: %s: %s", work.ID, err) + } + + var uploads sync.WaitGroup + defaultLogger := pipeline.LogFunc(func(proc *backend.Step, rc multipart.Reader) error { + part, rerr := rc.NextPart() + if rerr != nil { + return rerr + } + uploads.Add(1) + writer := rpc.NewLineWriter(client, work.ID, proc.Alias) + rlimit := io.LimitReader(part, maxLogsUpload) + io.Copy(writer, rlimit) + + defer func() { + log.Printf("pipeline: finish uploading logs: %s: step %s", work.ID, proc.Alias) + uploads.Done() + }() + + part, rerr = rc.NextPart() + if rerr != nil { + return nil + } + rlimit = io.LimitReader(part, maxFileUpload) + mime := part.Header().Get("Content-Type") + if serr := client.Upload(context.Background(), work.ID, mime, rlimit); serr != nil { + log.Printf("pipeline: cannot upload artifact: %s: %s: %s", work.ID, mime, serr) + } + return nil + }) + + err = pipeline.New(work.Config, + pipeline.WithContext(ctx), + pipeline.WithLogger(defaultLogger), + pipeline.WithTracer(pipeline.DefaultTracer), + pipeline.WithEngine(engine), + ).Run() + + state.Finished = time.Now().Unix() + state.Exited = true + if err != nil { + state.Error = err.Error() + if xerr, ok := err.(*pipeline.ExitError); ok { + state.ExitCode = xerr.Code + } + if xerr, ok := err.(*pipeline.OomError); ok { + state.ExitCode = xerr.Code + } + if cancelled.IsSet() { + state.ExitCode = 137 + } else if state.ExitCode == 0 { + state.ExitCode = 1 + } + } + + log.Printf("pipeline: execution complete: %s", work.ID) + + uploads.Wait() + err = client.Update(context.Background(), work.ID, state) + if err != nil { + log.Printf("Pipeine: error updating pipeline status: %s: %s", work.ID, err) + } + + return nil } diff --git a/drone/agent/exec.go b/drone/agent/exec.go deleted file mode 100644 index 268253420..000000000 --- a/drone/agent/exec.go +++ /dev/null @@ -1,85 +0,0 @@ -package agent - -import ( - "time" - - "github.com/Sirupsen/logrus" - "github.com/drone/drone/agent" - "github.com/drone/drone/build/docker" - "github.com/drone/drone/model" - "github.com/drone/mq/stomp" - - "github.com/samalba/dockerclient" -) - -type config struct { - platform string - namespace string - privileged []string - pull bool - logs int64 - timeout time.Duration - extension []string -} - -type pipelinet struct { - drone *stomp.Client - docker dockerclient.Client - config config -} - -func (r *pipelinet) run(w *model.Work) { - - // defer func() { - // // r.drone.Ack(id, opts) - // }() - - logrus.Infof("Starting build %s/%s#%d.%d", - w.Repo.Owner, w.Repo.Name, w.Build.Number, w.Job.Number) - - cancel := make(chan bool, 1) - engine := docker.NewClient(r.docker) - - a := agent.Agent{ - Update: agent.NewClientUpdater(r.drone), - Logger: agent.NewClientLogger(r.drone, w.Job.ID, r.config.logs), - Engine: engine, - Timeout: r.config.timeout, - Platform: r.config.platform, - Namespace: r.config.namespace, - Escalate: r.config.privileged, - Extension: r.config.extension, - Pull: r.config.pull, - } - - cancelFunc := func(m *stomp.Message) { - defer m.Release() - - id := m.Header.GetInt64("job-id") - if id == w.Job.ID { - cancel <- true - logrus.Infof("Cancel build %s/%s#%d.%d", - w.Repo.Owner, w.Repo.Name, w.Build.Number, w.Job.Number) - } - } - - // signal for canceling the build. - sub, err := r.drone.Subscribe("/topic/cancel", stomp.HandlerFunc(cancelFunc)) - if err != nil { - logrus.Errorf("Error subscribing to /topic/cancel. %s", err) - } - defer func() { - r.drone.Unsubscribe(sub) - }() - - a.Run(w, cancel) - - // if err := r.drone.LogPost(w.Job.ID, ioutil.NopCloser(&buf)); err != nil { - // logrus.Errorf("Error sending logs for %s/%s#%d.%d", - // w.Repo.Owner, w.Repo.Name, w.Build.Number, w.Job.Number) - // } - // stream.Close() - - logrus.Infof("Finished build %s/%s#%d.%d", - w.Repo.Owner, w.Repo.Name, w.Build.Number, w.Job.Number) -} diff --git a/drone/agent/exp.go b/drone/agent/exp.go deleted file mode 100644 index 133be9a89..000000000 --- a/drone/agent/exp.go +++ /dev/null @@ -1,206 +0,0 @@ -package agent - -import ( - "context" - "io" - "log" - "net/url" - "sync" - "time" - - "github.com/cncd/pipeline/pipeline" - "github.com/cncd/pipeline/pipeline/backend" - "github.com/cncd/pipeline/pipeline/backend/docker" - "github.com/cncd/pipeline/pipeline/interrupt" - "github.com/cncd/pipeline/pipeline/multipart" - "github.com/cncd/pipeline/pipeline/rpc" - - "github.com/codegangsta/cli" - "github.com/tevino/abool" -) - -func loop(c *cli.Context) error { - endpoint, err := url.Parse( - c.String("drone-server"), - ) - if err != nil { - return err - } - filter := rpc.Filter{ - Labels: map[string]string{ - "platform": c.String("platform"), - }, - } - - client, err := rpc.NewClient( - endpoint.String(), - rpc.WithRetryLimit( - c.Int("retry-limit"), - ), - rpc.WithBackoff( - c.Duration("backoff"), - ), - rpc.WithToken( - c.String("drone-secret"), - ), - ) - if err != nil { - return err - } - defer client.Close() - - sigterm := abool.New() - ctx := context.Background() - ctx = interrupt.WithContextFunc(ctx, func() { - println("ctrl+c received, terminating process") - sigterm.Set() - }) - - var wg sync.WaitGroup - parallel := c.Int("max-procs") - wg.Add(parallel) - - for i := 0; i < parallel; i++ { - go func() { - defer wg.Done() - for { - if sigterm.IsSet() { - return - } - if err := run(ctx, client, filter); err != nil { - log.Printf("build runner encountered error: exiting: %s", err) - return - } - } - }() - } - - wg.Wait() - return nil -} - -const ( - maxFileUpload = 5000000 - maxLogsUpload = 5000000 -) - -func run(ctx context.Context, client rpc.Peer, filter rpc.Filter) error { - log.Println("pipeline: request next execution") - - // get the next job from the queue - work, err := client.Next(ctx, filter) - if err != nil { - return err - } - if work == nil { - return nil - } - log.Printf("pipeline: received next execution: %s", work.ID) - - // new docker engine - engine, err := docker.NewEnv() - if err != nil { - return err - } - - timeout := time.Hour - if minutes := work.Timeout; minutes != 0 { - timeout = time.Duration(minutes) * time.Minute - } - - ctx, cancel := context.WithTimeout(context.Background(), timeout) - defer cancel() - - cancelled := abool.New() - go func() { - if werr := client.Wait(ctx, work.ID); werr != nil { - cancelled.SetTo(true) - log.Printf("pipeline: cancel signal received: %s: %s", work.ID, werr) - cancel() - } else { - log.Printf("pipeline: cancel channel closed: %s", work.ID) - } - }() - - go func() { - for { - select { - case <-ctx.Done(): - log.Printf("pipeline: cancel ping loop: %s", work.ID) - return - case <-time.After(time.Minute): - log.Printf("pipeline: ping queue: %s", work.ID) - client.Extend(ctx, work.ID) - } - } - }() - - state := rpc.State{} - state.Started = time.Now().Unix() - err = client.Update(context.Background(), work.ID, state) - if err != nil { - log.Printf("pipeline: error updating pipeline status: %s: %s", work.ID, err) - } - - var uploads sync.WaitGroup - defaultLogger := pipeline.LogFunc(func(proc *backend.Step, rc multipart.Reader) error { - part, rerr := rc.NextPart() - if rerr != nil { - return rerr - } - uploads.Add(1) - writer := rpc.NewLineWriter(client, work.ID, proc.Alias) - rlimit := io.LimitReader(part, maxLogsUpload) - io.Copy(writer, rlimit) - - defer func() { - log.Printf("pipeline: finish uploading logs: %s: step %s", work.ID, proc.Alias) - uploads.Done() - }() - - part, rerr = rc.NextPart() - if rerr != nil { - return nil - } - rlimit = io.LimitReader(part, maxFileUpload) - mime := part.Header().Get("Content-Type") - if serr := client.Upload(context.Background(), work.ID, mime, rlimit); serr != nil { - log.Printf("pipeline: cannot upload artifact: %s: %s: %s", work.ID, mime, serr) - } - return nil - }) - - err = pipeline.New(work.Config, - pipeline.WithContext(ctx), - pipeline.WithLogger(defaultLogger), - pipeline.WithTracer(pipeline.DefaultTracer), - pipeline.WithEngine(engine), - ).Run() - - state.Finished = time.Now().Unix() - state.Exited = true - if err != nil { - state.Error = err.Error() - if xerr, ok := err.(*pipeline.ExitError); ok { - state.ExitCode = xerr.Code - } - if xerr, ok := err.(*pipeline.OomError); ok { - state.ExitCode = xerr.Code - } - if cancelled.IsSet() { - state.ExitCode = 137 - } else if state.ExitCode == 0 { - state.ExitCode = 1 - } - } - - log.Printf("pipeline: execution complete: %s", work.ID) - - uploads.Wait() - err = client.Update(context.Background(), work.ID, state) - if err != nil { - log.Printf("Pipeine: error updating pipeline status: %s: %s", work.ID, err) - } - - return nil -} diff --git a/drone/agent_list.go b/drone/agent_list.go deleted file mode 100644 index 4bb435b49..000000000 --- a/drone/agent_list.go +++ /dev/null @@ -1,64 +0,0 @@ -package main - -import ( - "html/template" - "log" - "os" - "time" - - "github.com/codegangsta/cli" -) - -var agentsCmd = cli.Command{ - Name: "agents", - Usage: "manage agents", - Action: func(c *cli.Context) { - if err := agentList(c); err != nil { - log.Fatalln(err) - } - }, - Flags: []cli.Flag{ - cli.StringFlag{ - Name: "format", - Usage: "format output", - Value: tmplAgentList, - }, - }, -} - -func agentList(c *cli.Context) error { - client, err := newClient(c) - if err != nil { - return err - } - - agents, err := client.AgentList() - if err != nil { - return err - } - - tmpl, err := template.New("_").Funcs(funcMap).Parse(c.String("format") + "\n") - if err != nil { - return err - } - - for _, agent := range agents { - tmpl.Execute(os.Stdout, agent) - } - return nil -} - -// template for build list information -var tmplAgentList = "\x1b[33m{{ .Address }} \x1b[0m" + ` -Platform: {{ .Platform }} -Capacity: {{ .Capacity }} concurrent build(s) -Pinged: {{ since .Updated }} ago -Uptime: {{ since .Created }} -` - -var funcMap = template.FuncMap{ - "since": func(t int64) string { - d := time.Now().Sub(time.Unix(t, 0)) - return d.String() - }, -} diff --git a/drone/build.go b/drone/build.go index 164e35e8d..65c1ece14 100644 --- a/drone/build.go +++ b/drone/build.go @@ -1,6 +1,6 @@ package main -import "github.com/codegangsta/cli" +import "github.com/urfave/cli" var buildCmd = cli.Command{ Name: "build", diff --git a/drone/build_info.go b/drone/build_info.go index d3baba5e6..17df7882e 100644 --- a/drone/build_info.go +++ b/drone/build_info.go @@ -1,22 +1,17 @@ package main import ( - "log" "os" "strconv" "text/template" - "github.com/codegangsta/cli" + "github.com/urfave/cli" ) var buildInfoCmd = cli.Command{ - Name: "info", - Usage: "show build details", - Action: func(c *cli.Context) { - if err := buildInfo(c); err != nil { - log.Fatalln(err) - } - }, + Name: "info", + Usage: "show build details", + Action: buildInfo, Flags: []cli.Flag{ cli.StringFlag{ Name: "format", diff --git a/drone/build_last.go b/drone/build_last.go index 1bf431fd0..b1eb294ad 100644 --- a/drone/build_last.go +++ b/drone/build_last.go @@ -1,21 +1,16 @@ package main import ( - "log" "os" "text/template" - "github.com/codegangsta/cli" + "github.com/urfave/cli" ) var buildLastCmd = cli.Command{ - Name: "last", - Usage: "show latest build details", - Action: func(c *cli.Context) { - if err := buildLast(c); err != nil { - log.Fatalln(err) - } - }, + Name: "last", + Usage: "show latest build details", + Action: buildLast, Flags: []cli.Flag{ cli.StringFlag{ Name: "format", diff --git a/drone/build_list.go b/drone/build_list.go index f3c16a188..9d8002b57 100644 --- a/drone/build_list.go +++ b/drone/build_list.go @@ -1,21 +1,16 @@ package main import ( - "log" "os" "text/template" - "github.com/codegangsta/cli" + "github.com/urfave/cli" ) var buildListCmd = cli.Command{ - Name: "list", - Usage: "show build history", - Action: func(c *cli.Context) { - if err := buildList(c); err != nil { - log.Fatalln(err) - } - }, + Name: "list", + Usage: "show build history", + Action: buildList, Flags: []cli.Flag{ cli.StringFlag{ Name: "format", diff --git a/drone/build_logs.go b/drone/build_logs.go index 7f9f4c64e..0ed821b6d 100644 --- a/drone/build_logs.go +++ b/drone/build_logs.go @@ -6,8 +6,8 @@ import ( "log" "strconv" - "github.com/codegangsta/cli" - "github.com/drone/drone/build" + "github.com/cncd/pipeline/pipeline/rpc" + "github.com/urfave/cli" ) var buildLogsCmd = cli.Command{ @@ -61,7 +61,7 @@ func buildLogs(c *cli.Context) error { dec := json.NewDecoder(r) fmt.Printf("Logs for build %s/%s#%d.%d\n", owner, name, number, job) - var line build.Line + var line rpc.Line _, err = dec.Token() if err != nil { diff --git a/drone/build_queue.go b/drone/build_queue.go index c9450ab8f..85c8596ec 100644 --- a/drone/build_queue.go +++ b/drone/build_queue.go @@ -2,21 +2,16 @@ package main import ( "fmt" - "log" "os" "text/template" - "github.com/codegangsta/cli" + "github.com/urfave/cli" ) var buildQueueCmd = cli.Command{ - Name: "queue", - Usage: "show build queue", - Action: func(c *cli.Context) { - if err := buildQueue(c); err != nil { - log.Fatalln(err) - } - }, + Name: "queue", + Usage: "show build queue", + Action: buildQueue, Flags: []cli.Flag{ cli.StringFlag{ Name: "format", diff --git a/drone/build_start.go b/drone/build_start.go index bf9ff3cf0..d965866a5 100644 --- a/drone/build_start.go +++ b/drone/build_start.go @@ -2,21 +2,16 @@ package main import ( "fmt" - "log" "strconv" - "github.com/codegangsta/cli" "github.com/drone/drone/model" + "github.com/urfave/cli" ) var buildStartCmd = cli.Command{ - Name: "start", - Usage: "start a build", - Action: func(c *cli.Context) { - if err := buildStart(c); err != nil { - log.Fatalln(err) - } - }, + Name: "start", + Usage: "start a build", + Action: buildStart, Flags: []cli.Flag{ cli.BoolFlag{ Name: "fork", diff --git a/drone/build_stop.go b/drone/build_stop.go index 6d81b4f0f..0edfed0aa 100644 --- a/drone/build_stop.go +++ b/drone/build_stop.go @@ -2,20 +2,15 @@ package main import ( "fmt" - "log" "strconv" - "github.com/codegangsta/cli" + "github.com/urfave/cli" ) var buildStopCmd = cli.Command{ - Name: "stop", - Usage: "stop a build", - Action: func(c *cli.Context) { - if err := buildStop(c); err != nil { - log.Fatalln(err) - } - }, + Name: "stop", + Usage: "stop a build", + Action: buildStop, } func buildStop(c *cli.Context) (err error) { diff --git a/drone/compile.go b/drone/compile.go deleted file mode 100644 index 06ab7d0f9..000000000 --- a/drone/compile.go +++ /dev/null @@ -1 +0,0 @@ -package main diff --git a/drone/deploy.go b/drone/deploy.go index dd17092d1..e804122ec 100644 --- a/drone/deploy.go +++ b/drone/deploy.go @@ -3,22 +3,17 @@ package main import ( "fmt" "html/template" - "log" "os" "strconv" - "github.com/codegangsta/cli" "github.com/drone/drone/model" + "github.com/urfave/cli" ) var deployCmd = cli.Command{ - Name: "deploy", - Usage: "deploy code", - Action: func(c *cli.Context) { - if err := deploy(c); err != nil { - log.Fatalln(err) - } - }, + Name: "deploy", + Usage: "deploy code", + Action: deploy, Flags: []cli.Flag{ cli.StringFlag{ Name: "format", diff --git a/drone/exec.go b/drone/exec.go index 37c636f81..71c954c2a 100644 --- a/drone/exec.go +++ b/drone/exec.go @@ -1,22 +1,26 @@ package main import ( - "fmt" - "io/ioutil" + "context" + "io" "log" "os" - "os/signal" + "path" "path/filepath" "strings" "time" - "github.com/drone/drone/agent" - "github.com/drone/drone/build/docker" - "github.com/drone/drone/model" - "github.com/drone/drone/yaml" + "github.com/cncd/pipeline/pipeline" + "github.com/cncd/pipeline/pipeline/backend" + "github.com/cncd/pipeline/pipeline/backend/docker" + "github.com/cncd/pipeline/pipeline/frontend" + "github.com/cncd/pipeline/pipeline/frontend/yaml" + "github.com/cncd/pipeline/pipeline/frontend/yaml/compiler" + "github.com/cncd/pipeline/pipeline/interrupt" + "github.com/cncd/pipeline/pipeline/multipart" + "github.com/drone/envsubst" - "github.com/codegangsta/cli" - "github.com/joho/godotenv" + "github.com/urfave/cli" ) var execCmd = cli.Command{ @@ -33,439 +37,387 @@ var execCmd = cli.Command{ Usage: "build from local directory", EnvVar: "DRONE_LOCAL", }, - cli.StringSliceFlag{ - Name: "secret", - Usage: "build secrets in KEY=VALUE format", - EnvVar: "DRONE_SECRET", - }, - cli.StringFlag{ - Name: "secrets-file", - Usage: "build secrets file in KEY=VALUE format", - EnvVar: "DRONE_SECRETS_FILE", - }, - cli.StringSliceFlag{ - Name: "matrix", - Usage: "build matrix in KEY=VALUE format", - EnvVar: "DRONE_MATRIX", - }, cli.DurationFlag{ Name: "timeout", Usage: "build timeout", Value: time.Hour, EnvVar: "DRONE_TIMEOUT", }, - cli.DurationFlag{ - Name: "timeout.inactivity", - Usage: "build timeout for inactivity", - Value: time.Minute * 15, - EnvVar: "DRONE_TIMEOUT_INACTIVITY", - }, - cli.BoolFlag{ - EnvVar: "DRONE_PLUGIN_PULL", - Name: "pull", - Usage: "always pull latest plugin images", + cli.StringSliceFlag{ + Name: "volumes", + Usage: "build volumes", + EnvVar: "DRONE_VOLUMES", }, cli.StringSliceFlag{ - EnvVar: "DRONE_PLUGIN_PRIVILEGED", - Name: "privileged", - Usage: "plugins that require privileged mode", + Name: "privileged", + Usage: "privileged plugins", Value: &cli.StringSlice{ "plugins/docker", - "plugins/docker:*", "plugins/gcr", - "plugins/gcr:*", "plugins/ecr", - "plugins/ecr:*", }, }, - // Docker daemon flags - - cli.StringFlag{ - EnvVar: "DOCKER_HOST", - Name: "docker-host", - Usage: "docker daemon address", - Value: "unix:///var/run/docker.sock", - }, - cli.BoolFlag{ - EnvVar: "DOCKER_TLS_VERIFY", - Name: "docker-tls-verify", - Usage: "docker daemon supports tlsverify", - }, - cli.StringFlag{ - EnvVar: "DOCKER_CERT_PATH", - Name: "docker-cert-path", - Usage: "docker certificate directory", - Value: "", - }, - // - // Please note the below flags are mirrored in the plugin starter kit and - // should be kept synchronized. - // https://github.com/drone/drone-plugin-starter + // Please note the below flags are mirrored in the pipec and + // should be kept synchronized. Do not edit directly + // https://github.com/cncd/pipeline/pipec // + // + // workspace default + // cli.StringFlag{ - Name: "repo.fullname", - Usage: "repository full name", - EnvVar: "DRONE_REPO", + Name: "workspace-base", + Value: "/pipeline", + EnvVar: "DRONE_WORKSPACE_BASE", }, cli.StringFlag{ - Name: "repo.owner", - Usage: "repository owner", - EnvVar: "DRONE_REPO_OWNER", + Name: "workspace-path", + Value: "src", + EnvVar: "DRONE_WORKSPACE_PATH", }, + // + // netrc parameters + // cli.StringFlag{ - Name: "repo.name", - Usage: "repository name", - EnvVar: "DRONE_REPO_NAME", - }, - cli.StringFlag{ - Name: "repo.type", - Value: "git", - Usage: "repository type", - EnvVar: "DRONE_REPO_SCM", - }, - cli.StringFlag{ - Name: "repo.link", - Usage: "repository link", - EnvVar: "DRONE_REPO_LINK", - }, - cli.StringFlag{ - Name: "repo.avatar", - Usage: "repository avatar", - EnvVar: "DRONE_REPO_AVATAR", - }, - cli.StringFlag{ - Name: "repo.branch", - Usage: "repository default branch", - EnvVar: "DRONE_REPO_BRANCH", - }, - cli.BoolFlag{ - Name: "repo.private", - Usage: "repository is private", - EnvVar: "DRONE_REPO_PRIVATE", - }, - cli.BoolTFlag{ - Name: "repo.trusted", - Usage: "repository is trusted", - EnvVar: "DRONE_REPO_TRUSTED", - }, - cli.StringFlag{ - Name: "remote.url", - Usage: "git remote url", - EnvVar: "DRONE_REMOTE_URL", - }, - cli.StringFlag{ - Name: "commit.sha", - Usage: "git commit sha", - EnvVar: "DRONE_COMMIT_SHA", - }, - cli.StringFlag{ - Name: "commit.ref", - Value: "refs/heads/master", - Usage: "git commit ref", - EnvVar: "DRONE_COMMIT_REF", - }, - cli.StringFlag{ - Name: "commit.branch", - Value: "master", - Usage: "git commit branch", - EnvVar: "DRONE_COMMIT_BRANCH", - }, - cli.StringFlag{ - Name: "commit.message", - Usage: "git commit message", - EnvVar: "DRONE_COMMIT_MESSAGE", - }, - cli.StringFlag{ - Name: "commit.link", - Usage: "git commit link", - EnvVar: "DRONE_COMMIT_LINK", - }, - cli.StringFlag{ - Name: "commit.author.name", - Usage: "git author name", - EnvVar: "DRONE_COMMIT_AUTHOR", - }, - cli.StringFlag{ - Name: "commit.author.email", - Usage: "git author email", - EnvVar: "DRONE_COMMIT_AUTHOR_EMAIL", - }, - cli.StringFlag{ - Name: "commit.author.avatar", - Usage: "git author avatar", - EnvVar: "DRONE_COMMIT_AUTHOR_AVATAR", - }, - cli.StringFlag{ - Name: "build.event", - Value: "push", - Usage: "build event", - EnvVar: "DRONE_BUILD_EVENT", - }, - cli.IntFlag{ - Name: "build.number", - Usage: "build number", - EnvVar: "DRONE_BUILD_NUMBER", - }, - cli.IntFlag{ - Name: "build.created", - Usage: "build created", - EnvVar: "DRONE_BUILD_CREATED", - }, - cli.IntFlag{ - Name: "build.started", - Usage: "build started", - EnvVar: "DRONE_BUILD_STARTED", - }, - cli.IntFlag{ - Name: "build.finished", - Usage: "build finished", - EnvVar: "DRONE_BUILD_FINISHED", - }, - cli.StringFlag{ - Name: "build.status", - Usage: "build status", - Value: "success", - EnvVar: "DRONE_BUILD_STATUS", - }, - cli.StringFlag{ - Name: "build.link", - Usage: "build link", - EnvVar: "DRONE_BUILD_LINK", - }, - cli.StringFlag{ - Name: "build.deploy", - Usage: "build deployment target", - EnvVar: "DRONE_DEPLOY_TO", - }, - cli.BoolTFlag{ - Name: "yaml.verified", - Usage: "build yaml is verified", - EnvVar: "DRONE_YAML_VERIFIED", - }, - cli.BoolTFlag{ - Name: "yaml.signed", - Usage: "build yaml is signed", - EnvVar: "DRONE_YAML_SIGNED", - }, - cli.IntFlag{ - Name: "prev.build.number", - Usage: "previous build number", - EnvVar: "DRONE_PREV_BUILD_NUMBER", - }, - cli.StringFlag{ - Name: "prev.build.status", - Usage: "previous build status", - EnvVar: "DRONE_PREV_BUILD_STATUS", - }, - cli.StringFlag{ - Name: "prev.commit.sha", - Usage: "previous build sha", - EnvVar: "DRONE_PREV_COMMIT_SHA", - }, - - cli.StringFlag{ - Name: "netrc.username", - Usage: "previous build sha", + Name: "netrc-username", EnvVar: "DRONE_NETRC_USERNAME", }, cli.StringFlag{ - Name: "netrc.password", - Usage: "previous build sha", + Name: "netrc-password", EnvVar: "DRONE_NETRC_PASSWORD", }, cli.StringFlag{ - Name: "netrc.machine", - Usage: "previous build sha", + Name: "netrc-machine", EnvVar: "DRONE_NETRC_MACHINE", }, + // + // metadata parameters + // + cli.StringFlag{ + Name: "system-arch", + Value: "linux/amd64", + EnvVar: "DRONE_SYSTEM_ARCH", + }, + cli.StringFlag{ + Name: "system-name", + Value: "pipec", + EnvVar: "DRONE_SYSTEM_NAME", + }, + cli.StringFlag{ + Name: "system-link", + Value: "https://github.com/cncd/pipec", + EnvVar: "DRONE_SYSTEM_LINK", + }, + cli.StringFlag{ + Name: "repo-name", + EnvVar: "DRONE_REPO_NAME", + }, + cli.StringFlag{ + Name: "repo-link", + EnvVar: "DRONE_REPO_LINK", + }, + cli.StringFlag{ + Name: "repo-remote-url", + EnvVar: "DRONE_REPO_REMOTE", + }, + cli.StringFlag{ + Name: "repo-private", + EnvVar: "DRONE_REPO_PRIVATE", + }, + cli.IntFlag{ + Name: "build-number", + EnvVar: "DRONE_BUILD_NUMBER", + }, + cli.Int64Flag{ + Name: "build-created", + EnvVar: "DRONE_BUILD_CREATED", + }, + cli.Int64Flag{ + Name: "build-started", + EnvVar: "DRONE_BUILD_STARTED", + }, + cli.Int64Flag{ + Name: "build-finished", + EnvVar: "DRONE_BUILD_FINISHED", + }, + cli.StringFlag{ + Name: "build-status", + EnvVar: "DRONE_BUILD_STATUS", + }, + cli.StringFlag{ + Name: "build-event", + EnvVar: "DRONE_BUILD_EVENT", + }, + cli.StringFlag{ + Name: "build-link", + EnvVar: "DRONE_BUILD_LINK", + }, + cli.StringFlag{ + Name: "build-target", + EnvVar: "DRONE_BUILD_TARGET", + }, + cli.StringFlag{ + Name: "commit-sha", + EnvVar: "DRONE_COMMIT_SHA", + }, + cli.StringFlag{ + Name: "commit-ref", + EnvVar: "DRONE_COMMIT_REF", + }, + cli.StringFlag{ + Name: "commit-refspec", + EnvVar: "DRONE_COMMIT_REFSPEC", + }, + cli.StringFlag{ + Name: "commit-branch", + EnvVar: "DRONE_COMMIT_BRANCH", + }, + cli.StringFlag{ + Name: "commit-message", + EnvVar: "DRONE_COMMIT_MESSAGE", + }, + cli.StringFlag{ + Name: "commit-author-name", + EnvVar: "DRONE_COMMIT_AUTHOR_NAME", + }, + cli.StringFlag{ + Name: "commit-author-avatar", + EnvVar: "DRONE_COMMIT_AUTHOR_AVATAR", + }, + cli.StringFlag{ + Name: "commit-author-email", + EnvVar: "DRONE_COMMIT_AUTHOR_EMAIL", + }, + cli.IntFlag{ + Name: "prev-build-number", + EnvVar: "DRONE_PREV_BUILD_NUMBER", + }, + cli.Int64Flag{ + Name: "prev-build-created", + EnvVar: "DRONE_PREV_BUILD_CREATED", + }, + cli.Int64Flag{ + Name: "prev-build-started", + EnvVar: "DRONE_PREV_BUILD_STARTED", + }, + cli.Int64Flag{ + Name: "prev-build-finished", + EnvVar: "DRONE_PREV_BUILD_FINISHED", + }, + cli.StringFlag{ + Name: "prev-build-status", + EnvVar: "DRONE_PREV_BUILD_STATUS", + }, + cli.StringFlag{ + Name: "prev-build-event", + EnvVar: "DRONE_PREV_BUILD_EVENT", + }, + cli.StringFlag{ + Name: "prev-build-link", + EnvVar: "DRONE_PREV_BUILD_LINK", + }, + cli.StringFlag{ + Name: "prev-commit-sha", + EnvVar: "DRONE_PREV_COMMIT_SHA", + }, + cli.StringFlag{ + Name: "prev-commit-ref", + EnvVar: "DRONE_PREV_COMMIT_REF", + }, + cli.StringFlag{ + Name: "prev-commit-refspec", + EnvVar: "DRONE_PREV_COMMIT_REFSPEC", + }, + cli.StringFlag{ + Name: "prev-commit-branch", + EnvVar: "DRONE_PREV_COMMIT_BRANCH", + }, + cli.StringFlag{ + Name: "prev-commit-message", + EnvVar: "DRONE_PREV_COMMIT_MESSAGE", + }, + cli.StringFlag{ + Name: "prev-commit-author-name", + EnvVar: "DRONE_PREV_COMMIT_AUTHOR_NAME", + }, + cli.StringFlag{ + Name: "prev-commit-author-avatar", + EnvVar: "DRONE_PREV_COMMIT_AUTHOR_AVATAR", + }, + cli.StringFlag{ + Name: "prev-commit-author-email", + EnvVar: "DRONE_PREV_COMMIT_AUTHOR_EMAIL", + }, + cli.IntFlag{ + Name: "job-number", + EnvVar: "DRONE_JOB_NUMBER", + }, }, } func exec(c *cli.Context) error { - sigterm := make(chan os.Signal, 1) - cancelc := make(chan bool, 1) - signal.Notify(sigterm, os.Interrupt) - go func() { - <-sigterm - cancelc <- true - }() - - path := c.Args().First() - if path == "" { - path = ".drone.yml" + file := c.Args().First() + if file == "" { + file = ".drone.yml" } - path, _ = filepath.Abs(path) - dir := filepath.Dir(path) - file, err := ioutil.ReadFile(path) + metadata := metadataFromContext(c) + environ := metadata.Environ() + for k, v := range metadata.EnvironDrone() { + environ[k] = v + } + for _, env := range os.Environ() { + k := strings.Split(env, "=")[0] + v := strings.Split(env, "=")[1] + environ[k] = v + } + + tmpl, err := envsubst.ParseFile(file) + if err != nil { + return err + } + confstr, err := tmpl.Execute(func(name string) string { + return environ[name] + }) if err != nil { return err } - engine, err := docker.New( - c.String("docker-host"), - c.String("docker-cert-path"), - c.Bool("docker-tls-verify"), - ) + conf, err := yaml.ParseString(confstr) if err != nil { return err } - a := agent.Agent{ - Update: agent.NoopUpdateFunc, - Logger: agent.TermLoggerFunc, - Engine: engine, - Timeout: c.Duration("timeout.inactivity"), - Platform: "linux/amd64", - Escalate: c.StringSlice("privileged"), - Netrc: []string{}, - Local: dir, - Pull: c.Bool("pull"), - } - - payload := &model.Work{ - Yaml: string(file), - Verified: c.BoolT("yaml.verified"), - Signed: c.BoolT("yaml.signed"), - Repo: &model.Repo{ - FullName: c.String("repo.fullname"), - Owner: c.String("repo.owner"), - Name: c.String("repo.name"), - Kind: c.String("repo.type"), - Link: c.String("repo.link"), - Branch: c.String("repo.branch"), - Avatar: c.String("repo.avatar"), - Timeout: int64(c.Duration("timeout").Minutes()), - IsPrivate: c.Bool("repo.private"), - IsTrusted: c.BoolT("repo.trusted"), - Clone: c.String("remote.url"), - }, - System: &model.System{ - Link: c.GlobalString("server"), - }, - Secrets: getSecrets(c), - Netrc: &model.Netrc{ - Login: c.String("netrc.username"), - Password: c.String("netrc.password"), - Machine: c.String("netrc.machine"), - }, - Build: &model.Build{ - Commit: c.String("commit.sha"), - Branch: c.String("commit.branch"), - Ref: c.String("commit.ref"), - Link: c.String("commit.link"), - Message: c.String("commit.message"), - Author: c.String("commit.author.name"), - Email: c.String("commit.author.email"), - Avatar: c.String("commit.author.avatar"), - Number: c.Int("build.number"), - Event: c.String("build.event"), - Deploy: c.String("build.deploy"), - }, - BuildLast: &model.Build{ - Number: c.Int("prev.build.number"), - Status: c.String("prev.build.status"), - Commit: c.String("prev.commit.sha"), - }, - } - - if len(c.StringSlice("matrix")) > 0 { - p := *payload - p.Job = &model.Job{ - Environment: getMatrix(c), + // configure volumes for local execution + volumes := c.StringSlice("volumes") + if c.Bool("local") { + var ( + workspaceBase = conf.Workspace.Base + workspacePath = conf.Workspace.Path + ) + if workspaceBase == "" { + workspaceBase = c.String("workspace-base") } - return a.Run(&p, cancelc) + if workspacePath == "" { + workspacePath = c.String("workspace-path") + } + dir, _ := filepath.Abs(filepath.Dir(file)) + volumes = append(volumes, dir+":"+path.Join(workspaceBase, workspacePath)) } - axes, err := yaml.ParseMatrix(file) + // compiles the yaml file + compiled := compiler.New( + compiler.WithEscalated( + c.StringSlice("privileged")..., + ), + compiler.WithVolumes(volumes...), + compiler.WithWorkspace( + c.String("workspace-base"), + c.String("workspace-path"), + ), + compiler.WithPrefix( + c.String("prefix"), + ), + compiler.WithProxy(), + compiler.WithLocal( + c.Bool("local"), + ), + compiler.WithNetrc( + c.String("netrc-username"), + c.String("netrc-password"), + c.String("netrc-machine"), + ), + compiler.WithMetadata(metadata), + ).Compile(conf) + + engine, err := docker.NewEnv() if err != nil { return err } - if len(axes) == 0 { - axes = append(axes, yaml.Axis{}) - } + ctx, cancel := context.WithTimeout(context.Background(), c.Duration("timeout")) + defer cancel() + ctx = interrupt.WithContext(ctx) - var jobs []*model.Job - count := 0 - for _, axis := range axes { - jobs = append(jobs, &model.Job{ - Number: count, - Environment: axis, - }) - count++ - } - - for _, job := range jobs { - fmt.Printf("Running Matrix job #%d\n", job.Number) - p := *payload - p.Job = job - if err := a.Run(&p, cancelc); err != nil { - return err - } - } - - return nil + return pipeline.New(compiled, + pipeline.WithContext(ctx), + pipeline.WithLogger(defaultLogger), + pipeline.WithTracer(pipeline.DefaultTracer), + pipeline.WithLogger(defaultLogger), + pipeline.WithEngine(engine), + ).Run() } -// helper function to retrieve matrix variables. -func getMatrix(c *cli.Context) map[string]string { - envs := map[string]string{} - for _, s := range c.StringSlice("matrix") { - parts := strings.SplitN(s, "=", 2) - if len(parts) != 2 { - continue - } - k := parts[0] - v := parts[1] - envs[k] = v - } - return envs -} - -// helper function to retrieve secret variables. -func getSecrets(c *cli.Context) []*model.Secret { - - var secrets []*model.Secret - - if c.String("secrets-file") != "" { - envs, _ := godotenv.Read(c.String("secrets-file")) - for k, v := range envs { - secret := &model.Secret{ - Name: k, - Value: v, - Events: []string{ - model.EventPull, - model.EventPush, - model.EventTag, - model.EventDeploy, +// return the metadata from the cli context. +func metadataFromContext(c *cli.Context) frontend.Metadata { + return frontend.Metadata{ + Repo: frontend.Repo{ + Name: c.String("repo-name"), + Link: c.String("repo-link"), + Remote: c.String("repo-remote-url"), + Private: c.Bool("repo-private"), + }, + Curr: frontend.Build{ + Number: c.Int("build-number"), + Created: c.Int64("build-created"), + Started: c.Int64("build-started"), + Finished: c.Int64("build-finished"), + Status: c.String("build-status"), + Event: c.String("build-event"), + Link: c.String("build-link"), + Target: c.String("build-target"), + Commit: frontend.Commit{ + Sha: c.String("commit-sha"), + Ref: c.String("commit-ref"), + Refspec: c.String("commit-refspec"), + Branch: c.String("commit-branch"), + Message: c.String("commit-message"), + Author: frontend.Author{ + Name: c.String("commit-author-name"), + Email: c.String("commit-author-email"), + Avatar: c.String("commit-author-avatar"), }, - Images: []string{"*"}, - } - secrets = append(secrets, secret) - } - } - - for _, s := range c.StringSlice("secret") { - parts := strings.SplitN(s, "=", 2) - if len(parts) != 2 { - continue - } - secret := &model.Secret{ - Name: parts[0], - Value: parts[1], - Events: []string{ - model.EventPull, - model.EventPush, - model.EventTag, - model.EventDeploy, }, - Images: []string{"*"}, - } - secrets = append(secrets, secret) + }, + Prev: frontend.Build{ + Number: c.Int("prev-build-number"), + Created: c.Int64("prev-build-created"), + Started: c.Int64("prev-build-started"), + Finished: c.Int64("prev-build-finished"), + Status: c.String("prev-build-status"), + Event: c.String("prev-build-event"), + Link: c.String("prev-build-link"), + Commit: frontend.Commit{ + Sha: c.String("prev-commit-sha"), + Ref: c.String("prev-commit-ref"), + Refspec: c.String("prev-commit-refspec"), + Branch: c.String("prev-commit-branch"), + Message: c.String("prev-commit-message"), + Author: frontend.Author{ + Name: c.String("prev-commit-author-name"), + Email: c.String("prev-commit-author-email"), + Avatar: c.String("prev-commit-author-avatar"), + }, + }, + }, + Job: frontend.Job{ + Number: c.Int("job-number"), + }, + Sys: frontend.System{ + Name: c.String("system-name"), + Link: c.String("system-link"), + Arch: c.String("system-arch"), + }, } - return secrets } + +var defaultLogger = pipeline.LogFunc(func(proc *backend.Step, rc multipart.Reader) error { + part, err := rc.NextPart() + if err != nil { + return err + } + io.Copy(os.Stderr, part) + return nil +}) diff --git a/drone/global.go b/drone/global.go index bb6de1f4e..6bf706df7 100644 --- a/drone/global.go +++ b/drone/global.go @@ -1,6 +1,6 @@ package main -import "github.com/codegangsta/cli" +import "github.com/urfave/cli" var globalCmd = cli.Command{ Name: "global", diff --git a/drone/global_secret.go b/drone/global_secret.go index e6b2f0682..a3d2e06ba 100644 --- a/drone/global_secret.go +++ b/drone/global_secret.go @@ -1,6 +1,6 @@ package main -import "github.com/codegangsta/cli" +import "github.com/urfave/cli" var globalSecretCmd = cli.Command{ Name: "secret", diff --git a/drone/global_secret_add.go b/drone/global_secret_add.go index 747ca34c8..3543fa197 100644 --- a/drone/global_secret_add.go +++ b/drone/global_secret_add.go @@ -1,21 +1,13 @@ package main -import ( - "log" - - "github.com/codegangsta/cli" -) +import "github.com/urfave/cli" var globalSecretAddCmd = cli.Command{ Name: "add", Usage: "adds a secret", ArgsUsage: "[key] [value]", - Action: func(c *cli.Context) { - if err := globalSecretAdd(c); err != nil { - log.Fatalln(err) - } - }, - Flags: secretAddFlags(), + Action: globalSecretAdd, + Flags: secretAddFlags(), } func globalSecretAdd(c *cli.Context) error { diff --git a/drone/global_secret_list.go b/drone/global_secret_list.go index 5af148d1b..81c049a40 100644 --- a/drone/global_secret_list.go +++ b/drone/global_secret_list.go @@ -1,20 +1,12 @@ package main -import ( - "log" - - "github.com/codegangsta/cli" -) +import "github.com/urfave/cli" var globalSecretListCmd = cli.Command{ - Name: "ls", - Usage: "list all secrets", - Action: func(c *cli.Context) { - if err := globalSecretList(c); err != nil { - log.Fatalln(err) - } - }, - Flags: secretListFlags(), + Name: "ls", + Usage: "list all secrets", + Action: globalSecretList, + Flags: secretListFlags(), } func globalSecretList(c *cli.Context) error { diff --git a/drone/global_secret_rm.go b/drone/global_secret_rm.go index 47ceec8aa..398cb72e7 100644 --- a/drone/global_secret_rm.go +++ b/drone/global_secret_rm.go @@ -1,19 +1,11 @@ package main -import ( - "log" - - "github.com/codegangsta/cli" -) +import "github.com/urfave/cli" var globalSecretRemoveCmd = cli.Command{ - Name: "rm", - Usage: "remove a secret", - Action: func(c *cli.Context) { - if err := globalSecretRemove(c); err != nil { - log.Fatalln(err) - } - }, + Name: "rm", + Usage: "remove a secret", + Action: globalSecretRemove, } func globalSecretRemove(c *cli.Context) error { diff --git a/drone/info.go b/drone/info.go index b7396228f..d5d288844 100644 --- a/drone/info.go +++ b/drone/info.go @@ -1,21 +1,16 @@ package main import ( - "log" "os" "text/template" - "github.com/codegangsta/cli" + "github.com/urfave/cli" ) var infoCmd = cli.Command{ - Name: "info", - Usage: "show information about the current user", - Action: func(c *cli.Context) { - if err := info(c); err != nil { - log.Fatalln(err) - } - }, + Name: "info", + Usage: "show information about the current user", + Action: info, Flags: []cli.Flag{ cli.StringFlag{ Name: "format", diff --git a/drone/main.go b/drone/main.go index d4867da1e..459cf24f9 100644 --- a/drone/main.go +++ b/drone/main.go @@ -1,14 +1,15 @@ package main import ( + "fmt" "os" "github.com/drone/drone/drone/agent" "github.com/drone/drone/version" - "github.com/codegangsta/cli" "github.com/ianschenck/envflag" _ "github.com/joho/godotenv/autoload" + "github.com/urfave/cli" ) func main() { @@ -32,7 +33,6 @@ func main() { } app.Commands = []cli.Command{ agent.AgentCmd, - agentsCmd, buildCmd, deployCmd, execCmd, @@ -46,5 +46,8 @@ func main() { globalCmd, } - app.Run(os.Args) + if err := app.Run(os.Args); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } } diff --git a/drone/org.go b/drone/org.go index d9ffaefe0..b4e78d60e 100644 --- a/drone/org.go +++ b/drone/org.go @@ -1,6 +1,6 @@ package main -import "github.com/codegangsta/cli" +import "github.com/urfave/cli" var orgCmd = cli.Command{ Name: "org", diff --git a/drone/org_secret.go b/drone/org_secret.go index 1bf4e812d..542d32e20 100644 --- a/drone/org_secret.go +++ b/drone/org_secret.go @@ -1,6 +1,6 @@ package main -import "github.com/codegangsta/cli" +import "github.com/urfave/cli" var orgSecretCmd = cli.Command{ Name: "secret", diff --git a/drone/org_secret_add.go b/drone/org_secret_add.go index 07570a03f..c41973641 100644 --- a/drone/org_secret_add.go +++ b/drone/org_secret_add.go @@ -1,21 +1,13 @@ package main -import ( - "log" - - "github.com/codegangsta/cli" -) +import "github.com/urfave/cli" var orgSecretAddCmd = cli.Command{ Name: "add", Usage: "adds a secret", ArgsUsage: "[org] [key] [value]", - Action: func(c *cli.Context) { - if err := orgSecretAdd(c); err != nil { - log.Fatalln(err) - } - }, - Flags: secretAddFlags(), + Action: orgSecretAdd, + Flags: secretAddFlags(), } func orgSecretAdd(c *cli.Context) error { diff --git a/drone/org_secret_list.go b/drone/org_secret_list.go index 1163a3d2c..6f2e95139 100644 --- a/drone/org_secret_list.go +++ b/drone/org_secret_list.go @@ -1,20 +1,12 @@ package main -import ( - "log" - - "github.com/codegangsta/cli" -) +import "github.com/urfave/cli" var orgSecretListCmd = cli.Command{ - Name: "ls", - Usage: "list all secrets", - Action: func(c *cli.Context) { - if err := orgSecretList(c); err != nil { - log.Fatalln(err) - } - }, - Flags: secretListFlags(), + Name: "ls", + Usage: "list all secrets", + Action: orgSecretList, + Flags: secretListFlags(), } func orgSecretList(c *cli.Context) error { diff --git a/drone/org_secret_rm.go b/drone/org_secret_rm.go index fc46d7b60..c8748f9bd 100644 --- a/drone/org_secret_rm.go +++ b/drone/org_secret_rm.go @@ -3,7 +3,7 @@ package main import ( "log" - "github.com/codegangsta/cli" + "github.com/urfave/cli" ) var orgSecretRemoveCmd = cli.Command{ diff --git a/drone/repo.go b/drone/repo.go index a59433c78..d3ded8b2f 100644 --- a/drone/repo.go +++ b/drone/repo.go @@ -1,6 +1,6 @@ package main -import "github.com/codegangsta/cli" +import "github.com/urfave/cli" var repoCmd = cli.Command{ Name: "repo", diff --git a/drone/repo_add.go b/drone/repo_add.go index d859fa8c0..41965dc84 100644 --- a/drone/repo_add.go +++ b/drone/repo_add.go @@ -2,19 +2,14 @@ package main import ( "fmt" - "log" - "github.com/codegangsta/cli" + "github.com/urfave/cli" ) var repoAddCmd = cli.Command{ - Name: "add", - Usage: "add a repository", - Action: func(c *cli.Context) { - if err := repoAdd(c); err != nil { - log.Fatalln(err) - } - }, + Name: "add", + Usage: "add a repository", + Action: repoAdd, } func repoAdd(c *cli.Context) error { diff --git a/drone/repo_chown.go b/drone/repo_chown.go index c6fd8fcf4..f74fdafb7 100644 --- a/drone/repo_chown.go +++ b/drone/repo_chown.go @@ -2,19 +2,14 @@ package main import ( "fmt" - "log" - "github.com/codegangsta/cli" + "github.com/urfave/cli" ) var repoChownCmd = cli.Command{ - Name: "chown", - Usage: "assume ownership of a repository", - Action: func(c *cli.Context) { - if err := repoChown(c); err != nil { - log.Fatalln(err) - } - }, + Name: "chown", + Usage: "assume ownership of a repository", + Action: repoChown, } func repoChown(c *cli.Context) error { diff --git a/drone/repo_info.go b/drone/repo_info.go index bbaf0137b..f96ee69b7 100644 --- a/drone/repo_info.go +++ b/drone/repo_info.go @@ -1,21 +1,16 @@ package main import ( - "log" "os" "text/template" - "github.com/codegangsta/cli" + "github.com/urfave/cli" ) var repoInfoCmd = cli.Command{ - Name: "info", - Usage: "show repository details", - Action: func(c *cli.Context) { - if err := repoInfo(c); err != nil { - log.Fatalln(err) - } - }, + Name: "info", + Usage: "show repository details", + Action: repoInfo, Flags: []cli.Flag{ cli.StringFlag{ Name: "format", diff --git a/drone/repo_list.go b/drone/repo_list.go index c29fcea10..5709d255a 100644 --- a/drone/repo_list.go +++ b/drone/repo_list.go @@ -1,21 +1,16 @@ package main import ( - "log" "os" "text/template" - "github.com/codegangsta/cli" + "github.com/urfave/cli" ) var repoListCmd = cli.Command{ - Name: "ls", - Usage: "list all repos", - Action: func(c *cli.Context) { - if err := repoList(c); err != nil { - log.Fatalln(err) - } - }, + Name: "ls", + Usage: "list all repos", + Action: repoList, Flags: []cli.Flag{ cli.StringFlag{ Name: "format", diff --git a/drone/repo_rm.go b/drone/repo_rm.go index a2aa3c71f..c2c5cbd28 100644 --- a/drone/repo_rm.go +++ b/drone/repo_rm.go @@ -2,19 +2,14 @@ package main import ( "fmt" - "log" - "github.com/codegangsta/cli" + "github.com/urfave/cli" ) var repoRemoveCmd = cli.Command{ - Name: "rm", - Usage: "remove a repository", - Action: func(c *cli.Context) { - if err := repoRemove(c); err != nil { - log.Fatalln(err) - } - }, + Name: "rm", + Usage: "remove a repository", + Action: repoRemove, } func repoRemove(c *cli.Context) error { diff --git a/drone/secret.go b/drone/secret.go index 9639665bf..2858e5b28 100644 --- a/drone/secret.go +++ b/drone/secret.go @@ -6,8 +6,8 @@ import ( "strings" "text/template" - "github.com/codegangsta/cli" "github.com/drone/drone/model" + "github.com/urfave/cli" ) var secretCmd = cli.Command{ diff --git a/drone/secret_add.go b/drone/secret_add.go index fe8a9197a..8ba78e65c 100644 --- a/drone/secret_add.go +++ b/drone/secret_add.go @@ -1,21 +1,13 @@ package main -import ( - "log" - - "github.com/codegangsta/cli" -) +import "github.com/urfave/cli" var secretAddCmd = cli.Command{ Name: "add", Usage: "adds a secret", ArgsUsage: "[repo] [key] [value]", - Action: func(c *cli.Context) { - if err := secretAdd(c); err != nil { - log.Fatalln(err) - } - }, - Flags: secretAddFlags(), + Action: secretAdd, + Flags: secretAddFlags(), } func secretAdd(c *cli.Context) error { diff --git a/drone/secret_list.go b/drone/secret_list.go index 2fc6c33e9..6a94938c4 100644 --- a/drone/secret_list.go +++ b/drone/secret_list.go @@ -1,20 +1,12 @@ package main -import ( - "log" - - "github.com/codegangsta/cli" -) +import "github.com/urfave/cli" var secretListCmd = cli.Command{ - Name: "ls", - Usage: "list all secrets", - Action: func(c *cli.Context) { - if err := secretList(c); err != nil { - log.Fatalln(err) - } - }, - Flags: secretListFlags(), + Name: "ls", + Usage: "list all secrets", + Action: secretList, + Flags: secretListFlags(), } func secretList(c *cli.Context) error { diff --git a/drone/secret_rm.go b/drone/secret_rm.go index a68f17b76..37538af13 100644 --- a/drone/secret_rm.go +++ b/drone/secret_rm.go @@ -1,19 +1,11 @@ package main -import ( - "log" - - "github.com/codegangsta/cli" -) +import "github.com/urfave/cli" var secretRemoveCmd = cli.Command{ - Name: "rm", - Usage: "remove a secret", - Action: func(c *cli.Context) { - if err := secretRemove(c); err != nil { - log.Fatalln(err) - } - }, + Name: "rm", + Usage: "remove a secret", + Action: secretRemove, } func secretRemove(c *cli.Context) error { diff --git a/drone/server.go b/drone/server.go index eab5f13d1..39900f808 100644 --- a/drone/server.go +++ b/drone/server.go @@ -8,18 +8,14 @@ import ( "github.com/drone/drone/router/middleware" "github.com/Sirupsen/logrus" - "github.com/codegangsta/cli" "github.com/gin-gonic/contrib/ginrus" + "github.com/urfave/cli" ) var serverCmd = cli.Command{ - Name: "server", - Usage: "starts the drone server daemon", - Action: func(c *cli.Context) { - if err := server(c); err != nil { - logrus.Fatal(err) - } - }, + Name: "server", + Usage: "starts the drone server daemon", + Action: server, Flags: []cli.Flag{ cli.BoolFlag{ EnvVar: "DRONE_DEBUG", @@ -301,8 +297,6 @@ func server(c *cli.Context) error { middleware.Cache(c), middleware.Store(c), middleware.Remote(c), - middleware.Agents(c), - middleware.Broker(c), ) // start the server with tls enabled diff --git a/drone/sign.go b/drone/sign.go index b3b03efcb..e9fc1f0f3 100644 --- a/drone/sign.go +++ b/drone/sign.go @@ -2,19 +2,14 @@ package main import ( "io/ioutil" - "log" - "github.com/codegangsta/cli" + "github.com/urfave/cli" ) var signCmd = cli.Command{ - Name: "sign", - Usage: "creates a secure yaml file", - Action: func(c *cli.Context) { - if err := sign(c); err != nil { - log.Fatalln(err) - } - }, + Name: "sign", + Usage: "creates a secure yaml file", + Action: sign, Flags: []cli.Flag{ cli.StringFlag{ Name: "in", diff --git a/drone/user.go b/drone/user.go index 9c5e7cf90..db0444708 100644 --- a/drone/user.go +++ b/drone/user.go @@ -1,6 +1,6 @@ package main -import "github.com/codegangsta/cli" +import "github.com/urfave/cli" var userCmd = cli.Command{ Name: "user", diff --git a/drone/user_add.go b/drone/user_add.go index f68b6919c..1bbe63e23 100644 --- a/drone/user_add.go +++ b/drone/user_add.go @@ -2,20 +2,15 @@ package main import ( "fmt" - "log" - "github.com/codegangsta/cli" "github.com/drone/drone/model" + "github.com/urfave/cli" ) var userAddCmd = cli.Command{ - Name: "add", - Usage: "adds a user", - Action: func(c *cli.Context) { - if err := userAdd(c); err != nil { - log.Fatalln(err) - } - }, + Name: "add", + Usage: "adds a user", + Action: userAdd, } func userAdd(c *cli.Context) error { diff --git a/drone/user_info.go b/drone/user_info.go index 0a2f0cd54..3927620b5 100644 --- a/drone/user_info.go +++ b/drone/user_info.go @@ -2,21 +2,16 @@ package main import ( "fmt" - "log" "os" "text/template" - "github.com/codegangsta/cli" + "github.com/urfave/cli" ) var userInfoCmd = cli.Command{ - Name: "info", - Usage: "show user details", - Action: func(c *cli.Context) { - if err := userInfo(c); err != nil { - log.Fatalln(err) - } - }, + Name: "info", + Usage: "show user details", + Action: userInfo, Flags: []cli.Flag{ cli.StringFlag{ Name: "format", diff --git a/drone/user_list.go b/drone/user_list.go index 68aa69fb8..f827bfa24 100644 --- a/drone/user_list.go +++ b/drone/user_list.go @@ -1,21 +1,16 @@ package main import ( - "log" "os" "text/template" - "github.com/codegangsta/cli" + "github.com/urfave/cli" ) var userListCmd = cli.Command{ - Name: "ls", - Usage: "list all users", - Action: func(c *cli.Context) { - if err := userList(c); err != nil { - log.Fatalln(err) - } - }, + Name: "ls", + Usage: "list all users", + Action: userList, Flags: []cli.Flag{ cli.StringFlag{ Name: "format", diff --git a/drone/user_rm.go b/drone/user_rm.go index f3aa74d36..60d94f3d2 100644 --- a/drone/user_rm.go +++ b/drone/user_rm.go @@ -2,19 +2,14 @@ package main import ( "fmt" - "log" - "github.com/codegangsta/cli" + "github.com/urfave/cli" ) var userRemoveCmd = cli.Command{ - Name: "rm", - Usage: "remove a user", - Action: func(c *cli.Context) { - if err := userRemove(c); err != nil { - log.Fatalln(err) - } - }, + Name: "rm", + Usage: "remove a user", + Action: userRemove, } func userRemove(c *cli.Context) error { diff --git a/drone/util.go b/drone/util.go index da3be2550..aa515f8f6 100644 --- a/drone/util.go +++ b/drone/util.go @@ -9,8 +9,8 @@ import ( "github.com/drone/drone/client" - "github.com/codegangsta/cli" "github.com/jackspirou/syscerts" + "github.com/urfave/cli" ) func newClient(c *cli.Context) (client.Client, error) { diff --git a/router/middleware/agent.go b/router/middleware/agent.go deleted file mode 100644 index a1526e865..000000000 --- a/router/middleware/agent.go +++ /dev/null @@ -1,33 +0,0 @@ -package middleware - -import ( - "github.com/codegangsta/cli" - "github.com/drone/drone/shared/token" - - "github.com/Sirupsen/logrus" - "github.com/gin-gonic/gin" -) - -const agentKey = "agent" - -// Agents is a middleware function that initializes the authorization middleware -// for agents to connect to the queue. -func Agents(cli *cli.Context) gin.HandlerFunc { - secret := cli.String("agent-secret") - if secret == "" { - logrus.Fatalf("failed to generate token from DRONE_AGENT_SECRET") - } - - t := token.New(token.AgentToken, secret) - s, err := t.Sign(secret) - if err != nil { - logrus.Fatalf("failed to generate token from DRONE_AGENT_SECRET. %s", err) - } - - logrus.Infof("using agent secret %s", secret) - logrus.Warnf("agents can connect with token %s", s) - - return func(c *gin.Context) { - c.Set(agentKey, secret) - } -} diff --git a/router/middleware/broker.go b/router/middleware/broker.go deleted file mode 100644 index 400b68ef6..000000000 --- a/router/middleware/broker.go +++ /dev/null @@ -1,63 +0,0 @@ -package middleware - -import ( - "os" - "sync" - - handlers "github.com/drone/drone/server" - - "github.com/codegangsta/cli" - "github.com/drone/mq/logger" - "github.com/drone/mq/server" - "github.com/drone/mq/stomp" - - "github.com/Sirupsen/logrus" - "github.com/gin-gonic/gin" - "github.com/tidwall/redlog" -) - -const ( - serverKey = "broker" - clientKey = "stomp.client" // mirrored from stomp/context -) - -// Broker is a middleware function that initializes the broker -// and adds the broker client to the request context. -func Broker(cli *cli.Context) gin.HandlerFunc { - secret := cli.String("agent-secret") - if secret == "" { - logrus.Fatalf("fatal error. please provide the DRONE_SECRET") - } - - // setup broker logging. - log := redlog.New(os.Stderr) - log.SetLevel(2) - logger.SetLogger(log) - if cli.Bool("broker-debug") { - log.SetLevel(1) - } - - broker := server.NewServer( - server.WithCredentials("x-token", secret), - ) - client := broker.Client() - - var once sync.Once - return func(c *gin.Context) { - c.Set(serverKey, broker) - c.Set(clientKey, client) - once.Do(func() { - // this is some really hacky stuff - // turns out I need to do some refactoring - // don't judge! - // will fix in 0.6 release - ctx := c.Copy() - client.Connect( - stomp.WithCredentials("x-token", secret), - ) - client.Subscribe("/queue/updates", stomp.HandlerFunc(func(m *stomp.Message) { - go handlers.HandleUpdate(ctx, m.Copy()) - })) - }) - } -} diff --git a/router/middleware/cache.go b/router/middleware/cache.go index 6f6dec465..7f09d5f8e 100644 --- a/router/middleware/cache.go +++ b/router/middleware/cache.go @@ -3,8 +3,8 @@ package middleware import ( "github.com/drone/drone/cache" - "github.com/codegangsta/cli" "github.com/gin-gonic/gin" + "github.com/urfave/cli" ) // Cache is a middleware function that initializes the Cache and attaches to diff --git a/router/middleware/config.go b/router/middleware/config.go index e2f65dd10..54b1cbb0b 100644 --- a/router/middleware/config.go +++ b/router/middleware/config.go @@ -3,8 +3,8 @@ package middleware import ( "github.com/drone/drone/model" - "github.com/codegangsta/cli" "github.com/gin-gonic/gin" + "github.com/urfave/cli" ) const configKey = "config" diff --git a/router/middleware/remote.go b/router/middleware/remote.go index 98e197037..da1c454cb 100644 --- a/router/middleware/remote.go +++ b/router/middleware/remote.go @@ -4,7 +4,6 @@ import ( "fmt" "github.com/Sirupsen/logrus" - "github.com/codegangsta/cli" "github.com/drone/drone/remote" "github.com/drone/drone/remote/bitbucket" "github.com/drone/drone/remote/bitbucketserver" @@ -12,6 +11,7 @@ import ( "github.com/drone/drone/remote/gitlab" "github.com/drone/drone/remote/gogs" "github.com/gin-gonic/gin" + "github.com/urfave/cli" ) // Remote is a middleware function that initializes the Remote and attaches to diff --git a/router/middleware/store.go b/router/middleware/store.go index 33b9731bf..c7d706d56 100644 --- a/router/middleware/store.go +++ b/router/middleware/store.go @@ -1,9 +1,9 @@ package middleware import ( - "github.com/codegangsta/cli" "github.com/drone/drone/store" "github.com/drone/drone/store/datastore" + "github.com/urfave/cli" "github.com/gin-gonic/gin" ) diff --git a/router/router.go b/router/router.go index b09b85f7a..55ecb84e6 100644 --- a/router/router.go +++ b/router/router.go @@ -2,7 +2,6 @@ package router import ( "net/http" - "os" "github.com/gin-gonic/gin" @@ -41,8 +40,6 @@ func Load(middleware ...gin.HandlerFunc) http.Handler { e.GET("/logout", server.GetLogout) e.NoRoute(server.ShowIndex) - // TODO above will Go away with React UI - user := e.Group("/api/user") { user.Use(session.MustUser()) @@ -121,46 +118,28 @@ func Load(middleware ...gin.HandlerFunc) http.Handler { badges.GET("/cc.xml", server.GetCC) } - if os.Getenv("DRONE_CANARY") == "" { - e.POST("/hook", server.PostHook) - e.POST("/api/hook", server.PostHook) - } else { - e.POST("/hook", server.PostHook2) - e.POST("/api/hook", server.PostHook2) + e.POST("/hook", server.PostHook) + e.POST("/api/hook", server.PostHook) + + ws := e.Group("/ws") + { + ws.GET("/broker", server.RPCHandler) + ws.GET("/rpc", server.RPCHandler) + ws.GET("/feed", server.EventStream) + ws.GET("/logs/:owner/:name/:build/:number", + session.SetRepo(), + session.SetPerm(), + session.MustPull, + server.LogStream, + ) } - if os.Getenv("DRONE_CANARY") == "" { - ws := e.Group("/ws") - { - ws.GET("/broker", server.Broker) - ws.GET("/feed", server.EventStream) - ws.GET("/logs/:owner/:name/:build/:number", - session.SetRepo(), - session.SetPerm(), - session.MustPull, - server.LogStream, - ) - } - } else { - ws := e.Group("/ws") - { - ws.GET("/broker", server.RPCHandler) - ws.GET("/rpc", server.RPCHandler) - ws.GET("/feed", server.EventStream2) - ws.GET("/logs/:owner/:name/:build/:number", - session.SetRepo(), - session.SetPerm(), - session.MustPull, - server.LogStream2, - ) - } - info := e.Group("/api/info") - { - info.GET("/queue", - session.MustAdmin(), - server.GetQueueInfo, - ) - } + info := e.Group("/api/info") + { + info.GET("/queue", + session.MustAdmin(), + server.GetQueueInfo, + ) } auth := e.Group("/authorize") @@ -191,12 +170,5 @@ func Load(middleware ...gin.HandlerFunc) http.Handler { debugger.GET("/pprof/trace", debug.TraceHandler()) } - // bots := e.Group("/bots") - // { - // bots.Use(session.MustUser()) - // bots.POST("/slack", Slack) - // bots.POST("/slack/:command", Slack) - // } - return e } diff --git a/server/build.go b/server/build.go index 501e4e3d1..755c8f176 100644 --- a/server/build.go +++ b/server/build.go @@ -7,7 +7,6 @@ import ( "fmt" "io" "net/http" - "os" "strconv" "time" @@ -18,13 +17,10 @@ import ( "github.com/drone/drone/remote" "github.com/drone/drone/shared/httputil" "github.com/drone/drone/store" - "github.com/drone/drone/yaml" "github.com/gin-gonic/gin" - "github.com/square/go-jose" "github.com/drone/drone/model" "github.com/drone/drone/router/middleware/session" - "github.com/drone/mq/stomp" ) func GetBuilds(c *gin.Context) { @@ -156,229 +152,10 @@ func DeleteBuild(c *gin.Context) { job.ExitCode = 137 store.UpdateBuildJob(c, build, job) - if os.Getenv("DRONE_CANARY") == "" { - client := stomp.MustFromContext(c) - client.SendJSON("/topic/cancel", model.Event{ - Type: model.Cancelled, - Repo: *repo, - Build: *build, - Job: *job, - }, stomp.WithHeader("job-id", strconv.FormatInt(job.ID, 10))) - } else { - config.queue.Error(context.Background(), fmt.Sprint(job.ID), queue.ErrCancel) - } + config.queue.Error(context.Background(), fmt.Sprint(job.ID), queue.ErrCancel) c.String(204, "") } -func PostBuild(c *gin.Context) { - - if os.Getenv("DRONE_CANARY") == "true" { - PostBuild2(c) - return - } - - remote_ := remote.FromContext(c) - repo := session.Repo(c) - fork := c.DefaultQuery("fork", "false") - - num, err := strconv.Atoi(c.Param("number")) - if err != nil { - c.AbortWithError(http.StatusBadRequest, err) - return - } - - user, err := store.GetUser(c, repo.UserID) - if err != nil { - log.Errorf("failure to find repo owner %s. %s", repo.FullName, err) - c.AbortWithError(500, err) - return - } - - build, err := store.GetBuildNumber(c, repo, num) - if err != nil { - log.Errorf("failure to get build %d. %s", num, err) - c.AbortWithError(404, err) - return - } - - // if the remote has a refresh token, the current access token - // may be stale. Therefore, we should refresh prior to dispatching - // the job. - if refresher, ok := remote_.(remote.Refresher); ok { - ok, _ := refresher.Refresh(user) - if ok { - store.UpdateUser(c, user) - } - } - - // fetch the .drone.yml file from the database - cfg := ToConfig(c) - raw, err := remote_.File(user, repo, build, cfg.Yaml) - if err != nil { - log.Errorf("failure to get build config for %s. %s", repo.FullName, err) - c.AbortWithError(404, err) - return - } - - // Fetch secrets file but don't exit on error as it's optional - sec, err := remote_.File(user, repo, build, cfg.Shasum) - if err != nil { - log.Debugf("cannot find build secrets for %s. %s", repo.FullName, err) - } - - netrc, err := remote_.Netrc(user, repo) - if err != nil { - log.Errorf("failure to generate netrc for %s. %s", repo.FullName, err) - c.AbortWithError(500, err) - return - } - - jobs, err := store.GetJobList(c, build) - if err != nil { - log.Errorf("failure to get build %d jobs. %s", build.Number, err) - c.AbortWithError(404, err) - return - } - - // must not restart a running build - if build.Status == model.StatusPending || build.Status == model.StatusRunning { - c.String(409, "Cannot re-start a started build") - return - } - - // forking the build creates a duplicate of the build - // and then executes. This retains prior build history. - if forkit, _ := strconv.ParseBool(fork); forkit { - build.ID = 0 - build.Number = 0 - build.Parent = num - for _, job := range jobs { - job.ID = 0 - job.NodeID = 0 - } - err := store.CreateBuild(c, build, jobs...) - if err != nil { - c.String(500, err.Error()) - return - } - - event := c.DefaultQuery("event", build.Event) - if event == model.EventPush || - event == model.EventPull || - event == model.EventTag || - event == model.EventDeploy { - build.Event = event - } - build.Deploy = c.DefaultQuery("deploy_to", build.Deploy) - } - - // Read query string parameters into buildParams, exclude reserved params - var buildParams = map[string]string{} - for key, val := range c.Request.URL.Query() { - switch key { - case "fork", "event", "deploy_to": - default: - // We only accept string literals, because build parameters will be - // injected as environment variables - buildParams[key] = val[0] - } - } - - // todo move this to database tier - // and wrap inside a transaction - build.Status = model.StatusPending - build.Started = 0 - build.Finished = 0 - build.Enqueued = time.Now().UTC().Unix() - build.Error = "" - for _, job := range jobs { - for k, v := range buildParams { - job.Environment[k] = v - } - job.Error = "" - job.Status = model.StatusPending - job.Started = 0 - job.Finished = 0 - job.ExitCode = 0 - job.NodeID = 0 - job.Enqueued = build.Enqueued - store.UpdateJob(c, job) - } - - err = store.UpdateBuild(c, build) - if err != nil { - c.AbortWithStatus(500) - return - } - - c.JSON(202, build) - - // get the previous build so that we can send - // on status change notifications - last, _ := store.GetBuildLastBefore(c, repo, build.Branch, build.ID) - secs, err := store.GetMergedSecretList(c, repo) - if err != nil { - log.Debugf("Error getting secrets for %s#%d. %s", repo.FullName, build.Number, err) - } - - var signed bool - var verified bool - - signature, err := jose.ParseSigned(string(sec)) - if err != nil { - log.Debugf("cannot parse .drone.yml.sig file. %s", err) - } else if len(sec) == 0 { - log.Debugf("cannot parse .drone.yml.sig file. empty file") - } else { - signed = true - output, err := signature.Verify([]byte(repo.Hash)) - if err != nil { - log.Debugf("cannot verify .drone.yml.sig file. %s", err) - } else if string(output) != string(raw) { - log.Debugf("cannot verify .drone.yml.sig file. no match. %q <> %q", string(output), string(raw)) - } else { - verified = true - } - } - - log.Debugf(".drone.yml is signed=%v and verified=%v", signed, verified) - - client := stomp.MustFromContext(c) - client.SendJSON("/topic/events", model.Event{ - Type: model.Enqueued, - Repo: *repo, - Build: *build, - }, - stomp.WithHeader("repo", repo.FullName), - stomp.WithHeader("private", strconv.FormatBool(repo.IsPrivate)), - ) - - for _, job := range jobs { - broker, _ := stomp.FromContext(c) - broker.SendJSON("/queue/pending", &model.Work{ - Signed: signed, - Verified: verified, - User: user, - Repo: repo, - Build: build, - BuildLast: last, - Job: job, - Netrc: netrc, - Yaml: string(raw), - Secrets: secs, - System: &model.System{Link: httputil.GetURL(c.Request)}, - }, - stomp.WithHeader( - "platform", - yaml.ParsePlatformDefault(raw, "linux/amd64"), - ), - stomp.WithHeaders( - yaml.ParseLabel(raw), - ), - ) - } -} - func GetBuildQueue(c *gin.Context) { out, err := store.GetBuildQueue(c) if err != nil { @@ -412,7 +189,7 @@ func copyLogs(dest io.Writer, src io.Reader) error { // // -func PostBuild2(c *gin.Context) { +func PostBuild(c *gin.Context) { remote_ := remote.FromContext(c) repo := session.Repo(c) diff --git a/server/hook.go b/server/hook.go index 8819bc30b..c50537d2f 100644 --- a/server/hook.go +++ b/server/hook.go @@ -1,28 +1,268 @@ package server import ( + "context" + "encoding/json" "fmt" + "regexp" "strconv" + "time" "github.com/gin-gonic/gin" "github.com/square/go-jose" - log "github.com/Sirupsen/logrus" + "github.com/Sirupsen/logrus" "github.com/drone/drone/model" "github.com/drone/drone/remote" "github.com/drone/drone/shared/httputil" "github.com/drone/drone/shared/token" "github.com/drone/drone/store" - "github.com/drone/drone/yaml" - "github.com/drone/mq/stomp" + "github.com/drone/envsubst" + + "github.com/cncd/pipeline/pipeline/backend" + "github.com/cncd/pipeline/pipeline/frontend" + "github.com/cncd/pipeline/pipeline/frontend/yaml" + "github.com/cncd/pipeline/pipeline/frontend/yaml/compiler" + "github.com/cncd/pipeline/pipeline/frontend/yaml/linter" + "github.com/cncd/pipeline/pipeline/frontend/yaml/matrix" + "github.com/cncd/pipeline/pipeline/rpc" + "github.com/cncd/pubsub" + "github.com/cncd/queue" ) +// +// CANARY IMPLEMENTATION +// +// This file is a complete disaster because I'm trying to wedge in some +// experimental code. Please pardon our appearance during renovations. +// + +var skipRe = regexp.MustCompile(`\[(?i:ci *skip|skip *ci)\]`) + +func GetQueueInfo(c *gin.Context) { + c.IndentedJSON(200, + config.queue.Info(c), + ) +} + +// return the metadata from the cli context. +func metadataFromStruct(repo *model.Repo, build, last *model.Build, job *model.Job, link string) frontend.Metadata { + return frontend.Metadata{ + Repo: frontend.Repo{ + Name: repo.Name, + Link: repo.Link, + Remote: repo.Clone, + Private: repo.IsPrivate, + }, + Curr: frontend.Build{ + Number: build.Number, + Created: build.Created, + Started: build.Started, + Finished: build.Finished, + Status: build.Status, + Event: build.Event, + Link: build.Link, + Target: build.Deploy, + Commit: frontend.Commit{ + Sha: build.Commit, + Ref: build.Ref, + Refspec: build.Refspec, + Branch: build.Branch, + Message: build.Message, + Author: frontend.Author{ + Name: build.Author, + Email: build.Email, + Avatar: build.Avatar, + }, + }, + }, + Prev: frontend.Build{ + Number: last.Number, + Created: last.Created, + Started: last.Started, + Finished: last.Finished, + Status: last.Status, + Event: last.Event, + Link: last.Link, + Target: last.Deploy, + Commit: frontend.Commit{ + Sha: last.Commit, + Ref: last.Ref, + Refspec: last.Refspec, + Branch: last.Branch, + Message: last.Message, + Author: frontend.Author{ + Name: last.Author, + Email: last.Email, + Avatar: last.Avatar, + }, + }, + }, + Job: frontend.Job{ + Number: job.Number, + Matrix: job.Environment, + }, + Sys: frontend.System{ + Name: "drone", + Link: link, + Arch: "linux/amd64", + }, + } +} + +type builder struct { + Repo *model.Repo + Curr *model.Build + Last *model.Build + Netrc *model.Netrc + Secs []*model.Secret + Link string + Yaml string +} + +type buildItem struct { + Job *model.Job + Platform string + Labels map[string]string + Config *backend.Config +} + +func (b *builder) Build() ([]*buildItem, error) { + + axes, err := matrix.ParseString(b.Yaml) + if err != nil { + return nil, err + } + if len(axes) == 0 { + axes = append(axes, matrix.Axis{}) + } + + var items []*buildItem + for i, axis := range axes { + job := &model.Job{ + BuildID: b.Curr.ID, + Number: i + 1, + Status: model.StatusPending, + Environment: axis, + Enqueued: b.Curr.Created, + } + + metadata := metadataFromStruct(b.Repo, b.Curr, b.Last, job, b.Link) + environ := metadata.Environ() + for k, v := range metadata.EnvironDrone() { + environ[k] = v + } + + secrets := map[string]string{} + for _, sec := range b.Secs { + if !sec.MatchEvent(b.Curr.Event) { + continue + } + if b.Curr.Verified || sec.SkipVerify { + secrets[sec.Name] = sec.Value + } + } + sub := func(name string) string { + if v, ok := environ[name]; ok { + return v + } + return secrets[name] + } + + y := b.Yaml + if s, err := envsubst.Eval(y, sub); err != nil { + y = s + } + + parsed, err := yaml.ParseString(y) + if err != nil { + return nil, err + } + metadata.Sys.Arch = parsed.Platform + if metadata.Sys.Arch == "" { + metadata.Sys.Arch = "linux/amd64" + } + + lerr := linter.New( + linter.WithTrusted(b.Repo.IsTrusted), + ).Lint(parsed) + if lerr != nil { + return nil, err + } + + ir := compiler.New( + compiler.WithEnviron(environ), + // TODO ability to customize the escalated plugins + compiler.WithEscalated("plugins/docker", "plugins/gcr", "plugins/ecr"), + compiler.WithLocal(false), + compiler.WithNetrc(b.Netrc.Login, b.Netrc.Password, b.Netrc.Machine), + compiler.WithPrefix( + fmt.Sprintf( + "%d_%d", + job.ID, + time.Now().Unix(), + ), + ), + compiler.WithEnviron(job.Environment), + compiler.WithProxy(), + // TODO ability to set global volumes for things like certs + compiler.WithVolumes(), + compiler.WithWorkspaceFromURL("/drone", b.Curr.Link), + ).Compile(parsed) + + for _, sec := range b.Secs { + if !sec.MatchEvent(b.Curr.Event) { + continue + } + if b.Curr.Verified || sec.SkipVerify { + ir.Secrets = append(ir.Secrets, &backend.Secret{ + Mask: sec.Conceal, + Name: sec.Name, + Value: sec.Value, + }) + } + } + + item := &buildItem{ + Job: job, + Config: ir, + Labels: parsed.Labels, + Platform: metadata.Sys.Arch, + } + if item.Labels == nil { + item.Labels = map[string]string{} + } + items = append(items, item) + } + + return items, nil +} + +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// +// + func PostHook(c *gin.Context) { remote_ := remote.FromContext(c) tmprepo, build, err := remote_.Hook(c.Request) if err != nil { - log.Errorf("failure to parse hook. %s", err) + logrus.Errorf("failure to parse hook. %s", err) c.AbortWithError(400, err) return } @@ -31,7 +271,7 @@ func PostHook(c *gin.Context) { return } if tmprepo == nil { - log.Errorf("failure to ascertain repo from hook.") + logrus.Errorf("failure to ascertain repo from hook.") c.Writer.WriteHeader(400) return } @@ -40,14 +280,14 @@ func PostHook(c *gin.Context) { // wrapped in square brackets appear in the commit message skipMatch := skipRe.FindString(build.Message) if len(skipMatch) > 0 { - log.Infof("ignoring hook. %s found in %s", skipMatch, build.Commit) + logrus.Infof("ignoring hook. %s found in %s", skipMatch, build.Commit) c.Writer.WriteHeader(204) return } repo, err := store.GetRepoOwnerName(c, tmprepo.Owner, tmprepo.Name) if err != nil { - log.Errorf("failure to find repo %s/%s from hook. %s", tmprepo.Owner, tmprepo.Name, err) + logrus.Errorf("failure to find repo %s/%s from hook. %s", tmprepo.Owner, tmprepo.Name, err) c.AbortWithError(404, err) return } @@ -57,18 +297,18 @@ func PostHook(c *gin.Context) { return repo.Hash, nil }) if err != nil { - log.Errorf("failure to parse token from hook for %s. %s", repo.FullName, err) + logrus.Errorf("failure to parse token from hook for %s. %s", repo.FullName, err) c.AbortWithError(400, err) return } if parsed.Text != repo.FullName { - log.Errorf("failure to verify token from hook. Expected %s, got %s", repo.FullName, parsed.Text) + logrus.Errorf("failure to verify token from hook. Expected %s, got %s", repo.FullName, parsed.Text) c.AbortWithStatus(403) return } if repo.UserID == 0 { - log.Warnf("ignoring hook. repo %s has no owner.", repo.FullName) + logrus.Warnf("ignoring hook. repo %s has no owner.", repo.FullName) c.Writer.WriteHeader(204) return } @@ -81,33 +321,18 @@ func PostHook(c *gin.Context) { } if skipped { - log.Infof("ignoring hook. repo %s is disabled for %s events.", repo.FullName, build.Event) + logrus.Infof("ignoring hook. repo %s is disabled for %s events.", repo.FullName, build.Event) c.Writer.WriteHeader(204) return } user, err := store.GetUser(c, repo.UserID) if err != nil { - log.Errorf("failure to find repo owner %s. %s", repo.FullName, err) + logrus.Errorf("failure to find repo owner %s. %s", repo.FullName, err) c.AbortWithError(500, err) return } - // if there is no email address associated with the pull request, - // we lookup the email address based on the authors github login. - // - // my initial hesitation with this code is that it has the ability - // to expose your email address. At the same time, your email address - // is already exposed in the public .git log. So while some people will - // a small number of people will probably be upset by this, I'm not sure - // it is actually that big of a deal. - if len(build.Email) == 0 { - author, uerr := store.GetUserLogin(c, build.Author) - if uerr == nil { - build.Email = author.Email - } - } - // if the remote has a refresh token, the current access token // may be stale. Therefore, we should refresh prior to dispatching // the job. @@ -119,26 +344,16 @@ func PostHook(c *gin.Context) { } // fetch the build file from the database - config := ToConfig(c) - raw, err := remote_.File(user, repo, build, config.Yaml) + cfg := ToConfig(c) + raw, err := remote_.File(user, repo, build, cfg.Yaml) if err != nil { - log.Errorf("failure to get build config for %s. %s", repo.FullName, err) + logrus.Errorf("failure to get build config for %s. %s", repo.FullName, err) c.AbortWithError(404, err) return } - sec, err := remote_.File(user, repo, build, config.Shasum) + sec, err := remote_.File(user, repo, build, cfg.Shasum) if err != nil { - log.Debugf("cannot find build secrets for %s. %s", repo.FullName, err) - // NOTE we don't exit on failure. The sec file is optional - } - - axes, err := yaml.ParseMatrix(raw) - if err != nil { - c.String(500, "Failed to parse yaml file or calculate matrix. %s", err) - return - } - if len(axes) == 0 { - axes = append(axes, yaml.Axis{}) + logrus.Debugf("cannot find yaml signature for %s. %s", repo.FullName, err) } netrc, err := remote_.Netrc(user, repo) @@ -148,24 +363,28 @@ func PostHook(c *gin.Context) { } // verify the branches can be built vs skipped - branches := yaml.ParseBranch(raw) - if !branches.Match(build.Branch) && build.Event != model.EventTag && build.Event != model.EventDeploy { + branches, err := yaml.ParseBytes(raw) + if err != nil { + c.String(500, "Failed to parse yaml file. %s", err) + return + } + if !branches.Branches.Match(build.Branch) && build.Event != model.EventTag && build.Event != model.EventDeploy { c.String(200, "Branch does not match restrictions defined in yaml") return } signature, err := jose.ParseSigned(string(sec)) if err != nil { - log.Debugf("cannot parse .drone.yml.sig file. %s", err) + logrus.Debugf("cannot parse .drone.yml.sig file. %s", err) } else if len(sec) == 0 { - log.Debugf("cannot parse .drone.yml.sig file. empty file") + logrus.Debugf("cannot parse .drone.yml.sig file. empty file") } else { build.Signed = true output, verr := signature.Verify([]byte(repo.Hash)) if verr != nil { - log.Debugf("cannot verify .drone.yml.sig file. %s", verr) + logrus.Debugf("cannot verify .drone.yml.sig file. %s", verr) } else if string(output) != string(raw) { - log.Debugf("cannot verify .drone.yml.sig file. no match") + logrus.Debugf("cannot verify .drone.yml.sig file. no match") } else { build.Verified = true } @@ -175,71 +394,94 @@ func PostHook(c *gin.Context) { build.Status = model.StatusPending build.RepoID = repo.ID - // and use a transaction - var jobs []*model.Job - for num, axis := range axes { - jobs = append(jobs, &model.Job{ - BuildID: build.ID, - Number: num + 1, - Status: model.StatusPending, - Environment: axis, - }) - } - err = store.CreateBuild(c, build, jobs...) - if err != nil { - log.Errorf("failure to save commit for %s. %s", repo.FullName, err) + if err := store.CreateBuild(c, build, build.Jobs...); err != nil { + logrus.Errorf("failure to save commit for %s. %s", repo.FullName, err) c.AbortWithError(500, err) return } c.JSON(200, build) - url := fmt.Sprintf("%s/%s/%d", httputil.GetURL(c.Request), repo.FullName, build.Number) - err = remote_.Status(user, repo, build, url) - if err != nil { - log.Errorf("error setting commit status for %s/%d", repo.FullName, build.Number) - } - // get the previous build so that we can send // on status change notifications last, _ := store.GetBuildLastBefore(c, repo, build.Branch, build.ID) secs, err := store.GetMergedSecretList(c, repo) if err != nil { - log.Debugf("Error getting secrets for %s#%d. %s", repo.FullName, build.Number, err) + logrus.Debugf("Error getting secrets for %s#%d. %s", repo.FullName, build.Number, err) } - client := stomp.MustFromContext(c) - client.SendJSON("topic/events", model.Event{ + // + // BELOW: NEW + // + + defer func() { + uri := fmt.Sprintf("%s/%s/%d", httputil.GetURL(c.Request), repo.FullName, build.Number) + err = remote_.Status(user, repo, build, uri) + if err != nil { + logrus.Errorf("error setting commit status for %s/%d", repo.FullName, build.Number) + } + }() + + b := builder{ + Repo: repo, + Curr: build, + Last: last, + Netrc: netrc, + Secs: secs, + Link: httputil.GetURL(c.Request), + Yaml: string(raw), + } + items, err := b.Build() + if err != nil { + build.Status = model.StatusError + build.Started = time.Now().Unix() + build.Finished = build.Started + build.Error = err.Error() + return + } + + for _, item := range items { + build.Jobs = append(build.Jobs, item.Job) + store.CreateJob(c, item.Job) + // TODO err + } + + // + // publish topic + // + message := pubsub.Message{ + Labels: map[string]string{ + "repo": repo.FullName, + "private": strconv.FormatBool(repo.IsPrivate), + }, + } + message.Data, _ = json.Marshal(model.Event{ Type: model.Enqueued, Repo: *repo, Build: *build, - }, - stomp.WithHeader("repo", repo.FullName), - stomp.WithHeader("private", strconv.FormatBool(repo.IsPrivate)), - ) + }) + // TODO remove global reference + config.pubsub.Publish(c, "topic/events", message) + // + // end publish topic + // - for _, job := range jobs { - broker, _ := stomp.FromContext(c) - broker.SendJSON("/queue/pending", &model.Work{ - Signed: build.Signed, - Verified: build.Verified, - User: user, - Repo: repo, - Build: build, - BuildLast: last, - Job: job, - Netrc: netrc, - Yaml: string(raw), - Secrets: secs, - System: &model.System{Link: httputil.GetURL(c.Request)}, - }, - stomp.WithHeader( - "platform", - yaml.ParsePlatformDefault(raw, "linux/amd64"), - ), - stomp.WithHeaders( - yaml.ParseLabel(raw), - ), - ) + for _, item := range items { + task := new(queue.Task) + task.ID = fmt.Sprint(item.Job.ID) + task.Labels = map[string]string{} + task.Labels["platform"] = item.Platform + for k, v := range item.Labels { + task.Labels[k] = v + } + + task.Data, _ = json.Marshal(rpc.Pipeline{ + ID: fmt.Sprint(item.Job.ID), + Config: item.Config, + Timeout: b.Repo.Timeout, + }) + + config.logger.Open(context.Background(), task.ID) + config.queue.Push(context.Background(), task) } } diff --git a/server/hook2.go b/server/hook2.go deleted file mode 100644 index eb28c1bf8..000000000 --- a/server/hook2.go +++ /dev/null @@ -1,869 +0,0 @@ -package server - -import ( - "context" - "encoding/json" - "fmt" - "regexp" - "strconv" - "time" - - "github.com/gin-gonic/gin" - "github.com/square/go-jose" - - "github.com/Sirupsen/logrus" - "github.com/drone/drone/model" - "github.com/drone/drone/remote" - "github.com/drone/drone/shared/httputil" - "github.com/drone/drone/shared/token" - "github.com/drone/drone/store" - "github.com/drone/envsubst" - - "github.com/cncd/pipeline/pipeline/backend" - "github.com/cncd/pipeline/pipeline/frontend" - "github.com/cncd/pipeline/pipeline/frontend/yaml" - "github.com/cncd/pipeline/pipeline/frontend/yaml/compiler" - "github.com/cncd/pipeline/pipeline/frontend/yaml/linter" - "github.com/cncd/pipeline/pipeline/frontend/yaml/matrix" - "github.com/cncd/pipeline/pipeline/rpc" - "github.com/cncd/pubsub" - "github.com/cncd/queue" -) - -// -// CANARY IMPLEMENTATION -// -// This file is a complete disaster because I'm trying to wedge in some -// experimental code. Please pardon our appearance during renovations. -// - -var skipRe = regexp.MustCompile(`\[(?i:ci *skip|skip *ci)\]`) - -func GetQueueInfo(c *gin.Context) { - c.IndentedJSON(200, - config.queue.Info(c), - ) -} - -// func PostHookOld(c *gin.Context) { -// remote_ := remote.FromContext(c) -// -// tmprepo, build, err := remote_.Hook(c.Request) -// if err != nil { -// logrus.Errorf("failure to parse hook. %s", err) -// c.AbortWithError(400, err) -// return -// } -// if build == nil { -// c.Writer.WriteHeader(200) -// return -// } -// if tmprepo == nil { -// logrus.Errorf("failure to ascertain repo from hook.") -// c.Writer.WriteHeader(400) -// return -// } -// -// // skip the build if any case-insensitive combination of the words "skip" and "ci" -// // wrapped in square brackets appear in the commit message -// skipMatch := skipRe.FindString(build.Message) -// if len(skipMatch) > 0 { -// logrus.Infof("ignoring hook. %s found in %s", skipMatch, build.Commit) -// c.Writer.WriteHeader(204) -// return -// } -// -// repo, err := store.GetRepoOwnerName(c, tmprepo.Owner, tmprepo.Name) -// if err != nil { -// logrus.Errorf("failure to find repo %s/%s from hook. %s", tmprepo.Owner, tmprepo.Name, err) -// c.AbortWithError(404, err) -// return -// } -// -// // get the token and verify the hook is authorized -// parsed, err := token.ParseRequest(c.Request, func(t *token.Token) (string, error) { -// return repo.Hash, nil -// }) -// if err != nil { -// logrus.Errorf("failure to parse token from hook for %s. %s", repo.FullName, err) -// c.AbortWithError(400, err) -// return -// } -// if parsed.Text != repo.FullName { -// logrus.Errorf("failure to verify token from hook. Expected %s, got %s", repo.FullName, parsed.Text) -// c.AbortWithStatus(403) -// return -// } -// -// if repo.UserID == 0 { -// logrus.Warnf("ignoring hook. repo %s has no owner.", repo.FullName) -// c.Writer.WriteHeader(204) -// return -// } -// var skipped = true -// if (build.Event == model.EventPush && repo.AllowPush) || -// (build.Event == model.EventPull && repo.AllowPull) || -// (build.Event == model.EventDeploy && repo.AllowDeploy) || -// (build.Event == model.EventTag && repo.AllowTag) { -// skipped = false -// } -// -// if skipped { -// logrus.Infof("ignoring hook. repo %s is disabled for %s events.", repo.FullName, build.Event) -// c.Writer.WriteHeader(204) -// return -// } -// -// user, err := store.GetUser(c, repo.UserID) -// if err != nil { -// logrus.Errorf("failure to find repo owner %s. %s", repo.FullName, err) -// c.AbortWithError(500, err) -// return -// } -// -// // if the remote has a refresh token, the current access token -// // may be stale. Therefore, we should refresh prior to dispatching -// // the job. -// if refresher, ok := remote_.(remote.Refresher); ok { -// ok, _ := refresher.Refresh(user) -// if ok { -// store.UpdateUser(c, user) -// } -// } -// -// // fetch the build file from the database -// cfg := ToConfig(c) -// raw, err := remote_.File(user, repo, build, cfg.Yaml) -// if err != nil { -// logrus.Errorf("failure to get build config for %s. %s", repo.FullName, err) -// c.AbortWithError(404, err) -// return -// } -// sec, err := remote_.File(user, repo, build, cfg.Shasum) -// if err != nil { -// logrus.Debugf("cannot find yaml signature for %s. %s", repo.FullName, err) -// // NOTE we don't exit on failure. The sec file is optional -// } -// -// axes, err := matrix.Parse(raw) -// if err != nil { -// c.String(500, "Failed to parse yaml file or calculate matrix. %s", err) -// return -// } -// if len(axes) == 0 { -// axes = append(axes, matrix.Axis{}) -// } -// -// netrc, err := remote_.Netrc(user, repo) -// if err != nil { -// c.String(500, "Failed to generate netrc file. %s", err) -// return -// } -// -// // verify the branches can be built vs skipped -// branches, err := yaml.ParseBytes(raw) -// if err != nil { -// c.String(500, "Failed to parse yaml file. %s", err) -// return -// } -// if !branches.Branches.Match(build.Branch) && build.Event != model.EventTag && build.Event != model.EventDeploy { -// c.String(200, "Branch does not match restrictions defined in yaml") -// return -// } -// -// signature, err := jose.ParseSigned(string(sec)) -// if err != nil { -// logrus.Debugf("cannot parse .drone.yml.sig file. %s", err) -// } else if len(sec) == 0 { -// logrus.Debugf("cannot parse .drone.yml.sig file. empty file") -// } else { -// build.Signed = true -// output, verr := signature.Verify([]byte(repo.Hash)) -// if verr != nil { -// logrus.Debugf("cannot verify .drone.yml.sig file. %s", verr) -// } else if string(output) != string(raw) { -// logrus.Debugf("cannot verify .drone.yml.sig file. no match") -// } else { -// build.Verified = true -// } -// } -// -// // update some build fields -// build.Status = model.StatusPending -// build.RepoID = repo.ID -// -// // and use a transaction -// var jobs []*model.Job -// for num, axis := range axes { -// jobs = append(jobs, &model.Job{ -// BuildID: build.ID, -// Number: num + 1, -// Status: model.StatusPending, -// Environment: axis, -// }) -// } -// err = store.CreateBuild(c, build, jobs...) -// if err != nil { -// logrus.Errorf("failure to save commit for %s. %s", repo.FullName, err) -// c.AbortWithError(500, err) -// return -// } -// -// c.JSON(200, build) -// -// uri := fmt.Sprintf("%s/%s/%d", httputil.GetURL(c.Request), repo.FullName, build.Number) -// err = remote_.Status(user, repo, build, uri) -// if err != nil { -// logrus.Errorf("error setting commit status for %s/%d", repo.FullName, build.Number) -// } -// -// // get the previous build so that we can send -// // on status change notifications -// last, _ := store.GetBuildLastBefore(c, repo, build.Branch, build.ID) -// secs, err := store.GetMergedSecretList(c, repo) -// if err != nil { -// logrus.Debugf("Error getting secrets for %s#%d. %s", repo.FullName, build.Number, err) -// } -// -// // -// // BELOW: NEW -// // -// -// b := builder{ -// Repo: repo, -// Curr: build, -// Last: last, -// Netrc: netrc, -// Secs: secs, -// Link: httputil.GetURL(c.Request), -// Yaml: string(raw), -// } -// items, err := b.Build() -// if err != nil { -// build.Status = model.StatusError -// build.Started = time.Now().Unix() -// build.Finished = build.Started -// build.Error = err.Error() -// store.CreateBuild(c, build, build.Jobs...) -// return -// } -// -// for _, item := range items { -// build.Jobs = append(build.Jobs, item.Job) -// } -// -// if err := store.CreateBuild(c, build, build.Jobs...); err != nil { -// logrus.Errorf("failure to save commit for %s. %s", repo.FullName, err) -// c.AbortWithError(500, err) -// return -// } -// -// for _, item := range items { -// -// task := new(queue.Task) -// task.ID = fmt.Sprint(item.Job.ID) -// task.Labels = map[string]string{} -// task.Labels["platform"] = item.Platform -// for k, v := range item.Labels { -// task.Labels[k] = v -// } -// -// task.Data, _ = json.Marshal(rpc.Pipeline{ -// ID: fmt.Sprint(item.Job.ID), -// Config: item.Config, -// Timeout: b.Repo.Timeout, -// }) -// -// config.logger.Open(context.Background(), task.ID) -// config.queue.Push(context.Background(), task) -// } -// -// // -// // new code here -// // -// -// message := pubsub.Message{ -// Labels: map[string]string{ -// "repo": repo.FullName, -// "private": strconv.FormatBool(repo.IsPrivate), -// }, -// } -// message.Data, _ = json.Marshal(model.Event{ -// Type: model.Enqueued, -// Repo: *repo, -// Build: *build, -// }) -// // TODO remove global reference -// config.pubsub.Publish(c, "topic/events", message) -// -// // -// // workspace -// // -// -// for _, job := range jobs { -// -// metadata := metadataFromStruct(repo, build, last, job, httputil.GetURL(c.Request)) -// environ := metadata.Environ() -// -// secrets := map[string]string{} -// for _, sec := range secs { -// if !sec.MatchEvent(build.Event) { -// continue -// } -// if build.Verified || sec.SkipVerify { -// secrets[sec.Name] = sec.Value -// } -// } -// sub := func(name string) string { -// if v, ok := environ[name]; ok { -// return v -// } -// return secrets[name] -// } -// if s, err := envsubst.Eval(string(raw), sub); err != nil { -// raw = []byte(s) -// } -// parsed, err := yaml.ParseBytes(raw) -// if err != nil { -// job.ExitCode = 255 -// job.Enqueued = time.Now().Unix() -// job.Started = time.Now().Unix() -// job.Finished = time.Now().Unix() -// job.Error = err.Error() -// store.UpdateBuildJob(c, build, job) -// continue -// } -// -// lerr := linter.New( -// linter.WithTrusted(repo.IsTrusted), -// ).Lint(parsed) -// if lerr != nil { -// job.ExitCode = 255 -// job.Enqueued = time.Now().Unix() -// job.Started = time.Now().Unix() -// job.Finished = time.Now().Unix() -// job.Error = lerr.Error() -// store.UpdateBuildJob(c, build, job) -// continue -// } -// -// ir := compiler.New( -// compiler.WithEnviron(environ), -// // TODO ability to customize the escalated plugins -// compiler.WithEscalated("plugins/docker", "plugins/gcr", "plugins/ecr"), -// compiler.WithLocal(false), -// compiler.WithNetrc(netrc.Login, netrc.Password, netrc.Machine), -// compiler.WithPrefix( -// fmt.Sprintf( -// "%d_%d", -// job.ID, -// time.Now().Unix(), -// ), -// ), -// compiler.WithEnviron(job.Environment), -// compiler.WithProxy(), -// // TODO ability to set global volumes for things like certs -// compiler.WithVolumes(), -// compiler.WithWorkspaceFromURL("/drone", repo.Link), -// ).Compile(parsed) -// -// // TODO there is a chicken and egg problem here because -// // the compiled yaml has a platform environment variable -// // that is not correctly set, because we are just about -// // to set it .... -// // TODO maybe we remove platform from metadata and let -// // the compiler set the value from the yaml itself. -// if parsed.Platform == "" { -// parsed.Platform = "linux/amd64" -// } -// -// for _, sec := range secs { -// if !sec.MatchEvent(build.Event) { -// continue -// } -// if build.Verified || sec.SkipVerify { -// ir.Secrets = append(ir.Secrets, &backend.Secret{ -// Mask: sec.Conceal, -// Name: sec.Name, -// Value: sec.Value, -// }) -// } -// } -// -// task := new(queue.Task) -// task.ID = fmt.Sprint(job.ID) -// task.Labels = map[string]string{} -// task.Labels["platform"] = parsed.Platform -// if parsed.Labels != nil { -// for k, v := range parsed.Labels { -// task.Labels[k] = v -// } -// } -// -// task.Data, _ = json.Marshal(rpc.Pipeline{ -// ID: fmt.Sprint(job.ID), -// Config: ir, -// Timeout: repo.Timeout, -// }) -// -// config.logger.Open(context.Background(), task.ID) -// config.queue.Push(context.Background(), task) -// } -// -// } - -// return the metadata from the cli context. -func metadataFromStruct(repo *model.Repo, build, last *model.Build, job *model.Job, link string) frontend.Metadata { - return frontend.Metadata{ - Repo: frontend.Repo{ - Name: repo.Name, - Link: repo.Link, - Remote: repo.Clone, - Private: repo.IsPrivate, - }, - Curr: frontend.Build{ - Number: build.Number, - Created: build.Created, - Started: build.Started, - Finished: build.Finished, - Status: build.Status, - Event: build.Event, - Link: build.Link, - Target: build.Deploy, - Commit: frontend.Commit{ - Sha: build.Commit, - Ref: build.Ref, - Refspec: build.Refspec, - Branch: build.Branch, - Message: build.Message, - Author: frontend.Author{ - Name: build.Author, - Email: build.Email, - Avatar: build.Avatar, - }, - }, - }, - Prev: frontend.Build{ - Number: last.Number, - Created: last.Created, - Started: last.Started, - Finished: last.Finished, - Status: last.Status, - Event: last.Event, - Link: last.Link, - Target: last.Deploy, - Commit: frontend.Commit{ - Sha: last.Commit, - Ref: last.Ref, - Refspec: last.Refspec, - Branch: last.Branch, - Message: last.Message, - Author: frontend.Author{ - Name: last.Author, - Email: last.Email, - Avatar: last.Avatar, - }, - }, - }, - Job: frontend.Job{ - Number: job.Number, - Matrix: job.Environment, - }, - Sys: frontend.System{ - Name: "drone", - Link: link, - Arch: "linux/amd64", - }, - } -} - -// use helper funciton to return ([]backend.Config, error) - -// 1. fetch everything from github -// 2. create and persist the build object -// -// 3. generate the build jobs [Launcher?] -// a. parse yaml -// b. lint yaml -// c. compile yaml -// -// 4. persist the build jobs (... what if I already have jobs, via re-start) -// 5. update github status -// 6. send to queue -// 7. trigger pubsub - -type builder struct { - Repo *model.Repo - Curr *model.Build - Last *model.Build - Netrc *model.Netrc - Secs []*model.Secret - Link string - Yaml string -} - -type buildItem struct { - Job *model.Job - Platform string - Labels map[string]string - Config *backend.Config -} - -func (b *builder) Build() ([]*buildItem, error) { - - axes, err := matrix.ParseString(b.Yaml) - if err != nil { - return nil, err - } - if len(axes) == 0 { - axes = append(axes, matrix.Axis{}) - } - - var items []*buildItem - for i, axis := range axes { - job := &model.Job{ - BuildID: b.Curr.ID, - Number: i + 1, - Status: model.StatusPending, - Environment: axis, - Enqueued: b.Curr.Created, - } - - metadata := metadataFromStruct(b.Repo, b.Curr, b.Last, job, b.Link) - environ := metadata.Environ() - for k, v := range metadata.EnvironDrone() { - environ[k] = v - } - - secrets := map[string]string{} - for _, sec := range b.Secs { - if !sec.MatchEvent(b.Curr.Event) { - continue - } - if b.Curr.Verified || sec.SkipVerify { - secrets[sec.Name] = sec.Value - } - } - sub := func(name string) string { - if v, ok := environ[name]; ok { - return v - } - return secrets[name] - } - - y := b.Yaml - if s, err := envsubst.Eval(y, sub); err != nil { - y = s - } - - parsed, err := yaml.ParseString(y) - if err != nil { - return nil, err - } - metadata.Sys.Arch = parsed.Platform - if metadata.Sys.Arch == "" { - metadata.Sys.Arch = "linux/amd64" - } - - lerr := linter.New( - linter.WithTrusted(b.Repo.IsTrusted), - ).Lint(parsed) - if lerr != nil { - return nil, err - } - - ir := compiler.New( - compiler.WithEnviron(environ), - // TODO ability to customize the escalated plugins - compiler.WithEscalated("plugins/docker", "plugins/gcr", "plugins/ecr"), - compiler.WithLocal(false), - compiler.WithNetrc(b.Netrc.Login, b.Netrc.Password, b.Netrc.Machine), - compiler.WithPrefix( - fmt.Sprintf( - "%d_%d", - job.ID, - time.Now().Unix(), - ), - ), - compiler.WithEnviron(job.Environment), - compiler.WithProxy(), - // TODO ability to set global volumes for things like certs - compiler.WithVolumes(), - compiler.WithWorkspaceFromURL("/drone", b.Curr.Link), - ).Compile(parsed) - - for _, sec := range b.Secs { - if !sec.MatchEvent(b.Curr.Event) { - continue - } - if b.Curr.Verified || sec.SkipVerify { - ir.Secrets = append(ir.Secrets, &backend.Secret{ - Mask: sec.Conceal, - Name: sec.Name, - Value: sec.Value, - }) - } - } - - item := &buildItem{ - Job: job, - Config: ir, - Labels: parsed.Labels, - Platform: metadata.Sys.Arch, - } - if item.Labels == nil { - item.Labels = map[string]string{} - } - items = append(items, item) - } - - return items, nil -} - -// -// -// -// -// -// -// -// -// -// -// -// -// -// -// -// -// -// - -func PostHook2(c *gin.Context) { - remote_ := remote.FromContext(c) - - tmprepo, build, err := remote_.Hook(c.Request) - if err != nil { - logrus.Errorf("failure to parse hook. %s", err) - c.AbortWithError(400, err) - return - } - if build == nil { - c.Writer.WriteHeader(200) - return - } - if tmprepo == nil { - logrus.Errorf("failure to ascertain repo from hook.") - c.Writer.WriteHeader(400) - return - } - - // skip the build if any case-insensitive combination of the words "skip" and "ci" - // wrapped in square brackets appear in the commit message - skipMatch := skipRe.FindString(build.Message) - if len(skipMatch) > 0 { - logrus.Infof("ignoring hook. %s found in %s", skipMatch, build.Commit) - c.Writer.WriteHeader(204) - return - } - - repo, err := store.GetRepoOwnerName(c, tmprepo.Owner, tmprepo.Name) - if err != nil { - logrus.Errorf("failure to find repo %s/%s from hook. %s", tmprepo.Owner, tmprepo.Name, err) - c.AbortWithError(404, err) - return - } - - // get the token and verify the hook is authorized - parsed, err := token.ParseRequest(c.Request, func(t *token.Token) (string, error) { - return repo.Hash, nil - }) - if err != nil { - logrus.Errorf("failure to parse token from hook for %s. %s", repo.FullName, err) - c.AbortWithError(400, err) - return - } - if parsed.Text != repo.FullName { - logrus.Errorf("failure to verify token from hook. Expected %s, got %s", repo.FullName, parsed.Text) - c.AbortWithStatus(403) - return - } - - if repo.UserID == 0 { - logrus.Warnf("ignoring hook. repo %s has no owner.", repo.FullName) - c.Writer.WriteHeader(204) - return - } - var skipped = true - if (build.Event == model.EventPush && repo.AllowPush) || - (build.Event == model.EventPull && repo.AllowPull) || - (build.Event == model.EventDeploy && repo.AllowDeploy) || - (build.Event == model.EventTag && repo.AllowTag) { - skipped = false - } - - if skipped { - logrus.Infof("ignoring hook. repo %s is disabled for %s events.", repo.FullName, build.Event) - c.Writer.WriteHeader(204) - return - } - - user, err := store.GetUser(c, repo.UserID) - if err != nil { - logrus.Errorf("failure to find repo owner %s. %s", repo.FullName, err) - c.AbortWithError(500, err) - return - } - - // if the remote has a refresh token, the current access token - // may be stale. Therefore, we should refresh prior to dispatching - // the job. - if refresher, ok := remote_.(remote.Refresher); ok { - ok, _ := refresher.Refresh(user) - if ok { - store.UpdateUser(c, user) - } - } - - // fetch the build file from the database - cfg := ToConfig(c) - raw, err := remote_.File(user, repo, build, cfg.Yaml) - if err != nil { - logrus.Errorf("failure to get build config for %s. %s", repo.FullName, err) - c.AbortWithError(404, err) - return - } - sec, err := remote_.File(user, repo, build, cfg.Shasum) - if err != nil { - logrus.Debugf("cannot find yaml signature for %s. %s", repo.FullName, err) - } - - netrc, err := remote_.Netrc(user, repo) - if err != nil { - c.String(500, "Failed to generate netrc file. %s", err) - return - } - - // verify the branches can be built vs skipped - branches, err := yaml.ParseBytes(raw) - if err != nil { - c.String(500, "Failed to parse yaml file. %s", err) - return - } - if !branches.Branches.Match(build.Branch) && build.Event != model.EventTag && build.Event != model.EventDeploy { - c.String(200, "Branch does not match restrictions defined in yaml") - return - } - - signature, err := jose.ParseSigned(string(sec)) - if err != nil { - logrus.Debugf("cannot parse .drone.yml.sig file. %s", err) - } else if len(sec) == 0 { - logrus.Debugf("cannot parse .drone.yml.sig file. empty file") - } else { - build.Signed = true - output, verr := signature.Verify([]byte(repo.Hash)) - if verr != nil { - logrus.Debugf("cannot verify .drone.yml.sig file. %s", verr) - } else if string(output) != string(raw) { - logrus.Debugf("cannot verify .drone.yml.sig file. no match") - } else { - build.Verified = true - } - } - - // update some build fields - build.Status = model.StatusPending - build.RepoID = repo.ID - - if err := store.CreateBuild(c, build, build.Jobs...); err != nil { - logrus.Errorf("failure to save commit for %s. %s", repo.FullName, err) - c.AbortWithError(500, err) - return - } - - c.JSON(200, build) - - // get the previous build so that we can send - // on status change notifications - last, _ := store.GetBuildLastBefore(c, repo, build.Branch, build.ID) - secs, err := store.GetMergedSecretList(c, repo) - if err != nil { - logrus.Debugf("Error getting secrets for %s#%d. %s", repo.FullName, build.Number, err) - } - - // - // BELOW: NEW - // - - defer func() { - uri := fmt.Sprintf("%s/%s/%d", httputil.GetURL(c.Request), repo.FullName, build.Number) - err = remote_.Status(user, repo, build, uri) - if err != nil { - logrus.Errorf("error setting commit status for %s/%d", repo.FullName, build.Number) - } - }() - - b := builder{ - Repo: repo, - Curr: build, - Last: last, - Netrc: netrc, - Secs: secs, - Link: httputil.GetURL(c.Request), - Yaml: string(raw), - } - items, err := b.Build() - if err != nil { - build.Status = model.StatusError - build.Started = time.Now().Unix() - build.Finished = build.Started - build.Error = err.Error() - return - } - - for _, item := range items { - build.Jobs = append(build.Jobs, item.Job) - store.CreateJob(c, item.Job) - // TODO err - } - - // - // publish topic - // - message := pubsub.Message{ - Labels: map[string]string{ - "repo": repo.FullName, - "private": strconv.FormatBool(repo.IsPrivate), - }, - } - message.Data, _ = json.Marshal(model.Event{ - Type: model.Enqueued, - Repo: *repo, - Build: *build, - }) - // TODO remove global reference - config.pubsub.Publish(c, "topic/events", message) - // - // end publish topic - // - - for _, item := range items { - task := new(queue.Task) - task.ID = fmt.Sprint(item.Job.ID) - task.Labels = map[string]string{} - task.Labels["platform"] = item.Platform - for k, v := range item.Labels { - task.Labels[k] = v - } - - task.Data, _ = json.Marshal(rpc.Pipeline{ - ID: fmt.Sprint(item.Job.ID), - Config: item.Config, - Timeout: b.Repo.Timeout, - }) - - config.logger.Open(context.Background(), task.ID) - config.queue.Push(context.Background(), task) - } -} diff --git a/server/queue.go b/server/queue.go deleted file mode 100644 index 43d5e08f6..000000000 --- a/server/queue.go +++ /dev/null @@ -1,163 +0,0 @@ -package server - -import ( - "bytes" - "fmt" - "net/http" - "strconv" - "time" - - "golang.org/x/net/context" - - "github.com/Sirupsen/logrus" - "github.com/drone/drone/model" - "github.com/drone/drone/remote" - "github.com/drone/drone/store" - "github.com/drone/mq/stomp" - "github.com/gorilla/websocket" -) - -// newline defines a newline constant to separate lines in the build output -var newline = []byte{'\n'} - -// upgrader defines the default behavior for upgrading the websocket. -var upgrader = websocket.Upgrader{ - ReadBufferSize: 1024, - WriteBufferSize: 1024, - CheckOrigin: func(r *http.Request) bool { - return true - }, -} - -// HandleUpdate handles build updates from the agent and persists to the database. -func HandleUpdate(c context.Context, message *stomp.Message) { - defer func() { - message.Release() - if r := recover(); r != nil { - err := r.(error) - logrus.Errorf("Panic recover: broker update handler: %s", err) - } - }() - - work := new(model.Work) - if err := message.Unmarshal(work); err != nil { - logrus.Errorf("Invalid input. %s", err) - return - } - - // TODO(bradrydzewski) it is really annoying that we have to do this lookup - // and I'd prefer not to. The reason we do this is because the Build and Job - // have fields that aren't serialized to json and would be reset to their - // empty values if we just saved what was coming in the http.Request body. - build, err := store.GetBuild(c, work.Build.ID) - if err != nil { - logrus.Errorf("Unable to find build. %s", err) - return - } - job, err := store.GetJob(c, work.Job.ID) - if err != nil { - logrus.Errorf("Unable to find job. %s", err) - return - } - build.Started = work.Build.Started - build.Finished = work.Build.Finished - build.Status = work.Build.Status - job.Started = work.Job.Started - job.Finished = work.Job.Finished - job.Status = work.Job.Status - job.ExitCode = work.Job.ExitCode - job.Error = work.Job.Error - - if build.Status == model.StatusPending { - build.Started = work.Job.Started - build.Status = model.StatusRunning - store.UpdateBuild(c, build) - } - - // if job.Status == model.StatusRunning { - // err := stream.Create(c, stream.ToKey(job.ID)) - // if err != nil { - // logrus.Errorf("Unable to create stream. %s", err) - // } - // } - - ok, err := store.UpdateBuildJob(c, build, job) - if err != nil { - logrus.Errorf("Unable to update job. %s", err) - return - } - - if ok { - // get the user because we transfer the user form the server to agent - // and back we lose the token which does not get serialized to json. - user, uerr := store.GetUser(c, work.User.ID) - if uerr != nil { - logrus.Errorf("Unable to find user. %s", err) - return - } - remote.Status(c, user, work.Repo, build, - fmt.Sprintf("%s/%s/%d", work.System.Link, work.Repo.FullName, work.Build.Number)) - } - - client := stomp.MustFromContext(c) - err = client.SendJSON("/topic/events", model.Event{ - Type: func() model.EventType { - // HACK we don't even really care about the event type. - // so we should just simplify how events are triggered. - if job.Status == model.StatusRunning { - return model.Started - } - return model.Finished - }(), - Repo: *work.Repo, - Build: *build, - Job: *job, - }, - stomp.WithHeader("repo", work.Repo.FullName), - stomp.WithHeader("private", strconv.FormatBool(work.Repo.IsPrivate)), - ) - if err != nil { - logrus.Errorf("Unable to publish to /topic/events. %s", err) - } - - if job.Status == model.StatusRunning { - return - } - - var buf bytes.Buffer - var sub []byte - - done := make(chan bool) - dest := fmt.Sprintf("/topic/logs.%d", job.ID) - sub, err = client.Subscribe(dest, stomp.HandlerFunc(func(m *stomp.Message) { - defer m.Release() - if m.Header.GetBool("eof") { - done <- true - return - } - buf.Write(m.Body) - buf.WriteByte('\n') - })) - - if err != nil { - logrus.Errorf("Unable to read logs from broker. %s", err) - return - } - - defer func() { - client.Send(dest, []byte{}, stomp.WithRetain("remove")) - client.Unsubscribe(sub) - }() - - select { - case <-done: - case <-time.After(30 * time.Second): - logrus.Errorf("Unable to read logs from broker. Timeout. %s", err) - return - } - - if err := store.WriteLog(c, job, &buf); err != nil { - logrus.Errorf("Unable to write logs to store. %s", err) - return - } -} diff --git a/server/stream.go b/server/stream.go index dad1b46b8..264aa23b8 100644 --- a/server/stream.go +++ b/server/stream.go @@ -3,6 +3,7 @@ package server import ( "context" "fmt" + "net/http" "strconv" "time" @@ -12,7 +13,6 @@ import ( "github.com/drone/drone/model" "github.com/drone/drone/router/middleware/session" "github.com/drone/drone/store" - "github.com/drone/mq/stomp" "github.com/Sirupsen/logrus" "github.com/gin-gonic/gin" @@ -28,160 +28,17 @@ var ( // Send pings to client with this period. Must be less than pongWait. pingPeriod = 30 * time.Second + + // upgrader defines the default behavior for upgrading the websocket. + upgrader = websocket.Upgrader{ + ReadBufferSize: 1024, + WriteBufferSize: 1024, + CheckOrigin: func(r *http.Request) bool { + return true + }, + } ) -// LogStream streams the build log output to the client. -func LogStream(c *gin.Context) { - repo := session.Repo(c) - buildn, _ := strconv.Atoi(c.Param("build")) - jobn, _ := strconv.Atoi(c.Param("number")) - - c.Writer.Header().Set("Content-Type", "text/event-stream") - - build, err := store.GetBuildNumber(c, repo, buildn) - if err != nil { - logrus.Debugln("stream cannot get build number.", err) - c.AbortWithError(404, err) - return - } - job, err := store.GetJobNumber(c, build, jobn) - if err != nil { - logrus.Debugln("stream cannot get job number.", err) - c.AbortWithError(404, err) - return - } - if job.Status != model.StatusRunning { - logrus.Debugln("stream not found.") - c.AbortWithStatus(404) - return - } - - ws, err := upgrader.Upgrade(c.Writer, c.Request, nil) - if err != nil { - if _, ok := err.(websocket.HandshakeError); !ok { - logrus.Errorf("Cannot upgrade websocket. %s", err) - } - return - } - logrus.Debugf("Successfull upgraded websocket") - - ticker := time.NewTicker(pingPeriod) - defer ticker.Stop() - - logs := make(chan []byte) - done := make(chan bool) - var eof bool - dest := fmt.Sprintf("/topic/logs.%d", job.ID) - client, _ := stomp.FromContext(c) - sub, err := client.Subscribe(dest, stomp.HandlerFunc(func(m *stomp.Message) { - if m.Header.GetBool("eof") { - eof = true - done <- true - } else if eof { - return - } else { - logs <- m.Body - } - m.Release() - })) - if err != nil { - logrus.Errorf("Unable to read logs from broker. %s", err) - return - } - defer func() { - client.Unsubscribe(sub) - close(done) - close(logs) - }() - - for { - select { - case buf := <-logs: - ws.SetWriteDeadline(time.Now().Add(writeWait)) - ws.WriteMessage(websocket.TextMessage, buf) - case <-done: - return - case <-ticker.C: - err := ws.WriteControl(websocket.PingMessage, []byte{}, time.Now().Add(writeWait)) - if err != nil { - return - } - } - } -} - -// EventStream produces the User event stream, sending all repository, build -// and agent events to the client. -func EventStream(c *gin.Context) { - ws, err := upgrader.Upgrade(c.Writer, c.Request, nil) - if err != nil { - if _, ok := err.(websocket.HandshakeError); !ok { - logrus.Errorf("Cannot upgrade websocket. %s", err) - } - return - } - logrus.Debugf("Successfull upgraded websocket") - - user := session.User(c) - repo := map[string]bool{} - if user != nil { - repo, _ = cache.GetRepoMap(c, user) - } - - eventc := make(chan []byte, 10) - quitc := make(chan bool) - tick := time.NewTicker(pingPeriod) - defer func() { - tick.Stop() - ws.Close() - logrus.Debug("Successfully closed websocket") - }() - - client := stomp.MustFromContext(c) - sub, err := client.Subscribe("/topic/events", stomp.HandlerFunc(func(m *stomp.Message) { - name := m.Header.GetString("repo") - priv := m.Header.GetBool("private") - if repo[name] || !priv { - eventc <- m.Body - } - m.Release() - })) - if err != nil { - logrus.Errorf("Unable to read logs from broker. %s", err) - return - } - defer func() { - client.Unsubscribe(sub) - close(quitc) - close(eventc) - }() - - go func() { - defer func() { - recover() - }() - for { - select { - case <-quitc: - return - case event, ok := <-eventc: - if !ok { - return - } - ws.SetWriteDeadline(time.Now().Add(writeWait)) - ws.WriteMessage(websocket.TextMessage, event) - case <-tick.C: - err := ws.WriteControl(websocket.PingMessage, []byte{}, time.Now().Add(writeWait)) - if err != nil { - return - } - } - } - }() - - reader(ws) -} - func reader(ws *websocket.Conn) { defer ws.Close() ws.SetReadLimit(512) @@ -198,14 +55,7 @@ func reader(ws *websocket.Conn) { } } -// -// CANARY IMPLEMENTATION -// -// This file is a complete disaster because I'm trying to wedge in some -// experimental code. Please pardon our appearance during renovations. -// - -func LogStream2(c *gin.Context) { +func LogStream(c *gin.Context) { repo := session.Repo(c) buildn, _ := strconv.Atoi(c.Param("build")) jobn, _ := strconv.Atoi(c.Param("number")) @@ -286,7 +136,7 @@ func LogStream2(c *gin.Context) { reader(ws) } -func EventStream2(c *gin.Context) { +func EventStream(c *gin.Context) { ws, err := upgrader.Upgrade(c.Writer, c.Request, nil) if err != nil { if _, ok := err.(websocket.HandshakeError); !ok { diff --git a/version/version.go b/version/version.go index 170e25b92..329bfbd0a 100644 --- a/version/version.go +++ b/version/version.go @@ -6,7 +6,7 @@ var ( // VersionMajor is for an API incompatible changes VersionMajor = 0 // VersionMinor is for functionality in a backwards-compatible manner - VersionMinor = 5 + VersionMinor = 6 // VersionPatch is for backwards-compatible bug fixes VersionPatch = 0 diff --git a/yaml/branch.go b/yaml/branch.go deleted file mode 100644 index dbb04fd01..000000000 --- a/yaml/branch.go +++ /dev/null @@ -1,18 +0,0 @@ -package yaml - -import "gopkg.in/yaml.v2" - -// ParseBranch parses the branch section of the Yaml document. -func ParseBranch(in []byte) Constraint { - out := struct { - Constraint Constraint `yaml:"branches"` - }{} - - yaml.Unmarshal(in, &out) - return out.Constraint -} - -// ParseBranchString parses the branch section of the Yaml document. -func ParseBranchString(in string) Constraint { - return ParseBranch([]byte(in)) -} diff --git a/yaml/branch_test.go b/yaml/branch_test.go deleted file mode 100644 index 32055bbac..000000000 --- a/yaml/branch_test.go +++ /dev/null @@ -1,44 +0,0 @@ -package yaml - -import ( - "testing" - - "github.com/franela/goblin" -) - -func TestBranch(t *testing.T) { - - g := goblin.Goblin(t) - g.Describe("Branch filter", func() { - - g.It("Should parse and match emtpy", func() { - branch := ParseBranchString("") - g.Assert(branch.Match("master")).IsTrue() - }) - - g.It("Should parse and match", func() { - branch := ParseBranchString("branches: { include: [ master, develop ] }") - g.Assert(branch.Match("master")).IsTrue() - }) - - g.It("Should parse and match shortand", func() { - branch := ParseBranchString("branches: [ master, develop ]") - g.Assert(branch.Match("master")).IsTrue() - }) - - g.It("Should parse and match shortand string", func() { - branch := ParseBranchString("branches: master") - g.Assert(branch.Match("master")).IsTrue() - }) - - g.It("Should parse and match exclude", func() { - branch := ParseBranchString("branches: { exclude: [ master, develop ] }") - g.Assert(branch.Match("master")).IsFalse() - }) - - g.It("Should parse and match exclude shorthand", func() { - branch := ParseBranchString("branches: { exclude: master }") - g.Assert(branch.Match("master")).IsFalse() - }) - }) -} diff --git a/yaml/build.go b/yaml/build.go deleted file mode 100644 index ec3892a07..000000000 --- a/yaml/build.go +++ /dev/null @@ -1,26 +0,0 @@ -package yaml - -// Build represents Docker image build instructions. -type Build struct { - Context string - Dockerfile string - Args map[string]string -} - -// UnmarshalYAML implements custom Yaml unmarshaling. -func (b *Build) UnmarshalYAML(unmarshal func(interface{}) error) error { - err := unmarshal(&b.Context) - if err == nil { - return nil - } - out := struct { - Context string - Dockerfile string - Args map[string]string - }{} - err = unmarshal(&out) - b.Context = out.Context - b.Args = out.Args - b.Dockerfile = out.Dockerfile - return err -} diff --git a/yaml/build_test.go b/yaml/build_test.go deleted file mode 100644 index 69c9a1fb9..000000000 --- a/yaml/build_test.go +++ /dev/null @@ -1,38 +0,0 @@ -package yaml - -import ( - "testing" - - "github.com/franela/goblin" - "gopkg.in/yaml.v2" -) - -func TestBuild(t *testing.T) { - g := goblin.Goblin(t) - - g.Describe("Build", func() { - g.Describe("given a yaml file", func() { - - g.It("should unmarshal", func() { - in := []byte(".") - out := Build{} - err := yaml.Unmarshal(in, &out) - if err != nil { - g.Fail(err) - } - g.Assert(out.Context).Equal(".") - }) - - g.It("should unmarshal shorthand", func() { - in := []byte("{ context: ., dockerfile: Dockerfile }") - out := Build{} - err := yaml.Unmarshal(in, &out) - if err != nil { - g.Fail(err) - } - g.Assert(out.Context).Equal(".") - g.Assert(out.Dockerfile).Equal("Dockerfile") - }) - }) - }) -} diff --git a/yaml/config.go b/yaml/config.go deleted file mode 100644 index 7f05cab55..000000000 --- a/yaml/config.go +++ /dev/null @@ -1,67 +0,0 @@ -package yaml - -import "gopkg.in/yaml.v2" - -// Workspace represents the build workspace. -type Workspace struct { - Base string - Path string -} - -// Config represents the build configuration Yaml document. -type Config struct { - Image string - Build *Build - Workspace *Workspace - Pipeline []*Container - Services []*Container - Volumes []*Volume - Networks []*Network -} - -// ParseString parses the Yaml configuration document. -func ParseString(data string) (*Config, error) { - return Parse([]byte(data)) -} - -// Parse parses Yaml configuration document. -func Parse(data []byte) (*Config, error) { - v := struct { - Image string - Build *Build - Workspace *Workspace - Services containerList - Pipeline containerList - Networks networkList - Volumes volumeList - }{} - - err := yaml.Unmarshal(data, &v) - if err != nil { - return nil, err - } - - for _, c := range v.Services.containers { - c.Detached = true - } - - return &Config{ - Image: v.Image, - Build: v.Build, - Workspace: v.Workspace, - Services: v.Services.containers, - Pipeline: v.Pipeline.containers, - Networks: v.Networks.networks, - Volumes: v.Volumes.volumes, - }, nil -} - -type config struct { - Image string - Build *Build - Workspace *Workspace - Services containerList - Pipeline containerList - Networks networkList - Volumes volumeList -} diff --git a/yaml/config_test.go b/yaml/config_test.go deleted file mode 100644 index e20add807..000000000 --- a/yaml/config_test.go +++ /dev/null @@ -1,108 +0,0 @@ -package yaml - -import ( - "testing" - - "github.com/franela/goblin" -) - -func TestParse(t *testing.T) { - g := goblin.Goblin(t) - - g.Describe("Parser", func() { - g.Describe("Given a yaml file", func() { - - g.It("Should unmarshal a string", func() { - out, err := ParseString(sampleYaml) - if err != nil { - g.Fail(err) - } - g.Assert(out.Image).Equal("hello-world") - g.Assert(out.Workspace.Base).Equal("/go") - g.Assert(out.Workspace.Path).Equal("src/github.com/octocat/hello-world") - g.Assert(out.Build.Context).Equal(".") - g.Assert(out.Build.Dockerfile).Equal("Dockerfile") - g.Assert(out.Volumes[0].Name).Equal("custom") - g.Assert(out.Volumes[0].Driver).Equal("blockbridge") - g.Assert(out.Networks[0].Name).Equal("custom") - g.Assert(out.Networks[0].Driver).Equal("overlay") - g.Assert(out.Services[0].Name).Equal("database") - g.Assert(out.Services[0].Image).Equal("mysql") - g.Assert(out.Pipeline[0].Name).Equal("test") - g.Assert(out.Pipeline[0].Image).Equal("golang") - g.Assert(out.Pipeline[0].Commands).Equal([]string{"go install", "go test"}) - g.Assert(out.Pipeline[1].Name).Equal("build") - g.Assert(out.Pipeline[1].Image).Equal("golang") - g.Assert(out.Pipeline[1].Commands).Equal([]string{"go build"}) - g.Assert(out.Pipeline[2].Name).Equal("notify") - g.Assert(out.Pipeline[2].Image).Equal("slack") - }) - // Check to make sure variable expansion works in yaml.MapSlice - g.It("Should unmarshal variables", func() { - out, err := ParseString(sampleVarYaml) - if err != nil { - g.Fail(err) - } - g.Assert(out.Pipeline[0].Name).Equal("notify_fail") - g.Assert(out.Pipeline[0].Image).Equal("plugins/slack") - g.Assert(len(out.Pipeline[0].Constraints.Event.Include)).Equal(0) - g.Assert(out.Pipeline[1].Name).Equal("notify_success") - g.Assert(out.Pipeline[1].Image).Equal("plugins/slack") - g.Assert(out.Pipeline[1].Constraints.Event.Include).Equal([]string{"success"}) - }) - }) - }) -} - -var sampleYaml = ` -image: hello-world -build: - context: . - dockerfile: Dockerfile - -workspace: - path: src/github.com/octocat/hello-world - base: /go - -pipeline: - test: - image: golang - commands: - - go install - - go test - build: - image: golang - commands: - - go build - when: - event: push - notify: - image: slack - channel: dev - when: - event: failure - -services: - database: - image: mysql - -networks: - custom: - driver: overlay - -volumes: - custom: - driver: blockbridge -` - -var sampleVarYaml = ` -_slack: &SLACK - image: plugins/slack - -pipeline: - notify_fail: *SLACK - notify_success: - << : *SLACK - when: - event: success -` diff --git a/yaml/constraint.go b/yaml/constraint.go deleted file mode 100644 index 07608a68f..000000000 --- a/yaml/constraint.go +++ /dev/null @@ -1,155 +0,0 @@ -package yaml - -import ( - "path/filepath" - - "github.com/drone/drone/yaml/types" -) - -// Constraints define constraints for container execution. -type Constraints struct { - Repo Constraint - Ref Constraint - Platform Constraint - Environment Constraint - Event Constraint - Branch Constraint - Status Constraint - Matrix ConstraintMap - Local types.BoolTrue -} - -// Match returns true if all constraints match the given input. If a single constraint -// fails a false value is returned. -func (c *Constraints) Match(arch, target, event, branch, status string, matrix map[string]string) bool { - return c.Platform.Match(arch) && - c.Environment.Match(target) && - c.Event.Match(event) && - c.Branch.Match(branch) && - c.Status.Match(status) && - c.Matrix.Match(matrix) -} - -// Constraint defines an individual constraint. -type Constraint struct { - Include []string - Exclude []string -} - -// Match returns true if the string matches the include patterns and does not -// match any of the exclude patterns. -func (c *Constraint) Match(v string) bool { - if c.Excludes(v) { - return false - } - if c.Includes(v) { - return true - } - if len(c.Include) == 0 { - return true - } - return false -} - -// Includes returns true if the string matches matches the include patterns. -func (c *Constraint) Includes(v string) bool { - for _, pattern := range c.Include { - if ok, _ := filepath.Match(pattern, v); ok { - return true - } - } - return false -} - -// Excludes returns true if the string matches matches the exclude patterns. -func (c *Constraint) Excludes(v string) bool { - for _, pattern := range c.Exclude { - if ok, _ := filepath.Match(pattern, v); ok { - return true - } - } - return false -} - -// UnmarshalYAML implements custom Yaml unmarshaling. -func (c *Constraint) UnmarshalYAML(unmarshal func(interface{}) error) error { - - var out1 = struct { - Include types.StringOrSlice - Exclude types.StringOrSlice - }{} - - var out2 types.StringOrSlice - - unmarshal(&out1) - unmarshal(&out2) - - c.Exclude = out1.Exclude.Slice() - c.Include = append( - out1.Include.Slice(), - out2.Slice()..., - ) - return nil -} - -// ConstraintMap defines an individual constraint for key value structures. -type ConstraintMap struct { - Include map[string]string - Exclude map[string]string -} - -// Match returns true if the params matches the include key values and does not -// match any of the exclude key values. -func (c *ConstraintMap) Match(params map[string]string) bool { - // when no includes or excludes automatically match - if len(c.Include) == 0 && len(c.Exclude) == 0 { - return true - } - - // exclusions are processed first. So we can include everything and then - // selectively include others. - if len(c.Exclude) != 0 { - var matches int - - for key, val := range c.Exclude { - if params[key] == val { - matches++ - } - } - if matches == len(c.Exclude) { - return false - } - } - - for key, val := range c.Include { - if params[key] != val { - return false - } - } - - return true -} - -// UnmarshalYAML implements custom Yaml unmarshaling. -func (c *ConstraintMap) UnmarshalYAML(unmarshal func(interface{}) error) error { - - out1 := struct { - Include map[string]string - Exclude map[string]string - }{ - Include: map[string]string{}, - Exclude: map[string]string{}, - } - - out2 := map[string]string{} - - unmarshal(&out1) - unmarshal(&out2) - - c.Include = out1.Include - c.Exclude = out1.Exclude - for k, v := range out2 { - c.Include[k] = v - } - return nil -} diff --git a/yaml/constraint_test.go b/yaml/constraint_test.go deleted file mode 100644 index 22630de68..000000000 --- a/yaml/constraint_test.go +++ /dev/null @@ -1,142 +0,0 @@ -package yaml - -import ( - "testing" - - "github.com/franela/goblin" - "gopkg.in/yaml.v2" -) - -func TestConstraint(t *testing.T) { - - g := goblin.Goblin(t) - g.Describe("Constraint", func() { - - g.It("Should parse and match emtpy", func() { - c := parseConstraint("") - g.Assert(c.Match("master")).IsTrue() - }) - - g.It("Should parse and match", func() { - c := parseConstraint("{ include: [ master, develop ] }") - g.Assert(c.Include[0]).Equal("master") - g.Assert(c.Include[1]).Equal("develop") - g.Assert(c.Match("master")).IsTrue() - }) - - g.It("Should parse and match shortand", func() { - c := parseConstraint("[ master, develop ]") - g.Assert(c.Include[0]).Equal("master") - g.Assert(c.Include[1]).Equal("develop") - g.Assert(c.Match("master")).IsTrue() - }) - - g.It("Should parse and match shortand string", func() { - c := parseConstraint("master") - g.Assert(c.Include[0]).Equal("master") - g.Assert(c.Match("master")).IsTrue() - }) - - g.It("Should parse and match exclude", func() { - c := parseConstraint("{ exclude: [ master, develop ] }") - g.Assert(c.Exclude[0]).Equal("master") - g.Assert(c.Exclude[1]).Equal("develop") - g.Assert(c.Match("master")).IsFalse() - }) - - g.It("Should parse and match exclude shorthand", func() { - c := parseConstraint("{ exclude: master }") - g.Assert(c.Exclude[0]).Equal("master") - g.Assert(c.Match("master")).IsFalse() - }) - - g.It("Should match include", func() { - b := Constraint{} - b.Include = []string{"master"} - g.Assert(b.Match("master")).IsTrue() - }) - - g.It("Should match include pattern", func() { - b := Constraint{} - b.Include = []string{"feature/*"} - g.Assert(b.Match("feature/foo")).IsTrue() - }) - - g.It("Should fail to match include pattern", func() { - b := Constraint{} - b.Include = []string{"feature/*"} - g.Assert(b.Match("master")).IsFalse() - }) - - g.It("Should match exclude", func() { - b := Constraint{} - b.Exclude = []string{"master"} - g.Assert(b.Match("master")).IsFalse() - }) - - g.It("Should match exclude pattern", func() { - b := Constraint{} - b.Exclude = []string{"feature/*"} - g.Assert(b.Match("feature/foo")).IsFalse() - }) - - g.It("Should match when eclude patterns mismatch", func() { - b := Constraint{} - b.Exclude = []string{"foo"} - g.Assert(b.Match("bar")).IsTrue() - }) - }) -} - -func TestConstraintMap(t *testing.T) { - - g := goblin.Goblin(t) - g.Describe("Constraint Map", func() { - g.It("Should parse and match emtpy", func() { - p := map[string]string{"golang": "1.5", "redis": "3.2"} - c := parseConstraintMap("") - g.Assert(c.Match(p)).IsTrue() - }) - - g.It("Should parse and match", func() { - p := map[string]string{"golang": "1.5", "redis": "3.2"} - c := parseConstraintMap("{ include: { golang: 1.5 } }") - g.Assert(c.Include["golang"]).Equal("1.5") - g.Assert(c.Match(p)).IsTrue() - }) - - g.It("Should parse and match shortand", func() { - p := map[string]string{"golang": "1.5", "redis": "3.2"} - c := parseConstraintMap("{ golang: 1.5 }") - g.Assert(c.Include["golang"]).Equal("1.5") - g.Assert(c.Match(p)).IsTrue() - }) - - g.It("Should parse and match exclude", func() { - p := map[string]string{"golang": "1.5", "redis": "3.2"} - c := parseConstraintMap("{ exclude: { golang: 1.5 } }") - g.Assert(c.Exclude["golang"]).Equal("1.5") - g.Assert(c.Match(p)).IsFalse() - }) - - g.It("Should parse and mismatch exclude", func() { - p := map[string]string{"golang": "1.5", "redis": "3.2"} - c := parseConstraintMap("{ exclude: { golang: 1.5, redis: 2.8 } }") - g.Assert(c.Exclude["golang"]).Equal("1.5") - g.Assert(c.Exclude["redis"]).Equal("2.8") - g.Assert(c.Match(p)).IsTrue() - }) - }) -} - -func parseConstraint(s string) *Constraint { - c := &Constraint{} - yaml.Unmarshal([]byte(s), c) - return c -} - -func parseConstraintMap(s string) *ConstraintMap { - c := &ConstraintMap{} - yaml.Unmarshal([]byte(s), c) - return c -} diff --git a/yaml/container.go b/yaml/container.go deleted file mode 100644 index 24380eaa0..000000000 --- a/yaml/container.go +++ /dev/null @@ -1,166 +0,0 @@ -package yaml - -import ( - "fmt" - - "github.com/drone/drone/yaml/types" - "gopkg.in/yaml.v2" -) - -// Auth defines Docker authentication credentials. -type Auth struct { - Username string - Password string - Email string -} - -// Container defines a Docker container. -type Container struct { - ID string - Name string - Image string - Build string - Pull bool - AuthConfig Auth - Detached bool - Disabled bool - Privileged bool - WorkingDir string - Environment map[string]string - Labels map[string]string - Entrypoint []string - Command []string - Commands []string - ExtraHosts []string - Volumes []string - VolumesFrom []string - Devices []string - Network string - DNS []string - DNSSearch []string - MemSwapLimit int64 - MemLimit int64 - ShmSize int64 - CPUQuota int64 - CPUShares int64 - CPUSet string - OomKillDisable bool - Constraints Constraints - - Vargs map[string]interface{} -} - -// container is an intermediate type used for decoding a container in a format -// compatible with docker-compose.yml. - -// this file has a bunch of custom types that are annoying to work with, which -// is why this is used for intermediate purposes and converted to something -// easier to work with. -type container struct { - Name string `yaml:"name"` - Image string `yaml:"image"` - Build string `yaml:"build"` - Pull bool `yaml:"pull"` - Detached bool `yaml:"detach"` - Privileged bool `yaml:"privileged"` - Environment types.MapEqualSlice `yaml:"environment"` - Labels types.MapEqualSlice `yaml:"labels"` - Entrypoint types.StringOrSlice `yaml:"entrypoint"` - Command types.StringOrSlice `yaml:"command"` - Commands types.StringOrSlice `yaml:"commands"` - ExtraHosts types.StringOrSlice `yaml:"extra_hosts"` - Volumes types.StringOrSlice `yaml:"volumes"` - VolumesFrom types.StringOrSlice `yaml:"volumes_from"` - Devices types.StringOrSlice `yaml:"devices"` - Network string `yaml:"network_mode"` - DNS types.StringOrSlice `yaml:"dns"` - DNSSearch types.StringOrSlice `yaml:"dns_search"` - MemSwapLimit int64 `yaml:"memswap_limit"` - MemLimit int64 `yaml:"mem_limit"` - ShmSize int64 `yaml:"shm_size"` - CPUQuota int64 `yaml:"cpu_quota"` - CPUShares int64 `yaml:"cpu_shares"` - CPUSet string `yaml:"cpuset"` - OomKillDisable bool `yaml:"oom_kill_disable"` - - AuthConfig struct { - Username string `yaml:"username"` - Password string `yaml:"password"` - Email string `yaml:"email"` - Token string `yaml:"registry_token"` - } `yaml:"auth_config"` - - Constraints Constraints `yaml:"when"` - - Vargs map[string]interface{} `yaml:",inline"` -} - -// containerList is an intermediate type used for decoding a slice of containers -// in a format compatible with docker-compose.yml -type containerList struct { - containers []*Container -} - -// UnmarshalYAML implements custom Yaml unmarshaling. -func (c *containerList) UnmarshalYAML(unmarshal func(interface{}) error) error { - slice := yaml.MapSlice{} - err := unmarshal(&slice) - if err != nil { - return err - } - - for _, s := range slice { - cc := container{} - - out, merr := yaml.Marshal(s.Value) - if err != nil { - return merr - } - - err = yaml.Unmarshal(out, &cc) - if err != nil { - return err - } - if cc.Name == "" { - cc.Name = fmt.Sprintf("%v", s.Key) - } - if cc.Image == "" { - cc.Image = fmt.Sprintf("%v", s.Key) - } - c.containers = append(c.containers, &Container{ - Name: cc.Name, - Image: cc.Image, - Build: cc.Build, - Pull: cc.Pull, - Detached: cc.Detached, - Privileged: cc.Privileged, - Environment: cc.Environment.Map(), - Labels: cc.Labels.Map(), - Entrypoint: cc.Entrypoint.Slice(), - Command: cc.Command.Slice(), - Commands: cc.Commands.Slice(), - ExtraHosts: cc.ExtraHosts.Slice(), - Volumes: cc.Volumes.Slice(), - VolumesFrom: cc.VolumesFrom.Slice(), - Devices: cc.Devices.Slice(), - Network: cc.Network, - DNS: cc.DNS.Slice(), - DNSSearch: cc.DNSSearch.Slice(), - MemSwapLimit: cc.MemSwapLimit, - MemLimit: cc.MemLimit, - ShmSize: cc.ShmSize, - CPUQuota: cc.CPUQuota, - CPUShares: cc.CPUShares, - CPUSet: cc.CPUSet, - OomKillDisable: cc.OomKillDisable, - Vargs: cc.Vargs, - AuthConfig: Auth{ - Username: cc.AuthConfig.Username, - Password: cc.AuthConfig.Password, - Email: cc.AuthConfig.Email, - }, - Constraints: cc.Constraints, - }) - } - return err -} diff --git a/yaml/container_test.go b/yaml/container_test.go deleted file mode 100644 index 623944bc3..000000000 --- a/yaml/container_test.go +++ /dev/null @@ -1,99 +0,0 @@ -package yaml - -import ( - "testing" - - "github.com/franela/goblin" - "gopkg.in/yaml.v2" -) - -func TestContainerNode(t *testing.T) { - g := goblin.Goblin(t) - - g.Describe("Containers", func() { - g.Describe("given a yaml file", func() { - - g.It("should unmarshal", func() { - in := []byte(sampleContainer) - out := containerList{} - err := yaml.Unmarshal(in, &out) - if err != nil { - g.Fail(err) - } - g.Assert(len(out.containers)).Equal(1) - - c := out.containers[0] - g.Assert(c.Name).Equal("foo") - g.Assert(c.Image).Equal("golang") - g.Assert(c.Build).Equal(".") - g.Assert(c.Pull).Equal(true) - g.Assert(c.Detached).Equal(true) - g.Assert(c.Privileged).Equal(true) - g.Assert(c.Entrypoint).Equal([]string{"/bin/sh"}) - g.Assert(c.Command).Equal([]string{"yes"}) - g.Assert(c.Commands).Equal([]string{"whoami"}) - g.Assert(c.ExtraHosts).Equal([]string{"foo.com"}) - g.Assert(c.Volumes).Equal([]string{"/foo:/bar"}) - g.Assert(c.VolumesFrom).Equal([]string{"foo"}) - g.Assert(c.Devices).Equal([]string{"/dev/tty0"}) - g.Assert(c.Network).Equal("bridge") - g.Assert(c.DNS).Equal([]string{"8.8.8.8"}) - g.Assert(c.MemSwapLimit).Equal(int64(1)) - g.Assert(c.MemLimit).Equal(int64(2)) - g.Assert(c.CPUQuota).Equal(int64(3)) - g.Assert(c.CPUSet).Equal("1,2") - g.Assert(c.OomKillDisable).Equal(true) - g.Assert(c.AuthConfig.Username).Equal("octocat") - g.Assert(c.AuthConfig.Password).Equal("password") - g.Assert(c.AuthConfig.Email).Equal("octocat@github.com") - g.Assert(c.Vargs["access_key"]).Equal("970d28f4dd477bc184fbd10b376de753") - g.Assert(c.Vargs["secret_key"]).Equal("9c5785d3ece6a9cdefa42eb99b58986f9095ff1c") - }) - - g.It("should unmarshal named", func() { - in := []byte("foo: { name: bar }") - out := containerList{} - err := yaml.Unmarshal(in, &out) - if err != nil { - g.Fail(err) - } - g.Assert(len(out.containers)).Equal(1) - g.Assert(out.containers[0].Name).Equal("bar") - }) - - }) - }) -} - -var sampleContainer = ` -foo: - image: golang - build: . - pull: true - detach: true - privileged: true - environment: - FOO: BAR - entrypoint: /bin/sh - command: "yes" - commands: whoami - extra_hosts: foo.com - volumes: /foo:/bar - volumes_from: foo - devices: /dev/tty0 - network_mode: bridge - dns: 8.8.8.8 - memswap_limit: 1 - mem_limit: 2 - cpu_quota: 3 - cpuset: 1,2 - oom_kill_disable: true - - auth_config: - username: octocat - password: password - email: octocat@github.com - - access_key: 970d28f4dd477bc184fbd10b376de753 - secret_key: 9c5785d3ece6a9cdefa42eb99b58986f9095ff1c -` diff --git a/yaml/label.go b/yaml/label.go deleted file mode 100644 index 445a9b6b4..000000000 --- a/yaml/label.go +++ /dev/null @@ -1,26 +0,0 @@ -package yaml - -import ( - "gopkg.in/yaml.v2" - - "github.com/drone/drone/yaml/types" -) - -// ParseLabel parses the labels section of the Yaml document. -func ParseLabel(in []byte) map[string]string { - out := struct { - Labels types.MapEqualSlice `yaml:"labels"` - }{} - - yaml.Unmarshal(in, &out) - labels := out.Labels.Map() - if labels == nil { - labels = make(map[string]string) - } - return labels -} - -// ParseLabelString parses the labels section of the Yaml document. -func ParseLabelString(in string) map[string]string { - return ParseLabel([]byte(in)) -} diff --git a/yaml/label_test.go b/yaml/label_test.go deleted file mode 100644 index 3032f33ed..000000000 --- a/yaml/label_test.go +++ /dev/null @@ -1,32 +0,0 @@ -package yaml - -import ( - "testing" - - "github.com/franela/goblin" -) - -func TestLabel(t *testing.T) { - - g := goblin.Goblin(t) - g.Describe("Label parser", func() { - - g.It("Should parse empty yaml", func() { - labels := ParseLabelString("") - g.Assert(len(labels)).Equal(0) - }) - - g.It("Should parse slice", func() { - labels := ParseLabelString("labels: [foo=bar, baz=boo]") - g.Assert(len(labels)).Equal(2) - g.Assert(labels["foo"]).Equal("bar") - g.Assert(labels["baz"]).Equal("boo") - }) - - g.It("Should parse map", func() { - labels := ParseLabelString("labels: {foo: bar, baz: boo}") - g.Assert(labels["foo"]).Equal("bar") - g.Assert(labels["baz"]).Equal("boo") - }) - }) -} diff --git a/yaml/matrix.go b/yaml/matrix.go deleted file mode 100644 index 47b6768f1..000000000 --- a/yaml/matrix.go +++ /dev/null @@ -1,117 +0,0 @@ -package yaml - -import ( - "strings" - - "gopkg.in/yaml.v2" -) - -const ( - limitTags = 10 - limitAxis = 25 -) - -// Matrix represents the build matrix. -type Matrix map[string][]string - -// Axis represents a single permutation of entries from the build matrix. -type Axis map[string]string - -// String returns a string representation of an Axis as a comma-separated list -// of environment variables. -func (a Axis) String() string { - var envs []string - for k, v := range a { - envs = append(envs, k+"="+v) - } - return strings.Join(envs, " ") -} - -// ParseMatrix parses the Yaml matrix definition. -func ParseMatrix(data []byte) ([]Axis, error) { - - axis, err := parseMatrixList(data) - if err == nil && len(axis) != 0 { - return axis, nil - } - - matrix, err := parseMatrix(data) - if err != nil { - return nil, err - } - - // if not a matrix build return an array with just the single axis. - if len(matrix) == 0 { - return nil, nil - } - - return calcMatrix(matrix), nil -} - -// ParseMatrixString parses the Yaml string matrix definition. -func ParseMatrixString(data string) ([]Axis, error) { - return ParseMatrix([]byte(data)) -} - -func calcMatrix(matrix Matrix) []Axis { - // calculate number of permutations and extract the list of tags - // (ie go_version, redis_version, etc) - var perm int - var tags []string - for k, v := range matrix { - perm *= len(v) - if perm == 0 { - perm = len(v) - } - tags = append(tags, k) - } - - // structure to hold the transformed result set - axisList := []Axis{} - - // for each axis calculate the uniqe set of values that should be used. - for p := 0; p < perm; p++ { - axis := map[string]string{} - decr := perm - for i, tag := range tags { - elems := matrix[tag] - decr = decr / len(elems) - elem := p / decr % len(elems) - axis[tag] = elems[elem] - - // enforce a maximum number of tags in the build matrix. - if i > limitTags { - break - } - } - - // append to the list of axis. - axisList = append(axisList, axis) - - // enforce a maximum number of axis that should be calculated. - if p > limitAxis { - break - } - } - - return axisList -} - -func parseMatrix(raw []byte) (Matrix, error) { - data := struct { - Matrix map[string][]string - }{} - err := yaml.Unmarshal(raw, &data) - return data.Matrix, err -} - -func parseMatrixList(raw []byte) ([]Axis, error) { - data := struct { - Matrix struct { - Include []Axis - } - }{} - - err := yaml.Unmarshal(raw, &data) - return data.Matrix.Include, err -} diff --git a/yaml/matrix_test.go b/yaml/matrix_test.go deleted file mode 100644 index d819cff13..000000000 --- a/yaml/matrix_test.go +++ /dev/null @@ -1,70 +0,0 @@ -package yaml - -import ( - "testing" - - "github.com/franela/goblin" -) - -func TestMatrix(t *testing.T) { - - g := goblin.Goblin(t) - g.Describe("Calculate matrix", func() { - - axis, _ := ParseMatrixString(fakeMatrix) - - g.It("Should calculate permutations", func() { - g.Assert(len(axis)).Equal(24) - }) - - g.It("Should not duplicate permutations", func() { - set := map[string]bool{} - for _, perm := range axis { - set[perm.String()] = true - } - g.Assert(len(set)).Equal(24) - }) - - g.It("Should return nil if no matrix", func() { - axis, err := ParseMatrixString("") - g.Assert(err == nil).IsTrue() - g.Assert(axis == nil).IsTrue() - }) - - g.It("Should return included axis", func() { - axis, err := ParseMatrixString(fakeMatrixInclude) - g.Assert(err == nil).IsTrue() - g.Assert(len(axis)).Equal(2) - g.Assert(axis[0]["go_version"]).Equal("1.5") - g.Assert(axis[1]["go_version"]).Equal("1.6") - g.Assert(axis[0]["python_version"]).Equal("3.4") - g.Assert(axis[1]["python_version"]).Equal("3.4") - }) - }) -} - -var fakeMatrix = ` -matrix: - go_version: - - go1 - - go1.2 - python_version: - - 3.2 - - 3.3 - django_version: - - 1.7 - - 1.7.1 - - 1.7.2 - redis_version: - - 2.6 - - 2.8 -` - -var fakeMatrixInclude = ` -matrix: - include: - - go_version: 1.5 - python_version: 3.4 - - go_version: 1.6 - python_version: 3.4 -` diff --git a/yaml/network.go b/yaml/network.go deleted file mode 100644 index d49e86e3e..000000000 --- a/yaml/network.go +++ /dev/null @@ -1,51 +0,0 @@ -package yaml - -import ( - "fmt" - - "gopkg.in/yaml.v2" -) - -// Network defines a Docker network. -type Network struct { - Name string - Driver string - DriverOpts map[string]string `yaml:"driver_opts"` -} - -// networkList is an intermediate type used for decoding a slice of networks -// in a format compatible with docker-compose.yml -type networkList struct { - networks []*Network -} - -// UnmarshalYAML implements custom Yaml unmarshaling. -func (n *networkList) UnmarshalYAML(unmarshal func(interface{}) error) error { - slice := yaml.MapSlice{} - err := unmarshal(&slice) - if err != nil { - return err - } - - for _, s := range slice { - nn := Network{} - - out, merr := yaml.Marshal(s.Value) - if merr != nil { - return merr - } - - err = yaml.Unmarshal(out, &nn) - if err != nil { - return err - } - if nn.Name == "" { - nn.Name = fmt.Sprintf("%v", s.Key) - } - if nn.Driver == "" { - nn.Driver = "bridge" - } - n.networks = append(n.networks, &nn) - } - return err -} diff --git a/yaml/network_test.go b/yaml/network_test.go deleted file mode 100644 index 4fe3c8636..000000000 --- a/yaml/network_test.go +++ /dev/null @@ -1,51 +0,0 @@ -package yaml - -import ( - "testing" - - "github.com/franela/goblin" - "gopkg.in/yaml.v2" -) - -func TestNetworks(t *testing.T) { - g := goblin.Goblin(t) - - g.Describe("Networks", func() { - g.Describe("given a yaml file", func() { - - g.It("should unmarshal", func() { - in := []byte("foo: { driver: overlay }") - out := networkList{} - err := yaml.Unmarshal(in, &out) - if err != nil { - g.Fail(err) - } - g.Assert(len(out.networks)).Equal(1) - g.Assert(out.networks[0].Name).Equal("foo") - g.Assert(out.networks[0].Driver).Equal("overlay") - }) - - g.It("should unmarshal named", func() { - in := []byte("foo: { name: bar }") - out := networkList{} - err := yaml.Unmarshal(in, &out) - if err != nil { - g.Fail(err) - } - g.Assert(len(out.networks)).Equal(1) - g.Assert(out.networks[0].Name).Equal("bar") - }) - - g.It("should unmarshal and use default driver", func() { - in := []byte("foo: { name: bar }") - out := networkList{} - err := yaml.Unmarshal(in, &out) - if err != nil { - g.Fail(err) - } - g.Assert(len(out.networks)).Equal(1) - g.Assert(out.networks[0].Driver).Equal("bridge") - }) - }) - }) -} diff --git a/yaml/platform.go b/yaml/platform.go deleted file mode 100644 index 69e1cccc2..000000000 --- a/yaml/platform.go +++ /dev/null @@ -1,26 +0,0 @@ -package yaml - -import "gopkg.in/yaml.v2" - -// ParsePlatform parses the platform section of the Yaml document. -func ParsePlatform(in []byte) string { - out := struct { - Platform string `yaml:"platform"` - }{} - - yaml.Unmarshal(in, &out) - return out.Platform -} - -// ParsePlatformString parses the platform section of the Yaml document. -func ParsePlatformString(in string) string { - return ParsePlatform([]byte(in)) -} - -// ParsePlatformDefault parses the platform section of the Yaml document. -func ParsePlatformDefault(in []byte, platform string) string { - if p := ParsePlatform([]byte(in)); p != "" { - return p - } - return platform -} diff --git a/yaml/transform/clone.go b/yaml/transform/clone.go deleted file mode 100644 index 60c1e2094..000000000 --- a/yaml/transform/clone.go +++ /dev/null @@ -1,32 +0,0 @@ -package transform - -import "github.com/drone/drone/yaml" - -const clone = "clone" - -// Clone transforms the Yaml to include a clone step. -func Clone(c *yaml.Config, plugin string) error { - switch plugin { - case "", "git": - plugin = "plugins/git:latest" - case "hg": - plugin = "plugins/hg:latest" - } - - for _, p := range c.Pipeline { - if p.Name == clone { - if p.Image == "" { - p.Image = plugin - } - return nil - } - } - - s := &yaml.Container{ - Image: plugin, - Name: clone, - } - - c.Pipeline = append([]*yaml.Container{s}, c.Pipeline...) - return nil -} diff --git a/yaml/transform/clone_test.go b/yaml/transform/clone_test.go deleted file mode 100644 index 5796f91c6..000000000 --- a/yaml/transform/clone_test.go +++ /dev/null @@ -1 +0,0 @@ -package transform diff --git a/yaml/transform/command.go b/yaml/transform/command.go deleted file mode 100644 index a90e8c2a1..000000000 --- a/yaml/transform/command.go +++ /dev/null @@ -1,83 +0,0 @@ -package transform - -import ( - "bytes" - "encoding/base64" - "fmt" - "strings" - - "github.com/drone/drone/yaml" -) - -// CommandTransform transforms the custom shell commands in the Yaml pipeline -// into a container ENTRYPOINT and and CMD for execution. -func CommandTransform(c *yaml.Config) error { - for _, p := range c.Pipeline { - - if isPlugin(p) { - continue - } - - p.Entrypoint = []string{ - "/bin/sh", "-c", - } - p.Command = []string{ - "echo $DRONE_SCRIPT | base64 -d | /bin/sh -e", - } - if p.Environment == nil { - p.Environment = map[string]string{} - } - p.Environment["HOME"] = "/root" - p.Environment["SHELL"] = "/bin/sh" - p.Environment["DRONE_SCRIPT"] = toScript( - p.Commands, - ) - } - return nil -} - -func toScript(commands []string) string { - var buf bytes.Buffer - for _, command := range commands { - escaped := fmt.Sprintf("%q", command) - escaped = strings.Replace(escaped, "$", `\$`, -1) - buf.WriteString(fmt.Sprintf( - traceScript, - escaped, - command, - )) - } - - script := fmt.Sprintf( - setupScript, - buf.String(), - ) - - return base64.StdEncoding.EncodeToString([]byte(script)) -} - -// setupScript is a helper script this is added to the build to ensure -// a minimum set of environment variables are set correctly. -const setupScript = ` -if [ -n "$DRONE_NETRC_MACHINE" ]; then -cat < $HOME/.netrc -machine $DRONE_NETRC_MACHINE -login $DRONE_NETRC_USERNAME -password $DRONE_NETRC_PASSWORD -EOF -chmod 0600 $HOME/.netrc -fi - -unset DRONE_NETRC_USERNAME -unset DRONE_NETRC_PASSWORD -unset DRONE_SCRIPT - -%s -` - -// traceScript is a helper script that is added to the build script -// to trace a command. -const traceScript = ` -echo + %s -%s -` diff --git a/yaml/transform/command_test.go b/yaml/transform/command_test.go deleted file mode 100644 index 791e64be0..000000000 --- a/yaml/transform/command_test.go +++ /dev/null @@ -1,47 +0,0 @@ -package transform - -import ( - "testing" - - "github.com/drone/drone/yaml" - - "github.com/franela/goblin" -) - -func Test_command(t *testing.T) { - - g := goblin.Goblin(t) - g.Describe("Command genration", func() { - - g.It("should ignore plugin steps", func() { - c := newConfig(&yaml.Container{ - Commands: []string{ - "go build", - "go test", - }, - Vargs: map[string]interface{}{ - "depth": 50, - }, - }) - - CommandTransform(c) - g.Assert(len(c.Pipeline[0].Entrypoint)).Equal(0) - g.Assert(len(c.Pipeline[0].Command)).Equal(0) - g.Assert(c.Pipeline[0].Environment["DRONE_SCRIPT"]).Equal("") - }) - - g.It("should set entrypoint, command and environment variables", func() { - c := newConfig(&yaml.Container{ - Commands: []string{ - "go build", - "go test", - }, - }) - - CommandTransform(c) - g.Assert(c.Pipeline[0].Entrypoint).Equal([]string{"/bin/sh", "-c"}) - g.Assert(c.Pipeline[0].Command).Equal([]string{"echo $DRONE_SCRIPT | base64 -d | /bin/sh -e"}) - g.Assert(c.Pipeline[0].Environment["DRONE_SCRIPT"] != "").IsTrue() - }) - }) -} diff --git a/yaml/transform/environ.go b/yaml/transform/environ.go deleted file mode 100644 index 222bbfef5..000000000 --- a/yaml/transform/environ.go +++ /dev/null @@ -1,46 +0,0 @@ -package transform - -import ( - "os" - - "github.com/drone/drone/yaml" -) - -var ( - httpProxy = os.Getenv("HTTP_PROXY") - httpsProxy = os.Getenv("HTTPS_PROXY") - noProxy = os.Getenv("NO_PROXY") -) - -// Environ transforms the steps in the Yaml pipeline to include runtime -// environment variables. -func Environ(c *yaml.Config, envs map[string]string) error { - var images []*yaml.Container - images = append(images, c.Pipeline...) - images = append(images, c.Services...) - - for _, p := range images { - if p.Environment == nil { - p.Environment = map[string]string{} - } - for k, v := range envs { - if v == "" { - continue - } - p.Environment[k] = v - } - if httpProxy != "" { - p.Environment["HTTP_PROXY"] = httpProxy - p.Environment["http_proxy"] = httpProxy - } - if httpsProxy != "" { - p.Environment["HTTPS_PROXY"] = httpsProxy - p.Environment["https_proxy"] = httpsProxy - } - if noProxy != "" { - p.Environment["NO_PROXY"] = noProxy - p.Environment["no_proxy"] = noProxy - } - } - return nil -} diff --git a/yaml/transform/environ_test.go b/yaml/transform/environ_test.go deleted file mode 100644 index 903e31eff..000000000 --- a/yaml/transform/environ_test.go +++ /dev/null @@ -1,27 +0,0 @@ -package transform - -import ( - "testing" - - "github.com/drone/drone/yaml" - - "github.com/franela/goblin" -) - -func Test_env(t *testing.T) { - - g := goblin.Goblin(t) - g.Describe("environment variables", func() { - - g.It("should be copied", func() { - envs := map[string]string{"CI": "drone"} - - c := newConfig(&yaml.Container{ - Environment: map[string]string{}, - }) - - Environ(c, envs) - g.Assert(c.Pipeline[0].Environment["CI"]).Equal("drone") - }) - }) -} diff --git a/yaml/transform/filter.go b/yaml/transform/filter.go deleted file mode 100644 index 1d1ecb478..000000000 --- a/yaml/transform/filter.go +++ /dev/null @@ -1,75 +0,0 @@ -package transform - -import ( - "github.com/drone/drone/model" - "github.com/drone/drone/yaml" -) - -// ChangeFilter is a transform function that alters status constraints set to -// change and replaces with the opposite of the prior status. -func ChangeFilter(conf *yaml.Config, prev string) { - for _, step := range conf.Pipeline { - for i, status := range step.Constraints.Status.Include { - if status != "change" && status != "changed" { - continue - } - if prev == model.StatusFailure { - step.Constraints.Status.Include[i] = model.StatusSuccess - } else { - step.Constraints.Status.Include[i] = model.StatusFailure - } - } - } -} - -// DefaultFilter is a transform function that applies default Filters to each -// step in the Yaml specification file. -func DefaultFilter(conf *yaml.Config) { - for _, step := range conf.Pipeline { - defaultStatus(step) - defaultEvent(step) - } -} - -// defaultStatus sets default status conditions. -func defaultStatus(c *yaml.Container) { - if !isEmpty(c.Constraints.Status) { - return - } - c.Constraints.Status.Include = []string{ - model.StatusSuccess, - } -} - -// defaultEvent sets default event conditions. -func defaultEvent(c *yaml.Container) { - if !isEmpty(c.Constraints.Event) { - return - } - - if isPlugin(c) && !isClone(c) { - c.Constraints.Event.Exclude = []string{ - model.EventPull, - } - } -} - -// helper function returns true if the step is a clone step. -func isEmpty(c yaml.Constraint) bool { - return len(c.Include) == 0 && len(c.Exclude) == 0 -} - -// helper function returns true if the step is a plugin step. -func isPlugin(c *yaml.Container) bool { - return len(c.Commands) == 0 || len(c.Vargs) != 0 -} - -// helper function returns true if the step is a command step. -func isCommand(c *yaml.Container) bool { - return len(c.Commands) != 0 -} - -// helper function returns true if the step is a clone step. -func isClone(c *yaml.Container) bool { - return c.Name == "clone" -} diff --git a/yaml/transform/identifier.go b/yaml/transform/identifier.go deleted file mode 100644 index b2df68bc7..000000000 --- a/yaml/transform/identifier.go +++ /dev/null @@ -1,30 +0,0 @@ -package transform - -import ( - "encoding/base64" - "fmt" - - "github.com/drone/drone/yaml" - - "github.com/gorilla/securecookie" -) - -// Identifier transforms the container steps in the Yaml and assigns a unique -// container identifier. -func Identifier(c *yaml.Config) error { - - // creates a random prefix for the build - rand := base64.RawURLEncoding.EncodeToString( - securecookie.GenerateRandomKey(8), - ) - - for i, step := range c.Services { - step.ID = fmt.Sprintf("drone_%s_%d", rand, i) - } - - for i, step := range c.Pipeline { - step.ID = fmt.Sprintf("drone_%s_%d", rand, i+len(c.Services)) - } - - return nil -} diff --git a/yaml/transform/image.go b/yaml/transform/image.go deleted file mode 100644 index 514f0d6c8..000000000 --- a/yaml/transform/image.go +++ /dev/null @@ -1,60 +0,0 @@ -package transform - -import ( - "fmt" - "path/filepath" - "strings" - - "github.com/drone/drone/yaml" -) - -// ImagePull transforms the Yaml to automatically pull the latest image. -func ImagePull(conf *yaml.Config, pull bool) error { - for _, plugin := range conf.Pipeline { - if !isPlugin(plugin) { - continue - } - plugin.Pull = pull - } - return nil -} - -// ImageTag transforms the Yaml to use the :latest image tag when empty. -func ImageTag(conf *yaml.Config) error { - for _, image := range conf.Pipeline { - if !strings.Contains(image.Image, ":") { - image.Image = image.Image + ":latest" - } - } - for _, image := range conf.Services { - if !strings.Contains(image.Image, ":") { - image.Image = image.Image + ":latest" - } - } - return nil -} - -// ImageEscalate transforms the Yaml to automatically enable privileged mode -// for a subset of white-listed plugins matching the given patterns. -func ImageEscalate(conf *yaml.Config, patterns []string) error { - for _, c := range conf.Pipeline { - for _, pattern := range patterns { - if ok, _ := filepath.Match(pattern, c.Image); ok { - if c.Detached { - return fmt.Errorf("Detached mode disabled for the %s plugin", c.Image) - } - if len(c.Entrypoint) != 0 { - return fmt.Errorf("Custom entrypoint disabled for the %s plugin", c.Image) - } - if len(c.Command) != 0 { - return fmt.Errorf("Custom command disabled for the %s plugin", c.Image) - } - if len(c.Commands) != 0 { - return fmt.Errorf("Custom commands disabled for the %s plugin", c.Image) - } - c.Privileged = true - } - } - } - return nil -} diff --git a/yaml/transform/image_test.go b/yaml/transform/image_test.go deleted file mode 100644 index 26ad37dc3..000000000 --- a/yaml/transform/image_test.go +++ /dev/null @@ -1,132 +0,0 @@ -package transform - -import ( - "testing" - - "github.com/drone/drone/yaml" - - "github.com/franela/goblin" -) - -func Test_pull(t *testing.T) { - g := goblin.Goblin(t) - g.Describe("pull image", func() { - - g.It("should be enabled for plugins", func() { - c := newConfig(&yaml.Container{}) - - ImagePull(c, true) - g.Assert(c.Pipeline[0].Pull).IsTrue() - }) - - g.It("should be disabled for plugins", func() { - c := newConfig(&yaml.Container{}) - - ImagePull(c, false) - g.Assert(c.Pipeline[0].Pull).IsFalse() - }) - - g.It("should not apply to commands", func() { - c := newConfig(&yaml.Container{ - Commands: []string{ - "go build", - "go test", - }, - }) - - ImagePull(c, true) - g.Assert(c.Pipeline[0].Pull).IsFalse() - }) - - g.It("should not apply to services", func() { - c := newConfigService(&yaml.Container{ - Image: "mysql", - }) - - ImagePull(c, true) - g.Assert(c.Services[0].Pull).IsFalse() - }) - }) -} - -func Test_escalate(t *testing.T) { - - g := goblin.Goblin(t) - g.Describe("privileged transform", func() { - - g.It("should handle matches", func() { - c := newConfig(&yaml.Container{ - Image: "plugins/docker", - }) - - ImageEscalate(c, []string{"plugins/docker"}) - g.Assert(c.Pipeline[0].Privileged).IsTrue() - }) - - g.It("should handle glob matches", func() { - c := newConfig(&yaml.Container{ - Image: "plugins/docker:latest", - }) - - ImageEscalate(c, []string{"plugins/docker:*"}) - g.Assert(c.Pipeline[0].Privileged).IsTrue() - }) - - g.It("should handle non matches", func() { - c := newConfig(&yaml.Container{ - Image: "plugins/git:latest", - }) - - ImageEscalate(c, []string{"plugins/docker:*"}) - g.Assert(c.Pipeline[0].Privileged).IsFalse() - }) - - g.It("should handle non glob matches", func() { - c := newConfig(&yaml.Container{ - Image: "plugins/docker:latest", - }) - - ImageEscalate(c, []string{"plugins/docker"}) - g.Assert(c.Pipeline[0].Privileged).IsFalse() - }) - - g.It("should not escalate plugin with commands", func() { - c := newConfig(&yaml.Container{ - Image: "docker", - Commands: []string{"echo foo"}, - }) - - err := ImageEscalate(c, []string{"docker"}) - g.Assert(c.Pipeline[0].Privileged).IsFalse() - g.Assert(err.Error()).Equal("Custom commands disabled for the docker plugin") - }) - }) -} - -func Test_normalize(t *testing.T) { - - g := goblin.Goblin(t) - g.Describe("normalizing", func() { - - g.Describe("images", func() { - - g.It("should append tag if empty", func() { - c := newConfig(&yaml.Container{ - Image: "golang", - }) - - ImageTag(c) - g.Assert(c.Pipeline[0].Image).Equal("golang:latest") - }) - - g.It("should not override existing tag", func() { - c := newConfig(&yaml.Container{ - Image: "golang:1.5", - }) - - ImageTag(c) - g.Assert(c.Pipeline[0].Image).Equal("golang:1.5") - }) - }) - }) -} diff --git a/yaml/transform/plugin.go b/yaml/transform/plugin.go deleted file mode 100644 index 7bc1463d5..000000000 --- a/yaml/transform/plugin.go +++ /dev/null @@ -1,38 +0,0 @@ -package transform - -import "github.com/drone/drone/yaml" - -// PluginDisable is a transform function that alters the Yaml configuration to -// disables plugins. This is intended for use when executing the pipeline -// locally on your own computer. -func PluginDisable(conf *yaml.Config, local bool) error { - for _, container := range conf.Pipeline { - if local && !container.Constraints.Local.Bool() { - container.Disabled = true - } - - if isClone(container) { - container.Disabled = true - continue - } - } - return nil -} - -// PluginParams is a transform function that alters the Yaml configuration to -// include plugin vargs parameters as environment variables. -func PluginParams(conf *yaml.Config) error { - for _, container := range conf.Pipeline { - if len(container.Vargs) == 0 { - continue - } - if container.Environment == nil { - container.Environment = map[string]string{} - } - err := argsToEnv(container.Vargs, container.Environment) - if err != nil { - return err - } - } - return nil -} diff --git a/yaml/transform/plugin_test.go b/yaml/transform/plugin_test.go deleted file mode 100644 index 5796f91c6..000000000 --- a/yaml/transform/plugin_test.go +++ /dev/null @@ -1 +0,0 @@ -package transform diff --git a/yaml/transform/pod.go b/yaml/transform/pod.go deleted file mode 100644 index dd92acbb8..000000000 --- a/yaml/transform/pod.go +++ /dev/null @@ -1,94 +0,0 @@ -package transform - -import ( - "encoding/base64" - "fmt" - - "github.com/drone/drone/yaml" - - "github.com/gorilla/securecookie" -) - -type ambassador struct { - image string - entrypoint []string - command []string -} - -// default linux amd64 ambassador -var defaultAmbassador = ambassador{ - image: "busybox:latest", - entrypoint: []string{"/bin/sleep"}, - command: []string{"86400"}, -} - -// lookup ambassador configuration by architecture and os -var lookupAmbassador = map[string]ambassador{ - "linux/amd64": { - image: "busybox:latest", - entrypoint: []string{"/bin/sleep"}, - command: []string{"86400"}, - }, - "linux/arm": { - image: "armhf/alpine:latest", - entrypoint: []string{"/bin/sleep"}, - command: []string{"86400"}, - }, -} - -// Pod transforms the containers in the Yaml to use Pod networking, where every -// container shares the localhost connection. -func Pod(c *yaml.Config, platform string) error { - - rand := base64.RawURLEncoding.EncodeToString( - securecookie.GenerateRandomKey(8), - ) - - // choose the ambassador configuration based on os and architecture - conf, ok := lookupAmbassador[platform] - if !ok { - conf = defaultAmbassador - } - - ambassador := &yaml.Container{ - ID: fmt.Sprintf("drone_ambassador_%s", rand), - Name: "ambassador", - Image: conf.image, - Detached: true, - Entrypoint: conf.entrypoint, - Command: conf.command, - Volumes: []string{c.Workspace.Path, c.Workspace.Base}, - Environment: map[string]string{}, - } - network := fmt.Sprintf("container:%s", ambassador.ID) - - var containers []*yaml.Container - containers = append(containers, c.Pipeline...) - containers = append(containers, c.Services...) - - for _, container := range containers { - container.VolumesFrom = append(container.VolumesFrom, ambassador.ID) - if container.Network == "" { - container.Network = network - } - } - - c.Services = append([]*yaml.Container{ambassador}, c.Services...) - return nil -} - -// func (v *podOp) VisitContainer(node *parse.ContainerNode) error { -// if node.Container.Network == "" { -// parent := fmt.Sprintf("container:%s", v.name) -// node.Container.Network = parent -// } -// node.Container.VolumesFrom = append(node.Container.VolumesFrom, v.name) -// return nil -// } -// -// func (v *podOp) VisitRoot(node *parse.RootNode) error { -// -// -// node.Pod = service -// return nil -// } diff --git a/yaml/transform/rpc.go b/yaml/transform/rpc.go deleted file mode 100644 index f45df75ad..000000000 --- a/yaml/transform/rpc.go +++ /dev/null @@ -1,53 +0,0 @@ -package transform - -import ( - "bytes" - "encoding/json" - "fmt" - "io/ioutil" - "net/http" - - "github.com/drone/drone/yaml" -) - -func convertTransform(c *yaml.Config, url string) error { - var buf bytes.Buffer - - // encode yaml in json format - if err := json.NewEncoder(&buf).Encode(c); err != nil { - return err - } - resp, err := http.Post(url, "application/json", &buf) - if err != nil { - return err - } - defer resp.Body.Close() - - // decode the updated yaml from the body - if resp.StatusCode == 200 { - err = json.NewDecoder(resp.Body).Decode(c) - return err - } - - // handle error response - out, err := ioutil.ReadAll(resp.Body) - if err != nil { - return err - } - return fmt.Errorf(string(out)) -} - -// RemoteTransform makes remote transform requests. -func RemoteTransform(c *yaml.Config, url []string) error { - if len(url) == 0 { - return nil - } - - for _, u := range url { - if err := convertTransform(c, u); err != nil { - return err - } - } - - return nil -} diff --git a/yaml/transform/rpc_test.go b/yaml/transform/rpc_test.go deleted file mode 100644 index c194676fa..000000000 --- a/yaml/transform/rpc_test.go +++ /dev/null @@ -1,126 +0,0 @@ -package transform - -import ( - "encoding/json" - "net/http" - "net/http/httptest" - "strings" - "testing" - - "github.com/drone/drone/yaml" - "github.com/franela/goblin" -) - -func handleNetrcRemoval(w http.ResponseWriter, r *http.Request) { - c := new(yaml.Config) - err := json.NewDecoder(r.Body).Decode(c) - if err != nil { - w.WriteHeader(http.StatusBadRequest) - w.Write([]byte(err.Error())) - return - } - for _, container := range c.Pipeline { - if strings.HasPrefix(container.Image, "plugins/git") { - continue - } - container.Environment["DRONE_NETRC_USERNAME"] = "" - container.Environment["DRONE_NETRC_PASSWORD"] = "" - container.Environment["DRONE_NETRC_MACHINE"] = "" - } - json.NewEncoder(w).Encode(c) -} - -func Test_rpc_transform(t *testing.T) { - g := goblin.Goblin(t) - g.Describe("rpc transform", func() { - - g.It("should mutate the yaml", func() { - c := newConfig(&yaml.Container{ - Image: "golang", - Environment: map[string]string{ - "DRONE_NETRC_USERNAME": "foo", - "DRONE_NETRC_PASSWORD": "bar", - "DRONE_BRANCH": "master", - }, - Commands: []string{ - "go build", - "go test", - }, - }) - - server := httptest.NewServer(http.HandlerFunc(handleNetrcRemoval)) - defer server.Close() - - err := RemoteTransform(c, []string{server.URL}) - g.Assert(err == nil).IsTrue() - g.Assert(c.Pipeline[0].Image).Equal("golang") - g.Assert(c.Pipeline[0].Environment["DRONE_BRANCH"]).Equal("master") - g.Assert(c.Pipeline[0].Environment["DRONE_NETRC_USERNAME"]).Equal("") - g.Assert(c.Pipeline[0].Environment["DRONE_NETRC_PASSWORD"]).Equal("") - g.Assert(c.Pipeline[0].Commands[0]).Equal("go build") - g.Assert(c.Pipeline[0].Commands[1]).Equal("go test") - }) - }) -} - -func handleGoogleRemoval(w http.ResponseWriter, r *http.Request) { - c := new(yaml.Config) - err := json.NewDecoder(r.Body).Decode(c) - if err != nil { - w.WriteHeader(http.StatusBadRequest) - w.Write([]byte(err.Error())) - return - } - for _, container := range c.Pipeline { - if strings.HasPrefix(container.Image, "plugins/git") { - continue - } - container.Environment["DRONE_GOOGLE_USERNAME"] = "" - container.Environment["DRONE_GOOGLE_PASSWORD"] = "" - } - json.NewEncoder(w).Encode(c) -} - -func Test_rpc_multiple_transform(t *testing.T) { - g := goblin.Goblin(t) - g.Describe("rpc transform", func() { - - g.It("should mutate the multiple yaml", func() { - c := newConfig(&yaml.Container{ - Image: "golang", - Environment: map[string]string{ - "DRONE_NETRC_USERNAME": "foo", - "DRONE_NETRC_PASSWORD": "bar", - "DRONE_BRANCH": "master", - "DRONE_GOOGLE_USERNAME": "foo", - "DRONE_GOOGLE_PASSWORD": "bar", - }, - Commands: []string{ - "go build", - "go test", - }, - }) - - server1 := httptest.NewServer(http.HandlerFunc(handleNetrcRemoval)) - defer server1.Close() - - server2 := httptest.NewServer(http.HandlerFunc(handleGoogleRemoval)) - defer server2.Close() - - // testing missing urls - err := RemoteTransform(c, []string{}) - g.Assert(err == nil).IsTrue() - - err = RemoteTransform(c, []string{server1.URL, server2.URL}) - g.Assert(err == nil).IsTrue() - g.Assert(c.Pipeline[0].Image).Equal("golang") - g.Assert(c.Pipeline[0].Environment["DRONE_BRANCH"]).Equal("master") - g.Assert(c.Pipeline[0].Environment["DRONE_NETRC_USERNAME"]).Equal("") - g.Assert(c.Pipeline[0].Environment["DRONE_NETRC_PASSWORD"]).Equal("") - g.Assert(c.Pipeline[0].Environment["DRONE_GOOGLE_USERNAME"]).Equal("") - g.Assert(c.Pipeline[0].Environment["DRONE_GOOGLE_PASSWORD"]).Equal("") - g.Assert(c.Pipeline[0].Commands[0]).Equal("go build") - g.Assert(c.Pipeline[0].Commands[1]).Equal("go test") - }) - }) -} diff --git a/yaml/transform/secret.go b/yaml/transform/secret.go deleted file mode 100644 index fb355f21b..000000000 --- a/yaml/transform/secret.go +++ /dev/null @@ -1,43 +0,0 @@ -package transform - -import ( - "github.com/drone/drone/model" - "github.com/drone/drone/yaml" -) - -// -// TODO remove -// - -func ImageSecrets(c *yaml.Config, secrets []*model.Secret, event string) error { - var images []*yaml.Container - images = append(images, c.Pipeline...) - images = append(images, c.Services...) - - for _, image := range images { - imageSecrets(image, secrets, event) - } - return nil -} - -func imageSecrets(c *yaml.Container, secrets []*model.Secret, event string) { - for _, secret := range secrets { - if !secret.Match(c.Image, event) { - continue - } - - switch secret.Name { - case "REGISTRY_USERNAME": - c.AuthConfig.Username = secret.Value - case "REGISTRY_PASSWORD": - c.AuthConfig.Password = secret.Value - case "REGISTRY_EMAIL": - c.AuthConfig.Email = secret.Value - default: - if c.Environment == nil { - c.Environment = map[string]string{} - } - c.Environment[secret.Name] = secret.Value - } - } -} diff --git a/yaml/transform/secret_test.go b/yaml/transform/secret_test.go deleted file mode 100644 index 5796f91c6..000000000 --- a/yaml/transform/secret_test.go +++ /dev/null @@ -1 +0,0 @@ -package transform diff --git a/yaml/transform/transform.go b/yaml/transform/transform.go deleted file mode 100644 index 61d735282..000000000 --- a/yaml/transform/transform.go +++ /dev/null @@ -1,6 +0,0 @@ -package transform - -import "github.com/drone/drone/yaml" - -// TransformFunc defines an operation for transforming the Yaml file. -type TransformFunc func(*yaml.Config) error diff --git a/yaml/transform/util.go b/yaml/transform/util.go deleted file mode 100644 index 1f4b336fe..000000000 --- a/yaml/transform/util.go +++ /dev/null @@ -1,67 +0,0 @@ -package transform - -import ( - "fmt" - "reflect" - "strconv" - "strings" - - json "github.com/ghodss/yaml" - "gopkg.in/yaml.v2" -) - -// argsToEnv uses reflection to convert a map[string]interface to a list -// of environment variables. -func argsToEnv(from map[string]interface{}, to map[string]string) error { - - for k, v := range from { - if v == nil { - to[k] = "" - continue - } - - t := reflect.TypeOf(v) - vv := reflect.ValueOf(v) - - k = "PLUGIN_" + strings.ToUpper(k) - - switch t.Kind() { - case reflect.Bool: - to[k] = strconv.FormatBool(vv.Bool()) - - case reflect.String: - to[k] = vv.String() - - case reflect.Int, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Int8: - to[k] = fmt.Sprintf("%v", vv.Int()) - - case reflect.Float32, reflect.Float64: - to[k] = fmt.Sprintf("%v", vv.Float()) - - case reflect.Map: - yml, _ := yaml.Marshal(vv.Interface()) - out, _ := json.YAMLToJSON(yml) - to[k] = string(out) - - case reflect.Slice: - out, err := yaml.Marshal(vv.Interface()) - if err != nil { - return err - } - - in := []string{} - err = yaml.Unmarshal(out, &in) - if err == nil { - to[k] = strings.Join(in, ",") - } else { - out, err = json.YAMLToJSON(out) - if err != nil { - return err - } - to[k] = string(out) - } - } - } - - return nil -} diff --git a/yaml/transform/validate.go b/yaml/transform/validate.go deleted file mode 100644 index efcd1710a..000000000 --- a/yaml/transform/validate.go +++ /dev/null @@ -1,85 +0,0 @@ -package transform - -import ( - "fmt" - - "github.com/drone/drone/yaml" -) - -func Check(c *yaml.Config, trusted bool) error { - var images []*yaml.Container - images = append(images, c.Pipeline...) - images = append(images, c.Services...) - - for _, image := range c.Pipeline { - if err := CheckEntrypoint(image); err != nil { - return err - } - if trusted { - continue - } - if err := CheckTrusted(image); err != nil { - return err - } - } - for _, image := range c.Services { - if trusted { - continue - } - if err := CheckTrusted(image); err != nil { - return err - } - } - return nil -} - -// validate the plugin command and entrypoint and return an error -// the user attempts to set or override these values. -func CheckEntrypoint(c *yaml.Container) error { - if c.Detached { - return nil - } - if len(c.Entrypoint) != 0 { - return fmt.Errorf("Cannot set plugin Entrypoint") - } - if len(c.Command) != 0 { - return fmt.Errorf("Cannot set plugin Command") - } - return nil -} - -// validate the container configuration and return an error if restricted -// configurations are used. -func CheckTrusted(c *yaml.Container) error { - if c.Privileged { - return fmt.Errorf("Insufficient privileges to use privileged mode") - } - if c.ShmSize != 0 { - return fmt.Errorf("Insufficient privileges to override shm_size") - } - if len(c.DNS) != 0 { - return fmt.Errorf("Insufficient privileges to use custom dns") - } - if len(c.DNSSearch) != 0 { - return fmt.Errorf("Insufficient privileges to use dns_search") - } - if len(c.Devices) != 0 { - return fmt.Errorf("Insufficient privileges to use devices") - } - if len(c.ExtraHosts) != 0 { - return fmt.Errorf("Insufficient privileges to use extra_hosts") - } - if len(c.Network) != 0 { - return fmt.Errorf("Insufficient privileges to override the network") - } - if c.OomKillDisable { - return fmt.Errorf("Insufficient privileges to disable oom_kill") - } - if len(c.Volumes) != 0 { - return fmt.Errorf("Insufficient privileges to use volumes") - } - if len(c.VolumesFrom) != 0 { - return fmt.Errorf("Insufficient privileges to use volumes_from") - } - return nil -} diff --git a/yaml/transform/validate_test.go b/yaml/transform/validate_test.go deleted file mode 100644 index eddbcdf24..000000000 --- a/yaml/transform/validate_test.go +++ /dev/null @@ -1,154 +0,0 @@ -package transform - -import ( - "testing" - - "github.com/drone/drone/yaml" - - "github.com/franela/goblin" -) - -func Test_validate(t *testing.T) { - - g := goblin.Goblin(t) - g.Describe("validating", func() { - - g.Describe("privileged attributes", func() { - - g.It("should not error when trusted build", func() { - c := newConfig(&yaml.Container{Privileged: true}) - err := Check(c, true) - - g.Assert(err == nil).IsTrue("error should be nil") - }) - - g.It("should error when privleged mode", func() { - c := newConfig(&yaml.Container{ - Privileged: true, - }) - err := Check(c, false) - g.Assert(err != nil).IsTrue("error should not be nil") - g.Assert(err.Error()).Equal("Insufficient privileges to use privileged mode") - }) - - g.It("should error when privleged service container", func() { - c := newConfigService(&yaml.Container{ - Privileged: true, - }) - err := Check(c, false) - g.Assert(err != nil).IsTrue("error should not be nil") - g.Assert(err.Error()).Equal("Insufficient privileges to use privileged mode") - }) - - g.It("should error when dns configured", func() { - c := newConfig(&yaml.Container{ - DNS: []string{"8.8.8.8"}, - }) - err := Check(c, false) - g.Assert(err != nil).IsTrue("error should not be nil") - g.Assert(err.Error()).Equal("Insufficient privileges to use custom dns") - }) - - g.It("should error when dns_search configured", func() { - c := newConfig(&yaml.Container{ - DNSSearch: []string{"8.8.8.8"}, - }) - err := Check(c, false) - g.Assert(err != nil).IsTrue("error should not be nil") - g.Assert(err.Error()).Equal("Insufficient privileges to use dns_search") - }) - - g.It("should error when devices configured", func() { - c := newConfig(&yaml.Container{ - Devices: []string{"/dev/foo"}, - }) - err := Check(c, false) - g.Assert(err != nil).IsTrue("error should not be nil") - g.Assert(err.Error()).Equal("Insufficient privileges to use devices") - }) - - g.It("should error when extra_hosts configured", func() { - c := newConfig(&yaml.Container{ - ExtraHosts: []string{"1.2.3.4 foo.com"}, - }) - err := Check(c, false) - g.Assert(err != nil).IsTrue("error should not be nil") - g.Assert(err.Error()).Equal("Insufficient privileges to use extra_hosts") - }) - - g.It("should error when network configured", func() { - c := newConfig(&yaml.Container{ - Network: "host", - }) - err := Check(c, false) - g.Assert(err != nil).IsTrue("error should not be nil") - g.Assert(err.Error()).Equal("Insufficient privileges to override the network") - }) - - g.It("should error when oom_kill_disabled configured", func() { - c := newConfig(&yaml.Container{ - OomKillDisable: true, - }) - err := Check(c, false) - g.Assert(err != nil).IsTrue("error should not be nil") - g.Assert(err.Error()).Equal("Insufficient privileges to disable oom_kill") - }) - - g.It("should error when volumes configured", func() { - c := newConfig(&yaml.Container{ - Volumes: []string{"/:/tmp"}, - }) - err := Check(c, false) - g.Assert(err != nil).IsTrue("error should not be nil") - g.Assert(err.Error()).Equal("Insufficient privileges to use volumes") - }) - - g.It("should error when volumes_from configured", func() { - c := newConfig(&yaml.Container{ - VolumesFrom: []string{"drone"}, - }) - err := Check(c, false) - g.Assert(err != nil).IsTrue("error should not be nil") - g.Assert(err.Error()).Equal("Insufficient privileges to use volumes_from") - }) - }) - - g.Describe("plugin configuration", func() { - g.It("should error when entrypoint is configured", func() { - c := newConfig(&yaml.Container{ - Entrypoint: []string{"/bin/sh"}, - }) - err := Check(c, false) - g.Assert(err != nil).IsTrue("error should not be nil") - g.Assert(err.Error()).Equal("Cannot set plugin Entrypoint") - }) - - g.It("should error when command is configured", func() { - c := newConfig(&yaml.Container{ - Command: []string{"cat", "/proc/1/status"}, - }) - err := Check(c, false) - g.Assert(err != nil).IsTrue("error should not be nil") - g.Assert(err.Error()).Equal("Cannot set plugin Command") - }) - - g.It("should not error when empty entrypoint, command", func() { - c := newConfig(&yaml.Container{}) - err := Check(c, false) - g.Assert(err == nil).IsTrue("error should be nil") - }) - }) - }) -} - -func newConfig(container *yaml.Container) *yaml.Config { - return &yaml.Config{ - Pipeline: []*yaml.Container{container}, - } -} - -func newConfigService(container *yaml.Container) *yaml.Config { - return &yaml.Config{ - Services: []*yaml.Container{container}, - } -} diff --git a/yaml/transform/volume.go b/yaml/transform/volume.go deleted file mode 100644 index 39678b102..000000000 --- a/yaml/transform/volume.go +++ /dev/null @@ -1,21 +0,0 @@ -package transform - -import "github.com/drone/drone/yaml" - -// ImageVolume mounts a default volume (used for drone exec) -func ImageVolume(conf *yaml.Config, volumes []string) error { - - if len(volumes) == 0 { - return nil - } - - var containers []*yaml.Container - containers = append(containers, conf.Pipeline...) - containers = append(containers, conf.Services...) - - for _, container := range containers { - container.Volumes = append(container.Volumes, volumes...) - } - - return nil -} diff --git a/yaml/transform/workspace.go b/yaml/transform/workspace.go deleted file mode 100644 index caba299cd..000000000 --- a/yaml/transform/workspace.go +++ /dev/null @@ -1,32 +0,0 @@ -package transform - -import ( - "path/filepath" - - "github.com/drone/drone/yaml" -) - -// WorkspaceTransform transforms ... -func WorkspaceTransform(c *yaml.Config, base, path string) error { - if c.Workspace == nil { - c.Workspace = &yaml.Workspace{} - } - - if c.Workspace.Base == "" { - c.Workspace.Base = base - } - if c.Workspace.Path == "" { - c.Workspace.Path = path - } - if !filepath.IsAbs(c.Workspace.Path) { - c.Workspace.Path = filepath.Join( - c.Workspace.Base, - c.Workspace.Path, - ) - } - - for _, p := range c.Pipeline { - p.WorkingDir = c.Workspace.Path - } - return nil -} diff --git a/yaml/transform/workspace_test.go b/yaml/transform/workspace_test.go deleted file mode 100644 index c16c1f41e..000000000 --- a/yaml/transform/workspace_test.go +++ /dev/null @@ -1,99 +0,0 @@ -package transform - -import ( - "testing" - - "github.com/franela/goblin" - - "github.com/drone/drone/yaml" -) - -func TestWorkspace(t *testing.T) { - g := goblin.Goblin(t) - - g.Describe("workspace", func() { - - defaultBase := "/go" - defaultPath := "src/github.com/octocat/hello-world" - - g.It("should not override user paths", func() { - base := "/drone" - path := "/drone/src/github.com/octocat/hello-world" - - conf := &yaml.Config{ - Workspace: &yaml.Workspace{ - Base: base, - Path: path, - }, - } - - WorkspaceTransform(conf, defaultBase, defaultPath) - g.Assert(conf.Workspace.Base).Equal(base) - g.Assert(conf.Workspace.Path).Equal(path) - }) - - g.It("should convert user paths to absolute", func() { - base := "/drone" - path := "src/github.com/octocat/hello-world" - abs := "/drone/src/github.com/octocat/hello-world" - - conf := &yaml.Config{ - Workspace: &yaml.Workspace{ - Base: base, - Path: path, - }, - } - - WorkspaceTransform(conf, defaultBase, defaultPath) - g.Assert(conf.Workspace.Base).Equal(base) - g.Assert(conf.Workspace.Path).Equal(abs) - }) - - g.It("should set the default path", func() { - var base = "/go" - var path = "/go/src/github.com/octocat/hello-world" - - conf := &yaml.Config{} - - WorkspaceTransform(conf, defaultBase, defaultPath) - g.Assert(conf.Workspace.Base).Equal(base) - g.Assert(conf.Workspace.Path).Equal(path) - }) - - g.It("should use workspace as working_dir", func() { - var base = "/drone" - var path = "/drone/src/github.com/octocat/hello-world" - - conf := &yaml.Config{ - Workspace: &yaml.Workspace{ - Base: base, - Path: path, - }, - Pipeline: []*yaml.Container{ - {}, - }, - } - - WorkspaceTransform(conf, defaultBase, defaultPath) - g.Assert(conf.Pipeline[0].WorkingDir).Equal(path) - }) - - g.It("should not use workspace as working_dir for services", func() { - var base = "/drone" - var path = "/drone/src/github.com/octocat/hello-world" - - conf := &yaml.Config{ - Workspace: &yaml.Workspace{ - Base: base, - Path: path, - }, - Services: []*yaml.Container{ - {}, - }, - } - - WorkspaceTransform(conf, defaultBase, defaultPath) - g.Assert(conf.Services[0].WorkingDir).Equal("") - }) - }) -} diff --git a/yaml/types/bool.go b/yaml/types/bool.go deleted file mode 100644 index 17a7aa9af..000000000 --- a/yaml/types/bool.go +++ /dev/null @@ -1,28 +0,0 @@ -package types - -import "strconv" - -// BoolTrue is a custom Yaml boolean type that defaults to true. -type BoolTrue struct { - value bool -} - -// UnmarshalYAML implements custom Yaml unmarshaling. -func (b *BoolTrue) UnmarshalYAML(unmarshal func(interface{}) error) error { - var s string - err := unmarshal(&s) - if err != nil { - return err - } - - value, err := strconv.ParseBool(s) - if err == nil { - b.value = !value - } - return nil -} - -// Bool returns the bool value. -func (b BoolTrue) Bool() bool { - return !b.value -} diff --git a/yaml/types/bool_test.go b/yaml/types/bool_test.go deleted file mode 100644 index b864349f8..000000000 --- a/yaml/types/bool_test.go +++ /dev/null @@ -1,54 +0,0 @@ -package types - -import ( - "testing" - - "github.com/franela/goblin" - "gopkg.in/yaml.v2" -) - -func TestBoolTrue(t *testing.T) { - g := goblin.Goblin(t) - - g.Describe("Yaml bool type", func() { - g.Describe("given a yaml file", func() { - - g.It("should unmarshal true", func() { - in := []byte("true") - out := BoolTrue{} - err := yaml.Unmarshal(in, &out) - if err != nil { - g.Fail(err) - } - g.Assert(out.Bool()).Equal(true) - }) - - g.It("should unmarshal false", func() { - in := []byte("false") - out := BoolTrue{} - err := yaml.Unmarshal(in, &out) - if err != nil { - g.Fail(err) - } - g.Assert(out.Bool()).Equal(false) - }) - - g.It("should unmarshal true when empty", func() { - in := []byte("") - out := BoolTrue{} - err := yaml.Unmarshal(in, &out) - if err != nil { - g.Fail(err) - } - g.Assert(out.Bool()).Equal(true) - }) - - g.It("should throw error when invalid", func() { - in := []byte("{ }") // string value should fail parse - out := BoolTrue{} - err := yaml.Unmarshal(in, &out) - g.Assert(err != nil).IsTrue("expects error") - }) - }) - }) -} diff --git a/yaml/types/map.go b/yaml/types/map.go deleted file mode 100644 index b74498d8d..000000000 --- a/yaml/types/map.go +++ /dev/null @@ -1,43 +0,0 @@ -package types - -import "strings" - -// MapEqualSlice is a custom Yaml type that can hold a map or slice of strings -// in key=value format. -type MapEqualSlice struct { - parts map[string]string -} - -// UnmarshalYAML implements custom Yaml unmarshaling. -func (s *MapEqualSlice) UnmarshalYAML(unmarshal func(interface{}) error) error { - s.parts = map[string]string{} - err := unmarshal(&s.parts) - if err == nil { - return nil - } - - var slice []string - err = unmarshal(&slice) - if err != nil { - return err - } - for _, v := range slice { - parts := strings.SplitN(v, "=", 2) - if len(parts) == 2 { - key := parts[0] - val := parts[1] - s.parts[key] = val - } - } - return nil -} - -// Map returns the Yaml information as a map. -func (s *MapEqualSlice) Map() map[string]string { - return s.parts -} - -// NewMapEqualSlice returns a new MapEqualSlice. -func NewMapEqualSlice(from map[string]string) *MapEqualSlice { - return &MapEqualSlice{from} -} diff --git a/yaml/types/map_test.go b/yaml/types/map_test.go deleted file mode 100644 index 7ed1e1c50..000000000 --- a/yaml/types/map_test.go +++ /dev/null @@ -1,44 +0,0 @@ -package types - -import ( - "testing" - - "github.com/franela/goblin" - "gopkg.in/yaml.v2" -) - -func TestMapEqualSlice(t *testing.T) { - g := goblin.Goblin(t) - - g.Describe("Yaml map equal slice", func() { - - g.It("should unmarshal a map", func() { - in := []byte("foo: bar") - out := MapEqualSlice{} - err := yaml.Unmarshal(in, &out) - if err != nil { - g.Fail(err) - } - g.Assert(len(out.Map())).Equal(1) - g.Assert(out.Map()["foo"]).Equal("bar") - }) - - g.It("should unmarshal a map equal slice", func() { - in := []byte("[ foo=bar ]") - out := MapEqualSlice{} - err := yaml.Unmarshal(in, &out) - if err != nil { - g.Fail(err) - } - g.Assert(len(out.parts)).Equal(1) - g.Assert(out.parts["foo"]).Equal("bar") - }) - - g.It("should throw error when invalid map equal slice", func() { - in := []byte("foo") // string value should fail parse - out := MapEqualSlice{} - err := yaml.Unmarshal(in, &out) - g.Assert(err != nil).IsTrue("expects error") - }) - }) -} diff --git a/yaml/types/slice.go b/yaml/types/slice.go deleted file mode 100644 index b39e43212..000000000 --- a/yaml/types/slice.go +++ /dev/null @@ -1,35 +0,0 @@ -package types - -// StringOrSlice is a custom Yaml type that can hold a string or slice of strings. -type StringOrSlice struct { - parts []string -} - -// UnmarshalYAML implements custom Yaml unmarshaling. -func (s *StringOrSlice) UnmarshalYAML(unmarshal func(interface{}) error) error { - var sliceType []string - err := unmarshal(&sliceType) - if err == nil { - s.parts = sliceType - return nil - } - - var stringType string - err = unmarshal(&stringType) - if err == nil { - sliceType = make([]string, 0, 1) - s.parts = append(sliceType, string(stringType)) - return nil - } - return err -} - -// Slice returns the slice of strings. -func (s StringOrSlice) Slice() []string { - return s.parts -} - -// NewStringOrSlice returns a new StringOrSlice. -func NewStringOrSlice(from []string) *StringOrSlice { - return &StringOrSlice{from} -} diff --git a/yaml/types/slice_test.go b/yaml/types/slice_test.go deleted file mode 100644 index dcf9a25f6..000000000 --- a/yaml/types/slice_test.go +++ /dev/null @@ -1,46 +0,0 @@ -package types - -import ( - "testing" - - "github.com/franela/goblin" - "gopkg.in/yaml.v2" -) - -func TestStringSlice(t *testing.T) { - g := goblin.Goblin(t) - - g.Describe("Yaml string slice", func() { - g.Describe("given a yaml file", func() { - - g.It("should unmarshal a string", func() { - in := []byte("foo") - out := StringOrSlice{} - err := yaml.Unmarshal(in, &out) - if err != nil { - g.Fail(err) - } - g.Assert(len(out.Slice())).Equal(1) - g.Assert(out.Slice()[0]).Equal("foo") - }) - - g.It("should unmarshal a string slice", func() { - in := []byte("[ foo ]") - out := StringOrSlice{} - err := yaml.Unmarshal(in, &out) - if err != nil { - g.Fail(err) - } - g.Assert(len(out.parts)).Equal(1) - g.Assert(out.parts[0]).Equal("foo") - }) - - g.It("should throw error when invalid string slice", func() { - in := []byte("{ }") // string value should fail parse - out := StringOrSlice{} - err := yaml.Unmarshal(in, &out) - g.Assert(err != nil).IsTrue("expects error") - }) - }) - }) -} diff --git a/yaml/volume.go b/yaml/volume.go deleted file mode 100644 index 20d297bf9..000000000 --- a/yaml/volume.go +++ /dev/null @@ -1,51 +0,0 @@ -package yaml - -import ( - "fmt" - - "gopkg.in/yaml.v2" -) - -// Volume defines a Docker volume. -type Volume struct { - Name string - Driver string - DriverOpts map[string]string `yaml:"driver_opts"` - External bool -} - -// volumeList is an intermediate type used for decoding a slice of volumes -// in a format compatible with docker-compose.yml -type volumeList struct { - volumes []*Volume -} - -// UnmarshalYAML implements custom Yaml unmarshaling. -func (v *volumeList) UnmarshalYAML(unmarshal func(interface{}) error) error { - slice := yaml.MapSlice{} - err := unmarshal(&slice) - if err != nil { - return err - } - - for _, s := range slice { - vv := Volume{} - out, merr := yaml.Marshal(s.Value) - if merr != nil { - return merr - } - - err = yaml.Unmarshal(out, &vv) - if err != nil { - return err - } - if vv.Name == "" { - vv.Name = fmt.Sprintf("%v", s.Key) - } - if vv.Driver == "" { - vv.Driver = "local" - } - v.volumes = append(v.volumes, &vv) - } - return err -} diff --git a/yaml/volume_test.go b/yaml/volume_test.go deleted file mode 100644 index ebeaa9ae1..000000000 --- a/yaml/volume_test.go +++ /dev/null @@ -1,51 +0,0 @@ -package yaml - -import ( - "testing" - - "github.com/franela/goblin" - "gopkg.in/yaml.v2" -) - -func TestVolumes(t *testing.T) { - g := goblin.Goblin(t) - - g.Describe("Volumes", func() { - g.Describe("given a yaml file", func() { - - g.It("should unmarshal", func() { - in := []byte("foo: { driver: blockbridge }") - out := volumeList{} - err := yaml.Unmarshal(in, &out) - if err != nil { - g.Fail(err) - } - g.Assert(len(out.volumes)).Equal(1) - g.Assert(out.volumes[0].Name).Equal("foo") - g.Assert(out.volumes[0].Driver).Equal("blockbridge") - }) - - g.It("should unmarshal named", func() { - in := []byte("foo: { name: bar }") - out := volumeList{} - err := yaml.Unmarshal(in, &out) - if err != nil { - g.Fail(err) - } - g.Assert(len(out.volumes)).Equal(1) - g.Assert(out.volumes[0].Name).Equal("bar") - }) - - g.It("should unmarshal and use default driver", func() { - in := []byte("foo: { name: bar }") - out := volumeList{} - err := yaml.Unmarshal(in, &out) - if err != nil { - g.Fail(err) - } - g.Assert(len(out.volumes)).Equal(1) - g.Assert(out.volumes[0].Driver).Equal("local") - }) - }) - }) -}