mirrors/woodpecker
1
0
Fork 0
mirror of https://github.com/woodpecker-ci/woodpecker.git synced 2024-11-26 20:01:02 +00:00
Code Issues Projects Releases Wiki Activity

removed insanely overly complex build runner, parser, compiler

Browse source
This commit is contained in:
Brad Rydzewski 2016-05-10 17:07:51 -07:00
parent 850c00dbba
commit c76dd06b27
82 changed files with 1 additions and 5303 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
agent/agent.go
View file

@ -9,7 +9,6 @@ import (
"time"
"github.com/drone/drone/build"
"github.com/drone/drone/engine/runner"
"github.com/drone/drone/model"
"github.com/drone/drone/queue"
"github.com/drone/drone/version"
@ -68,7 +67,7 @@ func (a *Agent) Run(payload *queue.Work, cancel <-chan bool) error {
if err != nil {
payload.Job.ExitCode = 255
}
if exitErr, ok := err.(*runner.ExitError); ok {
if exitErr, ok := err.(*build.ExitError); ok {
payload.Job.ExitCode = exitErr.Code
}

29
engine/compiler/builtin/alias.go
View file

@ -1,29 +0,0 @@
package builtin
import (
"fmt"
"github.com/drone/drone/engine/compiler/parse"
)
type aliasOp struct {
visitor
index map[string]string
prefix string
suffix int
}
func NewAliasOp(prefix string) Visitor {
return &aliasOp{
index: map[string]string{},
prefix: prefix,
}
}
func (v *aliasOp) VisitContainer(node *parse.ContainerNode) error {
v.suffix++
node.Container.Alias = node.Container.Name
node.Container.Name = fmt.Sprintf("%s_%d", v.prefix, v.suffix)
return nil
}

90
engine/compiler/builtin/args.go
View file

@ -1,90 +0,0 @@
package builtin
import (
"fmt"
"reflect"
"strconv"
"strings"
"github.com/drone/drone/engine/compiler/parse"
json "github.com/ghodss/yaml"
"gopkg.in/yaml.v2"
)
type argsOps struct {
visitor
}
// NewArgsOp returns a transformer that provides the plugin node
// with the custom arguments from the Yaml file.
func NewArgsOp() Visitor {
return &argsOps{}
}
func (v *argsOps) VisitContainer(node *parse.ContainerNode) error {
switch node.NodeType {
case parse.NodePlugin, parse.NodeCache, parse.NodeClone:
break // no-op
default:
return nil
}
if node.Container.Environment == nil {
node.Container.Environment = map[string]string{}
}
return argsToEnv(node.Vargs, node.Container.Environment)
}
// argsToEnv uses reflection to convert a map[string]interface to a list
// of environment variables.
func argsToEnv(from map[string]interface{}, to map[string]string) error {
for k, v := range from {
t := reflect.TypeOf(v)
vv := reflect.ValueOf(v)
k = "PLUGIN_" + strings.ToUpper(k)
switch t.Kind() {
case reflect.Bool:
to[k] = strconv.FormatBool(vv.Bool())
case reflect.String:
to[k] = vv.String()
case reflect.Int, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Int8:
to[k] = fmt.Sprintf("%v", vv.Int())
case reflect.Float32, reflect.Float64:
to[k] = fmt.Sprintf("%v", vv.Float())
// case reflect.Int, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Int8:
// to[k] = strconv.FormatInt(vv.Int(), 16)
// case reflect.Float32, reflect.Float64:
// to[k] = strconv.FormatFloat(vv.Float(), 'E', -1, 64)
case reflect.Map:
yml, _ := yaml.Marshal(vv.Interface())
out, _ := json.YAMLToJSON(yml)
to[k] = string(out)
case reflect.Slice:
out, _ := yaml.Marshal(vv.Interface())
in := []string{}
err := yaml.Unmarshal(out, &in)
if err == nil {
to[k] = strings.Join(in, ",")
} else {
out, err = json.YAMLToJSON(out)
if err != nil {
// return err TODO(bradrydzewski) unit test coverage for possible errors
}
to[k] = string(out)
}
}
}
return nil
}

46
engine/compiler/builtin/args_test.go
View file

@ -1,46 +0,0 @@
package builtin
import (
"testing"
"github.com/drone/drone/engine/compiler/parse"
"github.com/drone/drone/engine/runner"
"github.com/franela/goblin"
)
func Test_args(t *testing.T) {
g := goblin.Goblin(t)
g.Describe("plugins arguments", func() {
g.It("should ignore non-plugin containers", func() {
root := parse.NewRootNode()
c := root.NewShellNode()
c.Container = runner.Container{}
c.Vargs = map[string]interface{}{
"depth": 50,
}
ops := NewArgsOp()
ops.VisitContainer(c)
g.Assert(c.Container.Environment["PLUGIN_DEPTH"]).Equal("")
})
g.It("should include args as environment variable", func() {
root := parse.NewRootNode()
c := root.NewPluginNode()
c.Container = runner.Container{}
c.Vargs = map[string]interface{}{
"depth": 50,
}
ops := NewArgsOp()
ops.VisitContainer(c)
g.Assert(c.Container.Environment["PLUGIN_DEPTH"]).Equal("50")
})
})
}

40
engine/compiler/builtin/build.go
View file

@ -1,40 +0,0 @@
package builtin
import (
"github.com/drone/drone/engine/compiler/parse"
"github.com/drone/drone/engine/runner"
)
// BuildOp is a transform operation that converts the build section of the Yaml
// to a step in the pipeline responsible for building the Docker image.
func BuildOp(node parse.Node) error {
build, ok := node.(*parse.BuildNode)
if !ok {
return nil
}
if build.Context == "" {
return nil
}
root := node.Root()
builder := root.NewContainerNode()
command := []string{
"build",
"--force-rm",
"-f", build.Dockerfile,
"-t", root.Image,
build.Context,
}
builder.Container = runner.Container{
Image: "docker:apline",
Volumes: []string{"/var/run/docker.sock:/var/run/docker.sock"},
Entrypoint: []string{"/usr/local/bin/docker"},
Command: command,
WorkingDir: root.Path,
}
root.Services = append(root.Services, builder)
return nil
}

45
engine/compiler/builtin/clone.go
View file

@ -1,45 +0,0 @@
package builtin
import (
"github.com/drone/drone/engine/compiler/parse"
"github.com/drone/drone/engine/runner"
)
type cloneOp struct {
visitor
plugin string
enable bool
}
// NewCloneOp returns a transformer that configures the default clone plugin.
func NewCloneOp(plugin string, enable bool) Visitor {
return &cloneOp{
enable: enable,
plugin: plugin,
}
}
func (v *cloneOp) VisitContainer(node *parse.ContainerNode) error {
if node.Type() != parse.NodeClone {
return nil
}
if v.enable == false {
node.Disabled = true
return nil
}
if node.Container.Name == "" {
node.Container.Name = "clone"
}
if node.Container.Image == "" {
node.Container.Image = v.plugin
}
// discard any other cache properties except the image name.
// everything else is discard for security reasons.
node.Container = runner.Container{
Name: node.Container.Name,
Image: node.Container.Image,
}
return nil
}

36
engine/compiler/builtin/clone_test.go
View file

@ -1,36 +0,0 @@
package builtin
// import (
// "testing"
// "github.com/libcd/libcd"
// "github.com/libcd/libyaml/parse"
// "github.com/franela/goblin"
// )
// func Test_clone(t *testing.T) {
// root := parse.NewRootNode()
// g := goblin.Goblin(t)
// g.Describe("clone", func() {
// g.It("should use default when nil", func() {
// op := NewCloneOp("plugins/git:latest")
// op.VisitRoot(root)
// g.Assert(root.Clone.(*parse.ContainerNode).Container.Image).Equal("plugins/git:latest")
// })
// g.It("should use user-defined clone plugin", func() {
// op := NewCloneOp("plugins/git:latest")
// clone := root.NewCloneNode()
// clone.Container = libcd.Container{}
// clone.Container.Image = "custom/hg:latest"
// root.Clone = clone
// op.VisitRoot(root)
// g.Assert(clone.Container.Image).Equal("custom/hg:latest")
// })
// })
// }

57
engine/compiler/builtin/envs.go
View file

@ -1,57 +0,0 @@
package builtin
import (
"os"
"strings"
"github.com/drone/drone/engine/compiler/parse"
)
var (
httpProxy = os.Getenv("HTTP_PROXY")
httpsProxy = os.Getenv("HTTPS_PROXY")
noProxy = os.Getenv("NO_PROXY")
)
type envOp struct {
visitor
envs map[string]string
}
// NewEnvOp returns a transformer that sets default environment variables
// for each container, service and plugin.
func NewEnvOp(envs map[string]string) Visitor {
return &envOp{
envs: envs,
}
}
func (v *envOp) VisitContainer(node *parse.ContainerNode) error {
if node.Container.Environment == nil {
node.Container.Environment = map[string]string{}
}
v.defaultEnv(node)
v.defaultEnvProxy(node)
return nil
}
func (v *envOp) defaultEnv(node *parse.ContainerNode) {
for k, v := range v.envs {
node.Container.Environment[k] = v
}
}
func (v *envOp) defaultEnvProxy(node *parse.ContainerNode) {
if httpProxy != "" {
node.Container.Environment["HTTP_PROXY"] = httpProxy
node.Container.Environment["http_proxy"] = strings.ToUpper(httpProxy)
}
if httpsProxy != "" {
node.Container.Environment["HTTPS_PROXY"] = httpsProxy
node.Container.Environment["https_proxy"] = strings.ToUpper(httpsProxy)
}
if noProxy != "" {
node.Container.Environment["NO_PROXY"] = noProxy
node.Container.Environment["no_proxy"] = strings.ToUpper(noProxy)
}
}

45
engine/compiler/builtin/envs_test.go
View file

@ -1,45 +0,0 @@
package builtin
import (
"testing"
"github.com/drone/drone/engine/compiler/parse"
"github.com/drone/drone/engine/runner"
"github.com/franela/goblin"
)
func Test_env(t *testing.T) {
root := parse.NewRootNode()
g := goblin.Goblin(t)
g.Describe("environment variables", func() {
g.It("should be copied", func() {
envs := map[string]string{"CI": "drone"}
c := root.NewContainerNode()
c.Container = runner.Container{}
op := NewEnvOp(envs)
op.VisitContainer(c)
g.Assert(c.Container.Environment["CI"]).Equal("drone")
})
g.It("should include http proxy variables", func() {
httpProxy = "foo"
httpsProxy = "bar"
noProxy = "baz"
c := root.NewContainerNode()
c.Container = runner.Container{}
op := NewEnvOp(map[string]string{})
op.VisitContainer(c)
g.Assert(c.Container.Environment["HTTP_PROXY"]).Equal("foo")
g.Assert(c.Container.Environment["HTTPS_PROXY"]).Equal("bar")
g.Assert(c.Container.Environment["NO_PROXY"]).Equal("baz")
})
})
}

30
engine/compiler/builtin/escalate.go
View file

@ -1,30 +0,0 @@
package builtin
import (
"path/filepath"
"github.com/drone/drone/engine/compiler/parse"
)
type escalateOp struct {
visitor
plugins []string
}
// NewEscalateOp returns a transformer that configures plugins to automatically
// execute in privileged mode. This is intended for plugins running dind.
func NewEscalateOp(plugins []string) Visitor {
return &escalateOp{
plugins: plugins,
}
}
func (v *escalateOp) VisitContainer(node *parse.ContainerNode) error {
for _, pattern := range v.plugins {
ok, _ := filepath.Match(pattern, node.Container.Image)
if ok {
node.Container.Privileged = true
}
}
return nil
}

54
engine/compiler/builtin/escalate_test.go
View file

@ -1,54 +0,0 @@
package builtin
import (
"testing"
"github.com/drone/drone/engine/compiler/parse"
"github.com/drone/drone/engine/runner"
"github.com/franela/goblin"
)
func Test_escalate(t *testing.T) {
root := parse.NewRootNode()
g := goblin.Goblin(t)
g.Describe("privileged transform", func() {
g.It("should handle matches", func() {
c := root.NewPluginNode()
c.Container = runner.Container{Image: "plugins/docker"}
op := NewEscalateOp([]string{"plugins/docker"})
op.VisitContainer(c)
g.Assert(c.Container.Privileged).IsTrue()
})
g.It("should handle glob matches", func() {
c := root.NewPluginNode()
c.Container = runner.Container{Image: "plugins/docker"}
op := NewEscalateOp([]string{"plugins/*"})
op.VisitContainer(c)
g.Assert(c.Container.Privileged).IsTrue()
})
g.It("should handle non matches", func() {
c := root.NewPluginNode()
c.Container = runner.Container{Image: "plugins/git"}
op := NewEscalateOp([]string{"plugins/docker"})
op.VisitContainer(c)
g.Assert(c.Container.Privileged).IsFalse()
})
g.It("should handle non glob matches", func() {
c := root.NewPluginNode()
c.Container = runner.Container{Image: "plugins/docker:develop"}
op := NewEscalateOp([]string{"plugins/docker"})
op.VisitContainer(c)
g.Assert(c.Container.Privileged).IsFalse()
})
})
}

128
engine/compiler/builtin/filter.go
View file

@ -1,128 +0,0 @@
package builtin
import (
"path/filepath"
"github.com/drone/drone/engine/compiler/parse"
)
type filterOp struct {
visitor
status string
branch string
event string
environ string
platform string
matrix map[string]string
}
// NewFilterOp returns a transformer that filters (ie removes) steps
// from the process based on conditional logic in the yaml.
func NewFilterOp(status, branch, event, env string, matrix map[string]string) Visitor {
return &filterOp{
status: status,
branch: branch,
event: event,
environ: env,
matrix: matrix,
}
}
func (v *filterOp) VisitContainer(node *parse.ContainerNode) error {
v.visitStatus(node)
v.visitBranch(node)
v.visitEvent(node)
v.visitMatrix(node)
v.visitPlatform(node)
return nil
}
// visitStatus is a helpfer function that converts an on_change status
// filter to either success or failure based on the prior build status.
func (v *filterOp) visitStatus(node *parse.ContainerNode) {
if len(node.Conditions.Status) == 0 {
node.Conditions.Status = []string{"success"}
return
}
for _, status := range node.Conditions.Status {
if status != "change" && status != "changed" && status != "changes" {
continue
}
var want []string
switch v.status {
case "success":
want = append(want, "failure")
case "failure", "error", "killed":
want = append(want, "success")
default:
want = []string{"success", "failure"}
}
node.Conditions.Status = append(node.Conditions.Status, want...)
break
}
}
// visitBranch is a helper function that disables container steps when
// the branch conditions are not satisfied.
func (v *filterOp) visitBranch(node *parse.ContainerNode) {
if len(node.Conditions.Branch) == 0 {
return
}
for _, pattern := range node.Conditions.Branch {
if ok, _ := filepath.Match(pattern, v.branch); ok {
return
}
}
node.Disabled = true
}
// visitEnvironment is a helper function that disables container steps
// when the deployment environment conditions are not satisfied.
func (v *filterOp) visitEnvironment(node *parse.ContainerNode) {
if len(node.Conditions.Environment) == 0 {
return
}
for _, pattern := range node.Conditions.Environment {
if ok, _ := filepath.Match(pattern, v.environ); ok {
return
}
}
node.Disabled = true
}
// visitEvent is a helper function that disables container steps
// when the build event conditions are not satisfied.
func (v *filterOp) visitEvent(node *parse.ContainerNode) {
if len(node.Conditions.Event) == 0 {
return
}
for _, pattern := range node.Conditions.Event {
if ok, _ := filepath.Match(pattern, v.event); ok {
return
}
}
node.Disabled = true
}
func (v *filterOp) visitMatrix(node *parse.ContainerNode) {
for key, val := range node.Conditions.Matrix {
if v.matrix[key] != val {
node.Disabled = true
break
}
}
}
// visitPlatform is a helper function that disables container steps
// when the build event conditions are not satisfied.
func (v *filterOp) visitPlatform(node *parse.ContainerNode) {
if len(node.Conditions.Platform) == 0 {
return
}
for _, pattern := range node.Conditions.Platform {
if ok, _ := filepath.Match(pattern, v.platform); ok {
return
}
}
node.Disabled = true
}

130
engine/compiler/builtin/filter_test.go
View file

@ -1,130 +0,0 @@
package builtin
// import (
// "testing"
// "github.com/franela/goblin"
// )
// func TestFilter(t *testing.T) {
// g := goblin.Goblin(t)
// g.Describe("Filters", func() {
// g.It("Should match no branch filter", func() {
// c := &Container{}
// FilterBranch("feature/foo")(nil, c)
// g.Assert(c.Disabled).IsFalse()
// })
// g.It("Should match branch", func() {
// c := &Container{}
// c.Conditions.Branch.parts = []string{"feature/*"}
// FilterBranch("feature/foo")(nil, c)
// g.Assert(c.Disabled).IsFalse()
// })
// g.It("Should match branch wildcard", func() {
// c := &Container{}
// c.Conditions.Branch.parts = []string{"feature/*"}
// FilterBranch("feature/foo")(nil, c)
// g.Assert(c.Disabled).IsFalse()
// })
// g.It("Should disable when branch filter doesn't match", func() {
// c := &Container{}
// c.Conditions.Branch.parts = []string{"feature/*", "develop"}
// FilterBranch("master")(nil, c)
// g.Assert(c.Disabled).IsTrue()
// })
// g.It("Should match no platform filter", func() {
// c := &Container{}
// FilterPlatform("linux_amd64")(nil, c)
// g.Assert(c.Disabled).IsFalse()
// })
// g.It("Should match platform", func() {
// c := &Container{}
// c.Conditions.Platform.parts = []string{"linux_amd64"}
// FilterPlatform("linux_amd64")(nil, c)
// g.Assert(c.Disabled).IsFalse()
// })
// g.It("Should disable when platform filter doesn't match", func() {
// c := &Container{}
// c.Conditions.Platform.parts = []string{"linux_arm", "linux_arm64"}
// FilterPlatform("linux_amd64")(nil, c)
// g.Assert(c.Disabled).IsTrue()
// })
// g.It("Should match no environment filter", func() {
// c := &Container{}
// FilterEnvironment("production")(nil, c)
// g.Assert(c.Disabled).IsFalse()
// })
// g.It("Should match environment", func() {
// c := &Container{}
// c.Conditions.Environment.parts = []string{"production"}
// FilterEnvironment("production")(nil, c)
// g.Assert(c.Disabled).IsFalse()
// })
// g.It("Should disable when environment filter doesn't match", func() {
// c := &Container{}
// c.Conditions.Environment.parts = []string{"develop", "staging"}
// FilterEnvironment("production")(nil, c)
// g.Assert(c.Disabled).IsTrue()
// })
// g.It("Should match no event filter", func() {
// c := &Container{}
// FilterEvent("push")(nil, c)
// g.Assert(c.Disabled).IsFalse()
// })
// g.It("Should match event", func() {
// c := &Container{}
// c.Conditions.Event.parts = []string{"push"}
// FilterEvent("push")(nil, c)
// g.Assert(c.Disabled).IsFalse()
// })
// g.It("Should disable when event filter doesn't match", func() {
// c := &Container{}
// c.Conditions.Event.parts = []string{"push", "tag"}
// FilterEvent("pull_request")(nil, c)
// g.Assert(c.Disabled).IsTrue()
// })
// g.It("Should match matrix", func() {
// c := &Container{}
// c.Conditions.Matrix = map[string]string{
// "go": "1.5",
// "redis": "3.0",
// }
// matrix := map[string]string{
// "go": "1.5",
// "redis": "3.0",
// "node": "5.0.0",
// }
// FilterMatrix(matrix)(nil, c)
// g.Assert(c.Disabled).IsFalse()
// })
// g.It("Should disable when event filter doesn't match", func() {
// c := &Container{}
// c.Conditions.Matrix = map[string]string{
// "go": "1.5",
// "redis": "3.0",
// }
// matrix := map[string]string{
// "go": "1.4.2",
// "redis": "3.0",
// "node": "5.0.0",
// }
// FilterMatrix(matrix)(nil, c)
// g.Assert(c.Disabled).IsTrue()
// })
// })
// }

66
engine/compiler/builtin/normalize.go
View file

@ -1,66 +0,0 @@
package builtin
import (
"path/filepath"
"strings"
"github.com/drone/drone/engine/compiler/parse"
)
type normalizeOp struct {
visitor
namespace string
}
// NewNormalizeOp returns a transformer that normalizes the container image
// names and plugin names to their fully qualified values.
func NewNormalizeOp(namespace string) Visitor {
return &normalizeOp{
namespace: namespace,
}
}
func (v *normalizeOp) VisitContainer(node *parse.ContainerNode) error {
v.normalizeName(node)
v.normalizeImage(node)
switch node.NodeType {
case parse.NodePlugin, parse.NodeCache, parse.NodeClone:
v.normalizePlugin(node)
}
return nil
}
// normalize the container image to the fully qualified name.
func (v *normalizeOp) normalizeImage(node *parse.ContainerNode) {
if strings.Contains(node.Container.Image, ":") {
return
}
node.Container.Image = node.Container.Image + ":latest"
}
// normalize the plugin entrypoint and command values.
func (v *normalizeOp) normalizePlugin(node *parse.ContainerNode) {
if strings.Contains(node.Container.Image, "/") {
return
}
if strings.Contains(node.Container.Image, "_") {
node.Container.Image = strings.Replace(node.Container.Image, "_", "-", -1)
}
node.Container.Image = filepath.Join(v.namespace, node.Container.Image)
}
// normalize the container name to ensrue a value is set.
func (v *normalizeOp) normalizeName(node *parse.ContainerNode) {
if node.Container.Name != "" {
return
}
parts := strings.Split(node.Container.Image, "/")
if len(parts) != 0 {
node.Container.Name = parts[len(parts)-1]
}
parts = strings.Split(node.Container.Image, ":")
if len(parts) != 0 {
node.Container.Name = parts[0]
}
}

78
engine/compiler/builtin/normalize_test.go
View file

@ -1,78 +0,0 @@
package builtin
import (
"testing"
"github.com/drone/drone/engine/compiler/parse"
"github.com/drone/drone/engine/runner"
"github.com/franela/goblin"
)
func Test_normalize(t *testing.T) {
root := parse.NewRootNode()
g := goblin.Goblin(t)
g.Describe("normalizing", func() {
g.Describe("images", func() {
g.It("should append tag if empty", func() {
c := root.NewContainerNode()
c.Container = runner.Container{Image: "golang"}
op := NewNormalizeOp("")
op.VisitContainer(c)
g.Assert(c.Container.Image).Equal("golang:latest")
})
g.It("should not override existing tag", func() {
c := root.NewContainerNode()
c.Container = runner.Container{Image: "golang:1.5"}
op := NewNormalizeOp("")
op.VisitContainer(c)
g.Assert(c.Container.Image).Equal("golang:1.5")
})
})
g.Describe("plugins", func() {
g.It("should prepend namespace", func() {
c := root.NewPluginNode()
c.Container = runner.Container{Image: "git"}
op := NewNormalizeOp("plugins")
op.VisitContainer(c)
g.Assert(c.Container.Image).Equal("plugins/git:latest")
})
g.It("should not override existing namespace", func() {
c := root.NewPluginNode()
c.Container = runner.Container{Image: "index.docker.io/drone/git"}
op := NewNormalizeOp("plugins")
op.VisitContainer(c)
g.Assert(c.Container.Image).Equal("index.docker.io/drone/git:latest")
})
g.It("should replace underscores with dashes", func() {
c := root.NewPluginNode()
c.Container = runner.Container{Image: "gh_pages"}
op := NewNormalizeOp("plugins")
op.VisitContainer(c)
g.Assert(c.Container.Image).Equal("plugins/gh-pages:latest")
})
g.It("should ignore shell or service types", func() {
c := root.NewShellNode()
c.Container = runner.Container{Image: "golang"}
op := NewNormalizeOp("plugins")
op.VisitContainer(c)
g.Assert(c.Container.Image).Equal("golang:latest")
})
})
})
}

50
engine/compiler/builtin/pod.go
View file

@ -1,50 +0,0 @@
package builtin
import (
"fmt"
"github.com/drone/drone/engine/compiler/parse"
"github.com/drone/drone/engine/runner"
)
type podOp struct {
visitor
name string
}
// NewPodOp returns a transformer that configures an ambassador container
// providing shared networking and container volumes.
func NewPodOp(name string) Visitor {
return &podOp{
name: name,
}
}
func (v *podOp) VisitContainer(node *parse.ContainerNode) error {
if node.Container.Network == "" {
parent := fmt.Sprintf("container:%s", v.name)
node.Container.Network = parent
}
node.Container.VolumesFrom = append(node.Container.VolumesFrom, v.name)
return nil
}
func (v *podOp) VisitRoot(node *parse.RootNode) error {
service := node.NewServiceNode()
service.Container = runner.Container{
Name: v.name,
Alias: "ambassador",
Image: "busybox:latest",
Entrypoint: []string{"/bin/sleep"},
Command: []string{"86400"},
Volumes: []string{node.Path, node.Base},
// Entrypoint: []string{"/bin/sh", "-c"},
// Volumes: []string{node.Base},
// Command: []string{
// fmt.Sprintf("mkdir -p %s; sleep 86400", node.Path),
// },
}
node.Pod = service
return nil
}

26
engine/compiler/builtin/pull.go
View file

@ -1,26 +0,0 @@
package builtin
import (
"github.com/drone/drone/engine/compiler/parse"
)
type pullOp struct {
visitor
pull bool
}
// NewPullOp returns a transformer that configures plugins to automatically
// pull the latest images at runtime.
func NewPullOp(pull bool) Visitor {
return &pullOp{
pull: pull,
}
}
func (v *pullOp) VisitContainer(node *parse.ContainerNode) error {
switch node.NodeType {
case parse.NodePlugin, parse.NodeCache, parse.NodeClone:
node.Container.Pull = v.pull
}
return nil
}

45
engine/compiler/builtin/pull_test.go
View file

@ -1,45 +0,0 @@
package builtin
import (
"testing"
"github.com/drone/drone/engine/compiler/parse"
"github.com/drone/drone/engine/runner"
"github.com/franela/goblin"
)
func Test_pull(t *testing.T) {
root := parse.NewRootNode()
g := goblin.Goblin(t)
g.Describe("pull image", func() {
g.It("should be enabled for plugins", func() {
c := root.NewPluginNode()
c.Container = runner.Container{}
op := NewPullOp(true)
op.VisitContainer(c)
g.Assert(c.Container.Pull).IsTrue()
})
g.It("should be disabled for plugins", func() {
c := root.NewPluginNode()
c.Container = runner.Container{}
op := NewPullOp(false)
op.VisitContainer(c)
g.Assert(c.Container.Pull).IsFalse()
})
g.It("should be disabled for non-plugins", func() {
c := root.NewShellNode()
c.Container = runner.Container{}
op := NewPullOp(true)
op.VisitContainer(c)
g.Assert(c.Container.Pull).IsFalse()
})
})
}

45
engine/compiler/builtin/secrets.go
View file

@ -1,45 +0,0 @@
package builtin
import (
"github.com/drone/drone/engine/compiler/parse"
"github.com/drone/drone/model"
)
type secretOp struct {
visitor
event string
secrets []*model.Secret
}
// NewSecretOp returns a transformer that configures plugin secrets.
func NewSecretOp(event string, secrets []*model.Secret) Visitor {
return &secretOp{
event: event,
secrets: secrets,
}
}
func (v *secretOp) VisitContainer(node *parse.ContainerNode) error {
for _, secret := range v.secrets {
if !secret.Match(node.Container.Image, v.event) {
continue
}
switch secret.Name {
case "REGISTRY_USERNAME":
node.Container.AuthConfig.Username = secret.Value
case "REGISTRY_PASSWORD":
node.Container.AuthConfig.Password = secret.Value
case "REGISTRY_EMAIL":
node.Container.AuthConfig.Email = secret.Value
case "REGISTRY_TOKEN":
node.Container.AuthConfig.Token = secret.Value
default:
if node.Container.Environment == nil {
node.Container.Environment = map[string]string{}
}
node.Container.Environment[secret.Name] = secret.Value
}
}
return nil
}

95
engine/compiler/builtin/shell.go
View file

@ -1,95 +0,0 @@
package builtin
import (
"bytes"
"encoding/base64"
"fmt"
"github.com/drone/drone/engine/compiler/parse"
)
const (
Freebsd_amd64 = "freebsd_amd64"
Linux_adm64 = "linux_amd64"
Windows_amd64 = "windows_amd64"
)
type shellOp struct {
visitor
platform string
}
// NewShellOp returns a transformer that converts the shell node to
// a runnable container.
func NewShellOp(platform string) Visitor {
return &shellOp{
platform: platform,
}
}
func (v *shellOp) VisitContainer(node *parse.ContainerNode) error {
if node.NodeType != parse.NodeShell {
return nil
}
node.Container.Entrypoint = []string{
"/bin/sh", "-c",
}
node.Container.Command = []string{
"echo $DRONE_SCRIPT | base64 -d | /bin/sh -e",
}
if node.Container.Environment == nil {
node.Container.Environment = map[string]string{}
}
node.Container.Environment["HOME"] = "/root"
node.Container.Environment["SHELL"] = "/bin/sh"
node.Container.Environment["DRONE_SCRIPT"] = toScript(
node.Root().Path,
node.Commands,
)
return nil
}
func toScript(base string, commands []string) string {
var buf bytes.Buffer
for _, command := range commands {
buf.WriteString(fmt.Sprintf(
traceScript,
"<command>"+command+"</command>",
command,
))
}
script := fmt.Sprintf(
setupScript,
buf.String(),
)
return base64.StdEncoding.EncodeToString([]byte(script))
}
// setupScript is a helper script this is added to the build to ensure
// a minimum set of environment variables are set correctly.
const setupScript = `
if [ -n "$DRONE_NETRC_MACHINE" ]; then
cat <<EOF > $HOME/.netrc
machine $DRONE_NETRC_MACHINE
login $DRONE_NETRC_USERNAME
password $DRONE_NETRC_PASSWORD
EOF
fi
unset DRONE_NETRC_USERNAME
unset DRONE_NETRC_PASSWORD
unset DRONE_SCRIPT
%s
`
// traceScript is a helper script that is added to the build script
// to trace a command.
const traceScript = `
echo %q
%s
`

44
engine/compiler/builtin/shell_test.go
View file

@ -1,44 +0,0 @@
package builtin
import (
"testing"
"github.com/drone/drone/engine/compiler/parse"
"github.com/drone/drone/engine/runner"
"github.com/franela/goblin"
)
func Test_shell(t *testing.T) {
g := goblin.Goblin(t)
g.Describe("shell containers", func() {
g.It("should ignore plugin steps", func() {
root := parse.NewRootNode()
c := root.NewPluginNode()
c.Container = runner.Container{}
ops := NewShellOp(Linux_adm64)
ops.VisitContainer(c)
g.Assert(len(c.Container.Entrypoint)).Equal(0)
g.Assert(len(c.Container.Command)).Equal(0)
g.Assert(c.Container.Environment["DRONE_SCRIPT"]).Equal("")
})
g.It("should set entrypoint, command and environment variables", func() {
root := parse.NewRootNode()
root.Base = "/go"
root.Path = "/go/src/github.com/octocat/hello-world"
c := root.NewShellNode()
c.Commands = []string{"go build"}
ops := NewShellOp(Linux_adm64)
ops.VisitContainer(c)
g.Assert(c.Container.Entrypoint).Equal([]string{"/bin/sh", "-c"})
g.Assert(c.Container.Command).Equal([]string{"echo $DRONE_SCRIPT | base64 -d | /bin/sh -e"})
g.Assert(c.Container.Environment["DRONE_SCRIPT"] != "").IsTrue()
})
})
}

120
engine/compiler/builtin/validate.go
View file

@ -1,120 +0,0 @@
package builtin
import (
"fmt"
"path/filepath"
"github.com/drone/drone/engine/compiler/parse"
)
type validateOp struct {
visitor
plugins []string
trusted bool
}
// NewValidateOp returns a linter that checks container configuration.
func NewValidateOp(trusted bool, plugins []string) Visitor {
return &validateOp{
trusted: trusted,
plugins: plugins,
}
}
func (v *validateOp) VisitContainer(node *parse.ContainerNode) error {
switch node.NodeType {
case parse.NodePlugin, parse.NodeCache, parse.NodeClone:
if err := v.validatePlugins(node); err != nil {
return err
}
}
if node.NodeType == parse.NodePlugin {
if err := v.validatePluginConfig(node); err != nil {
return err
}
}
return v.validateConfig(node)
}
// validate the plugin image and return an error if the plugin
// image does not match the whitelist.
func (v *validateOp) validatePlugins(node *parse.ContainerNode) error {
match := false
for _, pattern := range v.plugins {
ok, err := filepath.Match(pattern, node.Container.Image)
if ok && err == nil {
match = true
break
}
}
if !match {
return fmt.Errorf(
"Plugin %s is not in the whitelist",
node.Container.Image,
)
}
return nil
}
// validate the plugin command and entrypoint and return an error
// the user attempts to set or override these values.
func (v *validateOp) validatePluginConfig(node *parse.ContainerNode) error {
if len(node.Container.Entrypoint) != 0 {
return fmt.Errorf("Cannot set plugin Entrypoint")
}
if len(node.Container.Command) != 0 {
return fmt.Errorf("Cannot set plugin Command")
}
return nil
}
// validate the container configuration and return an error if
// restricted configurations are used.
func (v *validateOp) validateConfig(node *parse.ContainerNode) error {
if v.trusted {
return nil
}
if node.Container.Privileged {
return fmt.Errorf("Insufficient privileges to use privileged mode")
}
if len(node.Container.DNS) != 0 {
return fmt.Errorf("Insufficient privileges to use custom dns")
}
if len(node.Container.DNSSearch) != 0 {
return fmt.Errorf("Insufficient privileges to use dns_search")
}
if len(node.Container.Devices) != 0 {
return fmt.Errorf("Insufficient privileges to use devices")
}
if len(node.Container.ExtraHosts) != 0 {
return fmt.Errorf("Insufficient privileges to use extra_hosts")
}
if len(node.Container.Network) != 0 {
return fmt.Errorf("Insufficient privileges to override the network")
}
if node.Container.OomKillDisable {
return fmt.Errorf("Insufficient privileges to disable oom_kill")
}
if len(node.Container.Volumes) != 0 && node.Type() != parse.NodeCache {
return fmt.Errorf("Insufficient privileges to use volumes")
}
if len(node.Container.VolumesFrom) != 0 {
return fmt.Errorf("Insufficient privileges to use volumes_from")
}
return nil
}
// validate the environment configuration and return an error if
// an attempt is made to override system environment variables.
// func (v *validateOp) validateEnvironment(node *parse.ContainerNode) error {
// for key := range node.Container.Environment {
// upper := strings.ToUpper(key)
// switch {
// case strings.HasPrefix(upper, "DRONE_"):
// return fmt.Errorf("Cannot set or override DRONE_ environment variables")
// case strings.HasPrefix(upper, "PLUGIN_"):
// return fmt.Errorf("Cannot set or override PLUGIN_ environment variables")
// }
// }
// return nil
// }

199
engine/compiler/builtin/validate_test.go
View file

@ -1,199 +0,0 @@
package builtin
import (
"testing"
"github.com/drone/drone/engine/compiler/parse"
"github.com/drone/drone/engine/runner"
"github.com/franela/goblin"
)
func Test_validate(t *testing.T) {
root := parse.NewRootNode()
g := goblin.Goblin(t)
g.Describe("validating", func() {
g.Describe("privileged attributes", func() {
g.It("should not error when trusted build", func() {
c := root.NewContainerNode()
c.Container = runner.Container{}
ops := NewValidateOp(true, []string{"plugins/*"})
err := ops.VisitContainer(c)
g.Assert(err == nil).IsTrue("error should be nil")
})
g.It("should error when privleged mode", func() {
c := root.NewContainerNode()
c.Container = runner.Container{}
c.Container.Privileged = true
ops := NewValidateOp(false, []string{"plugins/*"})
err := ops.VisitContainer(c)
g.Assert(err != nil).IsTrue("error should not be nil")
g.Assert(err.Error()).Equal("Insufficient privileges to use privileged mode")
})
g.It("should error when dns configured", func() {
c := root.NewContainerNode()
c.Container = runner.Container{}
c.Container.DNS = []string{"8.8.8.8"}
ops := NewValidateOp(false, []string{"plugins/*"})
err := ops.VisitContainer(c)
g.Assert(err != nil).IsTrue("error should not be nil")
g.Assert(err.Error()).Equal("Insufficient privileges to use custom dns")
})
g.It("should error when dns_search configured", func() {
c := root.NewContainerNode()
c.Container = runner.Container{}
c.Container.DNSSearch = []string{"8.8.8.8"}
ops := NewValidateOp(false, []string{"plugins/*"})
err := ops.VisitContainer(c)
g.Assert(err != nil).IsTrue("error should not be nil")
g.Assert(err.Error()).Equal("Insufficient privileges to use dns_search")
})
g.It("should error when devices configured", func() {
c := root.NewContainerNode()
c.Container = runner.Container{}
c.Container.Devices = []string{"/dev/foo"}
ops := NewValidateOp(false, []string{"plugins/*"})
err := ops.VisitContainer(c)
g.Assert(err != nil).IsTrue("error should not be nil")
g.Assert(err.Error()).Equal("Insufficient privileges to use devices")
})
g.It("should error when extra_hosts configured", func() {
c := root.NewContainerNode()
c.Container = runner.Container{}
c.Container.ExtraHosts = []string{"1.2.3.4 foo.com"}
ops := NewValidateOp(false, []string{"plugins/*"})
err := ops.VisitContainer(c)
g.Assert(err != nil).IsTrue("error should not be nil")
g.Assert(err.Error()).Equal("Insufficient privileges to use extra_hosts")
})
g.It("should error when network configured", func() {
c := root.NewContainerNode()
c.Container = runner.Container{}
c.Container.Network = "host"
ops := NewValidateOp(false, []string{"plugins/*"})
err := ops.VisitContainer(c)
g.Assert(err != nil).IsTrue("error should not be nil")
g.Assert(err.Error()).Equal("Insufficient privileges to override the network")
})
g.It("should error when oom_kill_disabled configured", func() {
c := root.NewContainerNode()
c.Container = runner.Container{}
c.Container.OomKillDisable = true
ops := NewValidateOp(false, []string{"plugins/*"})
err := ops.VisitContainer(c)
g.Assert(err != nil).IsTrue("error should not be nil")
g.Assert(err.Error()).Equal("Insufficient privileges to disable oom_kill")
})
g.It("should error when volumes configured", func() {
c := root.NewContainerNode()
c.Container = runner.Container{}
c.Container.Volumes = []string{"/:/tmp"}
ops := NewValidateOp(false, []string{"plugins/*"})
err := ops.VisitContainer(c)
g.Assert(err != nil).IsTrue("error should not be nil")
g.Assert(err.Error()).Equal("Insufficient privileges to use volumes")
})
g.It("should error when volumes_from configured", func() {
c := root.NewContainerNode()
c.Container = runner.Container{}
c.Container.VolumesFrom = []string{"drone"}
ops := NewValidateOp(false, []string{"plugins/*"})
err := ops.VisitContainer(c)
g.Assert(err != nil).IsTrue("error should not be nil")
g.Assert(err.Error()).Equal("Insufficient privileges to use volumes_from")
})
})
g.Describe("plugin configuration", func() {
g.It("should error when entrypoint is configured", func() {
c := root.NewPluginNode()
c.Container = runner.Container{Image: "plugins/git"}
c.Container.Entrypoint = []string{"/bin/sh"}
ops := NewValidateOp(false, []string{"plugins/*"})
err := ops.VisitContainer(c)
g.Assert(err != nil).IsTrue("error should not be nil")
g.Assert(err.Error()).Equal("Cannot set plugin Entrypoint")
})
g.It("should error when command is configured", func() {
c := root.NewPluginNode()
c.Container = runner.Container{Image: "plugins/git"}
c.Container.Command = []string{"cat", "/proc/1/status"}
ops := NewValidateOp(false, []string{"plugins/*"})
err := ops.VisitContainer(c)
g.Assert(err != nil).IsTrue("error should not be nil")
g.Assert(err.Error()).Equal("Cannot set plugin Command")
})
g.It("should not error when empty entrypoint, command", func() {
c := root.NewPluginNode()
c.Container = runner.Container{Image: "plugins/git"}
ops := NewValidateOp(false, []string{"plugins/*"})
err := ops.VisitContainer(c)
g.Assert(err == nil).IsTrue("error should be nil")
})
})
g.Describe("plugin whitelist", func() {
g.It("should error when no match found", func() {
c := root.NewPluginNode()
c.Container = runner.Container{}
c.Container.Image = "custom/git"
ops := NewValidateOp(false, []string{"plugins/*"})
err := ops.VisitContainer(c)
g.Assert(err != nil).IsTrue("error should be nil")
g.Assert(err.Error()).Equal("Plugin custom/git is not in the whitelist")
})
g.It("should not error when match found", func() {
c := root.NewPluginNode()
c.Container = runner.Container{}
c.Container.Image = "plugins/git"
ops := NewValidateOp(false, []string{"plugins/*"})
err := ops.VisitContainer(c)
g.Assert(err == nil).IsTrue("error should be nil")
})
g.It("should ignore build images", func() {
c := root.NewShellNode()
c.Container = runner.Container{}
c.Container.Image = "google/golang"
ops := NewValidateOp(false, []string{"plugins/*"})
err := ops.VisitContainer(c)
g.Assert(err == nil).IsTrue("error should be nil")
})
})
})
}

23
engine/compiler/builtin/visitor.go
View file

@ -1,23 +0,0 @@
package builtin
import "github.com/drone/drone/engine/compiler/parse"
// Visitor interface for walking the Yaml file.
type Visitor interface {
VisitRoot(*parse.RootNode) error
VisitVolume(*parse.VolumeNode) error
VisitNetwork(*parse.NetworkNode) error
VisitBuild(*parse.BuildNode) error
VisitContainer(*parse.ContainerNode) error
}
// visitor provides an easy default implementation of a Visitor interface with
// stubbed methods. This can be embedded in transforms to meet the basic
// requirements.
type visitor struct{}
func (visitor) VisitRoot(*parse.RootNode) error { return nil }
func (visitor) VisitVolume(*parse.VolumeNode) error { return nil }
func (visitor) VisitNetwork(*parse.NetworkNode) error { return nil }
func (visitor) VisitBuild(*parse.BuildNode) error { return nil }
func (visitor) VisitContainer(*parse.ContainerNode) error { return nil }

51
engine/compiler/builtin/workspace.go
View file

@ -1,51 +0,0 @@
package builtin
import (
"path/filepath"
"github.com/drone/drone/engine/compiler/parse"
)
type workspaceOp struct {
visitor
base string
path string
}
// NewWorkspaceOp returns a transformer that provides a default workspace paths,
// including the base path (mounted as a volume) and absolute path where the
// code is cloned.
func NewWorkspaceOp(base, path string) Visitor {
return &workspaceOp{
base: base,
path: path,
}
}
func (v *workspaceOp) VisitRoot(node *parse.RootNode) error {
if node.Base == "" {
node.Base = v.base
}
if node.Path == "" {
node.Path = v.path
}
if !filepath.IsAbs(node.Path) {
node.Path = filepath.Join(
node.Base,
node.Path,
)
}
return nil
}
func (v *workspaceOp) VisitContainer(node *parse.ContainerNode) error {
if node.NodeType == parse.NodeService {
// we must not override the default working
// directory of service containers. All other
// container should launch in the workspace
return nil
}
root := node.Root()
node.Container.WorkingDir = root.Path
return nil
}

89
engine/compiler/builtin/workspace_test.go
View file

@ -1,89 +0,0 @@
package builtin
import (
"testing"
"github.com/franela/goblin"
"github.com/drone/drone/engine/compiler/parse"
)
func Test_workspace(t *testing.T) {
g := goblin.Goblin(t)
g.Describe("workspace", func() {
var defaultBase = "/go"
var defaultPath = "src/github.com/octocat/hello-world"
g.It("should not override user paths", func() {
var base = "/drone"
var path = "/drone/src/github.com/octocat/hello-world"
op := NewWorkspaceOp(defaultBase, defaultPath)
root := parse.NewRootNode()
root.Base = base
root.Path = path
op.VisitRoot(root)
g.Assert(root.Base).Equal(base)
g.Assert(root.Path).Equal(path)
})
g.It("should convert user paths to absolute", func() {
var base = "/drone"
var path = "src/github.com/octocat/hello-world"
var abs = "/drone/src/github.com/octocat/hello-world"
op := NewWorkspaceOp(defaultBase, defaultPath)
root := parse.NewRootNode()
root.Base = base
root.Path = path
op.VisitRoot(root)
g.Assert(root.Base).Equal(base)
g.Assert(root.Path).Equal(abs)
})
g.It("should set the default path", func() {
var base = "/go"
var path = "/go/src/github.com/octocat/hello-world"
op := NewWorkspaceOp(defaultBase, defaultPath)
root := parse.NewRootNode()
op.VisitRoot(root)
g.Assert(root.Base).Equal(base)
g.Assert(root.Path).Equal(path)
})
g.It("should use workspace as working_dir", func() {
var base = "/drone"
var path = "/drone/src/github.com/octocat/hello-world"
root := parse.NewRootNode()
root.Base = base
root.Path = path
c := root.NewContainerNode()
op := NewWorkspaceOp(defaultBase, defaultPath)
op.VisitContainer(c)
g.Assert(c.Container.WorkingDir).Equal(root.Path)
})
g.It("should not use workspace as working_dir for services", func() {
var base = "/drone"
var path = "/drone/src/github.com/octocat/hello-world"
root := parse.NewRootNode()
root.Base = base
root.Path = path
c := root.NewServiceNode()
op := NewWorkspaceOp(defaultBase, defaultPath)
op.VisitContainer(c)
g.Assert(c.Container.WorkingDir).Equal("")
})
})
}

137
engine/compiler/compile.go
View file

@ -1,137 +0,0 @@
package compiler
import (
"github.com/drone/drone/engine/runner"
"github.com/drone/drone/engine/runner/parse"
yaml "github.com/drone/drone/engine/compiler/parse"
)
// Compiler compiles the Yaml file to the intermediate representation.
type Compiler struct {
trans []Transform
}
func New() *Compiler {
return &Compiler{}
}
// Transforms sets the compiler transforms use to transform the intermediate
// representation during compilation.
func (c *Compiler) Transforms(trans []Transform) *Compiler {
c.trans = append(c.trans, trans...)
return c
}
// CompileString compiles the Yaml configuration string and returns
// the intermediate representation for the interpreter.
func (c *Compiler) CompileString(in string) (*runner.Spec, error) {
return c.Compile([]byte(in))
}
// CompileString compiles the Yaml configuration file and returns
// the intermediate representation for the interpreter.
func (c *Compiler) Compile(in []byte) (*runner.Spec, error) {
root, err := yaml.Parse(in)
if err != nil {
return nil, err
}
if err := root.Walk(c.walk); err != nil {
return nil, err
}
config := &runner.Spec{}
tree := parse.NewTree()
// pod section
if root.Pod != nil {
node, ok := root.Pod.(*yaml.ContainerNode)
if ok {
config.Containers = append(config.Containers, &node.Container)
tree.Append(parse.NewRunNode().SetName(node.Container.Name).SetDetach(true))
}
}
// clone section
if root.Clone != nil {
node, ok := root.Clone.(*yaml.ContainerNode)
if ok && !node.Disabled {
config.Containers = append(config.Containers, &node.Container)
tree.Append(parse.NewRunNode().SetName(node.Container.Name))
}
}
// services section
for _, container := range root.Services {
node, ok := container.(*yaml.ContainerNode)
if !ok || node.Disabled {
continue
}
config.Containers = append(config.Containers, &node.Container)
tree.Append(parse.NewRunNode().SetName(node.Container.Name).SetDetach(true))
}
// pipeline section
for i, container := range root.Script {
node, ok := container.(*yaml.ContainerNode)
if !ok || node.Disabled {
continue
}
config.Containers = append(config.Containers, &node.Container)
// step 1: lookahead to see if any status=failure exist
list := parse.NewListNode()
for ii, next := range root.Script {
if i >= ii {
continue
}
node, ok := next.(*yaml.ContainerNode)
if !ok || node.Disabled || !node.OnFailure() {
continue
}
list.Append(
parse.NewRecoverNode().SetBody(
parse.NewRunNode().SetName(
node.Container.Name,
),
),
)
}
// step 2: if yes, collect these and append to "error" node
if len(list.Body) == 0 {
tree.Append(parse.NewRunNode().SetName(node.Container.Name))
} else {
errorNode := parse.NewErrorNode()
errorNode.SetBody(parse.NewRunNode().SetName(node.Container.Name))
errorNode.SetDefer(list)
tree.Append(errorNode)
}
}
config.Nodes = tree
return config, nil
}
func (c *Compiler) walk(node yaml.Node) (err error) {
for _, trans := range c.trans {
switch v := node.(type) {
case *yaml.BuildNode:
err = trans.VisitBuild(v)
case *yaml.ContainerNode:
err = trans.VisitContainer(v)
case *yaml.NetworkNode:
err = trans.VisitNetwork(v)
case *yaml.VolumeNode:
err = trans.VisitVolume(v)
case *yaml.RootNode:
err = trans.VisitRoot(v)
}
if err != nil {
break
}
}
return err
}

1
engine/compiler/compile_test.go
View file

@ -1 +0,0 @@
package compiler

34
engine/compiler/parse/node.go
View file

@ -1,34 +0,0 @@
package parse
const (
NodeBuild = "build"
NodeCache = "cache"
NodeClone = "clone"
NodeContainer = "container"
NodeNetwork = "network"
NodePlugin = "plugin"
NodeRoot = "root"
NodeService = "service"
NodeShell = "shell"
NodeVolume = "volume"
)
// NodeType identifies the type of parse tree node.
type NodeType string
// Type returns itself an provides an easy default implementation.
// for embedding in a Node. Embedded in all non-trivial Nodes.
func (t NodeType) Type() NodeType {
return t
}
// String returns the string value of the Node type.
func (t NodeType) String() string {
return string(t)
}
// A Node is an element in the parse tree.
type Node interface {
Type() NodeType
Root() *RootNode
}

42
engine/compiler/parse/node_build.go
View file

@ -1,42 +0,0 @@
package parse
// BuildNode represents Docker image build instructions.
type BuildNode struct {
NodeType
Context string
Dockerfile string
Args map[string]string
root *RootNode
}
// Root returns the root node.
func (n *BuildNode) Root() *RootNode { return n.root }
//
// intermediate types for yaml decoding.
//
type build struct {
Context string
Dockerfile string
Args map[string]string
}
func (b *build) UnmarshalYAML(unmarshal func(interface{}) error) error {
err := unmarshal(&b.Context)
if err == nil {
return nil
}
out := struct {
Context string
Dockerfile string
Args map[string]string
}{}
err = unmarshal(&out)
b.Context = out.Context
b.Args = out.Args
b.Dockerfile = out.Dockerfile
return err
}

38
engine/compiler/parse/node_build_test.go
View file

@ -1,38 +0,0 @@
package parse
import (
"testing"
"github.com/franela/goblin"
"gopkg.in/yaml.v2"
)
func TestBuildNode(t *testing.T) {
g := goblin.Goblin(t)
g.Describe("Build", func() {
g.Describe("given a yaml file", func() {
g.It("should unmarshal", func() {
in := []byte(".")
out := build{}
err := yaml.Unmarshal(in, &out)
if err != nil {
g.Fail(err)
}
g.Assert(out.Context).Equal(".")
})
g.It("should unmarshal shorthand", func() {
in := []byte("{ context: ., dockerfile: Dockerfile }")
out := build{}
err := yaml.Unmarshal(in, &out)
if err != nil {
g.Fail(err)
}
g.Assert(out.Context).Equal(".")
g.Assert(out.Dockerfile).Equal("Dockerfile")
})
})
})
}

180
engine/compiler/parse/node_container.go
View file

@ -1,180 +0,0 @@
package parse
import (
"fmt"
"github.com/drone/drone/engine/runner"
"gopkg.in/yaml.v2"
)
type Conditions struct {
Platform []string
Environment []string
Event []string
Branch []string
Status []string
Matrix map[string]string
}
// ContainerNode represents a Docker container.
type ContainerNode struct {
NodeType
// Container represents the container configuration.
Container runner.Container
Conditions Conditions
Disabled bool
Commands []string
Vargs map[string]interface{}
root *RootNode
}
// Root returns the root node.
func (n *ContainerNode) Root() *RootNode { return n.root }
// OnSuccess returns true if the container should be executed
// when the exit code of the previous step is 0.
func (n *ContainerNode) OnSuccess() bool {
for _, status := range n.Conditions.Status {
if status == "success" {
return true
}
}
return false
}
// OnFailure returns true if the container should be executed
// even when the exit code of the previous step != 0.
func (n *ContainerNode) OnFailure() bool {
for _, status := range n.Conditions.Status {
if status == "failure" {
return true
}
}
return false
}
//
// intermediate types for yaml decoding.
//
type container struct {
Name string `yaml:"name"`
Image string `yaml:"image"`
Build string `yaml:"build"`
Pull bool `yaml:"pull"`
Privileged bool `yaml:"privileged"`
Environment mapEqualSlice `yaml:"environment"`
Entrypoint stringOrSlice `yaml:"entrypoint"`
Command stringOrSlice `yaml:"command"`
Commands stringOrSlice `yaml:"commands"`
ExtraHosts stringOrSlice `yaml:"extra_hosts"`
Volumes stringOrSlice `yaml:"volumes"`
VolumesFrom stringOrSlice `yaml:"volumes_from"`
Devices stringOrSlice `yaml:"devices"`
Network string `yaml:"network_mode"`
DNS stringOrSlice `yaml:"dns"`
DNSSearch stringOrSlice `yaml:"dns_search"`
MemSwapLimit int64 `yaml:"memswap_limit"`
MemLimit int64 `yaml:"mem_limit"`
CPUQuota int64 `yaml:"cpu_quota"`
CPUShares int64 `yaml:"cpu_shares"`
CPUSet string `yaml:"cpuset"`
OomKillDisable bool `yaml:"oom_kill_disable"`
AuthConfig struct {
Username string `yaml:"username"`
Password string `yaml:"password"`
Email string `yaml:"email"`
Token string `yaml:"registry_token"`
} `yaml:"auth_config"`
Conditions struct {
Platform stringOrSlice `yaml:"platform"`
Environment stringOrSlice `yaml:"environment"`
Event stringOrSlice `yaml:"event"`
Branch stringOrSlice `yaml:"branch"`
Status stringOrSlice `yaml:"status"`
Matrix map[string]string `yaml:"matrix"`
} `yaml:"when"`
Vargs map[string]interface{} `yaml:",inline"`
}
func (c *container) ToContainer() runner.Container {
return runner.Container{
Name: c.Name,
Image: c.Image,
Pull: c.Pull,
Privileged: c.Privileged,
Environment: c.Environment.parts,
Entrypoint: c.Entrypoint.parts,
Command: c.Command.parts,
ExtraHosts: c.ExtraHosts.parts,
Volumes: c.Volumes.parts,
VolumesFrom: c.VolumesFrom.parts,
Devices: c.Devices.parts,
Network: c.Network,
DNS: c.DNS.parts,
DNSSearch: c.DNSSearch.parts,
MemSwapLimit: c.MemSwapLimit,
MemLimit: c.MemLimit,
CPUQuota: c.CPUQuota,
CPUShares: c.CPUShares,
CPUSet: c.CPUSet,
OomKillDisable: c.OomKillDisable,
AuthConfig: runner.Auth{
Username: c.AuthConfig.Username,
Password: c.AuthConfig.Password,
Email: c.AuthConfig.Email,
Token: c.AuthConfig.Token,
},
}
}
func (c *container) ToConditions() Conditions {
return Conditions{
Platform: c.Conditions.Platform.parts,
Environment: c.Conditions.Environment.parts,
Event: c.Conditions.Event.parts,
Branch: c.Conditions.Branch.parts,
Status: c.Conditions.Status.parts,
Matrix: c.Conditions.Matrix,
}
}
type containerList struct {
containers []*container
}
func (c *containerList) UnmarshalYAML(unmarshal func(interface{}) error) error {
slice := yaml.MapSlice{}
err := unmarshal(&slice)
if err != nil {
return err
}
for _, s := range slice {
cc := container{}
out, err := yaml.Marshal(s.Value)
if err != nil {
return err
}
err = yaml.Unmarshal(out, &cc)
if err != nil {
return err
}
if cc.Name == "" {
cc.Name = fmt.Sprintf("%v", s.Key)
}
if cc.Image == "" {
cc.Image = fmt.Sprintf("%v", s.Key)
}
c.containers = append(c.containers, &cc)
}
return err
}

97
engine/compiler/parse/node_container_test.go
View file

@ -1,97 +0,0 @@
package parse
import (
"testing"
"github.com/franela/goblin"
"gopkg.in/yaml.v2"
)
func TestContainerNode(t *testing.T) {
g := goblin.Goblin(t)
g.Describe("Containers", func() {
g.Describe("given a yaml file", func() {
g.It("should unmarshal", func() {
in := []byte(sampleContainer)
out := containerList{}
err := yaml.Unmarshal(in, &out)
if err != nil {
g.Fail(err)
}
g.Assert(len(out.containers)).Equal(1)
c := out.containers[0]
g.Assert(c.Name).Equal("foo")
g.Assert(c.Image).Equal("golang")
g.Assert(c.Build).Equal(".")
g.Assert(c.Pull).Equal(true)
g.Assert(c.Privileged).Equal(true)
g.Assert(c.Entrypoint.parts).Equal([]string{"/bin/sh"})
g.Assert(c.Command.parts).Equal([]string{"yes"})
g.Assert(c.Commands.parts).Equal([]string{"whoami"})
g.Assert(c.ExtraHosts.parts).Equal([]string{"foo.com"})
g.Assert(c.Volumes.parts).Equal([]string{"/foo:/bar"})
g.Assert(c.VolumesFrom.parts).Equal([]string{"foo"})
g.Assert(c.Devices.parts).Equal([]string{"/dev/tty0"})
g.Assert(c.Network).Equal("bridge")
g.Assert(c.DNS.parts).Equal([]string{"8.8.8.8"})
g.Assert(c.MemSwapLimit).Equal(int64(1))
g.Assert(c.MemLimit).Equal(int64(2))
g.Assert(c.CPUQuota).Equal(int64(3))
g.Assert(c.CPUSet).Equal("1,2")
g.Assert(c.OomKillDisable).Equal(true)
g.Assert(c.AuthConfig.Username).Equal("octocat")
g.Assert(c.AuthConfig.Password).Equal("password")
g.Assert(c.AuthConfig.Email).Equal("octocat@github.com")
g.Assert(c.Vargs["access_key"]).Equal("970d28f4dd477bc184fbd10b376de753")
g.Assert(c.Vargs["secret_key"]).Equal("9c5785d3ece6a9cdefa42eb99b58986f9095ff1c")
})
g.It("should unmarshal named", func() {
in := []byte("foo: { name: bar }")
out := containerList{}
err := yaml.Unmarshal(in, &out)
if err != nil {
g.Fail(err)
}
g.Assert(len(out.containers)).Equal(1)
g.Assert(out.containers[0].Name).Equal("bar")
})
})
})
}
var sampleContainer = `
foo:
image: golang
build: .
pull: true
privileged: true
environment:
FOO: BAR
entrypoint: /bin/sh
command: "yes"
commands: whoami
extra_hosts: foo.com
volumes: /foo:/bar
volumes_from: foo
devices: /dev/tty0
network_mode: bridge
dns: 8.8.8.8
memswap_limit: 1
mem_limit: 2
cpu_quota: 3
cpuset: 1,2
oom_kill_disable: true
auth_config:
username: octocat
password: password
email: octocat@github.com
access_key: 970d28f4dd477bc184fbd10b376de753
secret_key: 9c5785d3ece6a9cdefa42eb99b58986f9095ff1c
`

68
engine/compiler/parse/node_network.go
View file

@ -1,68 +0,0 @@
package parse
import (
"fmt"
"gopkg.in/yaml.v2"
)
// NetworkNode represents a Docker network.
type NetworkNode struct {
NodeType
root *RootNode
Name string
Driver string
DriverOpts map[string]string
}
// Root returns the root node.
func (n *NetworkNode) Root() *RootNode { return n.root }
//
// intermediate types for yaml decoding.
//
// network is an intermediate type used for decoding a networks in a format
// compatible with docker-compose.yml
type network struct {
Name string
Driver string
DriverOpts map[string]string `yaml:"driver_opts"`
}
// networkList is an intermediate type used for decoding a slice of networks
// in a format compatible with docker-compose.yml
type networkList struct {
networks []*network
}
func (n *networkList) UnmarshalYAML(unmarshal func(interface{}) error) error {
slice := yaml.MapSlice{}
err := unmarshal(&slice)
if err != nil {
return err
}
for _, s := range slice {
nn := network{}
out, err := yaml.Marshal(s.Value)
if err != nil {
return err
}
err = yaml.Unmarshal(out, &nn)
if err != nil {
return err
}
if nn.Name == "" {
nn.Name = fmt.Sprintf("%v", s.Key)
}
if nn.Driver == "" {
nn.Driver = "bridge"
}
n.networks = append(n.networks, &nn)
}
return err
}

51
engine/compiler/parse/node_network_test.go
View file

@ -1,51 +0,0 @@
package parse
import (
"testing"
"github.com/franela/goblin"
"gopkg.in/yaml.v2"
)
func TestNetworkNode(t *testing.T) {
g := goblin.Goblin(t)
g.Describe("Networks", func() {
g.Describe("given a yaml file", func() {
g.It("should unmarshal", func() {
in := []byte("foo: { driver: overlay }")
out := networkList{}
err := yaml.Unmarshal(in, &out)
if err != nil {
g.Fail(err)
}
g.Assert(len(out.networks)).Equal(1)
g.Assert(out.networks[0].Name).Equal("foo")
g.Assert(out.networks[0].Driver).Equal("overlay")
})
g.It("should unmarshal named", func() {
in := []byte("foo: { name: bar }")
out := networkList{}
err := yaml.Unmarshal(in, &out)
if err != nil {
g.Fail(err)
}
g.Assert(len(out.networks)).Equal(1)
g.Assert(out.networks[0].Name).Equal("bar")
})
g.It("should unmarshal and use default driver", func() {
in := []byte("foo: { name: bar }")
out := volumeList{}
err := yaml.Unmarshal(in, &out)
if err != nil {
g.Fail(err)
}
g.Assert(len(out.volumes)).Equal(1)
g.Assert(out.volumes[0].Driver).Equal("local")
})
})
})
}

146
engine/compiler/parse/node_root.go
View file

@ -1,146 +0,0 @@
package parse
// RootNode is the root node in the parsed Yaml file.
type RootNode struct {
NodeType
Platform string
Base string
Path string
Image string
Pod Node
Build Node
Clone Node
Script []Node
Volumes []Node
Networks []Node
Services []Node
}
// NewRootNode returns a new root node.
func NewRootNode() *RootNode {
return &RootNode{
NodeType: NodeRoot,
}
}
// Root returns the root node.
func (n *RootNode) Root() *RootNode { return n }
// Returns a new Volume Node.
func (n *RootNode) NewVolumeNode(name string) *VolumeNode {
return &VolumeNode{
NodeType: NodeVolume,
Name: name,
root: n,
}
}
// Returns a new Network Node.
func (n *RootNode) NewNetworkNode(name string) *NetworkNode {
return &NetworkNode{
NodeType: NodeNetwork,
Name: name,
root: n,
}
}
// Returns a new Network Node.
func (n *RootNode) NewBuildNode(context string) *BuildNode {
return &BuildNode{
NodeType: NodeBuild,
Context: context,
root: n,
}
}
// Returns a new Container Plugin Node.
func (n *RootNode) NewPluginNode() *ContainerNode {
return &ContainerNode{
NodeType: NodePlugin,
root: n,
}
}
// Returns a new Container Shell Node.
func (n *RootNode) NewShellNode() *ContainerNode {
return &ContainerNode{
NodeType: NodeShell,
root: n,
}
}
// Returns a new Container Service Node.
func (n *RootNode) NewServiceNode() *ContainerNode {
return &ContainerNode{
NodeType: NodeService,
root: n,
}
}
// Returns a new Container Clone Node.
func (n *RootNode) NewCloneNode() *ContainerNode {
return &ContainerNode{
NodeType: NodeClone,
root: n,
}
}
// Returns a new Container Cache Node.
func (n *RootNode) NewCacheNode() *ContainerNode {
return &ContainerNode{
NodeType: NodeCache,
root: n,
}
}
// Returns a new Container Node.
func (n *RootNode) NewContainerNode() *ContainerNode {
return &ContainerNode{
NodeType: NodeContainer,
root: n,
}
}
// Walk is a function that walk through all child nodes of the RootNode
// and invokes the Walk callback function for each Node.
func (n *RootNode) Walk(fn WalkFunc) (err error) {
var nodes []Node
nodes = append(nodes, n)
nodes = append(nodes, n.Build)
nodes = append(nodes, n.Clone)
nodes = append(nodes, n.Script...)
nodes = append(nodes, n.Volumes...)
nodes = append(nodes, n.Networks...)
nodes = append(nodes, n.Services...)
for _, node := range nodes {
err = fn(node)
if err != nil {
return
}
}
return
}
type WalkFunc func(Node) error
//
// intermediate types for yaml decoding.
//
type root struct {
Workspace struct {
Path string
Base string
}
Image string
Platform string
Volumes volumeList
Networks networkList
Services containerList
Script containerList `yaml:"pipeline"`
Cache container
Clone container
Build build
}

85
engine/compiler/parse/node_root_test.go
View file

@ -1,85 +0,0 @@
package parse
import (
"testing"
"github.com/franela/goblin"
)
func TestRootNode(t *testing.T) {
g := goblin.Goblin(t)
r := &RootNode{}
g.Describe("Root Node", func() {
g.It("should return self as root", func() {
g.Assert(r).Equal(r.Root())
})
g.It("should create a Volume Node", func() {
n := r.NewVolumeNode("foo")
g.Assert(n.Root()).Equal(r)
g.Assert(n.Name).Equal("foo")
g.Assert(n.String()).Equal(NodeVolume)
g.Assert(n.Type()).Equal(NodeType(NodeVolume))
})
g.It("should create a Network Node", func() {
n := r.NewNetworkNode("foo")
g.Assert(n.Root()).Equal(r)
g.Assert(n.Name).Equal("foo")
g.Assert(n.String()).Equal(NodeNetwork)
g.Assert(n.Type()).Equal(NodeType(NodeNetwork))
})
g.It("should create a Plugin Node", func() {
n := r.NewPluginNode()
g.Assert(n.Root()).Equal(r)
g.Assert(n.String()).Equal(NodePlugin)
g.Assert(n.Type()).Equal(NodeType(NodePlugin))
})
g.It("should create a Shell Node", func() {
n := r.NewShellNode()
g.Assert(n.Root()).Equal(r)
g.Assert(n.String()).Equal(NodeShell)
g.Assert(n.Type()).Equal(NodeType(NodeShell))
})
g.It("should create a Service Node", func() {
n := r.NewServiceNode()
g.Assert(n.Root()).Equal(r)
g.Assert(n.String()).Equal(NodeService)
g.Assert(n.Type()).Equal(NodeType(NodeService))
})
g.It("should create a Build Node", func() {
n := r.NewBuildNode(".")
g.Assert(n.Root()).Equal(r)
g.Assert(n.Context).Equal(".")
g.Assert(n.String()).Equal(NodeBuild)
g.Assert(n.Type()).Equal(NodeType(NodeBuild))
})
g.It("should create a Cache Node", func() {
n := r.NewCacheNode()
g.Assert(n.Root()).Equal(r)
g.Assert(n.String()).Equal(NodeCache)
g.Assert(n.Type()).Equal(NodeType(NodeCache))
})
g.It("should create a Clone Node", func() {
n := r.NewCloneNode()
g.Assert(n.Root()).Equal(r)
g.Assert(n.String()).Equal(NodeClone)
g.Assert(n.Type()).Equal(NodeType(NodeClone))
})
g.It("should create a Container Node", func() {
n := r.NewContainerNode()
g.Assert(n.Root()).Equal(r)
g.Assert(n.String()).Equal(NodeContainer)
g.Assert(n.Type()).Equal(NodeType(NodeContainer))
})
})
}

69
engine/compiler/parse/node_volume.go
View file

@ -1,69 +0,0 @@
package parse
import (
"fmt"
"gopkg.in/yaml.v2"
)
// VolumeNode represents a Docker volume.
type VolumeNode struct {
NodeType
root *RootNode
Name string
Driver string
DriverOpts map[string]string
External bool
}
// Root returns the root node.
func (n *VolumeNode) Root() *RootNode { return n.root }
//
// intermediate types for yaml decoding.
//
// volume is an intermediate type used for decoding a volumes in a format
// compatible with docker-compose.yml
type volume struct {
Name string
Driver string
DriverOpts map[string]string `yaml:"driver_opts"`
}
// volumeList is an intermediate type used for decoding a slice of volumes
// in a format compatible with docker-compose.yml
type volumeList struct {
volumes []*volume
}
func (v *volumeList) UnmarshalYAML(unmarshal func(interface{}) error) error {
slice := yaml.MapSlice{}
err := unmarshal(&slice)
if err != nil {
return err
}
for _, s := range slice {
vv := volume{}
out, err := yaml.Marshal(s.Value)
if err != nil {
return err
}
err = yaml.Unmarshal(out, &vv)
if err != nil {
return err
}
if vv.Name == "" {
vv.Name = fmt.Sprintf("%v", s.Key)
}
if vv.Driver == "" {
vv.Driver = "local"
}
v.volumes = append(v.volumes, &vv)
}
return err
}

51
engine/compiler/parse/node_volume_test.go
View file

@ -1,51 +0,0 @@
package parse
import (
"testing"
"github.com/franela/goblin"
"gopkg.in/yaml.v2"
)
func TestVolumeNode(t *testing.T) {
g := goblin.Goblin(t)
g.Describe("Volumes", func() {
g.Describe("given a yaml file", func() {
g.It("should unmarshal", func() {
in := []byte("foo: { driver: blockbridge }")
out := volumeList{}
err := yaml.Unmarshal(in, &out)
if err != nil {
g.Fail(err)
}
g.Assert(len(out.volumes)).Equal(1)
g.Assert(out.volumes[0].Name).Equal("foo")
g.Assert(out.volumes[0].Driver).Equal("blockbridge")
})
g.It("should unmarshal named", func() {
in := []byte("foo: { name: bar }")
out := volumeList{}
err := yaml.Unmarshal(in, &out)
if err != nil {
g.Fail(err)
}
g.Assert(len(out.volumes)).Equal(1)
g.Assert(out.volumes[0].Name).Equal("bar")
})
g.It("should unmarshal and use default driver", func() {
in := []byte("foo: { name: bar }")
out := volumeList{}
err := yaml.Unmarshal(in, &out)
if err != nil {
g.Fail(err)
}
g.Assert(len(out.volumes)).Equal(1)
g.Assert(out.volumes[0].Driver).Equal("local")
})
})
})
}

90
engine/compiler/parse/parse.go
View file

@ -1,90 +0,0 @@
package parse
import (
"gopkg.in/yaml.v2"
)
// Parse parses a Yaml file and returns a Tree structure.
func Parse(in []byte) (*RootNode, error) {
out := root{}
err := yaml.Unmarshal(in, &out)
if err != nil {
return nil, err
}
root := NewRootNode()
root.Platform = out.Platform
root.Path = out.Workspace.Path
root.Base = out.Workspace.Base
root.Image = out.Image
// append volume nodes to tree
for _, v := range out.Volumes.volumes {
vv := root.NewVolumeNode(v.Name)
vv.Driver = v.Driver
vv.DriverOpts = v.DriverOpts
root.Volumes = append(root.Volumes, vv)
}
// append network nodes to tree
for _, n := range out.Networks.networks {
nn := root.NewNetworkNode(n.Name)
nn.Driver = n.Driver
nn.DriverOpts = n.DriverOpts
root.Networks = append(root.Networks, nn)
}
// add the build section
if out.Build.Context != "" {
root.Build = &BuildNode{
NodeType: NodeBuild,
Context: out.Build.Context,
Dockerfile: out.Build.Dockerfile,
Args: out.Build.Args,
root: root,
}
}
// add the clone section
{
cc := root.NewCloneNode()
cc.Conditions = out.Clone.ToConditions()
cc.Container = out.Clone.ToContainer()
cc.Container.Name = "clone"
cc.Vargs = out.Clone.Vargs
root.Clone = cc
}
// append services
for _, c := range out.Services.containers {
if c.Build != "" {
continue
}
cc := root.NewServiceNode()
cc.Conditions = c.ToConditions()
cc.Container = c.ToContainer()
root.Services = append(root.Services, cc)
}
// append scripts
for _, c := range out.Script.containers {
var cc *ContainerNode
if len(c.Commands.parts) == 0 {
cc = root.NewPluginNode()
} else {
cc = root.NewShellNode()
}
cc.Commands = c.Commands.parts
cc.Vargs = c.Vargs
cc.Container = c.ToContainer()
cc.Conditions = c.ToConditions()
root.Script = append(root.Script, cc)
}
return root, nil
}
// ParseString parses a Yaml string and returns a Tree structure.
func ParseString(in string) (*RootNode, error) {
return Parse([]byte(in))
}

95
engine/compiler/parse/parse_test.go
View file

@ -1,95 +0,0 @@
package parse
import (
"testing"
"github.com/franela/goblin"
)
func TestParse(t *testing.T) {
g := goblin.Goblin(t)
g.Describe("Parser", func() {
g.Describe("given a yaml file", func() {
g.It("should unmarshal a string", func() {
out, err := ParseString(sampleYaml)
if err != nil {
g.Fail(err)
}
g.Assert(out.Image).Equal("hello-world")
g.Assert(out.Base).Equal("/go")
g.Assert(out.Path).Equal("src/github.com/octocat/hello-world")
g.Assert(out.Build.(*BuildNode).Context).Equal(".")
g.Assert(out.Build.(*BuildNode).Dockerfile).Equal("Dockerfile")
g.Assert(out.Clone.(*ContainerNode).Container.Image).Equal("git")
g.Assert(out.Clone.(*ContainerNode).Vargs["depth"]).Equal(1)
g.Assert(out.Volumes[0].(*VolumeNode).Name).Equal("custom")
g.Assert(out.Volumes[0].(*VolumeNode).Driver).Equal("blockbridge")
g.Assert(out.Networks[0].(*NetworkNode).Name).Equal("custom")
g.Assert(out.Networks[0].(*NetworkNode).Driver).Equal("overlay")
g.Assert(out.Services[0].(*ContainerNode).Container.Name).Equal("database")
g.Assert(out.Services[0].(*ContainerNode).Container.Image).Equal("mysql")
g.Assert(out.Script[0].(*ContainerNode).Container.Name).Equal("test")
g.Assert(out.Script[0].(*ContainerNode).Container.Image).Equal("golang")
g.Assert(out.Script[0].(*ContainerNode).Commands).Equal([]string{"go install", "go test"})
g.Assert(out.Script[0].(*ContainerNode).String()).Equal(NodeShell)
g.Assert(out.Script[1].(*ContainerNode).Container.Name).Equal("build")
g.Assert(out.Script[1].(*ContainerNode).Container.Image).Equal("golang")
g.Assert(out.Script[1].(*ContainerNode).Commands).Equal([]string{"go build"})
g.Assert(out.Script[1].(*ContainerNode).String()).Equal(NodeShell)
g.Assert(out.Script[2].(*ContainerNode).Container.Name).Equal("notify")
g.Assert(out.Script[2].(*ContainerNode).Container.Image).Equal("slack")
g.Assert(out.Script[2].(*ContainerNode).String()).Equal(NodePlugin)
})
})
})
}
var sampleYaml = `
image: hello-world
build:
context: .
dockerfile: Dockerfile
workspace:
path: src/github.com/octocat/hello-world
base: /go
clone:
image: git
depth: 1
cache:
mount: node_modules
pipeline:
test:
image: golang
commands:
- go install
- go test
build:
image: golang
commands:
- go build
when:
event: push
notify:
image: slack
channel: dev
when:
event: failure
services:
database:
image: mysql
networks:
custom:
driver: overlay
volumes:
custom:
driver: blockbridge
`

55
engine/compiler/parse/types.go
View file

@ -1,55 +0,0 @@
package parse
import "strings"
// mapEqualSlice represents a map[string]string or a slice of
// strings in key=value format.
type mapEqualSlice struct {
parts map[string]string
}
func (s *mapEqualSlice) UnmarshalYAML(unmarshal func(interface{}) error) error {
s.parts = map[string]string{}
err := unmarshal(&s.parts)
if err == nil {
return nil
}
var slice []string
err = unmarshal(&slice)
if err != nil {
return err
}
for _, v := range slice {
parts := strings.SplitN(v, "=", 2)
if len(parts) == 2 {
key := parts[0]
val := parts[1]
s.parts[key] = val
}
}
return nil
}
// stringOrSlice represents a string or an array of strings.
type stringOrSlice struct {
parts []string
}
func (s *stringOrSlice) UnmarshalYAML(unmarshal func(interface{}) error) error {
var sliceType []string
err := unmarshal(&sliceType)
if err == nil {
s.parts = sliceType
return nil
}
var stringType string
err = unmarshal(&stringType)
if err == nil {
sliceType = make([]string, 0, 1)
s.parts = append(sliceType, string(stringType))
return nil
}
return err
}

75
engine/compiler/parse/types_test.go
View file

@ -1,75 +0,0 @@
package parse
import (
"testing"
"github.com/franela/goblin"
"gopkg.in/yaml.v2"
)
func TestTypes(t *testing.T) {
g := goblin.Goblin(t)
g.Describe("Yaml types", func() {
g.Describe("given a yaml file", func() {
g.It("should unmarshal a string", func() {
in := []byte("foo")
out := stringOrSlice{}
err := yaml.Unmarshal(in, &out)
if err != nil {
g.Fail(err)
}
g.Assert(len(out.parts)).Equal(1)
g.Assert(out.parts[0]).Equal("foo")
})
g.It("should unmarshal a string slice", func() {
in := []byte("[ foo ]")
out := stringOrSlice{}
err := yaml.Unmarshal(in, &out)
if err != nil {
g.Fail(err)
}
g.Assert(len(out.parts)).Equal(1)
g.Assert(out.parts[0]).Equal("foo")
})
g.It("should throw error when invalid string slice", func() {
in := []byte("{ }") // string value should fail parse
out := stringOrSlice{}
err := yaml.Unmarshal(in, &out)
g.Assert(err != nil).IsTrue("expects error")
})
g.It("should unmarshal a map", func() {
in := []byte("foo: bar")
out := mapEqualSlice{}
err := yaml.Unmarshal(in, &out)
if err != nil {
g.Fail(err)
}
g.Assert(len(out.parts)).Equal(1)
g.Assert(out.parts["foo"]).Equal("bar")
})
g.It("should unmarshal a map equal slice", func() {
in := []byte("[ foo=bar ]")
out := mapEqualSlice{}
err := yaml.Unmarshal(in, &out)
if err != nil {
g.Fail(err)
}
g.Assert(len(out.parts)).Equal(1)
g.Assert(out.parts["foo"]).Equal("bar")
})
g.It("should throw error when invalid map equal slice", func() {
in := []byte("foo") // string value should fail parse
out := mapEqualSlice{}
err := yaml.Unmarshal(in, &out)
g.Assert(err != nil).IsTrue("expects error")
})
})
})
}

13
engine/compiler/transform.go
View file

@ -1,13 +0,0 @@
package compiler
import "github.com/drone/drone/engine/compiler/parse"
// Transform is used to transform nodes from the parsed Yaml file during the
// compilation process. A Transform may be used to add, disable or alter nodes.
type Transform interface {
VisitRoot(*parse.RootNode) error
VisitVolume(*parse.VolumeNode) error
VisitNetwork(*parse.NetworkNode) error
VisitBuild(*parse.BuildNode) error
VisitContainer(*parse.ContainerNode) error
}

72
engine/runner/container.go
View file

@ -1,72 +0,0 @@
package runner
import "fmt"
// Container defines the container configuration.
type Container struct {
Name string `json:"name"`
Alias string `json:"alias"`
Image string `json:"image"`
Pull bool `json:"pull,omitempty"`
AuthConfig Auth `json:"auth_config,omitempty"`
Privileged bool `json:"privileged,omitempty"`
WorkingDir string `json:"working_dir,omitempty"`
Environment map[string]string `json:"environment,omitempty"`
Entrypoint []string `json:"entrypoint,omitempty"`
Command []string `json:"command,omitempty"`
ExtraHosts []string `json:"extra_hosts,omitempty"`
Volumes []string `json:"volumes,omitempty"`
VolumesFrom []string `json:"volumes_from,omitempty"`
Devices []string `json:"devices,omitempty"`
Network string `json:"network_mode,omitempty"`
DNS []string `json:"dns,omitempty"`
DNSSearch []string `json:"dns_search,omitempty"`
MemSwapLimit int64 `json:"memswap_limit,omitempty"`
MemLimit int64 `json:"mem_limit,omitempty"`
CPUQuota int64 `json:"cpu_quota,omitempty"`
CPUShares int64 `json:"cpu_shares,omitempty"`
CPUSet string `json:"cpuset,omitempty"`
OomKillDisable bool `json:"oom_kill_disable,omitempty"`
}
// Validate validates the container configuration details and returns an error
// if the validation fails.
func (c *Container) Validate() error {
switch {
case c.Name == "":
return fmt.Errorf("Missing container name")
case c.Image == "":
return fmt.Errorf("Missing container image")
default:
return nil
}
}
// Auth provides authentication parameters to authenticate to a remote
// container registry for image download.
type Auth struct {
Username string `json:"username,omitempty"`
Password string `json:"password,omitempty"`
Email string `json:"email,omitempty"`
Token string `json:"registry_token,omitempty"`
}
// Volume defines a container volume.
type Volume struct {
Name string `json:"name,omitempty"`
Alias string `json:"alias,omitempty"`
Driver string `json:"driver,omitempty"`
DriverOpts map[string]string `json:"driver_opts,omitempty"`
External bool `json:"external,omitempty"`
}
// Network defines a container network.
type Network struct {
Name string `json:"name,omitempty"`
Alias string `json:"alias,omitempty"`
Driver string `json:"driver,omitempty"`
DriverOpts map[string]string `json:"driver_opts,omitempty"`
External bool `json:"external,omitempty"`
}

40
engine/runner/container_test.go
View file

@ -1,40 +0,0 @@
package runner
import (
"testing"
"github.com/franela/goblin"
)
func TestContainer(t *testing.T) {
g := goblin.Goblin(t)
g.Describe("Container validation", func() {
g.It("fails with an invalid name", func() {
c := Container{
Image: "golang:1.5",
}
err := c.Validate()
g.Assert(err != nil).IsTrue()
g.Assert(err.Error()).Equal("Missing container name")
})
g.It("fails with an invalid image", func() {
c := Container{
Name: "container_0",
}
err := c.Validate()
g.Assert(err != nil).IsTrue()
g.Assert(err.Error()).Equal("Missing container image")
})
g.It("passes with valid attributes", func() {
c := Container{
Name: "container_0",
Image: "golang:1.5",
}
g.Assert(c.Validate() == nil).IsTrue()
})
})
}

24
engine/runner/docker/context.go
View file

@ -1,24 +0,0 @@
package docker
import (
"github.com/drone/drone/engine/runner"
"golang.org/x/net/context"
)
const key = "docker"
// Setter defines a context that enables setting values.
type Setter interface {
Set(string, interface{})
}
// FromContext returns the Engine associated with this context.
func FromContext(c context.Context) runner.Engine {
return c.Value(key).(runner.Engine)
}
// ToContext adds the Engine to this context if it supports the
// Setter interface.
func ToContext(c Setter, d runner.Engine) {
c.Set(key, d)
}

111
engine/runner/docker/docker.go
View file

@ -1,111 +0,0 @@
package docker
import (
"io"
"github.com/drone/drone/engine/runner"
"github.com/drone/drone/engine/runner/docker/internal"
"github.com/samalba/dockerclient"
)
type dockerEngine struct {
client dockerclient.Client
}
func (e *dockerEngine) ContainerStart(container *runner.Container) (string, error) {
conf := toContainerConfig(container)
auth := toAuthConfig(container)
// pull the image if it does not exists or if the Container
// is configured to always pull a new image.
_, err := e.client.InspectImage(container.Image)
if err != nil || container.Pull {
e.client.PullImage(container.Image, auth)
}
// create and start the container and return the Container ID.
id, err := e.client.CreateContainer(conf, container.Name, auth)
if err != nil {
return id, err
}
err = e.client.StartContainer(id, &conf.HostConfig)
if err != nil {
// remove the container if it cannot be started
e.client.RemoveContainer(id, true, true)
return id, err
}
return id, nil
}
func (e *dockerEngine) ContainerStop(id string) error {
e.client.StopContainer(id, 1)
e.client.KillContainer(id, "9")
return nil
}
func (e *dockerEngine) ContainerRemove(id string) error {
e.client.StopContainer(id, 1)
e.client.KillContainer(id, "9")
e.client.RemoveContainer(id, true, true)
return nil
}
func (e *dockerEngine) ContainerWait(id string) (*runner.State, error) {
// wait for the container to exit
//
// TODO(bradrydzewski) we should have a for loop here
// to re-connect and wait if this channel returns a
// result even though the container is still running.
//
<-e.client.Wait(id)
v, err := e.client.InspectContainer(id)
if err != nil {
return nil, err
}
return &runner.State{
ExitCode: v.State.ExitCode,
OOMKilled: v.State.OOMKilled,
}, nil
}
func (e *dockerEngine) ContainerLogs(id string) (io.ReadCloser, error) {
opts := &dockerclient.LogOptions{
Follow: true,
Stdout: true,
Stderr: true,
}
piper, pipew := io.Pipe()
go func() {
defer pipew.Close()
// sometimes the docker logs fails due to parsing errors. this
// routine will check for such a failure and attempt to resume
// if necessary.
for i := 0; i < 5; i++ {
if i > 0 {
opts.Tail = 1
}
rc, err := e.client.ContainerLogs(id, opts)
if err != nil {
return
}
defer rc.Close()
// use Docker StdCopy
internal.StdCopy(pipew, pipew, rc)
// check to see if the container is still running. If not,
// we can safely exit and assume there are no more logs left
// to stream.
v, err := e.client.InspectContainer(id)
if err != nil || !v.State.Running {
return
}
}
}()
return piper, nil
}

1
engine/runner/docker/docker_test.go
View file

@ -1 +0,0 @@
package docker

49
engine/runner/docker/helper.go
View file

@ -1,49 +0,0 @@
package docker
import (
"os"
"github.com/drone/drone/engine/runner"
"github.com/samalba/dockerclient"
)
var (
dockerHost = os.Getenv("DOCKER_HOST")
dockerCert = os.Getenv("DOCKER_CERT_PATH")
dockerTLS = os.Getenv("DOCKER_TLS_VERIFY")
)
func init() {
if dockerHost == "" {
dockerHost = "unix:///var/run/docker.sock"
}
}
// New returns a new Docker engine using the provided Docker client.
func New(client dockerclient.Client) runner.Engine {
return &dockerEngine{client}
}
// NewEnv returns a new Docker engine from the DOCKER_HOST and DOCKER_CERT_PATH
// environment variables.
func NewEnv() (runner.Engine, error) {
config, err := dockerclient.TLSConfigFromCertPath(dockerCert)
if err == nil && dockerTLS != "1" {
config.InsecureSkipVerify = true
}
client, err := dockerclient.NewDockerClient(dockerHost, config)
if err != nil {
return nil, err
}
return New(client), nil
}
// MustEnv returns a new Docker engine from the DOCKER_HOST and DOCKER_CERT_PATH
// environment variables. Errors creating the Docker engine will panic.
func MustEnv() runner.Engine {
engine, err := NewEnv()
if err != nil {
panic(err)
}
return engine
}

1
engine/runner/docker/helper_test.go
View file

@ -1 +0,0 @@
package docker

1
engine/runner/docker/internal/README
View file

@ -1 +0,0 @@
This is an internal copy of the Docker stdcopy package that removes the logrus debug logging. The original package is found at https://github.com/docker/docker/tree/master/pkg/stdcopy

167
engine/runner/docker/internal/stdcopy.go
View file

@ -1,167 +0,0 @@
package internal
import (
"encoding/binary"
"errors"
"fmt"
"io"
)
// StdType is the type of standard stream
// a writer can multiplex to.
type StdType byte
const (
// Stdin represents standard input stream type.
Stdin StdType = iota
// Stdout represents standard output stream type.
Stdout
// Stderr represents standard error steam type.
Stderr
stdWriterPrefixLen = 8
stdWriterFdIndex = 0
stdWriterSizeIndex = 4
startingBufLen = 32*1024 + stdWriterPrefixLen + 1
)
// stdWriter is wrapper of io.Writer with extra customized info.
type stdWriter struct {
io.Writer
prefix byte
}
// Write sends the buffer to the underneath writer.
// It insert the prefix header before the buffer,
// so stdcopy.StdCopy knows where to multiplex the output.
// It makes stdWriter to implement io.Writer.
func (w *stdWriter) Write(buf []byte) (n int, err error) {
if w == nil || w.Writer == nil {
return 0, errors.New("Writer not instantiated")
}
if buf == nil {
return 0, nil
}
header := [stdWriterPrefixLen]byte{stdWriterFdIndex: w.prefix}
binary.BigEndian.PutUint32(header[stdWriterSizeIndex:], uint32(len(buf)))
line := append(header[:], buf...)
n, err = w.Writer.Write(line)
n -= stdWriterPrefixLen
if n < 0 {
n = 0
}
return
}
// NewStdWriter instantiates a new Writer.
// Everything written to it will be encapsulated using a custom format,
// and written to the underlying `w` stream.
// This allows multiple write streams (e.g. stdout and stderr) to be muxed into a single connection.
// `t` indicates the id of the stream to encapsulate.
// It can be stdcopy.Stdin, stdcopy.Stdout, stdcopy.Stderr.
func NewStdWriter(w io.Writer, t StdType) io.Writer {
return &stdWriter{
Writer: w,
prefix: byte(t),
}
}
// StdCopy is a modified version of io.Copy.
//
// StdCopy will demultiplex `src`, assuming that it contains two streams,
// previously multiplexed together using a StdWriter instance.
// As it reads from `src`, StdCopy will write to `dstout` and `dsterr`.
//
// StdCopy will read until it hits EOF on `src`. It will then return a nil error.
// In other words: if `err` is non nil, it indicates a real underlying error.
//
// `written` will hold the total number of bytes written to `dstout` and `dsterr`.
func StdCopy(dstout, dsterr io.Writer, src io.Reader) (written int64, err error) {
var (
buf = make([]byte, startingBufLen)
bufLen = len(buf)
nr, nw int
er, ew error
out io.Writer
frameSize int
)
for {
// Make sure we have at least a full header
for nr < stdWriterPrefixLen {
var nr2 int
nr2, er = src.Read(buf[nr:])
nr += nr2
if er == io.EOF {
if nr < stdWriterPrefixLen {
return written, nil
}
break
}
if er != nil {
return 0, er
}
}
// Check the first byte to know where to write
switch StdType(buf[stdWriterFdIndex]) {
case Stdin:
fallthrough
case Stdout:
// Write on stdout
out = dstout
case Stderr:
// Write on stderr
out = dsterr
default:
return 0, fmt.Errorf("Unrecognized input header: %d", buf[stdWriterFdIndex])
}
// Retrieve the size of the frame
frameSize = int(binary.BigEndian.Uint32(buf[stdWriterSizeIndex : stdWriterSizeIndex+4]))
// Check if the buffer is big enough to read the frame.
// Extend it if necessary.
if frameSize+stdWriterPrefixLen > bufLen {
buf = append(buf, make([]byte, frameSize+stdWriterPrefixLen-bufLen+1)...)
bufLen = len(buf)
}
// While the amount of bytes read is less than the size of the frame + header, we keep reading
for nr < frameSize+stdWriterPrefixLen {
var nr2 int
nr2, er = src.Read(buf[nr:])
nr += nr2
if er == io.EOF {
if nr < frameSize+stdWriterPrefixLen {
return written, nil
}
break
}
if er != nil {
return 0, er
}
}
// Write the retrieved frame (without header)
nw, ew = out.Write(buf[stdWriterPrefixLen : frameSize+stdWriterPrefixLen])
if ew != nil {
return 0, ew
}
// If the frame has not been fully written: error
if nw != frameSize {
return 0, io.ErrShortWrite
}
written += int64(nw)
// Move the rest of the buffer to the beginning
copy(buf, buf[frameSize+stdWriterPrefixLen:])
// Move the index
nr -= frameSize + stdWriterPrefixLen
}
}

260
engine/runner/docker/internal/stdcopy_test.go
View file

@ -1,260 +0,0 @@
package internal
import (
"bytes"
"errors"
"io"
"io/ioutil"
"strings"
"testing"
)
func TestNewStdWriter(t *testing.T) {
writer := NewStdWriter(ioutil.Discard, Stdout)
if writer == nil {
t.Fatalf("NewStdWriter with an invalid StdType should not return nil.")
}
}
func TestWriteWithUnitializedStdWriter(t *testing.T) {
writer := stdWriter{
Writer: nil,
prefix: byte(Stdout),
}
n, err := writer.Write([]byte("Something here"))
if n != 0 || err == nil {
t.Fatalf("Should fail when given an uncomplete or uninitialized StdWriter")
}
}
func TestWriteWithNilBytes(t *testing.T) {
writer := NewStdWriter(ioutil.Discard, Stdout)
n, err := writer.Write(nil)
if err != nil {
t.Fatalf("Shouldn't have fail when given no data")
}
if n > 0 {
t.Fatalf("Write should have written 0 byte, but has written %d", n)
}
}
func TestWrite(t *testing.T) {
writer := NewStdWriter(ioutil.Discard, Stdout)
data := []byte("Test StdWrite.Write")
n, err := writer.Write(data)
if err != nil {
t.Fatalf("Error while writing with StdWrite")
}
if n != len(data) {
t.Fatalf("Write should have written %d byte but wrote %d.", len(data), n)
}
}
type errWriter struct {
n int
err error
}
func (f *errWriter) Write(buf []byte) (int, error) {
return f.n, f.err
}
func TestWriteWithWriterError(t *testing.T) {
expectedError := errors.New("expected")
expectedReturnedBytes := 10
writer := NewStdWriter(&errWriter{
n: stdWriterPrefixLen + expectedReturnedBytes,
err: expectedError}, Stdout)
data := []byte("This won't get written, sigh")
n, err := writer.Write(data)
if err != expectedError {
t.Fatalf("Didn't get expected error.")
}
if n != expectedReturnedBytes {
t.Fatalf("Didn't get expected written bytes %d, got %d.",
expectedReturnedBytes, n)
}
}
func TestWriteDoesNotReturnNegativeWrittenBytes(t *testing.T) {
writer := NewStdWriter(&errWriter{n: -1}, Stdout)
data := []byte("This won't get written, sigh")
actual, _ := writer.Write(data)
if actual != 0 {
t.Fatalf("Expected returned written bytes equal to 0, got %d", actual)
}
}
func getSrcBuffer(stdOutBytes, stdErrBytes []byte) (buffer *bytes.Buffer, err error) {
buffer = new(bytes.Buffer)
dstOut := NewStdWriter(buffer, Stdout)
_, err = dstOut.Write(stdOutBytes)
if err != nil {
return
}
dstErr := NewStdWriter(buffer, Stderr)
_, err = dstErr.Write(stdErrBytes)
return
}
func TestStdCopyWriteAndRead(t *testing.T) {
stdOutBytes := []byte(strings.Repeat("o", startingBufLen))
stdErrBytes := []byte(strings.Repeat("e", startingBufLen))
buffer, err := getSrcBuffer(stdOutBytes, stdErrBytes)
if err != nil {
t.Fatal(err)
}
written, err := StdCopy(ioutil.Discard, ioutil.Discard, buffer)
if err != nil {
t.Fatal(err)
}
expectedTotalWritten := len(stdOutBytes) + len(stdErrBytes)
if written != int64(expectedTotalWritten) {
t.Fatalf("Expected to have total of %d bytes written, got %d", expectedTotalWritten, written)
}
}
type customReader struct {
n int
err error
totalCalls int
correctCalls int
src *bytes.Buffer
}
func (f *customReader) Read(buf []byte) (int, error) {
f.totalCalls++
if f.totalCalls <= f.correctCalls {
return f.src.Read(buf)
}
return f.n, f.err
}
func TestStdCopyReturnsErrorReadingHeader(t *testing.T) {
expectedError := errors.New("error")
reader := &customReader{
err: expectedError}
written, err := StdCopy(ioutil.Discard, ioutil.Discard, reader)
if written != 0 {
t.Fatalf("Expected 0 bytes read, got %d", written)
}
if err != expectedError {
t.Fatalf("Didn't get expected error")
}
}
func TestStdCopyReturnsErrorReadingFrame(t *testing.T) {
expectedError := errors.New("error")
stdOutBytes := []byte(strings.Repeat("o", startingBufLen))
stdErrBytes := []byte(strings.Repeat("e", startingBufLen))
buffer, err := getSrcBuffer(stdOutBytes, stdErrBytes)
if err != nil {
t.Fatal(err)
}
reader := &customReader{
correctCalls: 1,
n: stdWriterPrefixLen + 1,
err: expectedError,
src: buffer}
written, err := StdCopy(ioutil.Discard, ioutil.Discard, reader)
if written != 0 {
t.Fatalf("Expected 0 bytes read, got %d", written)
}
if err != expectedError {
t.Fatalf("Didn't get expected error")
}
}
func TestStdCopyDetectsCorruptedFrame(t *testing.T) {
stdOutBytes := []byte(strings.Repeat("o", startingBufLen))
stdErrBytes := []byte(strings.Repeat("e", startingBufLen))
buffer, err := getSrcBuffer(stdOutBytes, stdErrBytes)
if err != nil {
t.Fatal(err)
}
reader := &customReader{
correctCalls: 1,
n: stdWriterPrefixLen + 1,
err: io.EOF,
src: buffer}
written, err := StdCopy(ioutil.Discard, ioutil.Discard, reader)
if written != startingBufLen {
t.Fatalf("Expected %d bytes read, got %d", startingBufLen, written)
}
if err != nil {
t.Fatal("Didn't get nil error")
}
}
func TestStdCopyWithInvalidInputHeader(t *testing.T) {
dstOut := NewStdWriter(ioutil.Discard, Stdout)
dstErr := NewStdWriter(ioutil.Discard, Stderr)
src := strings.NewReader("Invalid input")
_, err := StdCopy(dstOut, dstErr, src)
if err == nil {
t.Fatal("StdCopy with invalid input header should fail.")
}
}
func TestStdCopyWithCorruptedPrefix(t *testing.T) {
data := []byte{0x01, 0x02, 0x03}
src := bytes.NewReader(data)
written, err := StdCopy(nil, nil, src)
if err != nil {
t.Fatalf("StdCopy should not return an error with corrupted prefix.")
}
if written != 0 {
t.Fatalf("StdCopy should have written 0, but has written %d", written)
}
}
func TestStdCopyReturnsWriteErrors(t *testing.T) {
stdOutBytes := []byte(strings.Repeat("o", startingBufLen))
stdErrBytes := []byte(strings.Repeat("e", startingBufLen))
buffer, err := getSrcBuffer(stdOutBytes, stdErrBytes)
if err != nil {
t.Fatal(err)
}
expectedError := errors.New("expected")
dstOut := &errWriter{err: expectedError}
written, err := StdCopy(dstOut, ioutil.Discard, buffer)
if written != 0 {
t.Fatalf("StdCopy should have written 0, but has written %d", written)
}
if err != expectedError {
t.Fatalf("Didn't get expected error, got %v", err)
}
}
func TestStdCopyDetectsNotFullyWrittenFrames(t *testing.T) {
stdOutBytes := []byte(strings.Repeat("o", startingBufLen))
stdErrBytes := []byte(strings.Repeat("e", startingBufLen))
buffer, err := getSrcBuffer(stdOutBytes, stdErrBytes)
if err != nil {
t.Fatal(err)
}
dstOut := &errWriter{n: startingBufLen - 10}
written, err := StdCopy(dstOut, ioutil.Discard, buffer)
if written != 0 {
t.Fatalf("StdCopy should have return 0 written bytes, but returned %d", written)
}
if err != io.ErrShortWrite {
t.Fatalf("Didn't get expected io.ErrShortWrite error")
}
}
func BenchmarkWrite(b *testing.B) {
w := NewStdWriter(ioutil.Discard, Stdout)
data := []byte("Test line for testing stdwriter performance\n")
data = bytes.Repeat(data, 100)
b.SetBytes(int64(len(data)))
b.ResetTimer()
for i := 0; i < b.N; i++ {
if _, err := w.Write(data); err != nil {
b.Fatal(err)
}
}
}

102
engine/runner/docker/util.go
View file

@ -1,102 +0,0 @@
package docker
import (
"fmt"
"strings"
"github.com/drone/drone/engine/runner"
"github.com/samalba/dockerclient"
)
// helper function that converts the Continer data structure to the exepcted
// dockerclient.ContainerConfig.
func toContainerConfig(c *runner.Container) *dockerclient.ContainerConfig {
config := &dockerclient.ContainerConfig{
Image: c.Image,
Env: toEnvironmentSlice(c.Environment),
Cmd: c.Command,
Entrypoint: c.Entrypoint,
WorkingDir: c.WorkingDir,
HostConfig: dockerclient.HostConfig{
Privileged: c.Privileged,
NetworkMode: c.Network,
Memory: c.MemLimit,
CpuShares: c.CPUShares,
CpuQuota: c.CPUQuota,
CpusetCpus: c.CPUSet,
MemorySwappiness: -1,
OomKillDisable: c.OomKillDisable,
},
}
if len(config.Entrypoint) == 0 {
config.Entrypoint = nil
}
if len(config.Cmd) == 0 {
config.Cmd = nil
}
if len(c.ExtraHosts) > 0 {
config.HostConfig.ExtraHosts = c.ExtraHosts
}
if len(c.DNS) != 0 {
config.HostConfig.Dns = c.DNS
}
if len(c.DNSSearch) != 0 {
config.HostConfig.DnsSearch = c.DNSSearch
}
if len(c.VolumesFrom) != 0 {
config.HostConfig.VolumesFrom = c.VolumesFrom
}
config.Volumes = map[string]struct{}{}
for _, path := range c.Volumes {
if strings.Index(path, ":") == -1 {
config.Volumes[path] = struct{}{}
continue
}
parts := strings.Split(path, ":")
config.Volumes[parts[1]] = struct{}{}
config.HostConfig.Binds = append(config.HostConfig.Binds, path)
}
for _, path := range c.Devices {
if strings.Index(path, ":") == -1 {
continue
}
parts := strings.Split(path, ":")
device := dockerclient.DeviceMapping{
PathOnHost: parts[0],
PathInContainer: parts[1],
CgroupPermissions: "rwm",
}
config.HostConfig.Devices = append(config.HostConfig.Devices, device)
}
return config
}
// helper function that converts the AuthConfig data structure to the exepcted
// dockerclient.AuthConfig.
func toAuthConfig(container *runner.Container) *dockerclient.AuthConfig {
if container.AuthConfig.Username == "" &&
container.AuthConfig.Password == "" &&
container.AuthConfig.Token == "" {
return nil
}
return &dockerclient.AuthConfig{
Email: container.AuthConfig.Email,
Username: container.AuthConfig.Username,
Password: container.AuthConfig.Password,
RegistryToken: container.AuthConfig.Token,
}
}
// helper function that converts a key value map of environment variables to a
// string slice in key=value format.
func toEnvironmentSlice(env map[string]string) []string {
var envs []string
for k, v := range env {
envs = append(envs, fmt.Sprintf("%s=%s", k, v))
}
return envs
}

24
engine/runner/docker/util_test.go
View file

@ -1,24 +0,0 @@
package docker
import (
"testing"
)
func Test_toContainerConfig(t *testing.T) {
t.Skip()
}
func Test_toAuthConfig(t *testing.T) {
t.Skip()
}
func Test_toEnvironmentSlice(t *testing.T) {
env := map[string]string{
"HOME": "/root",
}
envs := toEnvironmentSlice(env)
want, got := "HOME=/root", envs[0]
if want != got {
t.Errorf("Wanted envar %s got %s", want, got)
}
}

22
engine/runner/engine.go
View file

@ -1,22 +0,0 @@
package runner
//go:generate mockery -name Engine -output mock -case=underscore
import "io"
// Engine defines the container runtime engine.
type Engine interface {
// VolumeCreate(*Volume) (string, error)
// VolumeRemove(string) error
ContainerStart(*Container) (string, error)
ContainerStop(string) error
ContainerRemove(string) error
ContainerWait(string) (*State, error)
ContainerLogs(string) (io.ReadCloser, error)
}
// State defines the state of the container.
type State struct {
ExitCode int // container exit code
OOMKilled bool // container exited due to oom error
}

37
engine/runner/error.go
View file

@ -1,37 +0,0 @@
package runner
import (
"errors"
"fmt"
)
var (
// ErrSkip is used as a return value when container execution should be
// skipped at runtime. It is not returned as an error by any function.
ErrSkip = errors.New("Skip")
// ErrTerm is used as a return value when the runner should terminate
// execution and exit. It is not returned as an error by any function.
ErrTerm = errors.New("Terminate")
)
// An ExitError reports an unsuccessful exit.
type ExitError struct {
Name string
Code int
}
// Error reteurns the error message in string format.
func (e *ExitError) Error() string {
return fmt.Sprintf("%s : exit code %d", e.Name, e.Code)
}
// An OomError reports the process received an OOMKill from the kernel.
type OomError struct {
Name string
}
// Error reteurns the error message in string format.
func (e *OomError) Error() string {
return fmt.Sprintf("%s : received oom kill", e.Name)
}

26
engine/runner/error_test.go
View file

@ -1,26 +0,0 @@
package runner
import (
"testing"
"github.com/franela/goblin"
)
func TestErrors(t *testing.T) {
g := goblin.Goblin(t)
g.Describe("Error messages", func() {
g.It("should include OOM details", func() {
err := OomError{Name: "golang"}
got, want := err.Error(), "golang : received oom kill"
g.Assert(got).Equal(want)
})
g.It("should include Exit code", func() {
err := ExitError{Name: "golang", Code: 255}
got, want := err.Error(), "golang : exit code 255"
g.Assert(got).Equal(want)
})
})
}

24
engine/runner/helper.go
View file

@ -1,24 +0,0 @@
package runner
import (
"encoding/json"
"io/ioutil"
)
// Parse parses a raw file containing a JSON encoded format of an intermediate
// representation of the pipeline.
func Parse(data []byte) (*Spec, error) {
v := &Spec{}
err := json.Unmarshal(data, v)
return v, err
}
// ParseFile parses a file containing a JSON encoded format of an intermediate
// representation of the pipeline.
func ParseFile(filename string) (*Spec, error) {
out, err := ioutil.ReadFile(filename)
if err != nil {
return nil, err
}
return Parse(out)
}

97
engine/runner/helper_test.go
View file

@ -1,97 +0,0 @@
package runner
import (
"io/ioutil"
"os"
"testing"
"github.com/franela/goblin"
)
func TestHelper(t *testing.T) {
g := goblin.Goblin(t)
g.Describe("Parsing", func() {
g.It("should unmarhsal file []byte", func() {
res, err := Parse(sample)
if err != nil {
t.Error(err)
return
}
g.Assert(err == nil).IsTrue("expect file parsed")
g.Assert(len(res.Containers)).Equal(2)
g.Assert(len(res.Volumes)).Equal(1)
})
g.It("should unmarshal from file", func() {
temp, _ := ioutil.TempFile("", "spec_")
defer os.Remove(temp.Name())
ioutil.WriteFile(temp.Name(), sample, 0700)
_, err := ParseFile(temp.Name())
if err != nil {
t.Error(err)
return
}
g.Assert(err == nil).IsTrue("expect file parsed")
})
g.It("should error when file not found", func() {
_, err := ParseFile("/tmp/foo/bar/dummy/file.json")
g.Assert(err == nil).IsFalse("expect file not found error")
})
})
}
// invalid json representation, simulate parsing error
var invalid = []byte(`[]`)
// valid json representation, verify parsing
var sample = []byte(`{
"containers": [
{
"name": "container_0",
"image": "node:latest"
},
{
"name": "container_1",
"image": "golang:latest"
}
],
"volumes": [
{
"name": "volume_0"
}
],
"program": {
"type": "list",
"body": [
{
"type": "defer",
"body": {
"type": "recover",
"body": {
"type": "run",
"name": "container_0"
}
},
"defer": {
"type": "parallel",
"body": [
{
"type": "run",
"name": "container_1"
},
{
"type": "run",
"name": "container_1"
}
],
"limit": 2
}
}
]
}
}`)

30
engine/runner/parse/node.go
View file

@ -1,30 +0,0 @@
package parse
const (
NodeList = "list"
NodeDefer = "defer"
NodeError = "error"
NodeRecover = "recover"
NodeParallel = "parallel"
NodeRun = "run"
)
// NodeType identifies the type of a parse tree node.
type NodeType string
// Type returns itself and provides an easy default implementation
// for embedding in a Node. Embedded in all non-trivial Nodes.
func (t NodeType) Type() NodeType {
return t
}
// String returns the string value of the Node type.
func (t NodeType) String() string {
return string(t)
}
// A Node is an element in the parse tree.
type Node interface {
Type() NodeType
Validate() error
}

40
engine/runner/parse/node_defer.go
View file

@ -1,40 +0,0 @@
package parse
import "fmt"
// DeferNode executes the child node, and then executes the deffered node.
// The deffered node is guaranteed to execute, even when the child node fails.
type DeferNode struct {
NodeType `json:"type"`
Body Node `json:"body"` // evaluate node
Defer Node `json:"defer"` // defer evaluation of node.
}
// NewDeferNode returns a new DeferNode.
func NewDeferNode() *DeferNode {
return &DeferNode{NodeType: NodeDefer}
}
func (n *DeferNode) SetBody(node Node) *DeferNode {
n.Body = node
return n
}
func (n *DeferNode) SetDefer(node Node) *DeferNode {
n.Defer = node
return n
}
func (n *DeferNode) Validate() error {
switch {
case n.NodeType != NodeDefer:
return fmt.Errorf("Defer Node uses an invalid type")
case n.Body == nil:
return fmt.Errorf("Defer Node body is empty")
case n.Defer == nil:
return fmt.Errorf("Defer Node defer is empty")
default:
return nil
}
}

56
engine/runner/parse/node_defer_test.go
View file

@ -1,56 +0,0 @@
package parse
import (
"testing"
"github.com/franela/goblin"
)
func TestDeferNode(t *testing.T) {
g := goblin.Goblin(t)
g.Describe("DeferNode", func() {
g.It("should set body and defer node", func() {
node0 := NewRunNode()
node1 := NewRunNode()
defer0 := NewDeferNode()
defer1 := defer0.SetBody(node0)
defer2 := defer0.SetDefer(node1)
g.Assert(defer0.Type().String()).Equal(NodeDefer)
g.Assert(defer0.Body).Equal(node0)
g.Assert(defer0.Defer).Equal(node1)
g.Assert(defer0).Equal(defer1)
g.Assert(defer0).Equal(defer2)
})
g.It("should fail validation when invalid type", func() {
defer0 := DeferNode{}
err := defer0.Validate()
g.Assert(err == nil).IsFalse()
g.Assert(err.Error()).Equal("Defer Node uses an invalid type")
})
g.It("should fail validation when empty body", func() {
defer0 := NewDeferNode()
err := defer0.Validate()
g.Assert(err == nil).IsFalse()
g.Assert(err.Error()).Equal("Defer Node body is empty")
})
g.It("should fail validation when empty defer", func() {
defer0 := NewDeferNode()
defer0.SetBody(NewRunNode())
err := defer0.Validate()
g.Assert(err == nil).IsFalse()
g.Assert(err.Error()).Equal("Defer Node defer is empty")
})
g.It("should pass validation", func() {
defer0 := NewDeferNode()
defer0.SetBody(NewRunNode())
defer0.SetDefer(NewRunNode())
g.Assert(defer0.Validate() == nil).IsTrue()
})
})
}

40
engine/runner/parse/node_error.go
View file

@ -1,40 +0,0 @@
package parse
import "fmt"
// ErrorNode executes the body node, and then executes the error node if
// the body node errors. This is similar to defer but only executes on error.
type ErrorNode struct {
NodeType `json:"type"`
Body Node `json:"body"` // evaluate node
Defer Node `json:"defer"` // defer evaluation of node on error.
}
// NewErrorNode returns a new ErrorNode.
func NewErrorNode() *ErrorNode {
return &ErrorNode{NodeType: NodeError}
}
func (n *ErrorNode) SetBody(node Node) *ErrorNode {
n.Body = node
return n
}
func (n *ErrorNode) SetDefer(node Node) *ErrorNode {
n.Defer = node
return n
}
func (n *ErrorNode) Validate() error {
switch {
case n.NodeType != NodeError:
return fmt.Errorf("Error Node uses an invalid type")
case n.Body == nil:
return fmt.Errorf("Error Node body is empty")
case n.Defer == nil:
return fmt.Errorf("Error Node defer is empty")
default:
return nil
}
}

56
engine/runner/parse/node_error_test.go
View file

@ -1,56 +0,0 @@
package parse
import (
"testing"
"github.com/franela/goblin"
)
func TestErrorNode(t *testing.T) {
g := goblin.Goblin(t)
g.Describe("ErrorNode", func() {
g.It("should set body and error node", func() {
node0 := NewRunNode()
node1 := NewRunNode()
error0 := NewErrorNode()
error1 := error0.SetBody(node0)
error2 := error0.SetDefer(node1)
g.Assert(error0.Type().String()).Equal(NodeError)
g.Assert(error0.Body).Equal(node0)
g.Assert(error0.Defer).Equal(node1)
g.Assert(error0).Equal(error1)
g.Assert(error0).Equal(error2)
})
g.It("should fail validation when invalid type", func() {
error0 := ErrorNode{}
err := error0.Validate()
g.Assert(err == nil).IsFalse()
g.Assert(err.Error()).Equal("Error Node uses an invalid type")
})
g.It("should fail validation when empty body", func() {
error0 := NewErrorNode()
err := error0.Validate()
g.Assert(err == nil).IsFalse()
g.Assert(err.Error()).Equal("Error Node body is empty")
})
g.It("should fail validation when empty error", func() {
error0 := NewErrorNode()
error0.SetBody(NewRunNode())
err := error0.Validate()
g.Assert(err == nil).IsFalse()
g.Assert(err.Error()).Equal("Error Node defer is empty")
})
g.It("should pass validation", func() {
error0 := NewErrorNode()
error0.SetBody(NewRunNode())
error0.SetDefer(NewRunNode())
g.Assert(error0.Validate() == nil).IsTrue()
})
})
}

33
engine/runner/parse/node_list.go
View file

@ -1,33 +0,0 @@
package parse
import "fmt"
// ListNode serially executes a list of child nodes.
type ListNode struct {
NodeType `json:"type"`
// Body is the list of child nodes
Body []Node `json:"body"`
}
// NewListNode returns a new ListNode.
func NewListNode() *ListNode {
return &ListNode{NodeType: NodeList}
}
// Append appens a child node to the list.
func (n *ListNode) Append(node Node) *ListNode {
n.Body = append(n.Body, node)
return n
}
func (n *ListNode) Validate() error {
switch {
case n.NodeType != NodeList:
return fmt.Errorf("List Node uses an invalid type")
case len(n.Body) == 0:
return fmt.Errorf("List Node body is empty")
default:
return nil
}
}

44
engine/runner/parse/node_list_test.go
View file

@ -1,44 +0,0 @@
package parse
import (
"testing"
"github.com/franela/goblin"
)
func TestListNode(t *testing.T) {
g := goblin.Goblin(t)
g.Describe("ListNode", func() {
g.It("should append nodes", func() {
node := NewRunNode()
list0 := NewListNode()
list1 := list0.Append(node)
g.Assert(list0.Type().String()).Equal(NodeList)
g.Assert(list0.Body[0]).Equal(node)
g.Assert(list0).Equal(list1)
})
g.It("should fail validation when invalid type", func() {
list := ListNode{}
err := list.Validate()
g.Assert(err == nil).IsFalse()
g.Assert(err.Error()).Equal("List Node uses an invalid type")
})
g.It("should fail validation when empty body", func() {
list := NewListNode()
err := list.Validate()
g.Assert(err == nil).IsFalse()
g.Assert(err.Error()).Equal("List Node body is empty")
})
g.It("should pass validation", func() {
node := NewRunNode()
list := NewListNode()
list.Append(node)
g.Assert(list.Validate() == nil).IsTrue()
})
})
}

36
engine/runner/parse/node_parallel.go
View file

@ -1,36 +0,0 @@
package parse
import "fmt"
// ParallelNode executes a list of child nodes in parallel.
type ParallelNode struct {
NodeType `json:"type"`
Body []Node `json:"body"` // nodes for parallel evaluation.
Limit int `json:"limit"` // limit for parallel evaluation.
}
func NewParallelNode() *ParallelNode {
return &ParallelNode{NodeType: NodeParallel}
}
func (n *ParallelNode) Append(node Node) *ParallelNode {
n.Body = append(n.Body, node)
return n
}
func (n *ParallelNode) SetLimit(limit int) *ParallelNode {
n.Limit = limit
return n
}
func (n *ParallelNode) Validate() error {
switch {
case n.NodeType != NodeParallel:
return fmt.Errorf("Parallel Node uses an invalid type")
case len(n.Body) == 0:
return fmt.Errorf("Parallel Node body is empty")
default:
return nil
}
}

42
engine/runner/parse/node_parallel_test.go
View file

@ -1,42 +0,0 @@
package parse
import (
"testing"
"github.com/franela/goblin"
)
func TestParallelNode(t *testing.T) {
g := goblin.Goblin(t)
g.Describe("ParallelNode", func() {
g.It("should append nodes", func() {
node := NewRunNode()
parallel0 := NewParallelNode()
parallel1 := parallel0.Append(node)
g.Assert(parallel0.Type().String()).Equal(NodeParallel)
g.Assert(parallel0.Body[0]).Equal(node)
g.Assert(parallel0).Equal(parallel1)
})
g.It("should fail validation when invalid type", func() {
node := ParallelNode{}
err := node.Validate()
g.Assert(err == nil).IsFalse()
g.Assert(err.Error()).Equal("Parallel Node uses an invalid type")
})
g.It("should fail validation when empty body", func() {
node := NewParallelNode()
err := node.Validate()
g.Assert(err == nil).IsFalse()
g.Assert(err.Error()).Equal("Parallel Node body is empty")
})
g.It("should pass validation", func() {
node := NewParallelNode().Append(NewRunNode())
g.Assert(node.Validate() == nil).IsTrue()
})
})
}

29
engine/runner/parse/node_recover.go
View file

@ -1,29 +0,0 @@
package parse
import "fmt"
type RecoverNode struct {
NodeType `json:"type"`
Body Node `json:"body"` // evaluate node and catch all errors.
}
func NewRecoverNode() *RecoverNode {
return &RecoverNode{NodeType: NodeRecover}
}
func (n *RecoverNode) SetBody(node Node) *RecoverNode {
n.Body = node
return n
}
func (n *RecoverNode) Validate() error {
switch {
case n.NodeType != NodeRecover:
return fmt.Errorf("Recover Node uses an invalid type")
case n.Body == nil:
return fmt.Errorf("Recover Node body is empty")
default:
return nil
}
}

43
engine/runner/parse/node_recover_test.go
View file

@ -1,43 +0,0 @@
package parse
import (
"testing"
"github.com/franela/goblin"
)
func TestRecoverNode(t *testing.T) {
g := goblin.Goblin(t)
g.Describe("RecoverNode", func() {
g.It("should set body", func() {
node0 := NewRunNode()
recover0 := NewRecoverNode()
recover1 := recover0.SetBody(node0)
g.Assert(recover0.Type().String()).Equal(NodeRecover)
g.Assert(recover0.Body).Equal(node0)
g.Assert(recover0).Equal(recover1)
})
g.It("should fail validation when invalid type", func() {
recover0 := RecoverNode{}
err := recover0.Validate()
g.Assert(err == nil).IsFalse()
g.Assert(err.Error()).Equal("Recover Node uses an invalid type")
})
g.It("should fail validation when empty body", func() {
recover0 := NewRecoverNode()
err := recover0.Validate()
g.Assert(err == nil).IsFalse()
g.Assert(err.Error()).Equal("Recover Node body is empty")
})
g.It("should pass validation", func() {
recover0 := NewRecoverNode()
recover0.SetBody(NewRunNode())
g.Assert(recover0.Validate() == nil).IsTrue()
})
})
}

41
engine/runner/parse/node_run.go
View file

@ -1,41 +0,0 @@
package parse
import "fmt"
type RunNode struct {
NodeType `json:"type"`
Name string `json:"name"`
Detach bool `json:"detach,omitempty"`
Silent bool `json:"silent,omitempty"`
}
func (n *RunNode) SetName(name string) *RunNode {
n.Name = name
return n
}
func (n *RunNode) SetDetach(detach bool) *RunNode {
n.Detach = detach
return n
}
func (n *RunNode) SetSilent(silent bool) *RunNode {
n.Silent = silent
return n
}
func NewRunNode() *RunNode {
return &RunNode{NodeType: NodeRun}
}
func (n *RunNode) Validate() error {
switch {
case n.NodeType != NodeRun:
return fmt.Errorf("Run Node uses an invalid type")
case n.Name == "":
return fmt.Errorf("Run Node has an invalid name")
default:
return nil
}
}

41
engine/runner/parse/node_run_test.go
View file

@ -1,41 +0,0 @@
package parse
import (
"testing"
"github.com/franela/goblin"
)
func TestRunNode(t *testing.T) {
g := goblin.Goblin(t)
g.Describe("RunNode", func() {
g.It("should set container name for lookup", func() {
node0 := NewRunNode()
node1 := node0.SetName("foo")
g.Assert(node0.Type().String()).Equal(NodeRun)
g.Assert(node0.Name).Equal("foo")
g.Assert(node0).Equal(node1)
})
g.It("should fail validation when invalid type", func() {
node := RunNode{}
err := node.Validate()
g.Assert(err == nil).IsFalse()
g.Assert(err.Error()).Equal("Run Node uses an invalid type")
})
g.It("should fail validation when invalid name", func() {
node := NewRunNode()
err := node.Validate()
g.Assert(err == nil).IsFalse()
g.Assert(err.Error()).Equal("Run Node has an invalid name")
})
g.It("should pass validation", func() {
node := NewRunNode().SetName("foo")
g.Assert(node.Validate() == nil).IsTrue()
})
})
}

221
engine/runner/parse/parse.go
View file

@ -1,221 +0,0 @@
package parse
import "encoding/json"
// Tree is the intermediate representation of a pipeline.
type Tree struct {
*ListNode // top-level Tree node
}
// New allocates a new Tree.
func NewTree() *Tree {
return &Tree{
NewListNode(),
}
}
// Parse parses a JSON encoded Tree.
func Parse(data []byte) (*Tree, error) {
tree := &Tree{}
err := tree.UnmarshalJSON(data)
return tree, err
}
// MarshalJSON implements the Marshaler interface and returns
// a JSON encoded representation of the Tree.
func (t *Tree) MarshalJSON() ([]byte, error) {
return json.Marshal(t.ListNode)
}
// UnmarshalJSON implements the Unmarshaler interface and returns
// a Tree from a JSON representation.
func (t *Tree) UnmarshalJSON(data []byte) error {
block, err := decodeList(data)
if err != nil {
return nil
}
t.ListNode = block.(*ListNode)
return nil
}
//
// below are custom decoding functions. We cannot use the default json
// decoder because the tree structure uses interfaces and the json decoder
// has difficulty ascertaining the interface type when decoding.
//
func decodeNode(data []byte) (Node, error) {
node := &nodeType{}
err := json.Unmarshal(data, node)
if err != nil {
return nil, err
}
switch node.Type {
case NodeList:
return decodeList(data)
case NodeDefer:
return decodeDefer(data)
case NodeError:
return decodeError(data)
case NodeRecover:
return decodeRecover(data)
case NodeParallel:
return decodeParallel(data)
case NodeRun:
return decodeRun(data)
}
return nil, nil
}
func decodeNodes(data []json.RawMessage) ([]Node, error) {
var nodes []Node
for _, d := range data {
node, err := decodeNode(d)
if err != nil {
return nil, err
}
nodes = append(nodes, node)
}
return nodes, nil
}
func decodeList(data []byte) (Node, error) {
v := &nodeList{}
err := json.Unmarshal(data, v)
if err != nil {
return nil, err
}
b, err := decodeNodes(v.Body)
if err != nil {
return nil, err
}
n := NewListNode()
n.Body = b
return n, nil
}
func decodeDefer(data []byte) (Node, error) {
v := &nodeDefer{}
err := json.Unmarshal(data, v)
if err != nil {
return nil, err
}
b, err := decodeNode(v.Body)
if err != nil {
return nil, err
}
d, err := decodeNode(v.Defer)
if err != nil {
return nil, err
}
n := NewDeferNode()
n.Body = b
n.Defer = d
return n, nil
}
func decodeError(data []byte) (Node, error) {
v := &nodeError{}
err := json.Unmarshal(data, v)
if err != nil {
return nil, err
}
b, err := decodeNode(v.Body)
if err != nil {
return nil, err
}
d, err := decodeNode(v.Defer)
if err != nil {
return nil, err
}
n := NewErrorNode()
n.Body = b
n.Defer = d
return n, nil
}
func decodeRecover(data []byte) (Node, error) {
v := &nodeRecover{}
err := json.Unmarshal(data, v)
if err != nil {
return nil, err
}
b, err := decodeNode(v.Body)
if err != nil {
return nil, err
}
n := NewRecoverNode()
n.Body = b
return n, nil
}
func decodeParallel(data []byte) (Node, error) {
v := &nodeParallel{}
err := json.Unmarshal(data, v)
if err != nil {
return nil, err
}
b, err := decodeNodes(v.Body)
if err != nil {
return nil, err
}
n := NewParallelNode()
n.Body = b
n.Limit = v.Limit
return n, nil
}
func decodeRun(data []byte) (Node, error) {
v := &nodeRun{}
err := json.Unmarshal(data, v)
if err != nil {
return nil, err
}
return &RunNode{NodeRun, v.Name, v.Detach, v.Silent}, nil
}
//
// below are intermediate representations of the node structures
// since we cannot simply encode / decode using the built-in json
// encoding and decoder.
//
type nodeType struct {
Type NodeType `json:"type"`
}
type nodeDefer struct {
Type NodeType `json:"type"`
Body json.RawMessage `json:"body"`
Defer json.RawMessage `json:"defer"`
}
type nodeError struct {
Type NodeType `json:"type"`
Body json.RawMessage `json:"body"`
Defer json.RawMessage `json:"defer"`
}
type nodeList struct {
Type NodeType `json:"type"`
Body []json.RawMessage `json:"body"`
}
type nodeRecover struct {
Type NodeType `json:"type"`
Body json.RawMessage `json:"body"`
}
type nodeParallel struct {
Type NodeType `json:"type"`
Body []json.RawMessage `json:"body"`
Limit int `json:"limit"`
}
type nodeRun struct {
Type NodeType `json:"type"`
Name string `json:"name"`
Detach bool `json:"detach,omitempty"`
Silent bool `json:"silent,omitempty"`
}

80
engine/runner/parse/parse_test.go
View file

@ -1,80 +0,0 @@
package parse
import (
"bytes"
"encoding/json"
"reflect"
"testing"
)
func TestUnmarshal(t *testing.T) {
node1 := NewRunNode().SetName("foo")
node2 := NewRecoverNode().SetBody(node1)
node3 := NewRunNode().SetName("bar")
node4 := NewRunNode().SetName("bar")
node5 := NewParallelNode().
Append(node3).
Append(node4).
SetLimit(2)
node6 := NewDeferNode().
SetBody(node2).
SetDefer(node5)
tree := NewTree()
tree.Append(node6)
encoded, err := json.MarshalIndent(tree, "", "\t")
if err != nil {
t.Error(err)
}
if !bytes.Equal(encoded, sample) {
t.Errorf("Want to marshal Tree to %s, got %s",
string(sample),
string(encoded),
)
}
parsed, err := Parse(encoded)
if err != nil {
t.Error(err)
}
if !reflect.DeepEqual(tree, parsed) {
t.Errorf("Want to marsnal and then unmarshal Tree")
}
}
var sample = []byte(`{
"type": "list",
"body": [
{
"type": "defer",
"body": {
"type": "recover",
"body": {
"type": "run",
"name": "foo"
}
},
"defer": {
"type": "parallel",
"body": [
{
"type": "run",
"name": "bar"
},
{
"type": "run",
"name": "bar"
}
],
"limit": 2
}
}
]
}`)

49
engine/runner/pipe.go
View file

@ -1,49 +0,0 @@
package runner
import "fmt"
// Pipe returns a buffered pipe that is connected to the console output.
type Pipe struct {
lines chan *Line
eof chan bool
}
// Next returns the next Line of console output.
func (p *Pipe) Next() *Line {
select {
case line := <-p.lines:
return line
case <-p.eof:
return nil
}
}
// Close closes the pipe of console output.
func (p *Pipe) Close() {
go func() {
p.eof <- true
}()
}
func newPipe(buffer int) *Pipe {
return &Pipe{
lines: make(chan *Line, buffer),
eof: make(chan bool),
}
}
// Line is a line of console output.
type Line struct {
Proc string `json:"proc,omitempty"`
Time int64 `json:"time,omitempty"`
Type int `json:"type,omitempty"`
Pos int `json:"pos,omityempty"`
Out string `json:"out,omitempty"`
}
func (l *Line) String() string {
return fmt.Sprintf("[%s:L%v:%vs] %s", l.Proc, l.Pos, l.Time, l.Out)
}
// TODO(bradrydzewski) consider an alternate buffer impelmentation based on the
// x.crypto ssh buffer https://github.com/golang/crypto/blob/master/ssh/buffer.go

54
engine/runner/pipe_test.go
View file

@ -1,54 +0,0 @@
package runner
import (
"sync"
"testing"
"github.com/franela/goblin"
)
func TestPipe(t *testing.T) {
g := goblin.Goblin(t)
g.Describe("Pipe", func() {
g.It("should get next line from buffer", func() {
line := &Line{
Proc: "redis",
Pos: 1,
Out: "starting redis server",
}
pipe := newPipe(10)
pipe.lines <- line
next := pipe.Next()
g.Assert(next).Equal(line)
})
g.It("should get null line on buffer closed", func() {
pipe := newPipe(10)
var wg sync.WaitGroup
wg.Add(1)
go func() {
next := pipe.Next()
g.Assert(next == nil).IsTrue("line should be nil")
wg.Done()
}()
pipe.Close()
wg.Wait()
})
g.Describe("Line output", func() {
g.It("should prefix string() with metadata", func() {
line := Line{
Proc: "redis",
Time: 60,
Pos: 1,
Out: "starting redis server",
}
g.Assert(line.String()).Equal("[redis:L1:60s] starting redis server")
})
})
})
}

245
engine/runner/runner.go
View file

@ -1,245 +0,0 @@
package runner
import (
"bufio"
"fmt"
"time"
"github.com/drone/drone/engine/runner/parse"
"golang.org/x/net/context"
)
// NoContext is the default context you should supply if not using your own
// context.Context
var NoContext = context.TODO()
// Tracer defines a tracing function that is invoked prior to creating and
// running the container.
type Tracer func(c *Container) error
// Config defines the configuration for creating the Runner.
type Config struct {
Tracer Tracer
Engine Engine
// Buffer defines the size of the buffer for the channel to which the
// console output is streamed.
Buffer uint
}
// Runner creates a build Runner using the specific configuration for the given
// Context and Specification.
func (c *Config) Runner(ctx context.Context, spec *Spec) *Runner {
// TODO(bradyrdzewski) we should make a copy of the configuration parameters
// instead of a direct reference. This helps avoid any race conditions or
//unexpected behavior if the Config changes.
return &Runner{
ctx: ctx,
conf: c,
spec: spec,
errc: make(chan error),
pipe: newPipe(int(c.Buffer) + 1),
}
}
type Runner struct {
ctx context.Context
conf *Config
spec *Spec
pipe *Pipe
errc chan (error)
containers []string
volumes []string
networks []string
}
// Run starts the build runner but does not wait for it to complete. The Wait
// method will return the exit code and release associated resources once the
// running containers exit.
func (r *Runner) Run() {
go func() {
r.setup()
err := r.exec(r.spec.Nodes.ListNode)
r.pipe.Close()
r.cancel()
r.teardown()
r.errc <- err
}()
go func() {
<-r.ctx.Done()
r.cancel()
}()
}
// Wait waits for the runner to exit.
func (r *Runner) Wait() error {
return <-r.errc
}
// Pipe returns a Pipe that is connected to the console output stream.
func (r *Runner) Pipe() *Pipe {
return r.pipe
}
func (r *Runner) exec(node parse.Node) error {
switch v := node.(type) {
case *parse.ListNode:
return r.execList(v)
case *parse.DeferNode:
return r.execDefer(v)
case *parse.ErrorNode:
return r.execError(v)
case *parse.RecoverNode:
return r.execRecover(v)
case *parse.ParallelNode:
return r.execParallel(v)
case *parse.RunNode:
return r.execRun(v)
}
return fmt.Errorf("runner: unexepected node %s", node)
}
func (r *Runner) execList(node *parse.ListNode) error {
for _, n := range node.Body {
err := r.exec(n)
if err != nil {
return err
}
}
return nil
}
func (r *Runner) execDefer(node *parse.DeferNode) error {
err1 := r.exec(node.Body)
err2 := r.exec(node.Defer)
if err1 != nil {
return err1
}
return err2
}
func (r *Runner) execError(node *parse.ErrorNode) error {
err := r.exec(node.Body)
if err != nil {
r.exec(node.Defer)
}
return err
}
func (r *Runner) execRecover(node *parse.RecoverNode) error {
r.exec(node.Body)
return nil
}
func (r *Runner) execParallel(node *parse.ParallelNode) error {
errc := make(chan error)
for _, n := range node.Body {
go func(node parse.Node) {
errc <- r.exec(node)
}(n)
}
var err error
for i := 0; i < len(node.Body); i++ {
select {
case cerr := <-errc:
if cerr != nil {
err = cerr
}
}
}
return err
}
func (r *Runner) execRun(node *parse.RunNode) error {
container, err := r.spec.lookupContainer(node.Name)
if err != nil {
return err
}
if r.conf.Tracer != nil {
err := r.conf.Tracer(container)
switch {
case err == ErrSkip:
return nil
case err != nil:
return err
}
}
// TODO(bradrydzewski) there is potential here for a race condition where
// the context is cancelled just after this line, resulting in the container
// still being started.
if r.ctx.Err() != nil {
return err
}
name, err := r.conf.Engine.ContainerStart(container)
if err != nil {
return err
}
r.containers = append(r.containers, name)
go func() {
if node.Silent {
return
}
rc, err := r.conf.Engine.ContainerLogs(name)
if err != nil {
return
}
defer rc.Close()
num := 0
now := time.Now().UTC()
scanner := bufio.NewScanner(rc)
for scanner.Scan() {
r.pipe.lines <- &Line{
Proc: container.Alias,
Time: int64(time.Since(now).Seconds()),
Pos: num,
Out: scanner.Text(),
}
num++
}
}()
// exit when running container in detached mode in background
if node.Detach {
return nil
}
state, err := r.conf.Engine.ContainerWait(name)
if err != nil {
return err
}
if state.OOMKilled {
return &OomError{name}
} else if state.ExitCode != 0 {
return &ExitError{name, state.ExitCode}
}
return nil
}
func (r *Runner) setup() {
// this is where we will setup network and volumes
}
func (r *Runner) teardown() {
// TODO(bradrydzewski) this is not yet thread safe.
for _, container := range r.containers {
r.conf.Engine.ContainerRemove(container)
}
}
func (r *Runner) cancel() {
// TODO(bradrydzewski) this is not yet thread safe.
for _, container := range r.containers {
r.conf.Engine.ContainerStop(container)
}
}

7
engine/runner/runner_test.go
View file

@ -1,7 +0,0 @@
package runner
import "testing"
func TestRunner(t *testing.T) {
t.Skip()
}

33
engine/runner/spec.go
View file

@ -1,33 +0,0 @@
package runner
import (
"fmt"
"github.com/drone/drone/engine/runner/parse"
)
// Spec defines the pipeline configuration and exeuction.
type Spec struct {
// Volumes defines a list of all container volumes.
Volumes []*Volume `json:"volumes,omitempty"`
// Networks defines a list of all container networks.
Networks []*Network `json:"networks,omitempty"`
// Containers defines a list of all containers in the pipeline.
Containers []*Container `json:"containers,omitempty"`
// Nodes defines the container execution tree.
Nodes *parse.Tree `json:"program,omitempty"`
}
// lookupContainer is a helper funciton that returns the named container from
// the slice of containers.
func (s *Spec) lookupContainer(name string) (*Container, error) {
for _, container := range s.Containers {
if container.Name == name {
return container, nil
}
}
return nil, fmt.Errorf("runner: unknown container %s", name)
}

35
engine/runner/spec_test.go
View file

@ -1,35 +0,0 @@
package runner
import (
"testing"
"github.com/franela/goblin"
)
func TestSpec(t *testing.T) {
g := goblin.Goblin(t)
g.Describe("Spec file", func() {
g.Describe("when looking up a container", func() {
spec := Spec{}
spec.Containers = append(spec.Containers, &Container{
Name: "golang",
})
g.It("should find and return the container", func() {
c, err := spec.lookupContainer("golang")
g.Assert(err == nil).IsTrue("error should be nil")
g.Assert(c).Equal(spec.Containers[0])
})
g.It("should return an error when not found", func() {
c, err := spec.lookupContainer("node")
g.Assert(err == nil).IsFalse("should return error")
g.Assert(c == nil).IsTrue("should return nil container")
})
})
})
}