From c76dd06b27cf106a021bb1e8c4a938e0cf98d48d Mon Sep 17 00:00:00 2001 From: Brad Rydzewski Date: Tue, 10 May 2016 17:07:51 -0700 Subject: [PATCH] removed insanely overly complex build runner, parser, compiler --- agent/agent.go | 3 +- engine/compiler/builtin/alias.go | 29 -- engine/compiler/builtin/args.go | 90 ------ engine/compiler/builtin/args_test.go | 46 ---- engine/compiler/builtin/build.go | 40 --- engine/compiler/builtin/clone.go | 45 --- engine/compiler/builtin/clone_test.go | 36 --- engine/compiler/builtin/envs.go | 57 ---- engine/compiler/builtin/envs_test.go | 45 --- engine/compiler/builtin/escalate.go | 30 -- engine/compiler/builtin/escalate_test.go | 54 ---- engine/compiler/builtin/filter.go | 128 --------- engine/compiler/builtin/filter_test.go | 130 --------- engine/compiler/builtin/normalize.go | 66 ----- engine/compiler/builtin/normalize_test.go | 78 ------ engine/compiler/builtin/pod.go | 50 ---- engine/compiler/builtin/pull.go | 26 -- engine/compiler/builtin/pull_test.go | 45 --- engine/compiler/builtin/secrets.go | 45 --- engine/compiler/builtin/shell.go | 95 ------- engine/compiler/builtin/shell_test.go | 44 --- engine/compiler/builtin/validate.go | 120 -------- engine/compiler/builtin/validate_test.go | 199 -------------- engine/compiler/builtin/visitor.go | 23 -- engine/compiler/builtin/workspace.go | 51 ---- engine/compiler/builtin/workspace_test.go | 89 ------ engine/compiler/compile.go | 137 --------- engine/compiler/compile_test.go | 1 - engine/compiler/parse/node.go | 34 --- engine/compiler/parse/node_build.go | 42 --- engine/compiler/parse/node_build_test.go | 38 --- engine/compiler/parse/node_container.go | 180 ------------ engine/compiler/parse/node_container_test.go | 97 ------- engine/compiler/parse/node_network.go | 68 ----- engine/compiler/parse/node_network_test.go | 51 ---- engine/compiler/parse/node_root.go | 146 ---------- engine/compiler/parse/node_root_test.go | 85 ------ engine/compiler/parse/node_volume.go | 69 ----- engine/compiler/parse/node_volume_test.go | 51 ---- engine/compiler/parse/parse.go | 90 ------ engine/compiler/parse/parse_test.go | 95 ------- engine/compiler/parse/types.go | 55 ---- engine/compiler/parse/types_test.go | 75 ----- engine/compiler/transform.go | 13 - engine/runner/container.go | 72 ----- engine/runner/container_test.go | 40 --- engine/runner/docker/context.go | 24 -- engine/runner/docker/docker.go | 111 -------- engine/runner/docker/docker_test.go | 1 - engine/runner/docker/helper.go | 49 ---- engine/runner/docker/helper_test.go | 1 - engine/runner/docker/internal/README | 1 - engine/runner/docker/internal/stdcopy.go | 167 ----------- engine/runner/docker/internal/stdcopy_test.go | 260 ------------------ engine/runner/docker/util.go | 102 ------- engine/runner/docker/util_test.go | 24 -- engine/runner/engine.go | 22 -- engine/runner/error.go | 37 --- engine/runner/error_test.go | 26 -- engine/runner/helper.go | 24 -- engine/runner/helper_test.go | 97 ------- engine/runner/parse/node.go | 30 -- engine/runner/parse/node_defer.go | 40 --- engine/runner/parse/node_defer_test.go | 56 ---- engine/runner/parse/node_error.go | 40 --- engine/runner/parse/node_error_test.go | 56 ---- engine/runner/parse/node_list.go | 33 --- engine/runner/parse/node_list_test.go | 44 --- engine/runner/parse/node_parallel.go | 36 --- engine/runner/parse/node_parallel_test.go | 42 --- engine/runner/parse/node_recover.go | 29 -- engine/runner/parse/node_recover_test.go | 43 --- engine/runner/parse/node_run.go | 41 --- engine/runner/parse/node_run_test.go | 41 --- engine/runner/parse/parse.go | 221 --------------- engine/runner/parse/parse_test.go | 80 ------ engine/runner/pipe.go | 49 ---- engine/runner/pipe_test.go | 54 ---- engine/runner/runner.go | 245 ----------------- engine/runner/runner_test.go | 7 - engine/runner/spec.go | 33 --- engine/runner/spec_test.go | 35 --- 82 files changed, 1 insertion(+), 5303 deletions(-) delete mode 100644 engine/compiler/builtin/alias.go delete mode 100644 engine/compiler/builtin/args.go delete mode 100644 engine/compiler/builtin/args_test.go delete mode 100644 engine/compiler/builtin/build.go delete mode 100644 engine/compiler/builtin/clone.go delete mode 100644 engine/compiler/builtin/clone_test.go delete mode 100644 engine/compiler/builtin/envs.go delete mode 100644 engine/compiler/builtin/envs_test.go delete mode 100644 engine/compiler/builtin/escalate.go delete mode 100644 engine/compiler/builtin/escalate_test.go delete mode 100644 engine/compiler/builtin/filter.go delete mode 100644 engine/compiler/builtin/filter_test.go delete mode 100644 engine/compiler/builtin/normalize.go delete mode 100644 engine/compiler/builtin/normalize_test.go delete mode 100644 engine/compiler/builtin/pod.go delete mode 100644 engine/compiler/builtin/pull.go delete mode 100644 engine/compiler/builtin/pull_test.go delete mode 100644 engine/compiler/builtin/secrets.go delete mode 100644 engine/compiler/builtin/shell.go delete mode 100644 engine/compiler/builtin/shell_test.go delete mode 100644 engine/compiler/builtin/validate.go delete mode 100644 engine/compiler/builtin/validate_test.go delete mode 100644 engine/compiler/builtin/visitor.go delete mode 100644 engine/compiler/builtin/workspace.go delete mode 100644 engine/compiler/builtin/workspace_test.go delete mode 100644 engine/compiler/compile.go delete mode 100644 engine/compiler/compile_test.go delete mode 100644 engine/compiler/parse/node.go delete mode 100644 engine/compiler/parse/node_build.go delete mode 100644 engine/compiler/parse/node_build_test.go delete mode 100644 engine/compiler/parse/node_container.go delete mode 100644 engine/compiler/parse/node_container_test.go delete mode 100644 engine/compiler/parse/node_network.go delete mode 100644 engine/compiler/parse/node_network_test.go delete mode 100644 engine/compiler/parse/node_root.go delete mode 100644 engine/compiler/parse/node_root_test.go delete mode 100644 engine/compiler/parse/node_volume.go delete mode 100644 engine/compiler/parse/node_volume_test.go delete mode 100644 engine/compiler/parse/parse.go delete mode 100644 engine/compiler/parse/parse_test.go delete mode 100644 engine/compiler/parse/types.go delete mode 100644 engine/compiler/parse/types_test.go delete mode 100644 engine/compiler/transform.go delete mode 100644 engine/runner/container.go delete mode 100644 engine/runner/container_test.go delete mode 100644 engine/runner/docker/context.go delete mode 100644 engine/runner/docker/docker.go delete mode 100644 engine/runner/docker/docker_test.go delete mode 100644 engine/runner/docker/helper.go delete mode 100644 engine/runner/docker/helper_test.go delete mode 100644 engine/runner/docker/internal/README delete mode 100644 engine/runner/docker/internal/stdcopy.go delete mode 100644 engine/runner/docker/internal/stdcopy_test.go delete mode 100644 engine/runner/docker/util.go delete mode 100644 engine/runner/docker/util_test.go delete mode 100644 engine/runner/engine.go delete mode 100644 engine/runner/error.go delete mode 100644 engine/runner/error_test.go delete mode 100644 engine/runner/helper.go delete mode 100644 engine/runner/helper_test.go delete mode 100644 engine/runner/parse/node.go delete mode 100644 engine/runner/parse/node_defer.go delete mode 100644 engine/runner/parse/node_defer_test.go delete mode 100644 engine/runner/parse/node_error.go delete mode 100644 engine/runner/parse/node_error_test.go delete mode 100644 engine/runner/parse/node_list.go delete mode 100644 engine/runner/parse/node_list_test.go delete mode 100644 engine/runner/parse/node_parallel.go delete mode 100644 engine/runner/parse/node_parallel_test.go delete mode 100644 engine/runner/parse/node_recover.go delete mode 100644 engine/runner/parse/node_recover_test.go delete mode 100644 engine/runner/parse/node_run.go delete mode 100644 engine/runner/parse/node_run_test.go delete mode 100644 engine/runner/parse/parse.go delete mode 100644 engine/runner/parse/parse_test.go delete mode 100644 engine/runner/pipe.go delete mode 100644 engine/runner/pipe_test.go delete mode 100644 engine/runner/runner.go delete mode 100644 engine/runner/runner_test.go delete mode 100644 engine/runner/spec.go delete mode 100644 engine/runner/spec_test.go diff --git a/agent/agent.go b/agent/agent.go index 6a3813930..c76eca72a 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -9,7 +9,6 @@ import ( "time" "github.com/drone/drone/build" - "github.com/drone/drone/engine/runner" "github.com/drone/drone/model" "github.com/drone/drone/queue" "github.com/drone/drone/version" @@ -68,7 +67,7 @@ func (a *Agent) Run(payload *queue.Work, cancel <-chan bool) error { if err != nil { payload.Job.ExitCode = 255 } - if exitErr, ok := err.(*runner.ExitError); ok { + if exitErr, ok := err.(*build.ExitError); ok { payload.Job.ExitCode = exitErr.Code } diff --git a/engine/compiler/builtin/alias.go b/engine/compiler/builtin/alias.go deleted file mode 100644 index 002cd8fae..000000000 --- a/engine/compiler/builtin/alias.go +++ /dev/null @@ -1,29 +0,0 @@ -package builtin - -import ( - "fmt" - - "github.com/drone/drone/engine/compiler/parse" -) - -type aliasOp struct { - visitor - index map[string]string - prefix string - suffix int -} - -func NewAliasOp(prefix string) Visitor { - return &aliasOp{ - index: map[string]string{}, - prefix: prefix, - } -} - -func (v *aliasOp) VisitContainer(node *parse.ContainerNode) error { - v.suffix++ - - node.Container.Alias = node.Container.Name - node.Container.Name = fmt.Sprintf("%s_%d", v.prefix, v.suffix) - return nil -} diff --git a/engine/compiler/builtin/args.go b/engine/compiler/builtin/args.go deleted file mode 100644 index 835a1ed48..000000000 --- a/engine/compiler/builtin/args.go +++ /dev/null @@ -1,90 +0,0 @@ -package builtin - -import ( - "fmt" - "reflect" - "strconv" - "strings" - - "github.com/drone/drone/engine/compiler/parse" - - json "github.com/ghodss/yaml" - "gopkg.in/yaml.v2" -) - -type argsOps struct { - visitor -} - -// NewArgsOp returns a transformer that provides the plugin node -// with the custom arguments from the Yaml file. -func NewArgsOp() Visitor { - return &argsOps{} -} - -func (v *argsOps) VisitContainer(node *parse.ContainerNode) error { - switch node.NodeType { - case parse.NodePlugin, parse.NodeCache, parse.NodeClone: - break // no-op - default: - return nil - } - if node.Container.Environment == nil { - node.Container.Environment = map[string]string{} - } - return argsToEnv(node.Vargs, node.Container.Environment) -} - -// argsToEnv uses reflection to convert a map[string]interface to a list -// of environment variables. -func argsToEnv(from map[string]interface{}, to map[string]string) error { - - for k, v := range from { - t := reflect.TypeOf(v) - vv := reflect.ValueOf(v) - - k = "PLUGIN_" + strings.ToUpper(k) - - switch t.Kind() { - case reflect.Bool: - to[k] = strconv.FormatBool(vv.Bool()) - - case reflect.String: - to[k] = vv.String() - - case reflect.Int, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Int8: - to[k] = fmt.Sprintf("%v", vv.Int()) - - case reflect.Float32, reflect.Float64: - to[k] = fmt.Sprintf("%v", vv.Float()) - - // case reflect.Int, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Int8: - // to[k] = strconv.FormatInt(vv.Int(), 16) - - // case reflect.Float32, reflect.Float64: - // to[k] = strconv.FormatFloat(vv.Float(), 'E', -1, 64) - - case reflect.Map: - yml, _ := yaml.Marshal(vv.Interface()) - out, _ := json.YAMLToJSON(yml) - to[k] = string(out) - - case reflect.Slice: - out, _ := yaml.Marshal(vv.Interface()) - - in := []string{} - err := yaml.Unmarshal(out, &in) - if err == nil { - to[k] = strings.Join(in, ",") - } else { - out, err = json.YAMLToJSON(out) - if err != nil { - // return err TODO(bradrydzewski) unit test coverage for possible errors - } - to[k] = string(out) - } - } - } - - return nil -} diff --git a/engine/compiler/builtin/args_test.go b/engine/compiler/builtin/args_test.go deleted file mode 100644 index 1669d48c7..000000000 --- a/engine/compiler/builtin/args_test.go +++ /dev/null @@ -1,46 +0,0 @@ -package builtin - -import ( - "testing" - - "github.com/drone/drone/engine/compiler/parse" - "github.com/drone/drone/engine/runner" - - "github.com/franela/goblin" -) - -func Test_args(t *testing.T) { - - g := goblin.Goblin(t) - g.Describe("plugins arguments", func() { - - g.It("should ignore non-plugin containers", func() { - root := parse.NewRootNode() - c := root.NewShellNode() - c.Container = runner.Container{} - c.Vargs = map[string]interface{}{ - "depth": 50, - } - - ops := NewArgsOp() - ops.VisitContainer(c) - - g.Assert(c.Container.Environment["PLUGIN_DEPTH"]).Equal("") - }) - - g.It("should include args as environment variable", func() { - root := parse.NewRootNode() - c := root.NewPluginNode() - c.Container = runner.Container{} - c.Vargs = map[string]interface{}{ - "depth": 50, - } - - ops := NewArgsOp() - ops.VisitContainer(c) - - g.Assert(c.Container.Environment["PLUGIN_DEPTH"]).Equal("50") - }) - }) - -} diff --git a/engine/compiler/builtin/build.go b/engine/compiler/builtin/build.go deleted file mode 100644 index 9d4d65163..000000000 --- a/engine/compiler/builtin/build.go +++ /dev/null @@ -1,40 +0,0 @@ -package builtin - -import ( - "github.com/drone/drone/engine/compiler/parse" - "github.com/drone/drone/engine/runner" -) - -// BuildOp is a transform operation that converts the build section of the Yaml -// to a step in the pipeline responsible for building the Docker image. -func BuildOp(node parse.Node) error { - build, ok := node.(*parse.BuildNode) - if !ok { - return nil - } - if build.Context == "" { - return nil - } - - root := node.Root() - builder := root.NewContainerNode() - - command := []string{ - "build", - "--force-rm", - "-f", build.Dockerfile, - "-t", root.Image, - build.Context, - } - - builder.Container = runner.Container{ - Image: "docker:apline", - Volumes: []string{"/var/run/docker.sock:/var/run/docker.sock"}, - Entrypoint: []string{"/usr/local/bin/docker"}, - Command: command, - WorkingDir: root.Path, - } - - root.Services = append(root.Services, builder) - return nil -} diff --git a/engine/compiler/builtin/clone.go b/engine/compiler/builtin/clone.go deleted file mode 100644 index 3b2c79c21..000000000 --- a/engine/compiler/builtin/clone.go +++ /dev/null @@ -1,45 +0,0 @@ -package builtin - -import ( - "github.com/drone/drone/engine/compiler/parse" - "github.com/drone/drone/engine/runner" -) - -type cloneOp struct { - visitor - plugin string - enable bool -} - -// NewCloneOp returns a transformer that configures the default clone plugin. -func NewCloneOp(plugin string, enable bool) Visitor { - return &cloneOp{ - enable: enable, - plugin: plugin, - } -} - -func (v *cloneOp) VisitContainer(node *parse.ContainerNode) error { - if node.Type() != parse.NodeClone { - return nil - } - if v.enable == false { - node.Disabled = true - return nil - } - - if node.Container.Name == "" { - node.Container.Name = "clone" - } - if node.Container.Image == "" { - node.Container.Image = v.plugin - } - - // discard any other cache properties except the image name. - // everything else is discard for security reasons. - node.Container = runner.Container{ - Name: node.Container.Name, - Image: node.Container.Image, - } - return nil -} diff --git a/engine/compiler/builtin/clone_test.go b/engine/compiler/builtin/clone_test.go deleted file mode 100644 index 98d869936..000000000 --- a/engine/compiler/builtin/clone_test.go +++ /dev/null @@ -1,36 +0,0 @@ -package builtin - -// import ( -// "testing" - -// "github.com/libcd/libcd" -// "github.com/libcd/libyaml/parse" - -// "github.com/franela/goblin" -// ) - -// func Test_clone(t *testing.T) { -// root := parse.NewRootNode() - -// g := goblin.Goblin(t) -// g.Describe("clone", func() { - -// g.It("should use default when nil", func() { -// op := NewCloneOp("plugins/git:latest") - -// op.VisitRoot(root) -// g.Assert(root.Clone.(*parse.ContainerNode).Container.Image).Equal("plugins/git:latest") -// }) - -// g.It("should use user-defined clone plugin", func() { -// op := NewCloneOp("plugins/git:latest") -// clone := root.NewCloneNode() -// clone.Container = libcd.Container{} -// clone.Container.Image = "custom/hg:latest" -// root.Clone = clone - -// op.VisitRoot(root) -// g.Assert(clone.Container.Image).Equal("custom/hg:latest") -// }) -// }) -// } diff --git a/engine/compiler/builtin/envs.go b/engine/compiler/builtin/envs.go deleted file mode 100644 index 42fb595e8..000000000 --- a/engine/compiler/builtin/envs.go +++ /dev/null @@ -1,57 +0,0 @@ -package builtin - -import ( - "os" - "strings" - - "github.com/drone/drone/engine/compiler/parse" -) - -var ( - httpProxy = os.Getenv("HTTP_PROXY") - httpsProxy = os.Getenv("HTTPS_PROXY") - noProxy = os.Getenv("NO_PROXY") -) - -type envOp struct { - visitor - envs map[string]string -} - -// NewEnvOp returns a transformer that sets default environment variables -// for each container, service and plugin. -func NewEnvOp(envs map[string]string) Visitor { - return &envOp{ - envs: envs, - } -} - -func (v *envOp) VisitContainer(node *parse.ContainerNode) error { - if node.Container.Environment == nil { - node.Container.Environment = map[string]string{} - } - v.defaultEnv(node) - v.defaultEnvProxy(node) - return nil -} - -func (v *envOp) defaultEnv(node *parse.ContainerNode) { - for k, v := range v.envs { - node.Container.Environment[k] = v - } -} - -func (v *envOp) defaultEnvProxy(node *parse.ContainerNode) { - if httpProxy != "" { - node.Container.Environment["HTTP_PROXY"] = httpProxy - node.Container.Environment["http_proxy"] = strings.ToUpper(httpProxy) - } - if httpsProxy != "" { - node.Container.Environment["HTTPS_PROXY"] = httpsProxy - node.Container.Environment["https_proxy"] = strings.ToUpper(httpsProxy) - } - if noProxy != "" { - node.Container.Environment["NO_PROXY"] = noProxy - node.Container.Environment["no_proxy"] = strings.ToUpper(noProxy) - } -} diff --git a/engine/compiler/builtin/envs_test.go b/engine/compiler/builtin/envs_test.go deleted file mode 100644 index aab72c50a..000000000 --- a/engine/compiler/builtin/envs_test.go +++ /dev/null @@ -1,45 +0,0 @@ -package builtin - -import ( - "testing" - - "github.com/drone/drone/engine/compiler/parse" - "github.com/drone/drone/engine/runner" - - "github.com/franela/goblin" -) - -func Test_env(t *testing.T) { - root := parse.NewRootNode() - - g := goblin.Goblin(t) - g.Describe("environment variables", func() { - - g.It("should be copied", func() { - envs := map[string]string{"CI": "drone"} - - c := root.NewContainerNode() - c.Container = runner.Container{} - op := NewEnvOp(envs) - - op.VisitContainer(c) - g.Assert(c.Container.Environment["CI"]).Equal("drone") - }) - - g.It("should include http proxy variables", func() { - httpProxy = "foo" - httpsProxy = "bar" - noProxy = "baz" - - c := root.NewContainerNode() - c.Container = runner.Container{} - op := NewEnvOp(map[string]string{}) - - op.VisitContainer(c) - g.Assert(c.Container.Environment["HTTP_PROXY"]).Equal("foo") - g.Assert(c.Container.Environment["HTTPS_PROXY"]).Equal("bar") - g.Assert(c.Container.Environment["NO_PROXY"]).Equal("baz") - }) - - }) -} diff --git a/engine/compiler/builtin/escalate.go b/engine/compiler/builtin/escalate.go deleted file mode 100644 index 78a0c1893..000000000 --- a/engine/compiler/builtin/escalate.go +++ /dev/null @@ -1,30 +0,0 @@ -package builtin - -import ( - "path/filepath" - - "github.com/drone/drone/engine/compiler/parse" -) - -type escalateOp struct { - visitor - plugins []string -} - -// NewEscalateOp returns a transformer that configures plugins to automatically -// execute in privileged mode. This is intended for plugins running dind. -func NewEscalateOp(plugins []string) Visitor { - return &escalateOp{ - plugins: plugins, - } -} - -func (v *escalateOp) VisitContainer(node *parse.ContainerNode) error { - for _, pattern := range v.plugins { - ok, _ := filepath.Match(pattern, node.Container.Image) - if ok { - node.Container.Privileged = true - } - } - return nil -} diff --git a/engine/compiler/builtin/escalate_test.go b/engine/compiler/builtin/escalate_test.go deleted file mode 100644 index e1374bedb..000000000 --- a/engine/compiler/builtin/escalate_test.go +++ /dev/null @@ -1,54 +0,0 @@ -package builtin - -import ( - "testing" - - "github.com/drone/drone/engine/compiler/parse" - "github.com/drone/drone/engine/runner" - - "github.com/franela/goblin" -) - -func Test_escalate(t *testing.T) { - root := parse.NewRootNode() - - g := goblin.Goblin(t) - g.Describe("privileged transform", func() { - - g.It("should handle matches", func() { - c := root.NewPluginNode() - c.Container = runner.Container{Image: "plugins/docker"} - op := NewEscalateOp([]string{"plugins/docker"}) - - op.VisitContainer(c) - g.Assert(c.Container.Privileged).IsTrue() - }) - - g.It("should handle glob matches", func() { - c := root.NewPluginNode() - c.Container = runner.Container{Image: "plugins/docker"} - op := NewEscalateOp([]string{"plugins/*"}) - - op.VisitContainer(c) - g.Assert(c.Container.Privileged).IsTrue() - }) - - g.It("should handle non matches", func() { - c := root.NewPluginNode() - c.Container = runner.Container{Image: "plugins/git"} - op := NewEscalateOp([]string{"plugins/docker"}) - - op.VisitContainer(c) - g.Assert(c.Container.Privileged).IsFalse() - }) - - g.It("should handle non glob matches", func() { - c := root.NewPluginNode() - c.Container = runner.Container{Image: "plugins/docker:develop"} - op := NewEscalateOp([]string{"plugins/docker"}) - - op.VisitContainer(c) - g.Assert(c.Container.Privileged).IsFalse() - }) - }) -} diff --git a/engine/compiler/builtin/filter.go b/engine/compiler/builtin/filter.go deleted file mode 100644 index 88f64283a..000000000 --- a/engine/compiler/builtin/filter.go +++ /dev/null @@ -1,128 +0,0 @@ -package builtin - -import ( - "path/filepath" - - "github.com/drone/drone/engine/compiler/parse" -) - -type filterOp struct { - visitor - status string - branch string - event string - environ string - platform string - matrix map[string]string -} - -// NewFilterOp returns a transformer that filters (ie removes) steps -// from the process based on conditional logic in the yaml. -func NewFilterOp(status, branch, event, env string, matrix map[string]string) Visitor { - return &filterOp{ - status: status, - branch: branch, - event: event, - environ: env, - matrix: matrix, - } -} - -func (v *filterOp) VisitContainer(node *parse.ContainerNode) error { - v.visitStatus(node) - v.visitBranch(node) - v.visitEvent(node) - v.visitMatrix(node) - v.visitPlatform(node) - return nil -} - -// visitStatus is a helpfer function that converts an on_change status -// filter to either success or failure based on the prior build status. -func (v *filterOp) visitStatus(node *parse.ContainerNode) { - if len(node.Conditions.Status) == 0 { - node.Conditions.Status = []string{"success"} - return - } - for _, status := range node.Conditions.Status { - if status != "change" && status != "changed" && status != "changes" { - continue - } - var want []string - switch v.status { - case "success": - want = append(want, "failure") - case "failure", "error", "killed": - want = append(want, "success") - default: - want = []string{"success", "failure"} - } - node.Conditions.Status = append(node.Conditions.Status, want...) - break - } -} - -// visitBranch is a helper function that disables container steps when -// the branch conditions are not satisfied. -func (v *filterOp) visitBranch(node *parse.ContainerNode) { - if len(node.Conditions.Branch) == 0 { - return - } - for _, pattern := range node.Conditions.Branch { - if ok, _ := filepath.Match(pattern, v.branch); ok { - return - } - } - node.Disabled = true -} - -// visitEnvironment is a helper function that disables container steps -// when the deployment environment conditions are not satisfied. -func (v *filterOp) visitEnvironment(node *parse.ContainerNode) { - if len(node.Conditions.Environment) == 0 { - return - } - for _, pattern := range node.Conditions.Environment { - if ok, _ := filepath.Match(pattern, v.environ); ok { - return - } - } - node.Disabled = true -} - -// visitEvent is a helper function that disables container steps -// when the build event conditions are not satisfied. -func (v *filterOp) visitEvent(node *parse.ContainerNode) { - if len(node.Conditions.Event) == 0 { - return - } - for _, pattern := range node.Conditions.Event { - if ok, _ := filepath.Match(pattern, v.event); ok { - return - } - } - node.Disabled = true -} - -func (v *filterOp) visitMatrix(node *parse.ContainerNode) { - for key, val := range node.Conditions.Matrix { - if v.matrix[key] != val { - node.Disabled = true - break - } - } -} - -// visitPlatform is a helper function that disables container steps -// when the build event conditions are not satisfied. -func (v *filterOp) visitPlatform(node *parse.ContainerNode) { - if len(node.Conditions.Platform) == 0 { - return - } - for _, pattern := range node.Conditions.Platform { - if ok, _ := filepath.Match(pattern, v.platform); ok { - return - } - } - node.Disabled = true -} diff --git a/engine/compiler/builtin/filter_test.go b/engine/compiler/builtin/filter_test.go deleted file mode 100644 index ae01fa3c5..000000000 --- a/engine/compiler/builtin/filter_test.go +++ /dev/null @@ -1,130 +0,0 @@ -package builtin - -// import ( -// "testing" - -// "github.com/franela/goblin" -// ) - -// func TestFilter(t *testing.T) { -// g := goblin.Goblin(t) -// g.Describe("Filters", func() { - -// g.It("Should match no branch filter", func() { -// c := &Container{} -// FilterBranch("feature/foo")(nil, c) -// g.Assert(c.Disabled).IsFalse() -// }) - -// g.It("Should match branch", func() { -// c := &Container{} -// c.Conditions.Branch.parts = []string{"feature/*"} -// FilterBranch("feature/foo")(nil, c) -// g.Assert(c.Disabled).IsFalse() -// }) - -// g.It("Should match branch wildcard", func() { -// c := &Container{} -// c.Conditions.Branch.parts = []string{"feature/*"} -// FilterBranch("feature/foo")(nil, c) -// g.Assert(c.Disabled).IsFalse() -// }) - -// g.It("Should disable when branch filter doesn't match", func() { -// c := &Container{} -// c.Conditions.Branch.parts = []string{"feature/*", "develop"} -// FilterBranch("master")(nil, c) -// g.Assert(c.Disabled).IsTrue() -// }) - -// g.It("Should match no platform filter", func() { -// c := &Container{} -// FilterPlatform("linux_amd64")(nil, c) -// g.Assert(c.Disabled).IsFalse() -// }) - -// g.It("Should match platform", func() { -// c := &Container{} -// c.Conditions.Platform.parts = []string{"linux_amd64"} -// FilterPlatform("linux_amd64")(nil, c) -// g.Assert(c.Disabled).IsFalse() -// }) - -// g.It("Should disable when platform filter doesn't match", func() { -// c := &Container{} -// c.Conditions.Platform.parts = []string{"linux_arm", "linux_arm64"} -// FilterPlatform("linux_amd64")(nil, c) -// g.Assert(c.Disabled).IsTrue() -// }) - -// g.It("Should match no environment filter", func() { -// c := &Container{} -// FilterEnvironment("production")(nil, c) -// g.Assert(c.Disabled).IsFalse() -// }) - -// g.It("Should match environment", func() { -// c := &Container{} -// c.Conditions.Environment.parts = []string{"production"} -// FilterEnvironment("production")(nil, c) -// g.Assert(c.Disabled).IsFalse() -// }) - -// g.It("Should disable when environment filter doesn't match", func() { -// c := &Container{} -// c.Conditions.Environment.parts = []string{"develop", "staging"} -// FilterEnvironment("production")(nil, c) -// g.Assert(c.Disabled).IsTrue() -// }) - -// g.It("Should match no event filter", func() { -// c := &Container{} -// FilterEvent("push")(nil, c) -// g.Assert(c.Disabled).IsFalse() -// }) - -// g.It("Should match event", func() { -// c := &Container{} -// c.Conditions.Event.parts = []string{"push"} -// FilterEvent("push")(nil, c) -// g.Assert(c.Disabled).IsFalse() -// }) - -// g.It("Should disable when event filter doesn't match", func() { -// c := &Container{} -// c.Conditions.Event.parts = []string{"push", "tag"} -// FilterEvent("pull_request")(nil, c) -// g.Assert(c.Disabled).IsTrue() -// }) - -// g.It("Should match matrix", func() { -// c := &Container{} -// c.Conditions.Matrix = map[string]string{ -// "go": "1.5", -// "redis": "3.0", -// } -// matrix := map[string]string{ -// "go": "1.5", -// "redis": "3.0", -// "node": "5.0.0", -// } -// FilterMatrix(matrix)(nil, c) -// g.Assert(c.Disabled).IsFalse() -// }) - -// g.It("Should disable when event filter doesn't match", func() { -// c := &Container{} -// c.Conditions.Matrix = map[string]string{ -// "go": "1.5", -// "redis": "3.0", -// } -// matrix := map[string]string{ -// "go": "1.4.2", -// "redis": "3.0", -// "node": "5.0.0", -// } -// FilterMatrix(matrix)(nil, c) -// g.Assert(c.Disabled).IsTrue() -// }) -// }) -// } diff --git a/engine/compiler/builtin/normalize.go b/engine/compiler/builtin/normalize.go deleted file mode 100644 index 4de12720d..000000000 --- a/engine/compiler/builtin/normalize.go +++ /dev/null @@ -1,66 +0,0 @@ -package builtin - -import ( - "path/filepath" - "strings" - - "github.com/drone/drone/engine/compiler/parse" -) - -type normalizeOp struct { - visitor - namespace string -} - -// NewNormalizeOp returns a transformer that normalizes the container image -// names and plugin names to their fully qualified values. -func NewNormalizeOp(namespace string) Visitor { - return &normalizeOp{ - namespace: namespace, - } -} - -func (v *normalizeOp) VisitContainer(node *parse.ContainerNode) error { - v.normalizeName(node) - v.normalizeImage(node) - switch node.NodeType { - case parse.NodePlugin, parse.NodeCache, parse.NodeClone: - v.normalizePlugin(node) - } - return nil -} - -// normalize the container image to the fully qualified name. -func (v *normalizeOp) normalizeImage(node *parse.ContainerNode) { - if strings.Contains(node.Container.Image, ":") { - return - } - node.Container.Image = node.Container.Image + ":latest" -} - -// normalize the plugin entrypoint and command values. -func (v *normalizeOp) normalizePlugin(node *parse.ContainerNode) { - if strings.Contains(node.Container.Image, "/") { - return - } - if strings.Contains(node.Container.Image, "_") { - node.Container.Image = strings.Replace(node.Container.Image, "_", "-", -1) - } - node.Container.Image = filepath.Join(v.namespace, node.Container.Image) -} - -// normalize the container name to ensrue a value is set. -func (v *normalizeOp) normalizeName(node *parse.ContainerNode) { - if node.Container.Name != "" { - return - } - - parts := strings.Split(node.Container.Image, "/") - if len(parts) != 0 { - node.Container.Name = parts[len(parts)-1] - } - parts = strings.Split(node.Container.Image, ":") - if len(parts) != 0 { - node.Container.Name = parts[0] - } -} diff --git a/engine/compiler/builtin/normalize_test.go b/engine/compiler/builtin/normalize_test.go deleted file mode 100644 index dbb24f2f6..000000000 --- a/engine/compiler/builtin/normalize_test.go +++ /dev/null @@ -1,78 +0,0 @@ -package builtin - -import ( - "testing" - - "github.com/drone/drone/engine/compiler/parse" - "github.com/drone/drone/engine/runner" - - "github.com/franela/goblin" -) - -func Test_normalize(t *testing.T) { - root := parse.NewRootNode() - - g := goblin.Goblin(t) - g.Describe("normalizing", func() { - - g.Describe("images", func() { - - g.It("should append tag if empty", func() { - c := root.NewContainerNode() - c.Container = runner.Container{Image: "golang"} - op := NewNormalizeOp("") - - op.VisitContainer(c) - g.Assert(c.Container.Image).Equal("golang:latest") - }) - - g.It("should not override existing tag", func() { - c := root.NewContainerNode() - c.Container = runner.Container{Image: "golang:1.5"} - op := NewNormalizeOp("") - - op.VisitContainer(c) - g.Assert(c.Container.Image).Equal("golang:1.5") - }) - }) - - g.Describe("plugins", func() { - - g.It("should prepend namespace", func() { - c := root.NewPluginNode() - c.Container = runner.Container{Image: "git"} - op := NewNormalizeOp("plugins") - - op.VisitContainer(c) - g.Assert(c.Container.Image).Equal("plugins/git:latest") - }) - - g.It("should not override existing namespace", func() { - c := root.NewPluginNode() - c.Container = runner.Container{Image: "index.docker.io/drone/git"} - op := NewNormalizeOp("plugins") - - op.VisitContainer(c) - g.Assert(c.Container.Image).Equal("index.docker.io/drone/git:latest") - }) - - g.It("should replace underscores with dashes", func() { - c := root.NewPluginNode() - c.Container = runner.Container{Image: "gh_pages"} - op := NewNormalizeOp("plugins") - - op.VisitContainer(c) - g.Assert(c.Container.Image).Equal("plugins/gh-pages:latest") - }) - - g.It("should ignore shell or service types", func() { - c := root.NewShellNode() - c.Container = runner.Container{Image: "golang"} - op := NewNormalizeOp("plugins") - - op.VisitContainer(c) - g.Assert(c.Container.Image).Equal("golang:latest") - }) - }) - }) -} diff --git a/engine/compiler/builtin/pod.go b/engine/compiler/builtin/pod.go deleted file mode 100644 index 791c2a6fd..000000000 --- a/engine/compiler/builtin/pod.go +++ /dev/null @@ -1,50 +0,0 @@ -package builtin - -import ( - "fmt" - - "github.com/drone/drone/engine/compiler/parse" - "github.com/drone/drone/engine/runner" -) - -type podOp struct { - visitor - name string -} - -// NewPodOp returns a transformer that configures an ambassador container -// providing shared networking and container volumes. -func NewPodOp(name string) Visitor { - return &podOp{ - name: name, - } -} - -func (v *podOp) VisitContainer(node *parse.ContainerNode) error { - if node.Container.Network == "" { - parent := fmt.Sprintf("container:%s", v.name) - node.Container.Network = parent - } - node.Container.VolumesFrom = append(node.Container.VolumesFrom, v.name) - return nil -} - -func (v *podOp) VisitRoot(node *parse.RootNode) error { - service := node.NewServiceNode() - service.Container = runner.Container{ - Name: v.name, - Alias: "ambassador", - Image: "busybox:latest", - Entrypoint: []string{"/bin/sleep"}, - Command: []string{"86400"}, - Volumes: []string{node.Path, node.Base}, - // Entrypoint: []string{"/bin/sh", "-c"}, - // Volumes: []string{node.Base}, - // Command: []string{ - // fmt.Sprintf("mkdir -p %s; sleep 86400", node.Path), - // }, - } - - node.Pod = service - return nil -} diff --git a/engine/compiler/builtin/pull.go b/engine/compiler/builtin/pull.go deleted file mode 100644 index 5796b5729..000000000 --- a/engine/compiler/builtin/pull.go +++ /dev/null @@ -1,26 +0,0 @@ -package builtin - -import ( - "github.com/drone/drone/engine/compiler/parse" -) - -type pullOp struct { - visitor - pull bool -} - -// NewPullOp returns a transformer that configures plugins to automatically -// pull the latest images at runtime. -func NewPullOp(pull bool) Visitor { - return &pullOp{ - pull: pull, - } -} - -func (v *pullOp) VisitContainer(node *parse.ContainerNode) error { - switch node.NodeType { - case parse.NodePlugin, parse.NodeCache, parse.NodeClone: - node.Container.Pull = v.pull - } - return nil -} diff --git a/engine/compiler/builtin/pull_test.go b/engine/compiler/builtin/pull_test.go deleted file mode 100644 index 882d32103..000000000 --- a/engine/compiler/builtin/pull_test.go +++ /dev/null @@ -1,45 +0,0 @@ -package builtin - -import ( - "testing" - - "github.com/drone/drone/engine/compiler/parse" - "github.com/drone/drone/engine/runner" - - "github.com/franela/goblin" -) - -func Test_pull(t *testing.T) { - root := parse.NewRootNode() - - g := goblin.Goblin(t) - g.Describe("pull image", func() { - - g.It("should be enabled for plugins", func() { - c := root.NewPluginNode() - c.Container = runner.Container{} - op := NewPullOp(true) - - op.VisitContainer(c) - g.Assert(c.Container.Pull).IsTrue() - }) - - g.It("should be disabled for plugins", func() { - c := root.NewPluginNode() - c.Container = runner.Container{} - op := NewPullOp(false) - - op.VisitContainer(c) - g.Assert(c.Container.Pull).IsFalse() - }) - - g.It("should be disabled for non-plugins", func() { - c := root.NewShellNode() - c.Container = runner.Container{} - op := NewPullOp(true) - - op.VisitContainer(c) - g.Assert(c.Container.Pull).IsFalse() - }) - }) -} diff --git a/engine/compiler/builtin/secrets.go b/engine/compiler/builtin/secrets.go deleted file mode 100644 index d5c418499..000000000 --- a/engine/compiler/builtin/secrets.go +++ /dev/null @@ -1,45 +0,0 @@ -package builtin - -import ( - "github.com/drone/drone/engine/compiler/parse" - "github.com/drone/drone/model" -) - -type secretOp struct { - visitor - event string - secrets []*model.Secret -} - -// NewSecretOp returns a transformer that configures plugin secrets. -func NewSecretOp(event string, secrets []*model.Secret) Visitor { - return &secretOp{ - event: event, - secrets: secrets, - } -} - -func (v *secretOp) VisitContainer(node *parse.ContainerNode) error { - for _, secret := range v.secrets { - if !secret.Match(node.Container.Image, v.event) { - continue - } - - switch secret.Name { - case "REGISTRY_USERNAME": - node.Container.AuthConfig.Username = secret.Value - case "REGISTRY_PASSWORD": - node.Container.AuthConfig.Password = secret.Value - case "REGISTRY_EMAIL": - node.Container.AuthConfig.Email = secret.Value - case "REGISTRY_TOKEN": - node.Container.AuthConfig.Token = secret.Value - default: - if node.Container.Environment == nil { - node.Container.Environment = map[string]string{} - } - node.Container.Environment[secret.Name] = secret.Value - } - } - return nil -} diff --git a/engine/compiler/builtin/shell.go b/engine/compiler/builtin/shell.go deleted file mode 100644 index a3dd32068..000000000 --- a/engine/compiler/builtin/shell.go +++ /dev/null @@ -1,95 +0,0 @@ -package builtin - -import ( - "bytes" - "encoding/base64" - "fmt" - - "github.com/drone/drone/engine/compiler/parse" -) - -const ( - Freebsd_amd64 = "freebsd_amd64" - Linux_adm64 = "linux_amd64" - Windows_amd64 = "windows_amd64" -) - -type shellOp struct { - visitor - platform string -} - -// NewShellOp returns a transformer that converts the shell node to -// a runnable container. -func NewShellOp(platform string) Visitor { - return &shellOp{ - platform: platform, - } -} - -func (v *shellOp) VisitContainer(node *parse.ContainerNode) error { - if node.NodeType != parse.NodeShell { - return nil - } - - node.Container.Entrypoint = []string{ - "/bin/sh", "-c", - } - node.Container.Command = []string{ - "echo $DRONE_SCRIPT | base64 -d | /bin/sh -e", - } - if node.Container.Environment == nil { - node.Container.Environment = map[string]string{} - } - node.Container.Environment["HOME"] = "/root" - node.Container.Environment["SHELL"] = "/bin/sh" - node.Container.Environment["DRONE_SCRIPT"] = toScript( - node.Root().Path, - node.Commands, - ) - - return nil -} - -func toScript(base string, commands []string) string { - var buf bytes.Buffer - for _, command := range commands { - buf.WriteString(fmt.Sprintf( - traceScript, - ""+command+"", - command, - )) - } - - script := fmt.Sprintf( - setupScript, - buf.String(), - ) - - return base64.StdEncoding.EncodeToString([]byte(script)) -} - -// setupScript is a helper script this is added to the build to ensure -// a minimum set of environment variables are set correctly. -const setupScript = ` -if [ -n "$DRONE_NETRC_MACHINE" ]; then -cat < $HOME/.netrc -machine $DRONE_NETRC_MACHINE -login $DRONE_NETRC_USERNAME -password $DRONE_NETRC_PASSWORD -EOF -fi - -unset DRONE_NETRC_USERNAME -unset DRONE_NETRC_PASSWORD -unset DRONE_SCRIPT - -%s -` - -// traceScript is a helper script that is added to the build script -// to trace a command. -const traceScript = ` -echo %q -%s -` diff --git a/engine/compiler/builtin/shell_test.go b/engine/compiler/builtin/shell_test.go deleted file mode 100644 index bc9dd8291..000000000 --- a/engine/compiler/builtin/shell_test.go +++ /dev/null @@ -1,44 +0,0 @@ -package builtin - -import ( - "testing" - - "github.com/drone/drone/engine/compiler/parse" - "github.com/drone/drone/engine/runner" - - "github.com/franela/goblin" -) - -func Test_shell(t *testing.T) { - - g := goblin.Goblin(t) - g.Describe("shell containers", func() { - - g.It("should ignore plugin steps", func() { - root := parse.NewRootNode() - c := root.NewPluginNode() - c.Container = runner.Container{} - ops := NewShellOp(Linux_adm64) - ops.VisitContainer(c) - - g.Assert(len(c.Container.Entrypoint)).Equal(0) - g.Assert(len(c.Container.Command)).Equal(0) - g.Assert(c.Container.Environment["DRONE_SCRIPT"]).Equal("") - }) - - g.It("should set entrypoint, command and environment variables", func() { - root := parse.NewRootNode() - root.Base = "/go" - root.Path = "/go/src/github.com/octocat/hello-world" - - c := root.NewShellNode() - c.Commands = []string{"go build"} - ops := NewShellOp(Linux_adm64) - ops.VisitContainer(c) - - g.Assert(c.Container.Entrypoint).Equal([]string{"/bin/sh", "-c"}) - g.Assert(c.Container.Command).Equal([]string{"echo $DRONE_SCRIPT | base64 -d | /bin/sh -e"}) - g.Assert(c.Container.Environment["DRONE_SCRIPT"] != "").IsTrue() - }) - }) -} diff --git a/engine/compiler/builtin/validate.go b/engine/compiler/builtin/validate.go deleted file mode 100644 index ec88953c5..000000000 --- a/engine/compiler/builtin/validate.go +++ /dev/null @@ -1,120 +0,0 @@ -package builtin - -import ( - "fmt" - "path/filepath" - - "github.com/drone/drone/engine/compiler/parse" -) - -type validateOp struct { - visitor - plugins []string - trusted bool -} - -// NewValidateOp returns a linter that checks container configuration. -func NewValidateOp(trusted bool, plugins []string) Visitor { - return &validateOp{ - trusted: trusted, - plugins: plugins, - } -} - -func (v *validateOp) VisitContainer(node *parse.ContainerNode) error { - switch node.NodeType { - case parse.NodePlugin, parse.NodeCache, parse.NodeClone: - if err := v.validatePlugins(node); err != nil { - return err - } - } - if node.NodeType == parse.NodePlugin { - if err := v.validatePluginConfig(node); err != nil { - return err - } - } - return v.validateConfig(node) -} - -// validate the plugin image and return an error if the plugin -// image does not match the whitelist. -func (v *validateOp) validatePlugins(node *parse.ContainerNode) error { - match := false - for _, pattern := range v.plugins { - ok, err := filepath.Match(pattern, node.Container.Image) - if ok && err == nil { - match = true - break - } - } - if !match { - return fmt.Errorf( - "Plugin %s is not in the whitelist", - node.Container.Image, - ) - } - return nil -} - -// validate the plugin command and entrypoint and return an error -// the user attempts to set or override these values. -func (v *validateOp) validatePluginConfig(node *parse.ContainerNode) error { - if len(node.Container.Entrypoint) != 0 { - return fmt.Errorf("Cannot set plugin Entrypoint") - } - if len(node.Container.Command) != 0 { - return fmt.Errorf("Cannot set plugin Command") - } - return nil -} - -// validate the container configuration and return an error if -// restricted configurations are used. -func (v *validateOp) validateConfig(node *parse.ContainerNode) error { - if v.trusted { - return nil - } - if node.Container.Privileged { - return fmt.Errorf("Insufficient privileges to use privileged mode") - } - if len(node.Container.DNS) != 0 { - return fmt.Errorf("Insufficient privileges to use custom dns") - } - if len(node.Container.DNSSearch) != 0 { - return fmt.Errorf("Insufficient privileges to use dns_search") - } - if len(node.Container.Devices) != 0 { - return fmt.Errorf("Insufficient privileges to use devices") - } - if len(node.Container.ExtraHosts) != 0 { - return fmt.Errorf("Insufficient privileges to use extra_hosts") - } - if len(node.Container.Network) != 0 { - return fmt.Errorf("Insufficient privileges to override the network") - } - if node.Container.OomKillDisable { - return fmt.Errorf("Insufficient privileges to disable oom_kill") - } - if len(node.Container.Volumes) != 0 && node.Type() != parse.NodeCache { - return fmt.Errorf("Insufficient privileges to use volumes") - } - if len(node.Container.VolumesFrom) != 0 { - return fmt.Errorf("Insufficient privileges to use volumes_from") - } - return nil -} - -// validate the environment configuration and return an error if -// an attempt is made to override system environment variables. -// func (v *validateOp) validateEnvironment(node *parse.ContainerNode) error { -// for key := range node.Container.Environment { -// upper := strings.ToUpper(key) -// switch { -// case strings.HasPrefix(upper, "DRONE_"): -// return fmt.Errorf("Cannot set or override DRONE_ environment variables") -// case strings.HasPrefix(upper, "PLUGIN_"): -// return fmt.Errorf("Cannot set or override PLUGIN_ environment variables") -// } -// } -// return nil -// } diff --git a/engine/compiler/builtin/validate_test.go b/engine/compiler/builtin/validate_test.go deleted file mode 100644 index 1744c6283..000000000 --- a/engine/compiler/builtin/validate_test.go +++ /dev/null @@ -1,199 +0,0 @@ -package builtin - -import ( - "testing" - - "github.com/drone/drone/engine/compiler/parse" - "github.com/drone/drone/engine/runner" - - "github.com/franela/goblin" -) - -func Test_validate(t *testing.T) { - root := parse.NewRootNode() - - g := goblin.Goblin(t) - g.Describe("validating", func() { - - g.Describe("privileged attributes", func() { - - g.It("should not error when trusted build", func() { - c := root.NewContainerNode() - c.Container = runner.Container{} - ops := NewValidateOp(true, []string{"plugins/*"}) - err := ops.VisitContainer(c) - - g.Assert(err == nil).IsTrue("error should be nil") - }) - - g.It("should error when privleged mode", func() { - c := root.NewContainerNode() - c.Container = runner.Container{} - c.Container.Privileged = true - ops := NewValidateOp(false, []string{"plugins/*"}) - err := ops.VisitContainer(c) - - g.Assert(err != nil).IsTrue("error should not be nil") - g.Assert(err.Error()).Equal("Insufficient privileges to use privileged mode") - }) - - g.It("should error when dns configured", func() { - c := root.NewContainerNode() - c.Container = runner.Container{} - c.Container.DNS = []string{"8.8.8.8"} - ops := NewValidateOp(false, []string{"plugins/*"}) - err := ops.VisitContainer(c) - - g.Assert(err != nil).IsTrue("error should not be nil") - g.Assert(err.Error()).Equal("Insufficient privileges to use custom dns") - }) - - g.It("should error when dns_search configured", func() { - c := root.NewContainerNode() - c.Container = runner.Container{} - c.Container.DNSSearch = []string{"8.8.8.8"} - ops := NewValidateOp(false, []string{"plugins/*"}) - err := ops.VisitContainer(c) - - g.Assert(err != nil).IsTrue("error should not be nil") - g.Assert(err.Error()).Equal("Insufficient privileges to use dns_search") - }) - - g.It("should error when devices configured", func() { - c := root.NewContainerNode() - c.Container = runner.Container{} - c.Container.Devices = []string{"/dev/foo"} - ops := NewValidateOp(false, []string{"plugins/*"}) - err := ops.VisitContainer(c) - - g.Assert(err != nil).IsTrue("error should not be nil") - g.Assert(err.Error()).Equal("Insufficient privileges to use devices") - }) - - g.It("should error when extra_hosts configured", func() { - c := root.NewContainerNode() - c.Container = runner.Container{} - c.Container.ExtraHosts = []string{"1.2.3.4 foo.com"} - ops := NewValidateOp(false, []string{"plugins/*"}) - err := ops.VisitContainer(c) - - g.Assert(err != nil).IsTrue("error should not be nil") - g.Assert(err.Error()).Equal("Insufficient privileges to use extra_hosts") - }) - - g.It("should error when network configured", func() { - c := root.NewContainerNode() - c.Container = runner.Container{} - c.Container.Network = "host" - ops := NewValidateOp(false, []string{"plugins/*"}) - err := ops.VisitContainer(c) - - g.Assert(err != nil).IsTrue("error should not be nil") - g.Assert(err.Error()).Equal("Insufficient privileges to override the network") - }) - - g.It("should error when oom_kill_disabled configured", func() { - c := root.NewContainerNode() - c.Container = runner.Container{} - c.Container.OomKillDisable = true - ops := NewValidateOp(false, []string{"plugins/*"}) - err := ops.VisitContainer(c) - - g.Assert(err != nil).IsTrue("error should not be nil") - g.Assert(err.Error()).Equal("Insufficient privileges to disable oom_kill") - }) - - g.It("should error when volumes configured", func() { - c := root.NewContainerNode() - c.Container = runner.Container{} - c.Container.Volumes = []string{"/:/tmp"} - ops := NewValidateOp(false, []string{"plugins/*"}) - err := ops.VisitContainer(c) - - g.Assert(err != nil).IsTrue("error should not be nil") - g.Assert(err.Error()).Equal("Insufficient privileges to use volumes") - }) - - g.It("should error when volumes_from configured", func() { - c := root.NewContainerNode() - c.Container = runner.Container{} - c.Container.VolumesFrom = []string{"drone"} - ops := NewValidateOp(false, []string{"plugins/*"}) - err := ops.VisitContainer(c) - - g.Assert(err != nil).IsTrue("error should not be nil") - g.Assert(err.Error()).Equal("Insufficient privileges to use volumes_from") - }) - }) - - g.Describe("plugin configuration", func() { - g.It("should error when entrypoint is configured", func() { - c := root.NewPluginNode() - c.Container = runner.Container{Image: "plugins/git"} - c.Container.Entrypoint = []string{"/bin/sh"} - ops := NewValidateOp(false, []string{"plugins/*"}) - err := ops.VisitContainer(c) - - g.Assert(err != nil).IsTrue("error should not be nil") - g.Assert(err.Error()).Equal("Cannot set plugin Entrypoint") - }) - - g.It("should error when command is configured", func() { - c := root.NewPluginNode() - c.Container = runner.Container{Image: "plugins/git"} - c.Container.Command = []string{"cat", "/proc/1/status"} - ops := NewValidateOp(false, []string{"plugins/*"}) - err := ops.VisitContainer(c) - - g.Assert(err != nil).IsTrue("error should not be nil") - g.Assert(err.Error()).Equal("Cannot set plugin Command") - }) - - g.It("should not error when empty entrypoint, command", func() { - c := root.NewPluginNode() - c.Container = runner.Container{Image: "plugins/git"} - ops := NewValidateOp(false, []string{"plugins/*"}) - err := ops.VisitContainer(c) - - g.Assert(err == nil).IsTrue("error should be nil") - }) - }) - - g.Describe("plugin whitelist", func() { - - g.It("should error when no match found", func() { - c := root.NewPluginNode() - c.Container = runner.Container{} - c.Container.Image = "custom/git" - - ops := NewValidateOp(false, []string{"plugins/*"}) - err := ops.VisitContainer(c) - - g.Assert(err != nil).IsTrue("error should be nil") - g.Assert(err.Error()).Equal("Plugin custom/git is not in the whitelist") - }) - - g.It("should not error when match found", func() { - c := root.NewPluginNode() - c.Container = runner.Container{} - c.Container.Image = "plugins/git" - - ops := NewValidateOp(false, []string{"plugins/*"}) - err := ops.VisitContainer(c) - - g.Assert(err == nil).IsTrue("error should be nil") - }) - - g.It("should ignore build images", func() { - c := root.NewShellNode() - c.Container = runner.Container{} - c.Container.Image = "google/golang" - - ops := NewValidateOp(false, []string{"plugins/*"}) - err := ops.VisitContainer(c) - - g.Assert(err == nil).IsTrue("error should be nil") - }) - }) - }) -} diff --git a/engine/compiler/builtin/visitor.go b/engine/compiler/builtin/visitor.go deleted file mode 100644 index bd84a8f5b..000000000 --- a/engine/compiler/builtin/visitor.go +++ /dev/null @@ -1,23 +0,0 @@ -package builtin - -import "github.com/drone/drone/engine/compiler/parse" - -// Visitor interface for walking the Yaml file. -type Visitor interface { - VisitRoot(*parse.RootNode) error - VisitVolume(*parse.VolumeNode) error - VisitNetwork(*parse.NetworkNode) error - VisitBuild(*parse.BuildNode) error - VisitContainer(*parse.ContainerNode) error -} - -// visitor provides an easy default implementation of a Visitor interface with -// stubbed methods. This can be embedded in transforms to meet the basic -// requirements. -type visitor struct{} - -func (visitor) VisitRoot(*parse.RootNode) error { return nil } -func (visitor) VisitVolume(*parse.VolumeNode) error { return nil } -func (visitor) VisitNetwork(*parse.NetworkNode) error { return nil } -func (visitor) VisitBuild(*parse.BuildNode) error { return nil } -func (visitor) VisitContainer(*parse.ContainerNode) error { return nil } diff --git a/engine/compiler/builtin/workspace.go b/engine/compiler/builtin/workspace.go deleted file mode 100644 index 84256f9cc..000000000 --- a/engine/compiler/builtin/workspace.go +++ /dev/null @@ -1,51 +0,0 @@ -package builtin - -import ( - "path/filepath" - - "github.com/drone/drone/engine/compiler/parse" -) - -type workspaceOp struct { - visitor - base string - path string -} - -// NewWorkspaceOp returns a transformer that provides a default workspace paths, -// including the base path (mounted as a volume) and absolute path where the -// code is cloned. -func NewWorkspaceOp(base, path string) Visitor { - return &workspaceOp{ - base: base, - path: path, - } -} - -func (v *workspaceOp) VisitRoot(node *parse.RootNode) error { - if node.Base == "" { - node.Base = v.base - } - if node.Path == "" { - node.Path = v.path - } - if !filepath.IsAbs(node.Path) { - node.Path = filepath.Join( - node.Base, - node.Path, - ) - } - return nil -} - -func (v *workspaceOp) VisitContainer(node *parse.ContainerNode) error { - if node.NodeType == parse.NodeService { - // we must not override the default working - // directory of service containers. All other - // container should launch in the workspace - return nil - } - root := node.Root() - node.Container.WorkingDir = root.Path - return nil -} diff --git a/engine/compiler/builtin/workspace_test.go b/engine/compiler/builtin/workspace_test.go deleted file mode 100644 index 523d2f019..000000000 --- a/engine/compiler/builtin/workspace_test.go +++ /dev/null @@ -1,89 +0,0 @@ -package builtin - -import ( - "testing" - - "github.com/franela/goblin" - "github.com/drone/drone/engine/compiler/parse" -) - -func Test_workspace(t *testing.T) { - g := goblin.Goblin(t) - - g.Describe("workspace", func() { - - var defaultBase = "/go" - var defaultPath = "src/github.com/octocat/hello-world" - - g.It("should not override user paths", func() { - var base = "/drone" - var path = "/drone/src/github.com/octocat/hello-world" - - op := NewWorkspaceOp(defaultBase, defaultPath) - root := parse.NewRootNode() - root.Base = base - root.Path = path - - op.VisitRoot(root) - g.Assert(root.Base).Equal(base) - g.Assert(root.Path).Equal(path) - }) - - g.It("should convert user paths to absolute", func() { - var base = "/drone" - var path = "src/github.com/octocat/hello-world" - var abs = "/drone/src/github.com/octocat/hello-world" - - op := NewWorkspaceOp(defaultBase, defaultPath) - root := parse.NewRootNode() - root.Base = base - root.Path = path - - op.VisitRoot(root) - g.Assert(root.Base).Equal(base) - g.Assert(root.Path).Equal(abs) - }) - - g.It("should set the default path", func() { - var base = "/go" - var path = "/go/src/github.com/octocat/hello-world" - - op := NewWorkspaceOp(defaultBase, defaultPath) - root := parse.NewRootNode() - - op.VisitRoot(root) - g.Assert(root.Base).Equal(base) - g.Assert(root.Path).Equal(path) - }) - - g.It("should use workspace as working_dir", func() { - var base = "/drone" - var path = "/drone/src/github.com/octocat/hello-world" - - root := parse.NewRootNode() - root.Base = base - root.Path = path - - c := root.NewContainerNode() - - op := NewWorkspaceOp(defaultBase, defaultPath) - op.VisitContainer(c) - g.Assert(c.Container.WorkingDir).Equal(root.Path) - }) - - g.It("should not use workspace as working_dir for services", func() { - var base = "/drone" - var path = "/drone/src/github.com/octocat/hello-world" - - root := parse.NewRootNode() - root.Base = base - root.Path = path - - c := root.NewServiceNode() - - op := NewWorkspaceOp(defaultBase, defaultPath) - op.VisitContainer(c) - g.Assert(c.Container.WorkingDir).Equal("") - }) - }) -} diff --git a/engine/compiler/compile.go b/engine/compiler/compile.go deleted file mode 100644 index 7d4ad665f..000000000 --- a/engine/compiler/compile.go +++ /dev/null @@ -1,137 +0,0 @@ -package compiler - -import ( - "github.com/drone/drone/engine/runner" - "github.com/drone/drone/engine/runner/parse" - - yaml "github.com/drone/drone/engine/compiler/parse" -) - -// Compiler compiles the Yaml file to the intermediate representation. -type Compiler struct { - trans []Transform -} - -func New() *Compiler { - return &Compiler{} -} - -// Transforms sets the compiler transforms use to transform the intermediate -// representation during compilation. -func (c *Compiler) Transforms(trans []Transform) *Compiler { - c.trans = append(c.trans, trans...) - return c -} - -// CompileString compiles the Yaml configuration string and returns -// the intermediate representation for the interpreter. -func (c *Compiler) CompileString(in string) (*runner.Spec, error) { - return c.Compile([]byte(in)) -} - -// CompileString compiles the Yaml configuration file and returns -// the intermediate representation for the interpreter. -func (c *Compiler) Compile(in []byte) (*runner.Spec, error) { - root, err := yaml.Parse(in) - if err != nil { - return nil, err - } - if err := root.Walk(c.walk); err != nil { - return nil, err - } - - config := &runner.Spec{} - tree := parse.NewTree() - - // pod section - if root.Pod != nil { - node, ok := root.Pod.(*yaml.ContainerNode) - if ok { - config.Containers = append(config.Containers, &node.Container) - tree.Append(parse.NewRunNode().SetName(node.Container.Name).SetDetach(true)) - } - } - - // clone section - if root.Clone != nil { - node, ok := root.Clone.(*yaml.ContainerNode) - if ok && !node.Disabled { - config.Containers = append(config.Containers, &node.Container) - tree.Append(parse.NewRunNode().SetName(node.Container.Name)) - } - } - - // services section - for _, container := range root.Services { - node, ok := container.(*yaml.ContainerNode) - if !ok || node.Disabled { - continue - } - - config.Containers = append(config.Containers, &node.Container) - tree.Append(parse.NewRunNode().SetName(node.Container.Name).SetDetach(true)) - } - - // pipeline section - for i, container := range root.Script { - node, ok := container.(*yaml.ContainerNode) - if !ok || node.Disabled { - continue - } - - config.Containers = append(config.Containers, &node.Container) - - // step 1: lookahead to see if any status=failure exist - list := parse.NewListNode() - for ii, next := range root.Script { - if i >= ii { - continue - } - node, ok := next.(*yaml.ContainerNode) - if !ok || node.Disabled || !node.OnFailure() { - continue - } - - list.Append( - parse.NewRecoverNode().SetBody( - parse.NewRunNode().SetName( - node.Container.Name, - ), - ), - ) - } - // step 2: if yes, collect these and append to "error" node - if len(list.Body) == 0 { - tree.Append(parse.NewRunNode().SetName(node.Container.Name)) - } else { - errorNode := parse.NewErrorNode() - errorNode.SetBody(parse.NewRunNode().SetName(node.Container.Name)) - errorNode.SetDefer(list) - tree.Append(errorNode) - } - } - - config.Nodes = tree - return config, nil -} - -func (c *Compiler) walk(node yaml.Node) (err error) { - for _, trans := range c.trans { - switch v := node.(type) { - case *yaml.BuildNode: - err = trans.VisitBuild(v) - case *yaml.ContainerNode: - err = trans.VisitContainer(v) - case *yaml.NetworkNode: - err = trans.VisitNetwork(v) - case *yaml.VolumeNode: - err = trans.VisitVolume(v) - case *yaml.RootNode: - err = trans.VisitRoot(v) - } - if err != nil { - break - } - } - return err -} diff --git a/engine/compiler/compile_test.go b/engine/compiler/compile_test.go deleted file mode 100644 index a20d4fea6..000000000 --- a/engine/compiler/compile_test.go +++ /dev/null @@ -1 +0,0 @@ -package compiler diff --git a/engine/compiler/parse/node.go b/engine/compiler/parse/node.go deleted file mode 100644 index 6a97159a1..000000000 --- a/engine/compiler/parse/node.go +++ /dev/null @@ -1,34 +0,0 @@ -package parse - -const ( - NodeBuild = "build" - NodeCache = "cache" - NodeClone = "clone" - NodeContainer = "container" - NodeNetwork = "network" - NodePlugin = "plugin" - NodeRoot = "root" - NodeService = "service" - NodeShell = "shell" - NodeVolume = "volume" -) - -// NodeType identifies the type of parse tree node. -type NodeType string - -// Type returns itself an provides an easy default implementation. -// for embedding in a Node. Embedded in all non-trivial Nodes. -func (t NodeType) Type() NodeType { - return t -} - -// String returns the string value of the Node type. -func (t NodeType) String() string { - return string(t) -} - -// A Node is an element in the parse tree. -type Node interface { - Type() NodeType - Root() *RootNode -} diff --git a/engine/compiler/parse/node_build.go b/engine/compiler/parse/node_build.go deleted file mode 100644 index 158529b88..000000000 --- a/engine/compiler/parse/node_build.go +++ /dev/null @@ -1,42 +0,0 @@ -package parse - -// BuildNode represents Docker image build instructions. -type BuildNode struct { - NodeType - - Context string - Dockerfile string - Args map[string]string - - root *RootNode -} - -// Root returns the root node. -func (n *BuildNode) Root() *RootNode { return n.root } - -// -// intermediate types for yaml decoding. -// - -type build struct { - Context string - Dockerfile string - Args map[string]string -} - -func (b *build) UnmarshalYAML(unmarshal func(interface{}) error) error { - err := unmarshal(&b.Context) - if err == nil { - return nil - } - out := struct { - Context string - Dockerfile string - Args map[string]string - }{} - err = unmarshal(&out) - b.Context = out.Context - b.Args = out.Args - b.Dockerfile = out.Dockerfile - return err -} diff --git a/engine/compiler/parse/node_build_test.go b/engine/compiler/parse/node_build_test.go deleted file mode 100644 index 223edbedd..000000000 --- a/engine/compiler/parse/node_build_test.go +++ /dev/null @@ -1,38 +0,0 @@ -package parse - -import ( - "testing" - - "github.com/franela/goblin" - "gopkg.in/yaml.v2" -) - -func TestBuildNode(t *testing.T) { - g := goblin.Goblin(t) - - g.Describe("Build", func() { - g.Describe("given a yaml file", func() { - - g.It("should unmarshal", func() { - in := []byte(".") - out := build{} - err := yaml.Unmarshal(in, &out) - if err != nil { - g.Fail(err) - } - g.Assert(out.Context).Equal(".") - }) - - g.It("should unmarshal shorthand", func() { - in := []byte("{ context: ., dockerfile: Dockerfile }") - out := build{} - err := yaml.Unmarshal(in, &out) - if err != nil { - g.Fail(err) - } - g.Assert(out.Context).Equal(".") - g.Assert(out.Dockerfile).Equal("Dockerfile") - }) - }) - }) -} diff --git a/engine/compiler/parse/node_container.go b/engine/compiler/parse/node_container.go deleted file mode 100644 index 8ffad21ab..000000000 --- a/engine/compiler/parse/node_container.go +++ /dev/null @@ -1,180 +0,0 @@ -package parse - -import ( - "fmt" - - "github.com/drone/drone/engine/runner" - - "gopkg.in/yaml.v2" -) - -type Conditions struct { - Platform []string - Environment []string - Event []string - Branch []string - Status []string - Matrix map[string]string -} - -// ContainerNode represents a Docker container. -type ContainerNode struct { - NodeType - - // Container represents the container configuration. - Container runner.Container - Conditions Conditions - Disabled bool - Commands []string - Vargs map[string]interface{} - - root *RootNode -} - -// Root returns the root node. -func (n *ContainerNode) Root() *RootNode { return n.root } - -// OnSuccess returns true if the container should be executed -// when the exit code of the previous step is 0. -func (n *ContainerNode) OnSuccess() bool { - for _, status := range n.Conditions.Status { - if status == "success" { - return true - } - } - return false -} - -// OnFailure returns true if the container should be executed -// even when the exit code of the previous step != 0. -func (n *ContainerNode) OnFailure() bool { - for _, status := range n.Conditions.Status { - if status == "failure" { - return true - } - } - return false -} - -// -// intermediate types for yaml decoding. -// - -type container struct { - Name string `yaml:"name"` - Image string `yaml:"image"` - Build string `yaml:"build"` - Pull bool `yaml:"pull"` - Privileged bool `yaml:"privileged"` - Environment mapEqualSlice `yaml:"environment"` - Entrypoint stringOrSlice `yaml:"entrypoint"` - Command stringOrSlice `yaml:"command"` - Commands stringOrSlice `yaml:"commands"` - ExtraHosts stringOrSlice `yaml:"extra_hosts"` - Volumes stringOrSlice `yaml:"volumes"` - VolumesFrom stringOrSlice `yaml:"volumes_from"` - Devices stringOrSlice `yaml:"devices"` - Network string `yaml:"network_mode"` - DNS stringOrSlice `yaml:"dns"` - DNSSearch stringOrSlice `yaml:"dns_search"` - MemSwapLimit int64 `yaml:"memswap_limit"` - MemLimit int64 `yaml:"mem_limit"` - CPUQuota int64 `yaml:"cpu_quota"` - CPUShares int64 `yaml:"cpu_shares"` - CPUSet string `yaml:"cpuset"` - OomKillDisable bool `yaml:"oom_kill_disable"` - - AuthConfig struct { - Username string `yaml:"username"` - Password string `yaml:"password"` - Email string `yaml:"email"` - Token string `yaml:"registry_token"` - } `yaml:"auth_config"` - - Conditions struct { - Platform stringOrSlice `yaml:"platform"` - Environment stringOrSlice `yaml:"environment"` - Event stringOrSlice `yaml:"event"` - Branch stringOrSlice `yaml:"branch"` - Status stringOrSlice `yaml:"status"` - Matrix map[string]string `yaml:"matrix"` - } `yaml:"when"` - - Vargs map[string]interface{} `yaml:",inline"` -} - -func (c *container) ToContainer() runner.Container { - return runner.Container{ - Name: c.Name, - Image: c.Image, - Pull: c.Pull, - Privileged: c.Privileged, - Environment: c.Environment.parts, - Entrypoint: c.Entrypoint.parts, - Command: c.Command.parts, - ExtraHosts: c.ExtraHosts.parts, - Volumes: c.Volumes.parts, - VolumesFrom: c.VolumesFrom.parts, - Devices: c.Devices.parts, - Network: c.Network, - DNS: c.DNS.parts, - DNSSearch: c.DNSSearch.parts, - MemSwapLimit: c.MemSwapLimit, - MemLimit: c.MemLimit, - CPUQuota: c.CPUQuota, - CPUShares: c.CPUShares, - CPUSet: c.CPUSet, - OomKillDisable: c.OomKillDisable, - AuthConfig: runner.Auth{ - Username: c.AuthConfig.Username, - Password: c.AuthConfig.Password, - Email: c.AuthConfig.Email, - Token: c.AuthConfig.Token, - }, - } -} - -func (c *container) ToConditions() Conditions { - return Conditions{ - Platform: c.Conditions.Platform.parts, - Environment: c.Conditions.Environment.parts, - Event: c.Conditions.Event.parts, - Branch: c.Conditions.Branch.parts, - Status: c.Conditions.Status.parts, - Matrix: c.Conditions.Matrix, - } -} - -type containerList struct { - containers []*container -} - -func (c *containerList) UnmarshalYAML(unmarshal func(interface{}) error) error { - slice := yaml.MapSlice{} - err := unmarshal(&slice) - if err != nil { - return err - } - - for _, s := range slice { - cc := container{} - - out, err := yaml.Marshal(s.Value) - if err != nil { - return err - } - - err = yaml.Unmarshal(out, &cc) - if err != nil { - return err - } - if cc.Name == "" { - cc.Name = fmt.Sprintf("%v", s.Key) - } - if cc.Image == "" { - cc.Image = fmt.Sprintf("%v", s.Key) - } - c.containers = append(c.containers, &cc) - } - return err -} diff --git a/engine/compiler/parse/node_container_test.go b/engine/compiler/parse/node_container_test.go deleted file mode 100644 index 352e98099..000000000 --- a/engine/compiler/parse/node_container_test.go +++ /dev/null @@ -1,97 +0,0 @@ -package parse - -import ( - "testing" - - "github.com/franela/goblin" - "gopkg.in/yaml.v2" -) - -func TestContainerNode(t *testing.T) { - g := goblin.Goblin(t) - - g.Describe("Containers", func() { - g.Describe("given a yaml file", func() { - - g.It("should unmarshal", func() { - in := []byte(sampleContainer) - out := containerList{} - err := yaml.Unmarshal(in, &out) - if err != nil { - g.Fail(err) - } - g.Assert(len(out.containers)).Equal(1) - - c := out.containers[0] - g.Assert(c.Name).Equal("foo") - g.Assert(c.Image).Equal("golang") - g.Assert(c.Build).Equal(".") - g.Assert(c.Pull).Equal(true) - g.Assert(c.Privileged).Equal(true) - g.Assert(c.Entrypoint.parts).Equal([]string{"/bin/sh"}) - g.Assert(c.Command.parts).Equal([]string{"yes"}) - g.Assert(c.Commands.parts).Equal([]string{"whoami"}) - g.Assert(c.ExtraHosts.parts).Equal([]string{"foo.com"}) - g.Assert(c.Volumes.parts).Equal([]string{"/foo:/bar"}) - g.Assert(c.VolumesFrom.parts).Equal([]string{"foo"}) - g.Assert(c.Devices.parts).Equal([]string{"/dev/tty0"}) - g.Assert(c.Network).Equal("bridge") - g.Assert(c.DNS.parts).Equal([]string{"8.8.8.8"}) - g.Assert(c.MemSwapLimit).Equal(int64(1)) - g.Assert(c.MemLimit).Equal(int64(2)) - g.Assert(c.CPUQuota).Equal(int64(3)) - g.Assert(c.CPUSet).Equal("1,2") - g.Assert(c.OomKillDisable).Equal(true) - g.Assert(c.AuthConfig.Username).Equal("octocat") - g.Assert(c.AuthConfig.Password).Equal("password") - g.Assert(c.AuthConfig.Email).Equal("octocat@github.com") - g.Assert(c.Vargs["access_key"]).Equal("970d28f4dd477bc184fbd10b376de753") - g.Assert(c.Vargs["secret_key"]).Equal("9c5785d3ece6a9cdefa42eb99b58986f9095ff1c") - }) - - g.It("should unmarshal named", func() { - in := []byte("foo: { name: bar }") - out := containerList{} - err := yaml.Unmarshal(in, &out) - if err != nil { - g.Fail(err) - } - g.Assert(len(out.containers)).Equal(1) - g.Assert(out.containers[0].Name).Equal("bar") - }) - - }) - }) -} - -var sampleContainer = ` -foo: - image: golang - build: . - pull: true - privileged: true - environment: - FOO: BAR - entrypoint: /bin/sh - command: "yes" - commands: whoami - extra_hosts: foo.com - volumes: /foo:/bar - volumes_from: foo - devices: /dev/tty0 - network_mode: bridge - dns: 8.8.8.8 - memswap_limit: 1 - mem_limit: 2 - cpu_quota: 3 - cpuset: 1,2 - oom_kill_disable: true - - auth_config: - username: octocat - password: password - email: octocat@github.com - - access_key: 970d28f4dd477bc184fbd10b376de753 - secret_key: 9c5785d3ece6a9cdefa42eb99b58986f9095ff1c -` diff --git a/engine/compiler/parse/node_network.go b/engine/compiler/parse/node_network.go deleted file mode 100644 index b78a4bb7e..000000000 --- a/engine/compiler/parse/node_network.go +++ /dev/null @@ -1,68 +0,0 @@ -package parse - -import ( - "fmt" - - "gopkg.in/yaml.v2" -) - -// NetworkNode represents a Docker network. -type NetworkNode struct { - NodeType - root *RootNode - - Name string - Driver string - DriverOpts map[string]string -} - -// Root returns the root node. -func (n *NetworkNode) Root() *RootNode { return n.root } - -// -// intermediate types for yaml decoding. -// - -// network is an intermediate type used for decoding a networks in a format -// compatible with docker-compose.yml -type network struct { - Name string - Driver string - DriverOpts map[string]string `yaml:"driver_opts"` -} - -// networkList is an intermediate type used for decoding a slice of networks -// in a format compatible with docker-compose.yml -type networkList struct { - networks []*network -} - -func (n *networkList) UnmarshalYAML(unmarshal func(interface{}) error) error { - slice := yaml.MapSlice{} - err := unmarshal(&slice) - if err != nil { - return err - } - - for _, s := range slice { - nn := network{} - - out, err := yaml.Marshal(s.Value) - if err != nil { - return err - } - - err = yaml.Unmarshal(out, &nn) - if err != nil { - return err - } - if nn.Name == "" { - nn.Name = fmt.Sprintf("%v", s.Key) - } - if nn.Driver == "" { - nn.Driver = "bridge" - } - n.networks = append(n.networks, &nn) - } - return err -} diff --git a/engine/compiler/parse/node_network_test.go b/engine/compiler/parse/node_network_test.go deleted file mode 100644 index c4b1ca4fe..000000000 --- a/engine/compiler/parse/node_network_test.go +++ /dev/null @@ -1,51 +0,0 @@ -package parse - -import ( - "testing" - - "github.com/franela/goblin" - "gopkg.in/yaml.v2" -) - -func TestNetworkNode(t *testing.T) { - g := goblin.Goblin(t) - - g.Describe("Networks", func() { - g.Describe("given a yaml file", func() { - - g.It("should unmarshal", func() { - in := []byte("foo: { driver: overlay }") - out := networkList{} - err := yaml.Unmarshal(in, &out) - if err != nil { - g.Fail(err) - } - g.Assert(len(out.networks)).Equal(1) - g.Assert(out.networks[0].Name).Equal("foo") - g.Assert(out.networks[0].Driver).Equal("overlay") - }) - - g.It("should unmarshal named", func() { - in := []byte("foo: { name: bar }") - out := networkList{} - err := yaml.Unmarshal(in, &out) - if err != nil { - g.Fail(err) - } - g.Assert(len(out.networks)).Equal(1) - g.Assert(out.networks[0].Name).Equal("bar") - }) - - g.It("should unmarshal and use default driver", func() { - in := []byte("foo: { name: bar }") - out := volumeList{} - err := yaml.Unmarshal(in, &out) - if err != nil { - g.Fail(err) - } - g.Assert(len(out.volumes)).Equal(1) - g.Assert(out.volumes[0].Driver).Equal("local") - }) - }) - }) -} diff --git a/engine/compiler/parse/node_root.go b/engine/compiler/parse/node_root.go deleted file mode 100644 index fc2ff615f..000000000 --- a/engine/compiler/parse/node_root.go +++ /dev/null @@ -1,146 +0,0 @@ -package parse - -// RootNode is the root node in the parsed Yaml file. -type RootNode struct { - NodeType - - Platform string - Base string - Path string - Image string - - Pod Node - Build Node - Clone Node - Script []Node - Volumes []Node - Networks []Node - Services []Node -} - -// NewRootNode returns a new root node. -func NewRootNode() *RootNode { - return &RootNode{ - NodeType: NodeRoot, - } -} - -// Root returns the root node. -func (n *RootNode) Root() *RootNode { return n } - -// Returns a new Volume Node. -func (n *RootNode) NewVolumeNode(name string) *VolumeNode { - return &VolumeNode{ - NodeType: NodeVolume, - Name: name, - root: n, - } -} - -// Returns a new Network Node. -func (n *RootNode) NewNetworkNode(name string) *NetworkNode { - return &NetworkNode{ - NodeType: NodeNetwork, - Name: name, - root: n, - } -} - -// Returns a new Network Node. -func (n *RootNode) NewBuildNode(context string) *BuildNode { - return &BuildNode{ - NodeType: NodeBuild, - Context: context, - root: n, - } -} - -// Returns a new Container Plugin Node. -func (n *RootNode) NewPluginNode() *ContainerNode { - return &ContainerNode{ - NodeType: NodePlugin, - root: n, - } -} - -// Returns a new Container Shell Node. -func (n *RootNode) NewShellNode() *ContainerNode { - return &ContainerNode{ - NodeType: NodeShell, - root: n, - } -} - -// Returns a new Container Service Node. -func (n *RootNode) NewServiceNode() *ContainerNode { - return &ContainerNode{ - NodeType: NodeService, - root: n, - } -} - -// Returns a new Container Clone Node. -func (n *RootNode) NewCloneNode() *ContainerNode { - return &ContainerNode{ - NodeType: NodeClone, - root: n, - } -} - -// Returns a new Container Cache Node. -func (n *RootNode) NewCacheNode() *ContainerNode { - return &ContainerNode{ - NodeType: NodeCache, - root: n, - } -} - -// Returns a new Container Node. -func (n *RootNode) NewContainerNode() *ContainerNode { - return &ContainerNode{ - NodeType: NodeContainer, - root: n, - } -} - -// Walk is a function that walk through all child nodes of the RootNode -// and invokes the Walk callback function for each Node. -func (n *RootNode) Walk(fn WalkFunc) (err error) { - var nodes []Node - nodes = append(nodes, n) - nodes = append(nodes, n.Build) - nodes = append(nodes, n.Clone) - nodes = append(nodes, n.Script...) - nodes = append(nodes, n.Volumes...) - nodes = append(nodes, n.Networks...) - nodes = append(nodes, n.Services...) - for _, node := range nodes { - err = fn(node) - if err != nil { - return - } - } - return -} - -type WalkFunc func(Node) error - -// -// intermediate types for yaml decoding. -// - -type root struct { - Workspace struct { - Path string - Base string - } - Image string - Platform string - Volumes volumeList - Networks networkList - Services containerList - Script containerList `yaml:"pipeline"` - Cache container - Clone container - Build build -} diff --git a/engine/compiler/parse/node_root_test.go b/engine/compiler/parse/node_root_test.go deleted file mode 100644 index f4760109a..000000000 --- a/engine/compiler/parse/node_root_test.go +++ /dev/null @@ -1,85 +0,0 @@ -package parse - -import ( - "testing" - - "github.com/franela/goblin" -) - -func TestRootNode(t *testing.T) { - g := goblin.Goblin(t) - r := &RootNode{} - - g.Describe("Root Node", func() { - - g.It("should return self as root", func() { - g.Assert(r).Equal(r.Root()) - }) - - g.It("should create a Volume Node", func() { - n := r.NewVolumeNode("foo") - g.Assert(n.Root()).Equal(r) - g.Assert(n.Name).Equal("foo") - g.Assert(n.String()).Equal(NodeVolume) - g.Assert(n.Type()).Equal(NodeType(NodeVolume)) - }) - - g.It("should create a Network Node", func() { - n := r.NewNetworkNode("foo") - g.Assert(n.Root()).Equal(r) - g.Assert(n.Name).Equal("foo") - g.Assert(n.String()).Equal(NodeNetwork) - g.Assert(n.Type()).Equal(NodeType(NodeNetwork)) - }) - - g.It("should create a Plugin Node", func() { - n := r.NewPluginNode() - g.Assert(n.Root()).Equal(r) - g.Assert(n.String()).Equal(NodePlugin) - g.Assert(n.Type()).Equal(NodeType(NodePlugin)) - }) - - g.It("should create a Shell Node", func() { - n := r.NewShellNode() - g.Assert(n.Root()).Equal(r) - g.Assert(n.String()).Equal(NodeShell) - g.Assert(n.Type()).Equal(NodeType(NodeShell)) - }) - - g.It("should create a Service Node", func() { - n := r.NewServiceNode() - g.Assert(n.Root()).Equal(r) - g.Assert(n.String()).Equal(NodeService) - g.Assert(n.Type()).Equal(NodeType(NodeService)) - }) - - g.It("should create a Build Node", func() { - n := r.NewBuildNode(".") - g.Assert(n.Root()).Equal(r) - g.Assert(n.Context).Equal(".") - g.Assert(n.String()).Equal(NodeBuild) - g.Assert(n.Type()).Equal(NodeType(NodeBuild)) - }) - - g.It("should create a Cache Node", func() { - n := r.NewCacheNode() - g.Assert(n.Root()).Equal(r) - g.Assert(n.String()).Equal(NodeCache) - g.Assert(n.Type()).Equal(NodeType(NodeCache)) - }) - - g.It("should create a Clone Node", func() { - n := r.NewCloneNode() - g.Assert(n.Root()).Equal(r) - g.Assert(n.String()).Equal(NodeClone) - g.Assert(n.Type()).Equal(NodeType(NodeClone)) - }) - - g.It("should create a Container Node", func() { - n := r.NewContainerNode() - g.Assert(n.Root()).Equal(r) - g.Assert(n.String()).Equal(NodeContainer) - g.Assert(n.Type()).Equal(NodeType(NodeContainer)) - }) - }) -} diff --git a/engine/compiler/parse/node_volume.go b/engine/compiler/parse/node_volume.go deleted file mode 100644 index 1aadfa1f7..000000000 --- a/engine/compiler/parse/node_volume.go +++ /dev/null @@ -1,69 +0,0 @@ -package parse - -import ( - "fmt" - - "gopkg.in/yaml.v2" -) - -// VolumeNode represents a Docker volume. -type VolumeNode struct { - NodeType - root *RootNode - - Name string - Driver string - DriverOpts map[string]string - External bool -} - -// Root returns the root node. -func (n *VolumeNode) Root() *RootNode { return n.root } - -// -// intermediate types for yaml decoding. -// - -// volume is an intermediate type used for decoding a volumes in a format -// compatible with docker-compose.yml -type volume struct { - Name string - Driver string - DriverOpts map[string]string `yaml:"driver_opts"` -} - -// volumeList is an intermediate type used for decoding a slice of volumes -// in a format compatible with docker-compose.yml -type volumeList struct { - volumes []*volume -} - -func (v *volumeList) UnmarshalYAML(unmarshal func(interface{}) error) error { - slice := yaml.MapSlice{} - err := unmarshal(&slice) - if err != nil { - return err - } - - for _, s := range slice { - vv := volume{} - - out, err := yaml.Marshal(s.Value) - if err != nil { - return err - } - - err = yaml.Unmarshal(out, &vv) - if err != nil { - return err - } - if vv.Name == "" { - vv.Name = fmt.Sprintf("%v", s.Key) - } - if vv.Driver == "" { - vv.Driver = "local" - } - v.volumes = append(v.volumes, &vv) - } - return err -} diff --git a/engine/compiler/parse/node_volume_test.go b/engine/compiler/parse/node_volume_test.go deleted file mode 100644 index 795880918..000000000 --- a/engine/compiler/parse/node_volume_test.go +++ /dev/null @@ -1,51 +0,0 @@ -package parse - -import ( - "testing" - - "github.com/franela/goblin" - "gopkg.in/yaml.v2" -) - -func TestVolumeNode(t *testing.T) { - g := goblin.Goblin(t) - - g.Describe("Volumes", func() { - g.Describe("given a yaml file", func() { - - g.It("should unmarshal", func() { - in := []byte("foo: { driver: blockbridge }") - out := volumeList{} - err := yaml.Unmarshal(in, &out) - if err != nil { - g.Fail(err) - } - g.Assert(len(out.volumes)).Equal(1) - g.Assert(out.volumes[0].Name).Equal("foo") - g.Assert(out.volumes[0].Driver).Equal("blockbridge") - }) - - g.It("should unmarshal named", func() { - in := []byte("foo: { name: bar }") - out := volumeList{} - err := yaml.Unmarshal(in, &out) - if err != nil { - g.Fail(err) - } - g.Assert(len(out.volumes)).Equal(1) - g.Assert(out.volumes[0].Name).Equal("bar") - }) - - g.It("should unmarshal and use default driver", func() { - in := []byte("foo: { name: bar }") - out := volumeList{} - err := yaml.Unmarshal(in, &out) - if err != nil { - g.Fail(err) - } - g.Assert(len(out.volumes)).Equal(1) - g.Assert(out.volumes[0].Driver).Equal("local") - }) - }) - }) -} diff --git a/engine/compiler/parse/parse.go b/engine/compiler/parse/parse.go deleted file mode 100644 index 61434d629..000000000 --- a/engine/compiler/parse/parse.go +++ /dev/null @@ -1,90 +0,0 @@ -package parse - -import ( - "gopkg.in/yaml.v2" -) - -// Parse parses a Yaml file and returns a Tree structure. -func Parse(in []byte) (*RootNode, error) { - out := root{} - err := yaml.Unmarshal(in, &out) - if err != nil { - return nil, err - } - - root := NewRootNode() - root.Platform = out.Platform - root.Path = out.Workspace.Path - root.Base = out.Workspace.Base - root.Image = out.Image - - // append volume nodes to tree - for _, v := range out.Volumes.volumes { - vv := root.NewVolumeNode(v.Name) - vv.Driver = v.Driver - vv.DriverOpts = v.DriverOpts - root.Volumes = append(root.Volumes, vv) - } - - // append network nodes to tree - for _, n := range out.Networks.networks { - nn := root.NewNetworkNode(n.Name) - nn.Driver = n.Driver - nn.DriverOpts = n.DriverOpts - root.Networks = append(root.Networks, nn) - } - - // add the build section - if out.Build.Context != "" { - root.Build = &BuildNode{ - NodeType: NodeBuild, - Context: out.Build.Context, - Dockerfile: out.Build.Dockerfile, - Args: out.Build.Args, - root: root, - } - } - - // add the clone section - { - cc := root.NewCloneNode() - cc.Conditions = out.Clone.ToConditions() - cc.Container = out.Clone.ToContainer() - cc.Container.Name = "clone" - cc.Vargs = out.Clone.Vargs - root.Clone = cc - } - - // append services - for _, c := range out.Services.containers { - if c.Build != "" { - continue - } - cc := root.NewServiceNode() - cc.Conditions = c.ToConditions() - cc.Container = c.ToContainer() - root.Services = append(root.Services, cc) - } - - // append scripts - for _, c := range out.Script.containers { - var cc *ContainerNode - if len(c.Commands.parts) == 0 { - cc = root.NewPluginNode() - } else { - cc = root.NewShellNode() - } - cc.Commands = c.Commands.parts - cc.Vargs = c.Vargs - cc.Container = c.ToContainer() - cc.Conditions = c.ToConditions() - root.Script = append(root.Script, cc) - } - - return root, nil -} - -// ParseString parses a Yaml string and returns a Tree structure. -func ParseString(in string) (*RootNode, error) { - return Parse([]byte(in)) -} diff --git a/engine/compiler/parse/parse_test.go b/engine/compiler/parse/parse_test.go deleted file mode 100644 index e6ece2c9b..000000000 --- a/engine/compiler/parse/parse_test.go +++ /dev/null @@ -1,95 +0,0 @@ -package parse - -import ( - "testing" - - "github.com/franela/goblin" -) - -func TestParse(t *testing.T) { - g := goblin.Goblin(t) - - g.Describe("Parser", func() { - g.Describe("given a yaml file", func() { - - g.It("should unmarshal a string", func() { - out, err := ParseString(sampleYaml) - if err != nil { - g.Fail(err) - } - g.Assert(out.Image).Equal("hello-world") - g.Assert(out.Base).Equal("/go") - g.Assert(out.Path).Equal("src/github.com/octocat/hello-world") - g.Assert(out.Build.(*BuildNode).Context).Equal(".") - g.Assert(out.Build.(*BuildNode).Dockerfile).Equal("Dockerfile") - g.Assert(out.Clone.(*ContainerNode).Container.Image).Equal("git") - g.Assert(out.Clone.(*ContainerNode).Vargs["depth"]).Equal(1) - g.Assert(out.Volumes[0].(*VolumeNode).Name).Equal("custom") - g.Assert(out.Volumes[0].(*VolumeNode).Driver).Equal("blockbridge") - g.Assert(out.Networks[0].(*NetworkNode).Name).Equal("custom") - g.Assert(out.Networks[0].(*NetworkNode).Driver).Equal("overlay") - g.Assert(out.Services[0].(*ContainerNode).Container.Name).Equal("database") - g.Assert(out.Services[0].(*ContainerNode).Container.Image).Equal("mysql") - g.Assert(out.Script[0].(*ContainerNode).Container.Name).Equal("test") - g.Assert(out.Script[0].(*ContainerNode).Container.Image).Equal("golang") - g.Assert(out.Script[0].(*ContainerNode).Commands).Equal([]string{"go install", "go test"}) - g.Assert(out.Script[0].(*ContainerNode).String()).Equal(NodeShell) - g.Assert(out.Script[1].(*ContainerNode).Container.Name).Equal("build") - g.Assert(out.Script[1].(*ContainerNode).Container.Image).Equal("golang") - g.Assert(out.Script[1].(*ContainerNode).Commands).Equal([]string{"go build"}) - g.Assert(out.Script[1].(*ContainerNode).String()).Equal(NodeShell) - g.Assert(out.Script[2].(*ContainerNode).Container.Name).Equal("notify") - g.Assert(out.Script[2].(*ContainerNode).Container.Image).Equal("slack") - g.Assert(out.Script[2].(*ContainerNode).String()).Equal(NodePlugin) - }) - }) - }) -} - -var sampleYaml = ` -image: hello-world -build: - context: . - dockerfile: Dockerfile - -workspace: - path: src/github.com/octocat/hello-world - base: /go - -clone: - image: git - depth: 1 - -cache: - mount: node_modules - -pipeline: - test: - image: golang - commands: - - go install - - go test - build: - image: golang - commands: - - go build - when: - event: push - notify: - image: slack - channel: dev - when: - event: failure - -services: - database: - image: mysql - -networks: - custom: - driver: overlay - -volumes: - custom: - driver: blockbridge -` diff --git a/engine/compiler/parse/types.go b/engine/compiler/parse/types.go deleted file mode 100644 index cf4596886..000000000 --- a/engine/compiler/parse/types.go +++ /dev/null @@ -1,55 +0,0 @@ -package parse - -import "strings" - -// mapEqualSlice represents a map[string]string or a slice of -// strings in key=value format. -type mapEqualSlice struct { - parts map[string]string -} - -func (s *mapEqualSlice) UnmarshalYAML(unmarshal func(interface{}) error) error { - s.parts = map[string]string{} - err := unmarshal(&s.parts) - if err == nil { - return nil - } - - var slice []string - err = unmarshal(&slice) - if err != nil { - return err - } - for _, v := range slice { - parts := strings.SplitN(v, "=", 2) - if len(parts) == 2 { - key := parts[0] - val := parts[1] - s.parts[key] = val - } - } - return nil -} - -// stringOrSlice represents a string or an array of strings. -type stringOrSlice struct { - parts []string -} - -func (s *stringOrSlice) UnmarshalYAML(unmarshal func(interface{}) error) error { - var sliceType []string - err := unmarshal(&sliceType) - if err == nil { - s.parts = sliceType - return nil - } - - var stringType string - err = unmarshal(&stringType) - if err == nil { - sliceType = make([]string, 0, 1) - s.parts = append(sliceType, string(stringType)) - return nil - } - return err -} diff --git a/engine/compiler/parse/types_test.go b/engine/compiler/parse/types_test.go deleted file mode 100644 index 463a72c75..000000000 --- a/engine/compiler/parse/types_test.go +++ /dev/null @@ -1,75 +0,0 @@ -package parse - -import ( - "testing" - - "github.com/franela/goblin" - "gopkg.in/yaml.v2" -) - -func TestTypes(t *testing.T) { - g := goblin.Goblin(t) - - g.Describe("Yaml types", func() { - g.Describe("given a yaml file", func() { - - g.It("should unmarshal a string", func() { - in := []byte("foo") - out := stringOrSlice{} - err := yaml.Unmarshal(in, &out) - if err != nil { - g.Fail(err) - } - g.Assert(len(out.parts)).Equal(1) - g.Assert(out.parts[0]).Equal("foo") - }) - - g.It("should unmarshal a string slice", func() { - in := []byte("[ foo ]") - out := stringOrSlice{} - err := yaml.Unmarshal(in, &out) - if err != nil { - g.Fail(err) - } - g.Assert(len(out.parts)).Equal(1) - g.Assert(out.parts[0]).Equal("foo") - }) - - g.It("should throw error when invalid string slice", func() { - in := []byte("{ }") // string value should fail parse - out := stringOrSlice{} - err := yaml.Unmarshal(in, &out) - g.Assert(err != nil).IsTrue("expects error") - }) - - g.It("should unmarshal a map", func() { - in := []byte("foo: bar") - out := mapEqualSlice{} - err := yaml.Unmarshal(in, &out) - if err != nil { - g.Fail(err) - } - g.Assert(len(out.parts)).Equal(1) - g.Assert(out.parts["foo"]).Equal("bar") - }) - - g.It("should unmarshal a map equal slice", func() { - in := []byte("[ foo=bar ]") - out := mapEqualSlice{} - err := yaml.Unmarshal(in, &out) - if err != nil { - g.Fail(err) - } - g.Assert(len(out.parts)).Equal(1) - g.Assert(out.parts["foo"]).Equal("bar") - }) - - g.It("should throw error when invalid map equal slice", func() { - in := []byte("foo") // string value should fail parse - out := mapEqualSlice{} - err := yaml.Unmarshal(in, &out) - g.Assert(err != nil).IsTrue("expects error") - }) - }) - }) -} diff --git a/engine/compiler/transform.go b/engine/compiler/transform.go deleted file mode 100644 index a61087e58..000000000 --- a/engine/compiler/transform.go +++ /dev/null @@ -1,13 +0,0 @@ -package compiler - -import "github.com/drone/drone/engine/compiler/parse" - -// Transform is used to transform nodes from the parsed Yaml file during the -// compilation process. A Transform may be used to add, disable or alter nodes. -type Transform interface { - VisitRoot(*parse.RootNode) error - VisitVolume(*parse.VolumeNode) error - VisitNetwork(*parse.NetworkNode) error - VisitBuild(*parse.BuildNode) error - VisitContainer(*parse.ContainerNode) error -} diff --git a/engine/runner/container.go b/engine/runner/container.go deleted file mode 100644 index e901e3b19..000000000 --- a/engine/runner/container.go +++ /dev/null @@ -1,72 +0,0 @@ -package runner - -import "fmt" - -// Container defines the container configuration. -type Container struct { - Name string `json:"name"` - Alias string `json:"alias"` - Image string `json:"image"` - Pull bool `json:"pull,omitempty"` - AuthConfig Auth `json:"auth_config,omitempty"` - Privileged bool `json:"privileged,omitempty"` - WorkingDir string `json:"working_dir,omitempty"` - Environment map[string]string `json:"environment,omitempty"` - Entrypoint []string `json:"entrypoint,omitempty"` - Command []string `json:"command,omitempty"` - ExtraHosts []string `json:"extra_hosts,omitempty"` - Volumes []string `json:"volumes,omitempty"` - VolumesFrom []string `json:"volumes_from,omitempty"` - Devices []string `json:"devices,omitempty"` - Network string `json:"network_mode,omitempty"` - DNS []string `json:"dns,omitempty"` - DNSSearch []string `json:"dns_search,omitempty"` - MemSwapLimit int64 `json:"memswap_limit,omitempty"` - MemLimit int64 `json:"mem_limit,omitempty"` - CPUQuota int64 `json:"cpu_quota,omitempty"` - CPUShares int64 `json:"cpu_shares,omitempty"` - CPUSet string `json:"cpuset,omitempty"` - OomKillDisable bool `json:"oom_kill_disable,omitempty"` -} - -// Validate validates the container configuration details and returns an error -// if the validation fails. -func (c *Container) Validate() error { - switch { - - case c.Name == "": - return fmt.Errorf("Missing container name") - case c.Image == "": - return fmt.Errorf("Missing container image") - default: - return nil - } - -} - -// Auth provides authentication parameters to authenticate to a remote -// container registry for image download. -type Auth struct { - Username string `json:"username,omitempty"` - Password string `json:"password,omitempty"` - Email string `json:"email,omitempty"` - Token string `json:"registry_token,omitempty"` -} - -// Volume defines a container volume. -type Volume struct { - Name string `json:"name,omitempty"` - Alias string `json:"alias,omitempty"` - Driver string `json:"driver,omitempty"` - DriverOpts map[string]string `json:"driver_opts,omitempty"` - External bool `json:"external,omitempty"` -} - -// Network defines a container network. -type Network struct { - Name string `json:"name,omitempty"` - Alias string `json:"alias,omitempty"` - Driver string `json:"driver,omitempty"` - DriverOpts map[string]string `json:"driver_opts,omitempty"` - External bool `json:"external,omitempty"` -} diff --git a/engine/runner/container_test.go b/engine/runner/container_test.go deleted file mode 100644 index 6fab60ee2..000000000 --- a/engine/runner/container_test.go +++ /dev/null @@ -1,40 +0,0 @@ -package runner - -import ( - "testing" - - "github.com/franela/goblin" -) - -func TestContainer(t *testing.T) { - g := goblin.Goblin(t) - - g.Describe("Container validation", func() { - - g.It("fails with an invalid name", func() { - c := Container{ - Image: "golang:1.5", - } - err := c.Validate() - g.Assert(err != nil).IsTrue() - g.Assert(err.Error()).Equal("Missing container name") - }) - - g.It("fails with an invalid image", func() { - c := Container{ - Name: "container_0", - } - err := c.Validate() - g.Assert(err != nil).IsTrue() - g.Assert(err.Error()).Equal("Missing container image") - }) - - g.It("passes with valid attributes", func() { - c := Container{ - Name: "container_0", - Image: "golang:1.5", - } - g.Assert(c.Validate() == nil).IsTrue() - }) - }) -} diff --git a/engine/runner/docker/context.go b/engine/runner/docker/context.go deleted file mode 100644 index e19ef84b2..000000000 --- a/engine/runner/docker/context.go +++ /dev/null @@ -1,24 +0,0 @@ -package docker - -import ( - "github.com/drone/drone/engine/runner" - "golang.org/x/net/context" -) - -const key = "docker" - -// Setter defines a context that enables setting values. -type Setter interface { - Set(string, interface{}) -} - -// FromContext returns the Engine associated with this context. -func FromContext(c context.Context) runner.Engine { - return c.Value(key).(runner.Engine) -} - -// ToContext adds the Engine to this context if it supports the -// Setter interface. -func ToContext(c Setter, d runner.Engine) { - c.Set(key, d) -} diff --git a/engine/runner/docker/docker.go b/engine/runner/docker/docker.go deleted file mode 100644 index 9e1bb4606..000000000 --- a/engine/runner/docker/docker.go +++ /dev/null @@ -1,111 +0,0 @@ -package docker - -import ( - "io" - - "github.com/drone/drone/engine/runner" - "github.com/drone/drone/engine/runner/docker/internal" - - "github.com/samalba/dockerclient" -) - -type dockerEngine struct { - client dockerclient.Client -} - -func (e *dockerEngine) ContainerStart(container *runner.Container) (string, error) { - conf := toContainerConfig(container) - auth := toAuthConfig(container) - - // pull the image if it does not exists or if the Container - // is configured to always pull a new image. - _, err := e.client.InspectImage(container.Image) - if err != nil || container.Pull { - e.client.PullImage(container.Image, auth) - } - - // create and start the container and return the Container ID. - id, err := e.client.CreateContainer(conf, container.Name, auth) - if err != nil { - return id, err - } - err = e.client.StartContainer(id, &conf.HostConfig) - if err != nil { - - // remove the container if it cannot be started - e.client.RemoveContainer(id, true, true) - return id, err - } - return id, nil -} - -func (e *dockerEngine) ContainerStop(id string) error { - e.client.StopContainer(id, 1) - e.client.KillContainer(id, "9") - return nil -} - -func (e *dockerEngine) ContainerRemove(id string) error { - e.client.StopContainer(id, 1) - e.client.KillContainer(id, "9") - e.client.RemoveContainer(id, true, true) - return nil -} - -func (e *dockerEngine) ContainerWait(id string) (*runner.State, error) { - // wait for the container to exit - // - // TODO(bradrydzewski) we should have a for loop here - // to re-connect and wait if this channel returns a - // result even though the container is still running. - // - <-e.client.Wait(id) - v, err := e.client.InspectContainer(id) - if err != nil { - return nil, err - } - return &runner.State{ - ExitCode: v.State.ExitCode, - OOMKilled: v.State.OOMKilled, - }, nil -} - -func (e *dockerEngine) ContainerLogs(id string) (io.ReadCloser, error) { - opts := &dockerclient.LogOptions{ - Follow: true, - Stdout: true, - Stderr: true, - } - - piper, pipew := io.Pipe() - go func() { - defer pipew.Close() - - // sometimes the docker logs fails due to parsing errors. this - // routine will check for such a failure and attempt to resume - // if necessary. - for i := 0; i < 5; i++ { - if i > 0 { - opts.Tail = 1 - } - - rc, err := e.client.ContainerLogs(id, opts) - if err != nil { - return - } - defer rc.Close() - - // use Docker StdCopy - internal.StdCopy(pipew, pipew, rc) - - // check to see if the container is still running. If not, - // we can safely exit and assume there are no more logs left - // to stream. - v, err := e.client.InspectContainer(id) - if err != nil || !v.State.Running { - return - } - } - }() - return piper, nil -} diff --git a/engine/runner/docker/docker_test.go b/engine/runner/docker/docker_test.go deleted file mode 100644 index 1cdc3ff91..000000000 --- a/engine/runner/docker/docker_test.go +++ /dev/null @@ -1 +0,0 @@ -package docker diff --git a/engine/runner/docker/helper.go b/engine/runner/docker/helper.go deleted file mode 100644 index 25b77f955..000000000 --- a/engine/runner/docker/helper.go +++ /dev/null @@ -1,49 +0,0 @@ -package docker - -import ( - "os" - - "github.com/drone/drone/engine/runner" - "github.com/samalba/dockerclient" -) - -var ( - dockerHost = os.Getenv("DOCKER_HOST") - dockerCert = os.Getenv("DOCKER_CERT_PATH") - dockerTLS = os.Getenv("DOCKER_TLS_VERIFY") -) - -func init() { - if dockerHost == "" { - dockerHost = "unix:///var/run/docker.sock" - } -} - -// New returns a new Docker engine using the provided Docker client. -func New(client dockerclient.Client) runner.Engine { - return &dockerEngine{client} -} - -// NewEnv returns a new Docker engine from the DOCKER_HOST and DOCKER_CERT_PATH -// environment variables. -func NewEnv() (runner.Engine, error) { - config, err := dockerclient.TLSConfigFromCertPath(dockerCert) - if err == nil && dockerTLS != "1" { - config.InsecureSkipVerify = true - } - client, err := dockerclient.NewDockerClient(dockerHost, config) - if err != nil { - return nil, err - } - return New(client), nil -} - -// MustEnv returns a new Docker engine from the DOCKER_HOST and DOCKER_CERT_PATH -// environment variables. Errors creating the Docker engine will panic. -func MustEnv() runner.Engine { - engine, err := NewEnv() - if err != nil { - panic(err) - } - return engine -} diff --git a/engine/runner/docker/helper_test.go b/engine/runner/docker/helper_test.go deleted file mode 100644 index 1cdc3ff91..000000000 --- a/engine/runner/docker/helper_test.go +++ /dev/null @@ -1 +0,0 @@ -package docker diff --git a/engine/runner/docker/internal/README b/engine/runner/docker/internal/README deleted file mode 100644 index 2bd3e9830..000000000 --- a/engine/runner/docker/internal/README +++ /dev/null @@ -1 +0,0 @@ -This is an internal copy of the Docker stdcopy package that removes the logrus debug logging. The original package is found at https://github.com/docker/docker/tree/master/pkg/stdcopy \ No newline at end of file diff --git a/engine/runner/docker/internal/stdcopy.go b/engine/runner/docker/internal/stdcopy.go deleted file mode 100644 index db61b0c88..000000000 --- a/engine/runner/docker/internal/stdcopy.go +++ /dev/null @@ -1,167 +0,0 @@ -package internal - -import ( - "encoding/binary" - "errors" - "fmt" - "io" -) - -// StdType is the type of standard stream -// a writer can multiplex to. -type StdType byte - -const ( - // Stdin represents standard input stream type. - Stdin StdType = iota - // Stdout represents standard output stream type. - Stdout - // Stderr represents standard error steam type. - Stderr - - stdWriterPrefixLen = 8 - stdWriterFdIndex = 0 - stdWriterSizeIndex = 4 - - startingBufLen = 32*1024 + stdWriterPrefixLen + 1 -) - -// stdWriter is wrapper of io.Writer with extra customized info. -type stdWriter struct { - io.Writer - prefix byte -} - -// Write sends the buffer to the underneath writer. -// It insert the prefix header before the buffer, -// so stdcopy.StdCopy knows where to multiplex the output. -// It makes stdWriter to implement io.Writer. -func (w *stdWriter) Write(buf []byte) (n int, err error) { - if w == nil || w.Writer == nil { - return 0, errors.New("Writer not instantiated") - } - if buf == nil { - return 0, nil - } - - header := [stdWriterPrefixLen]byte{stdWriterFdIndex: w.prefix} - binary.BigEndian.PutUint32(header[stdWriterSizeIndex:], uint32(len(buf))) - - line := append(header[:], buf...) - - n, err = w.Writer.Write(line) - n -= stdWriterPrefixLen - - if n < 0 { - n = 0 - } - return -} - -// NewStdWriter instantiates a new Writer. -// Everything written to it will be encapsulated using a custom format, -// and written to the underlying `w` stream. -// This allows multiple write streams (e.g. stdout and stderr) to be muxed into a single connection. -// `t` indicates the id of the stream to encapsulate. -// It can be stdcopy.Stdin, stdcopy.Stdout, stdcopy.Stderr. -func NewStdWriter(w io.Writer, t StdType) io.Writer { - return &stdWriter{ - Writer: w, - prefix: byte(t), - } -} - -// StdCopy is a modified version of io.Copy. -// -// StdCopy will demultiplex `src`, assuming that it contains two streams, -// previously multiplexed together using a StdWriter instance. -// As it reads from `src`, StdCopy will write to `dstout` and `dsterr`. -// -// StdCopy will read until it hits EOF on `src`. It will then return a nil error. -// In other words: if `err` is non nil, it indicates a real underlying error. -// -// `written` will hold the total number of bytes written to `dstout` and `dsterr`. -func StdCopy(dstout, dsterr io.Writer, src io.Reader) (written int64, err error) { - var ( - buf = make([]byte, startingBufLen) - bufLen = len(buf) - nr, nw int - er, ew error - out io.Writer - frameSize int - ) - - for { - // Make sure we have at least a full header - for nr < stdWriterPrefixLen { - var nr2 int - nr2, er = src.Read(buf[nr:]) - nr += nr2 - if er == io.EOF { - if nr < stdWriterPrefixLen { - return written, nil - } - break - } - if er != nil { - return 0, er - } - } - - // Check the first byte to know where to write - switch StdType(buf[stdWriterFdIndex]) { - case Stdin: - fallthrough - case Stdout: - // Write on stdout - out = dstout - case Stderr: - // Write on stderr - out = dsterr - default: - return 0, fmt.Errorf("Unrecognized input header: %d", buf[stdWriterFdIndex]) - } - - // Retrieve the size of the frame - frameSize = int(binary.BigEndian.Uint32(buf[stdWriterSizeIndex : stdWriterSizeIndex+4])) - - // Check if the buffer is big enough to read the frame. - // Extend it if necessary. - if frameSize+stdWriterPrefixLen > bufLen { - buf = append(buf, make([]byte, frameSize+stdWriterPrefixLen-bufLen+1)...) - bufLen = len(buf) - } - - // While the amount of bytes read is less than the size of the frame + header, we keep reading - for nr < frameSize+stdWriterPrefixLen { - var nr2 int - nr2, er = src.Read(buf[nr:]) - nr += nr2 - if er == io.EOF { - if nr < frameSize+stdWriterPrefixLen { - return written, nil - } - break - } - if er != nil { - return 0, er - } - } - - // Write the retrieved frame (without header) - nw, ew = out.Write(buf[stdWriterPrefixLen : frameSize+stdWriterPrefixLen]) - if ew != nil { - return 0, ew - } - // If the frame has not been fully written: error - if nw != frameSize { - return 0, io.ErrShortWrite - } - written += int64(nw) - - // Move the rest of the buffer to the beginning - copy(buf, buf[frameSize+stdWriterPrefixLen:]) - // Move the index - nr -= frameSize + stdWriterPrefixLen - } -} diff --git a/engine/runner/docker/internal/stdcopy_test.go b/engine/runner/docker/internal/stdcopy_test.go deleted file mode 100644 index 7a443bb8b..000000000 --- a/engine/runner/docker/internal/stdcopy_test.go +++ /dev/null @@ -1,260 +0,0 @@ -package internal - -import ( - "bytes" - "errors" - "io" - "io/ioutil" - "strings" - "testing" -) - -func TestNewStdWriter(t *testing.T) { - writer := NewStdWriter(ioutil.Discard, Stdout) - if writer == nil { - t.Fatalf("NewStdWriter with an invalid StdType should not return nil.") - } -} - -func TestWriteWithUnitializedStdWriter(t *testing.T) { - writer := stdWriter{ - Writer: nil, - prefix: byte(Stdout), - } - n, err := writer.Write([]byte("Something here")) - if n != 0 || err == nil { - t.Fatalf("Should fail when given an uncomplete or uninitialized StdWriter") - } -} - -func TestWriteWithNilBytes(t *testing.T) { - writer := NewStdWriter(ioutil.Discard, Stdout) - n, err := writer.Write(nil) - if err != nil { - t.Fatalf("Shouldn't have fail when given no data") - } - if n > 0 { - t.Fatalf("Write should have written 0 byte, but has written %d", n) - } -} - -func TestWrite(t *testing.T) { - writer := NewStdWriter(ioutil.Discard, Stdout) - data := []byte("Test StdWrite.Write") - n, err := writer.Write(data) - if err != nil { - t.Fatalf("Error while writing with StdWrite") - } - if n != len(data) { - t.Fatalf("Write should have written %d byte but wrote %d.", len(data), n) - } -} - -type errWriter struct { - n int - err error -} - -func (f *errWriter) Write(buf []byte) (int, error) { - return f.n, f.err -} - -func TestWriteWithWriterError(t *testing.T) { - expectedError := errors.New("expected") - expectedReturnedBytes := 10 - writer := NewStdWriter(&errWriter{ - n: stdWriterPrefixLen + expectedReturnedBytes, - err: expectedError}, Stdout) - data := []byte("This won't get written, sigh") - n, err := writer.Write(data) - if err != expectedError { - t.Fatalf("Didn't get expected error.") - } - if n != expectedReturnedBytes { - t.Fatalf("Didn't get expected written bytes %d, got %d.", - expectedReturnedBytes, n) - } -} - -func TestWriteDoesNotReturnNegativeWrittenBytes(t *testing.T) { - writer := NewStdWriter(&errWriter{n: -1}, Stdout) - data := []byte("This won't get written, sigh") - actual, _ := writer.Write(data) - if actual != 0 { - t.Fatalf("Expected returned written bytes equal to 0, got %d", actual) - } -} - -func getSrcBuffer(stdOutBytes, stdErrBytes []byte) (buffer *bytes.Buffer, err error) { - buffer = new(bytes.Buffer) - dstOut := NewStdWriter(buffer, Stdout) - _, err = dstOut.Write(stdOutBytes) - if err != nil { - return - } - dstErr := NewStdWriter(buffer, Stderr) - _, err = dstErr.Write(stdErrBytes) - return -} - -func TestStdCopyWriteAndRead(t *testing.T) { - stdOutBytes := []byte(strings.Repeat("o", startingBufLen)) - stdErrBytes := []byte(strings.Repeat("e", startingBufLen)) - buffer, err := getSrcBuffer(stdOutBytes, stdErrBytes) - if err != nil { - t.Fatal(err) - } - written, err := StdCopy(ioutil.Discard, ioutil.Discard, buffer) - if err != nil { - t.Fatal(err) - } - expectedTotalWritten := len(stdOutBytes) + len(stdErrBytes) - if written != int64(expectedTotalWritten) { - t.Fatalf("Expected to have total of %d bytes written, got %d", expectedTotalWritten, written) - } -} - -type customReader struct { - n int - err error - totalCalls int - correctCalls int - src *bytes.Buffer -} - -func (f *customReader) Read(buf []byte) (int, error) { - f.totalCalls++ - if f.totalCalls <= f.correctCalls { - return f.src.Read(buf) - } - return f.n, f.err -} - -func TestStdCopyReturnsErrorReadingHeader(t *testing.T) { - expectedError := errors.New("error") - reader := &customReader{ - err: expectedError} - written, err := StdCopy(ioutil.Discard, ioutil.Discard, reader) - if written != 0 { - t.Fatalf("Expected 0 bytes read, got %d", written) - } - if err != expectedError { - t.Fatalf("Didn't get expected error") - } -} - -func TestStdCopyReturnsErrorReadingFrame(t *testing.T) { - expectedError := errors.New("error") - stdOutBytes := []byte(strings.Repeat("o", startingBufLen)) - stdErrBytes := []byte(strings.Repeat("e", startingBufLen)) - buffer, err := getSrcBuffer(stdOutBytes, stdErrBytes) - if err != nil { - t.Fatal(err) - } - reader := &customReader{ - correctCalls: 1, - n: stdWriterPrefixLen + 1, - err: expectedError, - src: buffer} - written, err := StdCopy(ioutil.Discard, ioutil.Discard, reader) - if written != 0 { - t.Fatalf("Expected 0 bytes read, got %d", written) - } - if err != expectedError { - t.Fatalf("Didn't get expected error") - } -} - -func TestStdCopyDetectsCorruptedFrame(t *testing.T) { - stdOutBytes := []byte(strings.Repeat("o", startingBufLen)) - stdErrBytes := []byte(strings.Repeat("e", startingBufLen)) - buffer, err := getSrcBuffer(stdOutBytes, stdErrBytes) - if err != nil { - t.Fatal(err) - } - reader := &customReader{ - correctCalls: 1, - n: stdWriterPrefixLen + 1, - err: io.EOF, - src: buffer} - written, err := StdCopy(ioutil.Discard, ioutil.Discard, reader) - if written != startingBufLen { - t.Fatalf("Expected %d bytes read, got %d", startingBufLen, written) - } - if err != nil { - t.Fatal("Didn't get nil error") - } -} - -func TestStdCopyWithInvalidInputHeader(t *testing.T) { - dstOut := NewStdWriter(ioutil.Discard, Stdout) - dstErr := NewStdWriter(ioutil.Discard, Stderr) - src := strings.NewReader("Invalid input") - _, err := StdCopy(dstOut, dstErr, src) - if err == nil { - t.Fatal("StdCopy with invalid input header should fail.") - } -} - -func TestStdCopyWithCorruptedPrefix(t *testing.T) { - data := []byte{0x01, 0x02, 0x03} - src := bytes.NewReader(data) - written, err := StdCopy(nil, nil, src) - if err != nil { - t.Fatalf("StdCopy should not return an error with corrupted prefix.") - } - if written != 0 { - t.Fatalf("StdCopy should have written 0, but has written %d", written) - } -} - -func TestStdCopyReturnsWriteErrors(t *testing.T) { - stdOutBytes := []byte(strings.Repeat("o", startingBufLen)) - stdErrBytes := []byte(strings.Repeat("e", startingBufLen)) - buffer, err := getSrcBuffer(stdOutBytes, stdErrBytes) - if err != nil { - t.Fatal(err) - } - expectedError := errors.New("expected") - - dstOut := &errWriter{err: expectedError} - - written, err := StdCopy(dstOut, ioutil.Discard, buffer) - if written != 0 { - t.Fatalf("StdCopy should have written 0, but has written %d", written) - } - if err != expectedError { - t.Fatalf("Didn't get expected error, got %v", err) - } -} - -func TestStdCopyDetectsNotFullyWrittenFrames(t *testing.T) { - stdOutBytes := []byte(strings.Repeat("o", startingBufLen)) - stdErrBytes := []byte(strings.Repeat("e", startingBufLen)) - buffer, err := getSrcBuffer(stdOutBytes, stdErrBytes) - if err != nil { - t.Fatal(err) - } - dstOut := &errWriter{n: startingBufLen - 10} - - written, err := StdCopy(dstOut, ioutil.Discard, buffer) - if written != 0 { - t.Fatalf("StdCopy should have return 0 written bytes, but returned %d", written) - } - if err != io.ErrShortWrite { - t.Fatalf("Didn't get expected io.ErrShortWrite error") - } -} - -func BenchmarkWrite(b *testing.B) { - w := NewStdWriter(ioutil.Discard, Stdout) - data := []byte("Test line for testing stdwriter performance\n") - data = bytes.Repeat(data, 100) - b.SetBytes(int64(len(data))) - b.ResetTimer() - for i := 0; i < b.N; i++ { - if _, err := w.Write(data); err != nil { - b.Fatal(err) - } - } -} diff --git a/engine/runner/docker/util.go b/engine/runner/docker/util.go deleted file mode 100644 index 2d35fee7c..000000000 --- a/engine/runner/docker/util.go +++ /dev/null @@ -1,102 +0,0 @@ -package docker - -import ( - "fmt" - "strings" - - "github.com/drone/drone/engine/runner" - "github.com/samalba/dockerclient" -) - -// helper function that converts the Continer data structure to the exepcted -// dockerclient.ContainerConfig. -func toContainerConfig(c *runner.Container) *dockerclient.ContainerConfig { - config := &dockerclient.ContainerConfig{ - Image: c.Image, - Env: toEnvironmentSlice(c.Environment), - Cmd: c.Command, - Entrypoint: c.Entrypoint, - WorkingDir: c.WorkingDir, - HostConfig: dockerclient.HostConfig{ - Privileged: c.Privileged, - NetworkMode: c.Network, - Memory: c.MemLimit, - CpuShares: c.CPUShares, - CpuQuota: c.CPUQuota, - CpusetCpus: c.CPUSet, - MemorySwappiness: -1, - OomKillDisable: c.OomKillDisable, - }, - } - - if len(config.Entrypoint) == 0 { - config.Entrypoint = nil - } - if len(config.Cmd) == 0 { - config.Cmd = nil - } - if len(c.ExtraHosts) > 0 { - config.HostConfig.ExtraHosts = c.ExtraHosts - } - if len(c.DNS) != 0 { - config.HostConfig.Dns = c.DNS - } - if len(c.DNSSearch) != 0 { - config.HostConfig.DnsSearch = c.DNSSearch - } - if len(c.VolumesFrom) != 0 { - config.HostConfig.VolumesFrom = c.VolumesFrom - } - - config.Volumes = map[string]struct{}{} - for _, path := range c.Volumes { - if strings.Index(path, ":") == -1 { - config.Volumes[path] = struct{}{} - continue - } - parts := strings.Split(path, ":") - config.Volumes[parts[1]] = struct{}{} - config.HostConfig.Binds = append(config.HostConfig.Binds, path) - } - - for _, path := range c.Devices { - if strings.Index(path, ":") == -1 { - continue - } - parts := strings.Split(path, ":") - device := dockerclient.DeviceMapping{ - PathOnHost: parts[0], - PathInContainer: parts[1], - CgroupPermissions: "rwm", - } - config.HostConfig.Devices = append(config.HostConfig.Devices, device) - } - - return config -} - -// helper function that converts the AuthConfig data structure to the exepcted -// dockerclient.AuthConfig. -func toAuthConfig(container *runner.Container) *dockerclient.AuthConfig { - if container.AuthConfig.Username == "" && - container.AuthConfig.Password == "" && - container.AuthConfig.Token == "" { - return nil - } - return &dockerclient.AuthConfig{ - Email: container.AuthConfig.Email, - Username: container.AuthConfig.Username, - Password: container.AuthConfig.Password, - RegistryToken: container.AuthConfig.Token, - } -} - -// helper function that converts a key value map of environment variables to a -// string slice in key=value format. -func toEnvironmentSlice(env map[string]string) []string { - var envs []string - for k, v := range env { - envs = append(envs, fmt.Sprintf("%s=%s", k, v)) - } - return envs -} diff --git a/engine/runner/docker/util_test.go b/engine/runner/docker/util_test.go deleted file mode 100644 index 1a4a8ce3c..000000000 --- a/engine/runner/docker/util_test.go +++ /dev/null @@ -1,24 +0,0 @@ -package docker - -import ( - "testing" -) - -func Test_toContainerConfig(t *testing.T) { - t.Skip() -} - -func Test_toAuthConfig(t *testing.T) { - t.Skip() -} - -func Test_toEnvironmentSlice(t *testing.T) { - env := map[string]string{ - "HOME": "/root", - } - envs := toEnvironmentSlice(env) - want, got := "HOME=/root", envs[0] - if want != got { - t.Errorf("Wanted envar %s got %s", want, got) - } -} diff --git a/engine/runner/engine.go b/engine/runner/engine.go deleted file mode 100644 index 5f24cc324..000000000 --- a/engine/runner/engine.go +++ /dev/null @@ -1,22 +0,0 @@ -package runner - -//go:generate mockery -name Engine -output mock -case=underscore - -import "io" - -// Engine defines the container runtime engine. -type Engine interface { - // VolumeCreate(*Volume) (string, error) - // VolumeRemove(string) error - ContainerStart(*Container) (string, error) - ContainerStop(string) error - ContainerRemove(string) error - ContainerWait(string) (*State, error) - ContainerLogs(string) (io.ReadCloser, error) -} - -// State defines the state of the container. -type State struct { - ExitCode int // container exit code - OOMKilled bool // container exited due to oom error -} diff --git a/engine/runner/error.go b/engine/runner/error.go deleted file mode 100644 index e10040cb6..000000000 --- a/engine/runner/error.go +++ /dev/null @@ -1,37 +0,0 @@ -package runner - -import ( - "errors" - "fmt" -) - -var ( - // ErrSkip is used as a return value when container execution should be - // skipped at runtime. It is not returned as an error by any function. - ErrSkip = errors.New("Skip") - - // ErrTerm is used as a return value when the runner should terminate - // execution and exit. It is not returned as an error by any function. - ErrTerm = errors.New("Terminate") -) - -// An ExitError reports an unsuccessful exit. -type ExitError struct { - Name string - Code int -} - -// Error reteurns the error message in string format. -func (e *ExitError) Error() string { - return fmt.Sprintf("%s : exit code %d", e.Name, e.Code) -} - -// An OomError reports the process received an OOMKill from the kernel. -type OomError struct { - Name string -} - -// Error reteurns the error message in string format. -func (e *OomError) Error() string { - return fmt.Sprintf("%s : received oom kill", e.Name) -} diff --git a/engine/runner/error_test.go b/engine/runner/error_test.go deleted file mode 100644 index 4bee938dd..000000000 --- a/engine/runner/error_test.go +++ /dev/null @@ -1,26 +0,0 @@ -package runner - -import ( - "testing" - - "github.com/franela/goblin" -) - -func TestErrors(t *testing.T) { - g := goblin.Goblin(t) - - g.Describe("Error messages", func() { - - g.It("should include OOM details", func() { - err := OomError{Name: "golang"} - got, want := err.Error(), "golang : received oom kill" - g.Assert(got).Equal(want) - }) - - g.It("should include Exit code", func() { - err := ExitError{Name: "golang", Code: 255} - got, want := err.Error(), "golang : exit code 255" - g.Assert(got).Equal(want) - }) - }) -} diff --git a/engine/runner/helper.go b/engine/runner/helper.go deleted file mode 100644 index 1b49caf22..000000000 --- a/engine/runner/helper.go +++ /dev/null @@ -1,24 +0,0 @@ -package runner - -import ( - "encoding/json" - "io/ioutil" -) - -// Parse parses a raw file containing a JSON encoded format of an intermediate -// representation of the pipeline. -func Parse(data []byte) (*Spec, error) { - v := &Spec{} - err := json.Unmarshal(data, v) - return v, err -} - -// ParseFile parses a file containing a JSON encoded format of an intermediate -// representation of the pipeline. -func ParseFile(filename string) (*Spec, error) { - out, err := ioutil.ReadFile(filename) - if err != nil { - return nil, err - } - return Parse(out) -} diff --git a/engine/runner/helper_test.go b/engine/runner/helper_test.go deleted file mode 100644 index 2a60efc2a..000000000 --- a/engine/runner/helper_test.go +++ /dev/null @@ -1,97 +0,0 @@ -package runner - -import ( - "io/ioutil" - "os" - "testing" - - "github.com/franela/goblin" -) - -func TestHelper(t *testing.T) { - g := goblin.Goblin(t) - - g.Describe("Parsing", func() { - - g.It("should unmarhsal file []byte", func() { - res, err := Parse(sample) - if err != nil { - t.Error(err) - return - } - g.Assert(err == nil).IsTrue("expect file parsed") - g.Assert(len(res.Containers)).Equal(2) - g.Assert(len(res.Volumes)).Equal(1) - }) - - g.It("should unmarshal from file", func() { - temp, _ := ioutil.TempFile("", "spec_") - defer os.Remove(temp.Name()) - - ioutil.WriteFile(temp.Name(), sample, 0700) - - _, err := ParseFile(temp.Name()) - if err != nil { - t.Error(err) - return - } - g.Assert(err == nil).IsTrue("expect file parsed") - }) - - g.It("should error when file not found", func() { - _, err := ParseFile("/tmp/foo/bar/dummy/file.json") - g.Assert(err == nil).IsFalse("expect file not found error") - }) - }) -} - -// invalid json representation, simulate parsing error -var invalid = []byte(`[]`) - -// valid json representation, verify parsing -var sample = []byte(`{ - "containers": [ - { - "name": "container_0", - "image": "node:latest" - }, - { - "name": "container_1", - "image": "golang:latest" - } - ], - "volumes": [ - { - "name": "volume_0" - } - ], - "program": { - "type": "list", - "body": [ - { - "type": "defer", - "body": { - "type": "recover", - "body": { - "type": "run", - "name": "container_0" - } - }, - "defer": { - "type": "parallel", - "body": [ - { - "type": "run", - "name": "container_1" - }, - { - "type": "run", - "name": "container_1" - } - ], - "limit": 2 - } - } - ] - } -}`) diff --git a/engine/runner/parse/node.go b/engine/runner/parse/node.go deleted file mode 100644 index 0c8b7050b..000000000 --- a/engine/runner/parse/node.go +++ /dev/null @@ -1,30 +0,0 @@ -package parse - -const ( - NodeList = "list" - NodeDefer = "defer" - NodeError = "error" - NodeRecover = "recover" - NodeParallel = "parallel" - NodeRun = "run" -) - -// NodeType identifies the type of a parse tree node. -type NodeType string - -// Type returns itself and provides an easy default implementation -// for embedding in a Node. Embedded in all non-trivial Nodes. -func (t NodeType) Type() NodeType { - return t -} - -// String returns the string value of the Node type. -func (t NodeType) String() string { - return string(t) -} - -// A Node is an element in the parse tree. -type Node interface { - Type() NodeType - Validate() error -} diff --git a/engine/runner/parse/node_defer.go b/engine/runner/parse/node_defer.go deleted file mode 100644 index bc6935f2a..000000000 --- a/engine/runner/parse/node_defer.go +++ /dev/null @@ -1,40 +0,0 @@ -package parse - -import "fmt" - -// DeferNode executes the child node, and then executes the deffered node. -// The deffered node is guaranteed to execute, even when the child node fails. -type DeferNode struct { - NodeType `json:"type"` - - Body Node `json:"body"` // evaluate node - Defer Node `json:"defer"` // defer evaluation of node. -} - -// NewDeferNode returns a new DeferNode. -func NewDeferNode() *DeferNode { - return &DeferNode{NodeType: NodeDefer} -} - -func (n *DeferNode) SetBody(node Node) *DeferNode { - n.Body = node - return n -} - -func (n *DeferNode) SetDefer(node Node) *DeferNode { - n.Defer = node - return n -} - -func (n *DeferNode) Validate() error { - switch { - case n.NodeType != NodeDefer: - return fmt.Errorf("Defer Node uses an invalid type") - case n.Body == nil: - return fmt.Errorf("Defer Node body is empty") - case n.Defer == nil: - return fmt.Errorf("Defer Node defer is empty") - default: - return nil - } -} diff --git a/engine/runner/parse/node_defer_test.go b/engine/runner/parse/node_defer_test.go deleted file mode 100644 index 9de1bf886..000000000 --- a/engine/runner/parse/node_defer_test.go +++ /dev/null @@ -1,56 +0,0 @@ -package parse - -import ( - "testing" - - "github.com/franela/goblin" -) - -func TestDeferNode(t *testing.T) { - g := goblin.Goblin(t) - - g.Describe("DeferNode", func() { - g.It("should set body and defer node", func() { - node0 := NewRunNode() - node1 := NewRunNode() - - defer0 := NewDeferNode() - defer1 := defer0.SetBody(node0) - defer2 := defer0.SetDefer(node1) - g.Assert(defer0.Type().String()).Equal(NodeDefer) - g.Assert(defer0.Body).Equal(node0) - g.Assert(defer0.Defer).Equal(node1) - g.Assert(defer0).Equal(defer1) - g.Assert(defer0).Equal(defer2) - }) - - g.It("should fail validation when invalid type", func() { - defer0 := DeferNode{} - err := defer0.Validate() - g.Assert(err == nil).IsFalse() - g.Assert(err.Error()).Equal("Defer Node uses an invalid type") - }) - - g.It("should fail validation when empty body", func() { - defer0 := NewDeferNode() - err := defer0.Validate() - g.Assert(err == nil).IsFalse() - g.Assert(err.Error()).Equal("Defer Node body is empty") - }) - - g.It("should fail validation when empty defer", func() { - defer0 := NewDeferNode() - defer0.SetBody(NewRunNode()) - err := defer0.Validate() - g.Assert(err == nil).IsFalse() - g.Assert(err.Error()).Equal("Defer Node defer is empty") - }) - - g.It("should pass validation", func() { - defer0 := NewDeferNode() - defer0.SetBody(NewRunNode()) - defer0.SetDefer(NewRunNode()) - g.Assert(defer0.Validate() == nil).IsTrue() - }) - }) -} diff --git a/engine/runner/parse/node_error.go b/engine/runner/parse/node_error.go deleted file mode 100644 index cb3f55e7f..000000000 --- a/engine/runner/parse/node_error.go +++ /dev/null @@ -1,40 +0,0 @@ -package parse - -import "fmt" - -// ErrorNode executes the body node, and then executes the error node if -// the body node errors. This is similar to defer but only executes on error. -type ErrorNode struct { - NodeType `json:"type"` - - Body Node `json:"body"` // evaluate node - Defer Node `json:"defer"` // defer evaluation of node on error. -} - -// NewErrorNode returns a new ErrorNode. -func NewErrorNode() *ErrorNode { - return &ErrorNode{NodeType: NodeError} -} - -func (n *ErrorNode) SetBody(node Node) *ErrorNode { - n.Body = node - return n -} - -func (n *ErrorNode) SetDefer(node Node) *ErrorNode { - n.Defer = node - return n -} - -func (n *ErrorNode) Validate() error { - switch { - case n.NodeType != NodeError: - return fmt.Errorf("Error Node uses an invalid type") - case n.Body == nil: - return fmt.Errorf("Error Node body is empty") - case n.Defer == nil: - return fmt.Errorf("Error Node defer is empty") - default: - return nil - } -} diff --git a/engine/runner/parse/node_error_test.go b/engine/runner/parse/node_error_test.go deleted file mode 100644 index f68cce858..000000000 --- a/engine/runner/parse/node_error_test.go +++ /dev/null @@ -1,56 +0,0 @@ -package parse - -import ( - "testing" - - "github.com/franela/goblin" -) - -func TestErrorNode(t *testing.T) { - g := goblin.Goblin(t) - - g.Describe("ErrorNode", func() { - g.It("should set body and error node", func() { - node0 := NewRunNode() - node1 := NewRunNode() - - error0 := NewErrorNode() - error1 := error0.SetBody(node0) - error2 := error0.SetDefer(node1) - g.Assert(error0.Type().String()).Equal(NodeError) - g.Assert(error0.Body).Equal(node0) - g.Assert(error0.Defer).Equal(node1) - g.Assert(error0).Equal(error1) - g.Assert(error0).Equal(error2) - }) - - g.It("should fail validation when invalid type", func() { - error0 := ErrorNode{} - err := error0.Validate() - g.Assert(err == nil).IsFalse() - g.Assert(err.Error()).Equal("Error Node uses an invalid type") - }) - - g.It("should fail validation when empty body", func() { - error0 := NewErrorNode() - err := error0.Validate() - g.Assert(err == nil).IsFalse() - g.Assert(err.Error()).Equal("Error Node body is empty") - }) - - g.It("should fail validation when empty error", func() { - error0 := NewErrorNode() - error0.SetBody(NewRunNode()) - err := error0.Validate() - g.Assert(err == nil).IsFalse() - g.Assert(err.Error()).Equal("Error Node defer is empty") - }) - - g.It("should pass validation", func() { - error0 := NewErrorNode() - error0.SetBody(NewRunNode()) - error0.SetDefer(NewRunNode()) - g.Assert(error0.Validate() == nil).IsTrue() - }) - }) -} diff --git a/engine/runner/parse/node_list.go b/engine/runner/parse/node_list.go deleted file mode 100644 index 514cd7bae..000000000 --- a/engine/runner/parse/node_list.go +++ /dev/null @@ -1,33 +0,0 @@ -package parse - -import "fmt" - -// ListNode serially executes a list of child nodes. -type ListNode struct { - NodeType `json:"type"` - - // Body is the list of child nodes - Body []Node `json:"body"` -} - -// NewListNode returns a new ListNode. -func NewListNode() *ListNode { - return &ListNode{NodeType: NodeList} -} - -// Append appens a child node to the list. -func (n *ListNode) Append(node Node) *ListNode { - n.Body = append(n.Body, node) - return n -} - -func (n *ListNode) Validate() error { - switch { - case n.NodeType != NodeList: - return fmt.Errorf("List Node uses an invalid type") - case len(n.Body) == 0: - return fmt.Errorf("List Node body is empty") - default: - return nil - } -} diff --git a/engine/runner/parse/node_list_test.go b/engine/runner/parse/node_list_test.go deleted file mode 100644 index 5c0ad3281..000000000 --- a/engine/runner/parse/node_list_test.go +++ /dev/null @@ -1,44 +0,0 @@ -package parse - -import ( - "testing" - - "github.com/franela/goblin" -) - -func TestListNode(t *testing.T) { - g := goblin.Goblin(t) - - g.Describe("ListNode", func() { - g.It("should append nodes", func() { - node := NewRunNode() - - list0 := NewListNode() - list1 := list0.Append(node) - g.Assert(list0.Type().String()).Equal(NodeList) - g.Assert(list0.Body[0]).Equal(node) - g.Assert(list0).Equal(list1) - }) - - g.It("should fail validation when invalid type", func() { - list := ListNode{} - err := list.Validate() - g.Assert(err == nil).IsFalse() - g.Assert(err.Error()).Equal("List Node uses an invalid type") - }) - - g.It("should fail validation when empty body", func() { - list := NewListNode() - err := list.Validate() - g.Assert(err == nil).IsFalse() - g.Assert(err.Error()).Equal("List Node body is empty") - }) - - g.It("should pass validation", func() { - node := NewRunNode() - list := NewListNode() - list.Append(node) - g.Assert(list.Validate() == nil).IsTrue() - }) - }) -} diff --git a/engine/runner/parse/node_parallel.go b/engine/runner/parse/node_parallel.go deleted file mode 100644 index a587235e5..000000000 --- a/engine/runner/parse/node_parallel.go +++ /dev/null @@ -1,36 +0,0 @@ -package parse - -import "fmt" - -// ParallelNode executes a list of child nodes in parallel. -type ParallelNode struct { - NodeType `json:"type"` - - Body []Node `json:"body"` // nodes for parallel evaluation. - Limit int `json:"limit"` // limit for parallel evaluation. -} - -func NewParallelNode() *ParallelNode { - return &ParallelNode{NodeType: NodeParallel} -} - -func (n *ParallelNode) Append(node Node) *ParallelNode { - n.Body = append(n.Body, node) - return n -} - -func (n *ParallelNode) SetLimit(limit int) *ParallelNode { - n.Limit = limit - return n -} - -func (n *ParallelNode) Validate() error { - switch { - case n.NodeType != NodeParallel: - return fmt.Errorf("Parallel Node uses an invalid type") - case len(n.Body) == 0: - return fmt.Errorf("Parallel Node body is empty") - default: - return nil - } -} diff --git a/engine/runner/parse/node_parallel_test.go b/engine/runner/parse/node_parallel_test.go deleted file mode 100644 index 9c0f0fb74..000000000 --- a/engine/runner/parse/node_parallel_test.go +++ /dev/null @@ -1,42 +0,0 @@ -package parse - -import ( - "testing" - - "github.com/franela/goblin" -) - -func TestParallelNode(t *testing.T) { - g := goblin.Goblin(t) - - g.Describe("ParallelNode", func() { - g.It("should append nodes", func() { - node := NewRunNode() - - parallel0 := NewParallelNode() - parallel1 := parallel0.Append(node) - g.Assert(parallel0.Type().String()).Equal(NodeParallel) - g.Assert(parallel0.Body[0]).Equal(node) - g.Assert(parallel0).Equal(parallel1) - }) - - g.It("should fail validation when invalid type", func() { - node := ParallelNode{} - err := node.Validate() - g.Assert(err == nil).IsFalse() - g.Assert(err.Error()).Equal("Parallel Node uses an invalid type") - }) - - g.It("should fail validation when empty body", func() { - node := NewParallelNode() - err := node.Validate() - g.Assert(err == nil).IsFalse() - g.Assert(err.Error()).Equal("Parallel Node body is empty") - }) - - g.It("should pass validation", func() { - node := NewParallelNode().Append(NewRunNode()) - g.Assert(node.Validate() == nil).IsTrue() - }) - }) -} diff --git a/engine/runner/parse/node_recover.go b/engine/runner/parse/node_recover.go deleted file mode 100644 index 9cac51a12..000000000 --- a/engine/runner/parse/node_recover.go +++ /dev/null @@ -1,29 +0,0 @@ -package parse - -import "fmt" - -type RecoverNode struct { - NodeType `json:"type"` - - Body Node `json:"body"` // evaluate node and catch all errors. -} - -func NewRecoverNode() *RecoverNode { - return &RecoverNode{NodeType: NodeRecover} -} - -func (n *RecoverNode) SetBody(node Node) *RecoverNode { - n.Body = node - return n -} - -func (n *RecoverNode) Validate() error { - switch { - case n.NodeType != NodeRecover: - return fmt.Errorf("Recover Node uses an invalid type") - case n.Body == nil: - return fmt.Errorf("Recover Node body is empty") - default: - return nil - } -} diff --git a/engine/runner/parse/node_recover_test.go b/engine/runner/parse/node_recover_test.go deleted file mode 100644 index 20248655e..000000000 --- a/engine/runner/parse/node_recover_test.go +++ /dev/null @@ -1,43 +0,0 @@ -package parse - -import ( - "testing" - - "github.com/franela/goblin" -) - -func TestRecoverNode(t *testing.T) { - g := goblin.Goblin(t) - - g.Describe("RecoverNode", func() { - g.It("should set body", func() { - node0 := NewRunNode() - - recover0 := NewRecoverNode() - recover1 := recover0.SetBody(node0) - g.Assert(recover0.Type().String()).Equal(NodeRecover) - g.Assert(recover0.Body).Equal(node0) - g.Assert(recover0).Equal(recover1) - }) - - g.It("should fail validation when invalid type", func() { - recover0 := RecoverNode{} - err := recover0.Validate() - g.Assert(err == nil).IsFalse() - g.Assert(err.Error()).Equal("Recover Node uses an invalid type") - }) - - g.It("should fail validation when empty body", func() { - recover0 := NewRecoverNode() - err := recover0.Validate() - g.Assert(err == nil).IsFalse() - g.Assert(err.Error()).Equal("Recover Node body is empty") - }) - - g.It("should pass validation", func() { - recover0 := NewRecoverNode() - recover0.SetBody(NewRunNode()) - g.Assert(recover0.Validate() == nil).IsTrue() - }) - }) -} diff --git a/engine/runner/parse/node_run.go b/engine/runner/parse/node_run.go deleted file mode 100644 index dedc90731..000000000 --- a/engine/runner/parse/node_run.go +++ /dev/null @@ -1,41 +0,0 @@ -package parse - -import "fmt" - -type RunNode struct { - NodeType `json:"type"` - - Name string `json:"name"` - Detach bool `json:"detach,omitempty"` - Silent bool `json:"silent,omitempty"` -} - -func (n *RunNode) SetName(name string) *RunNode { - n.Name = name - return n -} - -func (n *RunNode) SetDetach(detach bool) *RunNode { - n.Detach = detach - return n -} - -func (n *RunNode) SetSilent(silent bool) *RunNode { - n.Silent = silent - return n -} - -func NewRunNode() *RunNode { - return &RunNode{NodeType: NodeRun} -} - -func (n *RunNode) Validate() error { - switch { - case n.NodeType != NodeRun: - return fmt.Errorf("Run Node uses an invalid type") - case n.Name == "": - return fmt.Errorf("Run Node has an invalid name") - default: - return nil - } -} diff --git a/engine/runner/parse/node_run_test.go b/engine/runner/parse/node_run_test.go deleted file mode 100644 index 9051249db..000000000 --- a/engine/runner/parse/node_run_test.go +++ /dev/null @@ -1,41 +0,0 @@ -package parse - -import ( - "testing" - - "github.com/franela/goblin" -) - -func TestRunNode(t *testing.T) { - g := goblin.Goblin(t) - - g.Describe("RunNode", func() { - g.It("should set container name for lookup", func() { - node0 := NewRunNode() - node1 := node0.SetName("foo") - - g.Assert(node0.Type().String()).Equal(NodeRun) - g.Assert(node0.Name).Equal("foo") - g.Assert(node0).Equal(node1) - }) - - g.It("should fail validation when invalid type", func() { - node := RunNode{} - err := node.Validate() - g.Assert(err == nil).IsFalse() - g.Assert(err.Error()).Equal("Run Node uses an invalid type") - }) - - g.It("should fail validation when invalid name", func() { - node := NewRunNode() - err := node.Validate() - g.Assert(err == nil).IsFalse() - g.Assert(err.Error()).Equal("Run Node has an invalid name") - }) - - g.It("should pass validation", func() { - node := NewRunNode().SetName("foo") - g.Assert(node.Validate() == nil).IsTrue() - }) - }) -} diff --git a/engine/runner/parse/parse.go b/engine/runner/parse/parse.go deleted file mode 100644 index b027cff34..000000000 --- a/engine/runner/parse/parse.go +++ /dev/null @@ -1,221 +0,0 @@ -package parse - -import "encoding/json" - -// Tree is the intermediate representation of a pipeline. -type Tree struct { - *ListNode // top-level Tree node -} - -// New allocates a new Tree. -func NewTree() *Tree { - return &Tree{ - NewListNode(), - } -} - -// Parse parses a JSON encoded Tree. -func Parse(data []byte) (*Tree, error) { - tree := &Tree{} - err := tree.UnmarshalJSON(data) - return tree, err -} - -// MarshalJSON implements the Marshaler interface and returns -// a JSON encoded representation of the Tree. -func (t *Tree) MarshalJSON() ([]byte, error) { - return json.Marshal(t.ListNode) -} - -// UnmarshalJSON implements the Unmarshaler interface and returns -// a Tree from a JSON representation. -func (t *Tree) UnmarshalJSON(data []byte) error { - block, err := decodeList(data) - if err != nil { - return nil - } - t.ListNode = block.(*ListNode) - return nil -} - -// -// below are custom decoding functions. We cannot use the default json -// decoder because the tree structure uses interfaces and the json decoder -// has difficulty ascertaining the interface type when decoding. -// - -func decodeNode(data []byte) (Node, error) { - node := &nodeType{} - - err := json.Unmarshal(data, node) - if err != nil { - return nil, err - } - switch node.Type { - case NodeList: - return decodeList(data) - case NodeDefer: - return decodeDefer(data) - case NodeError: - return decodeError(data) - case NodeRecover: - return decodeRecover(data) - case NodeParallel: - return decodeParallel(data) - case NodeRun: - return decodeRun(data) - } - return nil, nil -} - -func decodeNodes(data []json.RawMessage) ([]Node, error) { - var nodes []Node - for _, d := range data { - node, err := decodeNode(d) - if err != nil { - return nil, err - } - nodes = append(nodes, node) - } - return nodes, nil -} - -func decodeList(data []byte) (Node, error) { - v := &nodeList{} - err := json.Unmarshal(data, v) - if err != nil { - return nil, err - } - b, err := decodeNodes(v.Body) - if err != nil { - return nil, err - } - n := NewListNode() - n.Body = b - return n, nil -} - -func decodeDefer(data []byte) (Node, error) { - v := &nodeDefer{} - err := json.Unmarshal(data, v) - if err != nil { - return nil, err - } - b, err := decodeNode(v.Body) - if err != nil { - return nil, err - } - d, err := decodeNode(v.Defer) - if err != nil { - return nil, err - } - n := NewDeferNode() - n.Body = b - n.Defer = d - return n, nil -} - -func decodeError(data []byte) (Node, error) { - v := &nodeError{} - err := json.Unmarshal(data, v) - if err != nil { - return nil, err - } - b, err := decodeNode(v.Body) - if err != nil { - return nil, err - } - d, err := decodeNode(v.Defer) - if err != nil { - return nil, err - } - n := NewErrorNode() - n.Body = b - n.Defer = d - return n, nil -} - -func decodeRecover(data []byte) (Node, error) { - v := &nodeRecover{} - err := json.Unmarshal(data, v) - if err != nil { - return nil, err - } - b, err := decodeNode(v.Body) - if err != nil { - return nil, err - } - n := NewRecoverNode() - n.Body = b - return n, nil -} - -func decodeParallel(data []byte) (Node, error) { - v := &nodeParallel{} - err := json.Unmarshal(data, v) - if err != nil { - return nil, err - } - b, err := decodeNodes(v.Body) - if err != nil { - return nil, err - } - n := NewParallelNode() - n.Body = b - n.Limit = v.Limit - return n, nil -} - -func decodeRun(data []byte) (Node, error) { - v := &nodeRun{} - err := json.Unmarshal(data, v) - if err != nil { - return nil, err - } - return &RunNode{NodeRun, v.Name, v.Detach, v.Silent}, nil -} - -// -// below are intermediate representations of the node structures -// since we cannot simply encode / decode using the built-in json -// encoding and decoder. -// - -type nodeType struct { - Type NodeType `json:"type"` -} - -type nodeDefer struct { - Type NodeType `json:"type"` - Body json.RawMessage `json:"body"` - Defer json.RawMessage `json:"defer"` -} - -type nodeError struct { - Type NodeType `json:"type"` - Body json.RawMessage `json:"body"` - Defer json.RawMessage `json:"defer"` -} - -type nodeList struct { - Type NodeType `json:"type"` - Body []json.RawMessage `json:"body"` -} - -type nodeRecover struct { - Type NodeType `json:"type"` - Body json.RawMessage `json:"body"` -} - -type nodeParallel struct { - Type NodeType `json:"type"` - Body []json.RawMessage `json:"body"` - Limit int `json:"limit"` -} - -type nodeRun struct { - Type NodeType `json:"type"` - Name string `json:"name"` - Detach bool `json:"detach,omitempty"` - Silent bool `json:"silent,omitempty"` -} diff --git a/engine/runner/parse/parse_test.go b/engine/runner/parse/parse_test.go deleted file mode 100644 index b384882d1..000000000 --- a/engine/runner/parse/parse_test.go +++ /dev/null @@ -1,80 +0,0 @@ -package parse - -import ( - "bytes" - "encoding/json" - "reflect" - "testing" -) - -func TestUnmarshal(t *testing.T) { - - node1 := NewRunNode().SetName("foo") - node2 := NewRecoverNode().SetBody(node1) - - node3 := NewRunNode().SetName("bar") - node4 := NewRunNode().SetName("bar") - - node5 := NewParallelNode(). - Append(node3). - Append(node4). - SetLimit(2) - - node6 := NewDeferNode(). - SetBody(node2). - SetDefer(node5) - - tree := NewTree() - tree.Append(node6) - - encoded, err := json.MarshalIndent(tree, "", "\t") - if err != nil { - t.Error(err) - } - - if !bytes.Equal(encoded, sample) { - t.Errorf("Want to marshal Tree to %s, got %s", - string(sample), - string(encoded), - ) - } - - parsed, err := Parse(encoded) - if err != nil { - t.Error(err) - } - - if !reflect.DeepEqual(tree, parsed) { - t.Errorf("Want to marsnal and then unmarshal Tree") - } -} - -var sample = []byte(`{ - "type": "list", - "body": [ - { - "type": "defer", - "body": { - "type": "recover", - "body": { - "type": "run", - "name": "foo" - } - }, - "defer": { - "type": "parallel", - "body": [ - { - "type": "run", - "name": "bar" - }, - { - "type": "run", - "name": "bar" - } - ], - "limit": 2 - } - } - ] -}`) diff --git a/engine/runner/pipe.go b/engine/runner/pipe.go deleted file mode 100644 index d49654297..000000000 --- a/engine/runner/pipe.go +++ /dev/null @@ -1,49 +0,0 @@ -package runner - -import "fmt" - -// Pipe returns a buffered pipe that is connected to the console output. -type Pipe struct { - lines chan *Line - eof chan bool -} - -// Next returns the next Line of console output. -func (p *Pipe) Next() *Line { - select { - case line := <-p.lines: - return line - case <-p.eof: - return nil - } -} - -// Close closes the pipe of console output. -func (p *Pipe) Close() { - go func() { - p.eof <- true - }() -} - -func newPipe(buffer int) *Pipe { - return &Pipe{ - lines: make(chan *Line, buffer), - eof: make(chan bool), - } -} - -// Line is a line of console output. -type Line struct { - Proc string `json:"proc,omitempty"` - Time int64 `json:"time,omitempty"` - Type int `json:"type,omitempty"` - Pos int `json:"pos,omityempty"` - Out string `json:"out,omitempty"` -} - -func (l *Line) String() string { - return fmt.Sprintf("[%s:L%v:%vs] %s", l.Proc, l.Pos, l.Time, l.Out) -} - -// TODO(bradrydzewski) consider an alternate buffer impelmentation based on the -// x.crypto ssh buffer https://github.com/golang/crypto/blob/master/ssh/buffer.go diff --git a/engine/runner/pipe_test.go b/engine/runner/pipe_test.go deleted file mode 100644 index d7be32945..000000000 --- a/engine/runner/pipe_test.go +++ /dev/null @@ -1,54 +0,0 @@ -package runner - -import ( - "sync" - "testing" - - "github.com/franela/goblin" -) - -func TestPipe(t *testing.T) { - g := goblin.Goblin(t) - - g.Describe("Pipe", func() { - g.It("should get next line from buffer", func() { - line := &Line{ - Proc: "redis", - Pos: 1, - Out: "starting redis server", - } - pipe := newPipe(10) - pipe.lines <- line - next := pipe.Next() - g.Assert(next).Equal(line) - }) - - g.It("should get null line on buffer closed", func() { - pipe := newPipe(10) - - var wg sync.WaitGroup - wg.Add(1) - - go func() { - next := pipe.Next() - g.Assert(next == nil).IsTrue("line should be nil") - wg.Done() - }() - - pipe.Close() - wg.Wait() - }) - - g.Describe("Line output", func() { - g.It("should prefix string() with metadata", func() { - line := Line{ - Proc: "redis", - Time: 60, - Pos: 1, - Out: "starting redis server", - } - g.Assert(line.String()).Equal("[redis:L1:60s] starting redis server") - }) - }) - }) -} diff --git a/engine/runner/runner.go b/engine/runner/runner.go deleted file mode 100644 index 7bad01840..000000000 --- a/engine/runner/runner.go +++ /dev/null @@ -1,245 +0,0 @@ -package runner - -import ( - "bufio" - "fmt" - "time" - - "github.com/drone/drone/engine/runner/parse" - - "golang.org/x/net/context" -) - -// NoContext is the default context you should supply if not using your own -// context.Context -var NoContext = context.TODO() - -// Tracer defines a tracing function that is invoked prior to creating and -// running the container. -type Tracer func(c *Container) error - -// Config defines the configuration for creating the Runner. -type Config struct { - Tracer Tracer - Engine Engine - - // Buffer defines the size of the buffer for the channel to which the - // console output is streamed. - Buffer uint -} - -// Runner creates a build Runner using the specific configuration for the given -// Context and Specification. -func (c *Config) Runner(ctx context.Context, spec *Spec) *Runner { - - // TODO(bradyrdzewski) we should make a copy of the configuration parameters - // instead of a direct reference. This helps avoid any race conditions or - //unexpected behavior if the Config changes. - return &Runner{ - ctx: ctx, - conf: c, - spec: spec, - errc: make(chan error), - pipe: newPipe(int(c.Buffer) + 1), - } -} - -type Runner struct { - ctx context.Context - conf *Config - spec *Spec - pipe *Pipe - errc chan (error) - - containers []string - volumes []string - networks []string -} - -// Run starts the build runner but does not wait for it to complete. The Wait -// method will return the exit code and release associated resources once the -// running containers exit. -func (r *Runner) Run() { - - go func() { - r.setup() - err := r.exec(r.spec.Nodes.ListNode) - r.pipe.Close() - r.cancel() - r.teardown() - r.errc <- err - }() - - go func() { - <-r.ctx.Done() - r.cancel() - }() -} - -// Wait waits for the runner to exit. -func (r *Runner) Wait() error { - return <-r.errc -} - -// Pipe returns a Pipe that is connected to the console output stream. -func (r *Runner) Pipe() *Pipe { - return r.pipe -} - -func (r *Runner) exec(node parse.Node) error { - switch v := node.(type) { - case *parse.ListNode: - return r.execList(v) - case *parse.DeferNode: - return r.execDefer(v) - case *parse.ErrorNode: - return r.execError(v) - case *parse.RecoverNode: - return r.execRecover(v) - case *parse.ParallelNode: - return r.execParallel(v) - case *parse.RunNode: - return r.execRun(v) - } - return fmt.Errorf("runner: unexepected node %s", node) -} - -func (r *Runner) execList(node *parse.ListNode) error { - for _, n := range node.Body { - err := r.exec(n) - if err != nil { - return err - } - } - return nil -} - -func (r *Runner) execDefer(node *parse.DeferNode) error { - err1 := r.exec(node.Body) - err2 := r.exec(node.Defer) - if err1 != nil { - return err1 - } - return err2 -} - -func (r *Runner) execError(node *parse.ErrorNode) error { - err := r.exec(node.Body) - if err != nil { - r.exec(node.Defer) - } - return err -} - -func (r *Runner) execRecover(node *parse.RecoverNode) error { - r.exec(node.Body) - return nil -} - -func (r *Runner) execParallel(node *parse.ParallelNode) error { - errc := make(chan error) - - for _, n := range node.Body { - go func(node parse.Node) { - errc <- r.exec(node) - }(n) - } - - var err error - for i := 0; i < len(node.Body); i++ { - select { - case cerr := <-errc: - if cerr != nil { - err = cerr - } - } - } - - return err -} - -func (r *Runner) execRun(node *parse.RunNode) error { - container, err := r.spec.lookupContainer(node.Name) - if err != nil { - return err - } - if r.conf.Tracer != nil { - err := r.conf.Tracer(container) - switch { - case err == ErrSkip: - return nil - case err != nil: - return err - } - } - // TODO(bradrydzewski) there is potential here for a race condition where - // the context is cancelled just after this line, resulting in the container - // still being started. - if r.ctx.Err() != nil { - return err - } - - name, err := r.conf.Engine.ContainerStart(container) - if err != nil { - return err - } - r.containers = append(r.containers, name) - - go func() { - if node.Silent { - return - } - rc, err := r.conf.Engine.ContainerLogs(name) - if err != nil { - return - } - defer rc.Close() - - num := 0 - now := time.Now().UTC() - scanner := bufio.NewScanner(rc) - for scanner.Scan() { - r.pipe.lines <- &Line{ - Proc: container.Alias, - Time: int64(time.Since(now).Seconds()), - Pos: num, - Out: scanner.Text(), - } - num++ - } - }() - - // exit when running container in detached mode in background - if node.Detach { - return nil - } - - state, err := r.conf.Engine.ContainerWait(name) - if err != nil { - return err - } - if state.OOMKilled { - return &OomError{name} - } else if state.ExitCode != 0 { - return &ExitError{name, state.ExitCode} - } - return nil -} - -func (r *Runner) setup() { - // this is where we will setup network and volumes -} - -func (r *Runner) teardown() { - // TODO(bradrydzewski) this is not yet thread safe. - for _, container := range r.containers { - r.conf.Engine.ContainerRemove(container) - } -} - -func (r *Runner) cancel() { - // TODO(bradrydzewski) this is not yet thread safe. - for _, container := range r.containers { - r.conf.Engine.ContainerStop(container) - } -} diff --git a/engine/runner/runner_test.go b/engine/runner/runner_test.go deleted file mode 100644 index 09a3ecd67..000000000 --- a/engine/runner/runner_test.go +++ /dev/null @@ -1,7 +0,0 @@ -package runner - -import "testing" - -func TestRunner(t *testing.T) { - t.Skip() -} diff --git a/engine/runner/spec.go b/engine/runner/spec.go deleted file mode 100644 index 1f78a001b..000000000 --- a/engine/runner/spec.go +++ /dev/null @@ -1,33 +0,0 @@ -package runner - -import ( - "fmt" - - "github.com/drone/drone/engine/runner/parse" -) - -// Spec defines the pipeline configuration and exeuction. -type Spec struct { - // Volumes defines a list of all container volumes. - Volumes []*Volume `json:"volumes,omitempty"` - - // Networks defines a list of all container networks. - Networks []*Network `json:"networks,omitempty"` - - // Containers defines a list of all containers in the pipeline. - Containers []*Container `json:"containers,omitempty"` - - // Nodes defines the container execution tree. - Nodes *parse.Tree `json:"program,omitempty"` -} - -// lookupContainer is a helper funciton that returns the named container from -// the slice of containers. -func (s *Spec) lookupContainer(name string) (*Container, error) { - for _, container := range s.Containers { - if container.Name == name { - return container, nil - } - } - return nil, fmt.Errorf("runner: unknown container %s", name) -} diff --git a/engine/runner/spec_test.go b/engine/runner/spec_test.go deleted file mode 100644 index ba627000f..000000000 --- a/engine/runner/spec_test.go +++ /dev/null @@ -1,35 +0,0 @@ -package runner - -import ( - "testing" - - "github.com/franela/goblin" -) - -func TestSpec(t *testing.T) { - g := goblin.Goblin(t) - - g.Describe("Spec file", func() { - - g.Describe("when looking up a container", func() { - - spec := Spec{} - spec.Containers = append(spec.Containers, &Container{ - Name: "golang", - }) - - g.It("should find and return the container", func() { - c, err := spec.lookupContainer("golang") - g.Assert(err == nil).IsTrue("error should be nil") - g.Assert(c).Equal(spec.Containers[0]) - }) - - g.It("should return an error when not found", func() { - c, err := spec.lookupContainer("node") - g.Assert(err == nil).IsFalse("should return error") - g.Assert(c == nil).IsTrue("should return nil container") - }) - - }) - }) -}