diff --git a/drone/secert_add.go b/drone/secert_add.go index 2840e6256..4ab115f83 100644 --- a/drone/secert_add.go +++ b/drone/secert_add.go @@ -34,6 +34,10 @@ var secretAddCmd = cli.Command{ Usage: "inject the secret for these image types", Value: &cli.StringSlice{}, }, + cli.StringFlag{ + Name: "input", + Usage: "input secret value from a file", + }, }, } @@ -60,8 +64,10 @@ func secretAdd(c *cli.Context) error { return fmt.Errorf("Please specify the --image parameter") } - // allow secret value to come from a file when prefixed with the @ symbol, - // similar to curl conventions. + // TODO(bradrydzewski) below we use an @ sybmol to denote that the secret + // value should be loaded from a file (inspired by curl). I'd prefer to use + // a --input flag to explicitly specify a filepath instead. + if strings.HasPrefix(secret.Value, "@") { path := secret.Value[1:] out, ferr := ioutil.ReadFile(path) diff --git a/engine/compiler/compile.go b/engine/compiler/compile.go index da4e56364..7d4ad665f 100644 --- a/engine/compiler/compile.go +++ b/engine/compiler/compile.go @@ -52,15 +52,6 @@ func (c *Compiler) Compile(in []byte) (*runner.Spec, error) { } } - // cache section - if root.Cache != nil { - node, ok := root.Cache.(*yaml.ContainerNode) - if ok && !node.Disabled { - config.Containers = append(config.Containers, &node.Container) - tree.Append(parse.NewRunNode().SetName(node.Container.Name)) - } - } - // clone section if root.Clone != nil { node, ok := root.Clone.(*yaml.ContainerNode) diff --git a/engine/compiler/parse/node_root.go b/engine/compiler/parse/node_root.go index 0288f5f47..fc2ff615f 100644 --- a/engine/compiler/parse/node_root.go +++ b/engine/compiler/parse/node_root.go @@ -11,7 +11,6 @@ type RootNode struct { Pod Node Build Node - Cache Node Clone Node Script []Node Volumes []Node @@ -110,7 +109,6 @@ func (n *RootNode) Walk(fn WalkFunc) (err error) { var nodes []Node nodes = append(nodes, n) nodes = append(nodes, n.Build) - nodes = append(nodes, n.Cache) nodes = append(nodes, n.Clone) nodes = append(nodes, n.Script...) nodes = append(nodes, n.Volumes...) diff --git a/engine/compiler/parse/parse.go b/engine/compiler/parse/parse.go index a3be5ed32..61434d629 100644 --- a/engine/compiler/parse/parse.go +++ b/engine/compiler/parse/parse.go @@ -45,16 +45,6 @@ func Parse(in []byte) (*RootNode, error) { } } - // add the cache section - { - cc := root.NewCacheNode() - cc.Container = out.Cache.ToContainer() - cc.Conditions = out.Cache.ToConditions() - cc.Container.Name = "cache" - cc.Vargs = out.Cache.Vargs - root.Cache = cc - } - // add the clone section { cc := root.NewCloneNode() diff --git a/engine/compiler/parse/parse_test.go b/engine/compiler/parse/parse_test.go index 02d17af93..e6ece2c9b 100644 --- a/engine/compiler/parse/parse_test.go +++ b/engine/compiler/parse/parse_test.go @@ -22,7 +22,6 @@ func TestParse(t *testing.T) { g.Assert(out.Path).Equal("src/github.com/octocat/hello-world") g.Assert(out.Build.(*BuildNode).Context).Equal(".") g.Assert(out.Build.(*BuildNode).Dockerfile).Equal("Dockerfile") - g.Assert(out.Cache.(*ContainerNode).Vargs["mount"]).Equal("node_modules") g.Assert(out.Clone.(*ContainerNode).Container.Image).Equal("git") g.Assert(out.Clone.(*ContainerNode).Vargs["depth"]).Equal(1) g.Assert(out.Volumes[0].(*VolumeNode).Name).Equal("custom") diff --git a/yaml/branch.go b/yaml/branch.go index ae426313a..a95af5958 100644 --- a/yaml/branch.go +++ b/yaml/branch.go @@ -3,12 +3,14 @@ package yaml import ( "path/filepath" + "github.com/drone/drone/yaml/types" + "gopkg.in/yaml.v2" ) type Branch struct { - Include []string `yaml:"include"` - Exclude []string `yaml:"exclude"` + Include []string + Exclude []string } // ParseBranch parses the branch section of the Yaml document. @@ -21,16 +23,16 @@ func ParseBranchString(in string) *Branch { return ParseBranch([]byte(in)) } -// Matches returns true if the branch matches the include patterns and -// does not match any of the exclude patterns. +// Matches returns true if the branch matches the include patterns and does not +// match any of the exclude patterns. func (b *Branch) Matches(branch string) bool { // when no includes or excludes automatically match if len(b.Include) == 0 && len(b.Exclude) == 0 { return true } - // exclusions are processed first. So we can include everything and - // then selectively exclude certain sub-patterns. + // exclusions are processed first. So we can include everything and then + // selectively exclude certain sub-patterns. for _, pattern := range b.Exclude { if pattern == branch { return false @@ -55,13 +57,13 @@ func (b *Branch) Matches(branch string) bool { func parseBranch(in []byte) *Branch { out1 := struct { Branch struct { - Include stringOrSlice `yaml:"include"` - Exclude stringOrSlice `yaml:"exclude"` + Include types.StringOrSlice `yaml:"include"` + Exclude types.StringOrSlice `yaml:"exclude"` } `yaml:"branches"` }{} out2 := struct { - Include stringOrSlice `yaml:"branches"` + Include types.StringOrSlice `yaml:"branches"` }{} yaml.Unmarshal(in, &out1) diff --git a/yaml/build.go b/yaml/build.go new file mode 100644 index 000000000..ec3892a07 --- /dev/null +++ b/yaml/build.go @@ -0,0 +1,26 @@ +package yaml + +// Build represents Docker image build instructions. +type Build struct { + Context string + Dockerfile string + Args map[string]string +} + +// UnmarshalYAML implements custom Yaml unmarshaling. +func (b *Build) UnmarshalYAML(unmarshal func(interface{}) error) error { + err := unmarshal(&b.Context) + if err == nil { + return nil + } + out := struct { + Context string + Dockerfile string + Args map[string]string + }{} + err = unmarshal(&out) + b.Context = out.Context + b.Args = out.Args + b.Dockerfile = out.Dockerfile + return err +} diff --git a/yaml/build_test.go b/yaml/build_test.go new file mode 100644 index 000000000..69c9a1fb9 --- /dev/null +++ b/yaml/build_test.go @@ -0,0 +1,38 @@ +package yaml + +import ( + "testing" + + "github.com/franela/goblin" + "gopkg.in/yaml.v2" +) + +func TestBuild(t *testing.T) { + g := goblin.Goblin(t) + + g.Describe("Build", func() { + g.Describe("given a yaml file", func() { + + g.It("should unmarshal", func() { + in := []byte(".") + out := Build{} + err := yaml.Unmarshal(in, &out) + if err != nil { + g.Fail(err) + } + g.Assert(out.Context).Equal(".") + }) + + g.It("should unmarshal shorthand", func() { + in := []byte("{ context: ., dockerfile: Dockerfile }") + out := Build{} + err := yaml.Unmarshal(in, &out) + if err != nil { + g.Fail(err) + } + g.Assert(out.Context).Equal(".") + g.Assert(out.Dockerfile).Equal("Dockerfile") + }) + }) + }) +} diff --git a/yaml/config.go b/yaml/config.go new file mode 100644 index 000000000..7f05cab55 --- /dev/null +++ b/yaml/config.go @@ -0,0 +1,67 @@ +package yaml + +import "gopkg.in/yaml.v2" + +// Workspace represents the build workspace. +type Workspace struct { + Base string + Path string +} + +// Config represents the build configuration Yaml document. +type Config struct { + Image string + Build *Build + Workspace *Workspace + Pipeline []*Container + Services []*Container + Volumes []*Volume + Networks []*Network +} + +// ParseString parses the Yaml configuration document. +func ParseString(data string) (*Config, error) { + return Parse([]byte(data)) +} + +// Parse parses Yaml configuration document. +func Parse(data []byte) (*Config, error) { + v := struct { + Image string + Build *Build + Workspace *Workspace + Services containerList + Pipeline containerList + Networks networkList + Volumes volumeList + }{} + + err := yaml.Unmarshal(data, &v) + if err != nil { + return nil, err + } + + for _, c := range v.Services.containers { + c.Detached = true + } + + return &Config{ + Image: v.Image, + Build: v.Build, + Workspace: v.Workspace, + Services: v.Services.containers, + Pipeline: v.Pipeline.containers, + Networks: v.Networks.networks, + Volumes: v.Volumes.volumes, + }, nil +} + +type config struct { + Image string + Build *Build + Workspace *Workspace + Services containerList + Pipeline containerList + Networks networkList + Volumes volumeList +} diff --git a/yaml/config_test.go b/yaml/config_test.go new file mode 100644 index 000000000..5e5e780cc --- /dev/null +++ b/yaml/config_test.go @@ -0,0 +1,83 @@ +package yaml + +import ( + "testing" + + "github.com/franela/goblin" +) + +func TestParse(t *testing.T) { + g := goblin.Goblin(t) + + g.Describe("Parser", func() { + g.Describe("Given a yaml file", func() { + + g.It("Should unmarshal a string", func() { + out, err := ParseString(sampleYaml) + if err != nil { + g.Fail(err) + } + g.Assert(out.Image).Equal("hello-world") + g.Assert(out.Workspace.Base).Equal("/go") + g.Assert(out.Workspace.Path).Equal("src/github.com/octocat/hello-world") + g.Assert(out.Build.Context).Equal(".") + g.Assert(out.Build.Dockerfile).Equal("Dockerfile") + g.Assert(out.Volumes[0].Name).Equal("custom") + g.Assert(out.Volumes[0].Driver).Equal("blockbridge") + g.Assert(out.Networks[0].Name).Equal("custom") + g.Assert(out.Networks[0].Driver).Equal("overlay") + g.Assert(out.Services[0].Name).Equal("database") + g.Assert(out.Services[0].Image).Equal("mysql") + g.Assert(out.Pipeline[0].Name).Equal("test") + g.Assert(out.Pipeline[0].Image).Equal("golang") + g.Assert(out.Pipeline[0].Commands).Equal([]string{"go install", "go test"}) + g.Assert(out.Pipeline[1].Name).Equal("build") + g.Assert(out.Pipeline[1].Image).Equal("golang") + g.Assert(out.Pipeline[1].Commands).Equal([]string{"go build"}) + g.Assert(out.Pipeline[2].Name).Equal("notify") + g.Assert(out.Pipeline[2].Image).Equal("slack") + }) + }) + }) +} + +var sampleYaml = ` +image: hello-world +build: + context: . + dockerfile: Dockerfile + +workspace: + path: src/github.com/octocat/hello-world + base: /go + +pipeline: + test: + image: golang + commands: + - go install + - go test + build: + image: golang + commands: + - go build + when: + event: push + notify: + image: slack + channel: dev + when: + event: failure + +services: + database: + image: mysql + +networks: + custom: + driver: overlay + +volumes: + custom: + driver: blockbridge +` diff --git a/yaml/constraint.go b/yaml/constraint.go new file mode 100644 index 000000000..9ad7b659c --- /dev/null +++ b/yaml/constraint.go @@ -0,0 +1,49 @@ +package yaml + +// Constraints define constraints for container execution. +type Constraints struct { + Platform []string + Environment []string + Event []string + Branch []string + Status []string + Matrix map[string]string +} + +// +// // Constraint defines an individual contraint. +// type Constraint struct { +// Include []string +// Exclude []string +// } +// +// // Match returns true if the branch matches the include patterns and does not +// // match any of the exclude patterns. +// func (c *Constraint) Match(v string) bool { +// // when no includes or excludes automatically match +// if len(c.Include) == 0 && len(c.Exclude) == 0 { +// return true +// } +// +// // exclusions are processed first. So we can include everything and then +// // selectively exclude certain sub-patterns. +// for _, pattern := range c.Exclude { +// if pattern == v { +// return false +// } +// if ok, _ := filepath.Match(pattern, v); ok { +// return false +// } +// } +// +// for _, pattern := range c.Include { +// if pattern == v { +// return true +// } +// if ok, _ := filepath.Match(pattern, v); ok { +// return true +// } +// } +// +// return false +// } diff --git a/yaml/container.go b/yaml/container.go new file mode 100644 index 000000000..4d3ba4140 --- /dev/null +++ b/yaml/container.go @@ -0,0 +1,171 @@ +package yaml + +import ( + "fmt" + + "github.com/drone/drone/yaml/types" + "gopkg.in/yaml.v2" +) + +// Auth defines Docker authentication credentials. +type Auth struct { + Username string + Password string + Email string +} + +// Container defines a Docker container. +type Container struct { + ID string + Name string + Image string + Build string + Pull bool + AuthConfig Auth + Detached bool + Privileged bool + WorkingDir string + Environment map[string]string + Entrypoint []string + Command []string + Commands []string + ExtraHosts []string + Volumes []string + VolumesFrom []string + Devices []string + Network string + DNS []string + DNSSearch []string + MemSwapLimit int64 + MemLimit int64 + CPUQuota int64 + CPUShares int64 + CPUSet string + OomKillDisable bool + Constraints Constraints + + Vargs map[string]interface{} +} + +// container is an intermediate type used for decoding a container in a format +// compatible with docker-compose.yml. + +// this file has a bunch of custom types that are annoying to work with, which +// is why this is used for intermediate purposes and converted to something +// easier to work with. +type container struct { + Name string `yaml:"name"` + Image string `yaml:"image"` + Build string `yaml:"build"` + Pull bool `yaml:"pull"` + Privileged bool `yaml:"privileged"` + Environment types.MapEqualSlice `yaml:"environment"` + Entrypoint types.StringOrSlice `yaml:"entrypoint"` + Command types.StringOrSlice `yaml:"command"` + Commands types.StringOrSlice `yaml:"commands"` + ExtraHosts types.StringOrSlice `yaml:"extra_hosts"` + Volumes types.StringOrSlice `yaml:"volumes"` + VolumesFrom types.StringOrSlice `yaml:"volumes_from"` + Devices types.StringOrSlice `yaml:"devices"` + Network string `yaml:"network_mode"` + DNS types.StringOrSlice `yaml:"dns"` + DNSSearch types.StringOrSlice `yaml:"dns_search"` + MemSwapLimit int64 `yaml:"memswap_limit"` + MemLimit int64 `yaml:"mem_limit"` + CPUQuota int64 `yaml:"cpu_quota"` + CPUShares int64 `yaml:"cpu_shares"` + CPUSet string `yaml:"cpuset"` + OomKillDisable bool `yaml:"oom_kill_disable"` + + AuthConfig struct { + Username string `yaml:"username"` + Password string `yaml:"password"` + Email string `yaml:"email"` + Token string `yaml:"registry_token"` + } `yaml:"auth_config"` + + Constraints struct { + Platform types.StringOrSlice `yaml:"platform"` + Environment types.StringOrSlice `yaml:"environment"` + Event types.StringOrSlice `yaml:"event"` + Branch types.StringOrSlice `yaml:"branch"` + Status types.StringOrSlice `yaml:"status"` + Matrix map[string]string `yaml:"matrix"` + } `yaml:"when"` + + Vargs map[string]interface{} `yaml:",inline"` +} + +// containerList is an intermediate type used for decoding a slice of containers +// in a format compatible with docker-compose.yml +type containerList struct { + containers []*Container +} + +// UnmarshalYAML implements custom Yaml unmarshaling. +func (c *containerList) UnmarshalYAML(unmarshal func(interface{}) error) error { + slice := yaml.MapSlice{} + err := unmarshal(&slice) + if err != nil { + return err + } + + for _, s := range slice { + cc := container{} + + out, merr := yaml.Marshal(s.Value) + if err != nil { + return merr + } + + err = yaml.Unmarshal(out, &cc) + if err != nil { + return err + } + if cc.Name == "" { + cc.Name = fmt.Sprintf("%v", s.Key) + } + if cc.Image == "" { + cc.Image = fmt.Sprintf("%v", s.Key) + } + c.containers = append(c.containers, &Container{ + Name: cc.Name, + Image: cc.Image, + Build: cc.Build, + Pull: cc.Pull, + Privileged: cc.Privileged, + Environment: cc.Environment.Map(), + Entrypoint: cc.Entrypoint.Slice(), + Command: cc.Command.Slice(), + Commands: cc.Commands.Slice(), + ExtraHosts: cc.ExtraHosts.Slice(), + Volumes: cc.Volumes.Slice(), + VolumesFrom: cc.VolumesFrom.Slice(), + Devices: cc.Devices.Slice(), + Network: cc.Network, + DNS: cc.DNS.Slice(), + DNSSearch: cc.DNSSearch.Slice(), + MemSwapLimit: cc.MemSwapLimit, + MemLimit: cc.MemLimit, + CPUQuota: cc.CPUQuota, + CPUShares: cc.CPUShares, + CPUSet: cc.CPUSet, + OomKillDisable: cc.OomKillDisable, + Vargs: cc.Vargs, + AuthConfig: Auth{ + Username: cc.AuthConfig.Username, + Password: cc.AuthConfig.Password, + Email: cc.AuthConfig.Email, + }, + Constraints: Constraints{ + Platform: cc.Constraints.Platform.Slice(), + Environment: cc.Constraints.Environment.Slice(), + Event: cc.Constraints.Event.Slice(), + Branch: cc.Constraints.Branch.Slice(), + Status: cc.Constraints.Status.Slice(), + Matrix: cc.Constraints.Matrix, + }, + }) + } + return err +} diff --git a/yaml/container_test.go b/yaml/container_test.go new file mode 100644 index 000000000..6d0af800b --- /dev/null +++ b/yaml/container_test.go @@ -0,0 +1,97 @@ +package yaml + +import ( + "testing" + + "github.com/franela/goblin" + "gopkg.in/yaml.v2" +) + +func TestContainerNode(t *testing.T) { + g := goblin.Goblin(t) + + g.Describe("Containers", func() { + g.Describe("given a yaml file", func() { + + g.It("should unmarshal", func() { + in := []byte(sampleContainer) + out := containerList{} + err := yaml.Unmarshal(in, &out) + if err != nil { + g.Fail(err) + } + g.Assert(len(out.containers)).Equal(1) + + c := out.containers[0] + g.Assert(c.Name).Equal("foo") + g.Assert(c.Image).Equal("golang") + g.Assert(c.Build).Equal(".") + g.Assert(c.Pull).Equal(true) + g.Assert(c.Privileged).Equal(true) + g.Assert(c.Entrypoint).Equal([]string{"/bin/sh"}) + g.Assert(c.Command).Equal([]string{"yes"}) + g.Assert(c.Commands).Equal([]string{"whoami"}) + g.Assert(c.ExtraHosts).Equal([]string{"foo.com"}) + g.Assert(c.Volumes).Equal([]string{"/foo:/bar"}) + g.Assert(c.VolumesFrom).Equal([]string{"foo"}) + g.Assert(c.Devices).Equal([]string{"/dev/tty0"}) + g.Assert(c.Network).Equal("bridge") + g.Assert(c.DNS).Equal([]string{"8.8.8.8"}) + g.Assert(c.MemSwapLimit).Equal(int64(1)) + g.Assert(c.MemLimit).Equal(int64(2)) + g.Assert(c.CPUQuota).Equal(int64(3)) + g.Assert(c.CPUSet).Equal("1,2") + g.Assert(c.OomKillDisable).Equal(true) + g.Assert(c.AuthConfig.Username).Equal("octocat") + g.Assert(c.AuthConfig.Password).Equal("password") + g.Assert(c.AuthConfig.Email).Equal("octocat@github.com") + g.Assert(c.Vargs["access_key"]).Equal("970d28f4dd477bc184fbd10b376de753") + g.Assert(c.Vargs["secret_key"]).Equal("9c5785d3ece6a9cdefa42eb99b58986f9095ff1c") + }) + + g.It("should unmarshal named", func() { + in := []byte("foo: { name: bar }") + out := containerList{} + err := yaml.Unmarshal(in, &out) + if err != nil { + g.Fail(err) + } + g.Assert(len(out.containers)).Equal(1) + g.Assert(out.containers[0].Name).Equal("bar") + }) + + }) + }) +} + +var sampleContainer = ` +foo: + image: golang + build: . + pull: true + privileged: true + environment: + FOO: BAR + entrypoint: /bin/sh + command: "yes" + commands: whoami + extra_hosts: foo.com + volumes: /foo:/bar + volumes_from: foo + devices: /dev/tty0 + network_mode: bridge + dns: 8.8.8.8 + memswap_limit: 1 + mem_limit: 2 + cpu_quota: 3 + cpuset: 1,2 + oom_kill_disable: true + + auth_config: + username: octocat + password: password + email: octocat@github.com + + access_key: 970d28f4dd477bc184fbd10b376de753 + secret_key: 9c5785d3ece6a9cdefa42eb99b58986f9095ff1c +` diff --git a/yaml/interpreter/error.go b/yaml/interpreter/error.go new file mode 100644 index 000000000..c3ea26c86 --- /dev/null +++ b/yaml/interpreter/error.go @@ -0,0 +1,37 @@ +package interpreter + +import ( + "errors" + "fmt" +) + +var ( + // ErrSkip is used as a return value when container execution should be + // skipped at runtime. It is not returned as an error by any function. + ErrSkip = errors.New("Skip") + + // ErrTerm is used as a return value when the runner should terminate + // execution and exit. It is not returned as an error by any function. + ErrTerm = errors.New("Terminate") +) + +// An ExitError reports an unsuccessful exit. +type ExitError struct { + Name string + Code int +} + +// Error reteurns the error message in string format. +func (e *ExitError) Error() string { + return fmt.Sprintf("%s : exit code %d", e.Name, e.Code) +} + +// An OomError reports the process received an OOMKill from the kernel. +type OomError struct { + Name string +} + +// Error reteurns the error message in string format. +func (e *OomError) Error() string { + return fmt.Sprintf("%s : received oom kill", e.Name) +} diff --git a/yaml/interpreter/error_test.go b/yaml/interpreter/error_test.go new file mode 100644 index 000000000..6c381bc34 --- /dev/null +++ b/yaml/interpreter/error_test.go @@ -0,0 +1,26 @@ +package interpreter + +import ( + "testing" + + "github.com/franela/goblin" +) + +func TestErrors(t *testing.T) { + g := goblin.Goblin(t) + + g.Describe("Error messages", func() { + + g.It("should include OOM details", func() { + err := OomError{Name: "golang"} + got, want := err.Error(), "golang : received oom kill" + g.Assert(got).Equal(want) + }) + + g.It("should include Exit code", func() { + err := ExitError{Name: "golang", Code: 255} + got, want := err.Error(), "golang : exit code 255" + g.Assert(got).Equal(want) + }) + }) +} diff --git a/yaml/interpreter/pipe.go b/yaml/interpreter/pipe.go new file mode 100644 index 000000000..b94dc0f8e --- /dev/null +++ b/yaml/interpreter/pipe.go @@ -0,0 +1,49 @@ +package interpreter + +import "fmt" + +// Pipe returns a buffered pipe that is connected to the console output. +type Pipe struct { + lines chan *Line + eof chan bool +} + +// Next returns the next Line of console output. +func (p *Pipe) Next() *Line { + select { + case line := <-p.lines: + return line + case <-p.eof: + return nil + } +} + +// Close closes the pipe of console output. +func (p *Pipe) Close() { + go func() { + p.eof <- true + }() +} + +func newPipe(buffer int) *Pipe { + return &Pipe{ + lines: make(chan *Line, buffer), + eof: make(chan bool), + } +} + +// Line is a line of console output. +type Line struct { + Proc string `json:"proc,omitempty"` + Time int64 `json:"time,omitempty"` + Type int `json:"type,omitempty"` + Pos int `json:"pos,omityempty"` + Out string `json:"out,omitempty"` +} + +func (l *Line) String() string { + return fmt.Sprintf("[%s:L%v:%vs] %s", l.Proc, l.Pos, l.Time, l.Out) +} + +// TODO(bradrydzewski) consider an alternate buffer impelmentation based on the +// x.crypto ssh buffer https://github.com/golang/crypto/blob/master/ssh/buffer.go diff --git a/yaml/interpreter/pipe_test.go b/yaml/interpreter/pipe_test.go new file mode 100644 index 000000000..ae17dbbb5 --- /dev/null +++ b/yaml/interpreter/pipe_test.go @@ -0,0 +1,54 @@ +package interpreter + +import ( + "sync" + "testing" + + "github.com/franela/goblin" +) + +func TestPipe(t *testing.T) { + g := goblin.Goblin(t) + + g.Describe("Pipe", func() { + g.It("should get next line from buffer", func() { + line := &Line{ + Proc: "redis", + Pos: 1, + Out: "starting redis server", + } + pipe := newPipe(10) + pipe.lines <- line + next := pipe.Next() + g.Assert(next).Equal(line) + }) + + g.It("should get null line on buffer closed", func() { + pipe := newPipe(10) + + var wg sync.WaitGroup + wg.Add(1) + + go func() { + next := pipe.Next() + g.Assert(next == nil).IsTrue("line should be nil") + wg.Done() + }() + + pipe.Close() + wg.Wait() + }) + + g.Describe("Line output", func() { + g.It("should prefix string() with metadata", func() { + line := Line{ + Proc: "redis", + Time: 60, + Pos: 1, + Out: "starting redis server", + } + g.Assert(line.String()).Equal("[redis:L1:60s] starting redis server") + }) + }) + }) +} diff --git a/yaml/interpreter/pipeline.go b/yaml/interpreter/pipeline.go new file mode 100644 index 000000000..bfe3b7a41 --- /dev/null +++ b/yaml/interpreter/pipeline.go @@ -0,0 +1,348 @@ +package interpreter + +import ( + "bufio" + "fmt" + "io" + "strings" + + "github.com/drone/drone/yaml" + + "github.com/samalba/dockerclient" +) + +// element represents a link in the linked list. +type element struct { + *yaml.Container + next *element +} + +// Pipeline represents a build pipeline. +type Pipeline struct { + conf *yaml.Config + head *element + tail *element + next chan (error) + done chan (error) + err error + + containers []string + volumes []string + networks []string + + client dockerclient.Client +} + +// Load loads the pipeline from the Yaml configuration file. +func Load(conf *yaml.Config) *Pipeline { + pipeline := Pipeline{ + conf: conf, + next: make(chan error), + done: make(chan error), + } + + var containers []*yaml.Container + containers = append(containers, conf.Services...) + containers = append(containers, conf.Pipeline...) + + for i, c := range containers { + next := &element{Container: c} + if i == 0 { + pipeline.head = next + pipeline.tail = next + } else { + pipeline.tail.next = next + pipeline.tail = next + } + } + + go func() { + pipeline.next <- nil + }() + + return &pipeline +} + +// Done returns when the process is done executing. +func (p *Pipeline) Done() <-chan error { + return p.done +} + +// Err returns the error for the current process. +func (p *Pipeline) Err() error { + return p.err +} + +// Next returns the next step in the process. +func (p *Pipeline) Next() <-chan error { + return p.next +} + +// Exec executes the current step. +func (p *Pipeline) Exec() { + err := p.exec(p.head.Container) + if err != nil { + p.err = err + } + p.step() +} + +// Skip skips the current step. +func (p *Pipeline) Skip() { + p.step() +} + +// Head returns the head item in the list. +func (p *Pipeline) Head() *yaml.Container { + return p.head.Container +} + +// Tail returns the tail item in the list. +func (p *Pipeline) Tail() *yaml.Container { + return p.tail.Container +} + +// Stop stops the pipeline. +func (p *Pipeline) Stop() { + p.close(ErrTerm) + return +} + +// Setup prepares the build pipeline environment. +func (p *Pipeline) Setup() error { + return nil +} + +// Teardown removes the pipeline environment. +func (p *Pipeline) Teardown() { + for _, id := range p.containers { + p.client.StopContainer(id, 1) + p.client.KillContainer(id, "9") + p.client.RemoveContainer(id, true, true) + } + for _, id := range p.networks { + p.client.RemoveNetwork(id) + } + for _, id := range p.volumes { + p.client.RemoveVolume(id) + } +} + +// step steps through the pipeline to head.next +func (p *Pipeline) step() { + if p.head == p.tail { + p.close(nil) + return + } + go func() { + p.head = p.head.next + p.next <- nil + }() +} + +// close closes open channels and signals the pipeline is done. +func (p *Pipeline) close(err error) { + go func() { + p.done <- nil + close(p.next) + close(p.done) + }() +} + +func (p *Pipeline) exec(c *yaml.Container) error { + conf := toContainerConfig(c) + auth := toAuthConfig(c) + + // check for the image and pull if not exists or if configured to always + // pull the latest version. + _, err := p.client.InspectImage(c.Image) + if err == nil || c.Pull { + err = p.client.PullImage(c.Image, auth) + if err != nil { + return err + } + } + + // creates and starts the container. + id, err := p.client.CreateContainer(conf, c.ID, auth) + if err != nil { + return err + } + p.containers = append(p.containers, id) + + err = p.client.StartContainer(c.ID, &conf.HostConfig) + if err != nil { + return err + } + + // stream the container logs + go func() { + rc, rerr := toLogs(p.client, c.ID) + if rerr != nil { + return + } + defer rc.Close() + + num := 0 + // now := time.Now().UTC() + scanner := bufio.NewScanner(rc) + for scanner.Scan() { + // r.pipe.lines <- &Line{ + // Proc: c.Name, + // Time: int64(time.Since(now).Seconds()), + // Pos: num, + // Out: scanner.Text(), + // } + num++ + } + }() + + // if the container is run in detached mode we can exit without waiting + // for execution to complete. + if c.Detached { + return nil + } + + <-p.client.Wait(c.ID) + + res, err := p.client.InspectContainer(c.ID) + if err != nil { + return err + } + + if res.State.OOMKilled { + return &OomError{c.Name} + } else if res.State.ExitCode != 0 { + return &ExitError{c.Name, res.State.ExitCode} + } + return nil +} + +func toLogs(client dockerclient.Client, id string) (io.ReadCloser, error) { + opts := &dockerclient.LogOptions{ + Follow: true, + Stdout: true, + Stderr: true, + } + + piper, pipew := io.Pipe() + go func() { + defer pipew.Close() + + // sometimes the docker logs fails due to parsing errors. this routine will + // check for such a failure and attempt to resume if necessary. + for i := 0; i < 5; i++ { + if i > 0 { + opts.Tail = 1 + } + + rc, err := client.ContainerLogs(id, opts) + if err != nil { + return + } + defer rc.Close() + + // use Docker StdCopy + // internal.StdCopy(pipew, pipew, rc) + + // check to see if the container is still running. If not, we can safely + // exit and assume there are no more logs left to stream. + v, err := client.InspectContainer(id) + if err != nil || !v.State.Running { + return + } + } + }() + return piper, nil +} + +// helper function that converts the Continer data structure to the exepcted +// dockerclient.ContainerConfig. +func toContainerConfig(c *yaml.Container) *dockerclient.ContainerConfig { + config := &dockerclient.ContainerConfig{ + Image: c.Image, + Env: toEnvironmentSlice(c.Environment), + Cmd: c.Command, + Entrypoint: c.Entrypoint, + WorkingDir: c.WorkingDir, + HostConfig: dockerclient.HostConfig{ + Privileged: c.Privileged, + NetworkMode: c.Network, + Memory: c.MemLimit, + CpuShares: c.CPUShares, + CpuQuota: c.CPUQuota, + CpusetCpus: c.CPUSet, + MemorySwappiness: -1, + OomKillDisable: c.OomKillDisable, + }, + } + + if len(config.Entrypoint) == 0 { + config.Entrypoint = nil + } + if len(config.Cmd) == 0 { + config.Cmd = nil + } + if len(c.ExtraHosts) > 0 { + config.HostConfig.ExtraHosts = c.ExtraHosts + } + if len(c.DNS) != 0 { + config.HostConfig.Dns = c.DNS + } + if len(c.DNSSearch) != 0 { + config.HostConfig.DnsSearch = c.DNSSearch + } + if len(c.VolumesFrom) != 0 { + config.HostConfig.VolumesFrom = c.VolumesFrom + } + + config.Volumes = map[string]struct{}{} + for _, path := range c.Volumes { + if strings.Index(path, ":") == -1 { + config.Volumes[path] = struct{}{} + continue + } + parts := strings.Split(path, ":") + config.Volumes[parts[1]] = struct{}{} + config.HostConfig.Binds = append(config.HostConfig.Binds, path) + } + + for _, path := range c.Devices { + if strings.Index(path, ":") == -1 { + continue + } + parts := strings.Split(path, ":") + device := dockerclient.DeviceMapping{ + PathOnHost: parts[0], + PathInContainer: parts[1], + CgroupPermissions: "rwm", + } + config.HostConfig.Devices = append(config.HostConfig.Devices, device) + } + + return config +} + +// helper function that converts the AuthConfig data structure to the exepcted +// dockerclient.AuthConfig. +func toAuthConfig(c *yaml.Container) *dockerclient.AuthConfig { + if c.AuthConfig.Username == "" && + c.AuthConfig.Password == "" { + return nil + } + return &dockerclient.AuthConfig{ + Email: c.AuthConfig.Email, + Username: c.AuthConfig.Username, + Password: c.AuthConfig.Password, + } +} + +// helper function that converts a key value map of environment variables to a +// string slice in key=value format. +func toEnvironmentSlice(env map[string]string) []string { + var envs []string + for k, v := range env { + envs = append(envs, fmt.Sprintf("%s=%s", k, v)) + } + return envs +} diff --git a/yaml/interpreter/pipeline_test.go b/yaml/interpreter/pipeline_test.go new file mode 100644 index 000000000..038cb47be --- /dev/null +++ b/yaml/interpreter/pipeline_test.go @@ -0,0 +1,70 @@ +package interpreter + +import ( + "fmt" + "testing" + + "github.com/drone/drone/yaml" +) + +func TestInterpreter(t *testing.T) { + + conf, err := yaml.ParseString(sampleYaml) + if err != nil { + t.Fatal(err) + } + + pipeline := Load(conf) + + for { + select { + case <-pipeline.Done(): + fmt.Println("GOT DONE") + return + + case <-pipeline.Next(): + pipeline.Exec() + } + } +} + +var sampleYaml = ` +image: hello-world +build: + context: . + dockerfile: Dockerfile + +workspace: + path: src/github.com/octocat/hello-world + base: /go + +pipeline: + test: + image: golang + commands: + - go install + - go test + build: + image: golang + commands: + - go build + when: + event: push + notify: + image: slack + channel: dev + when: + event: failure + +services: + database: + image: mysql + +networks: + custom: + driver: overlay + +volumes: + custom: + driver: blockbridge +` diff --git a/yaml/network.go b/yaml/network.go new file mode 100644 index 000000000..d49e86e3e --- /dev/null +++ b/yaml/network.go @@ -0,0 +1,51 @@ +package yaml + +import ( + "fmt" + + "gopkg.in/yaml.v2" +) + +// Network defines a Docker network. +type Network struct { + Name string + Driver string + DriverOpts map[string]string `yaml:"driver_opts"` +} + +// networkList is an intermediate type used for decoding a slice of networks +// in a format compatible with docker-compose.yml +type networkList struct { + networks []*Network +} + +// UnmarshalYAML implements custom Yaml unmarshaling. +func (n *networkList) UnmarshalYAML(unmarshal func(interface{}) error) error { + slice := yaml.MapSlice{} + err := unmarshal(&slice) + if err != nil { + return err + } + + for _, s := range slice { + nn := Network{} + + out, merr := yaml.Marshal(s.Value) + if merr != nil { + return merr + } + + err = yaml.Unmarshal(out, &nn) + if err != nil { + return err + } + if nn.Name == "" { + nn.Name = fmt.Sprintf("%v", s.Key) + } + if nn.Driver == "" { + nn.Driver = "bridge" + } + n.networks = append(n.networks, &nn) + } + return err +} diff --git a/yaml/network_test.go b/yaml/network_test.go new file mode 100644 index 000000000..4fe3c8636 --- /dev/null +++ b/yaml/network_test.go @@ -0,0 +1,51 @@ +package yaml + +import ( + "testing" + + "github.com/franela/goblin" + "gopkg.in/yaml.v2" +) + +func TestNetworks(t *testing.T) { + g := goblin.Goblin(t) + + g.Describe("Networks", func() { + g.Describe("given a yaml file", func() { + + g.It("should unmarshal", func() { + in := []byte("foo: { driver: overlay }") + out := networkList{} + err := yaml.Unmarshal(in, &out) + if err != nil { + g.Fail(err) + } + g.Assert(len(out.networks)).Equal(1) + g.Assert(out.networks[0].Name).Equal("foo") + g.Assert(out.networks[0].Driver).Equal("overlay") + }) + + g.It("should unmarshal named", func() { + in := []byte("foo: { name: bar }") + out := networkList{} + err := yaml.Unmarshal(in, &out) + if err != nil { + g.Fail(err) + } + g.Assert(len(out.networks)).Equal(1) + g.Assert(out.networks[0].Name).Equal("bar") + }) + + g.It("should unmarshal and use default driver", func() { + in := []byte("foo: { name: bar }") + out := networkList{} + err := yaml.Unmarshal(in, &out) + if err != nil { + g.Fail(err) + } + g.Assert(len(out.networks)).Equal(1) + g.Assert(out.networks[0].Driver).Equal("bridge") + }) + }) + }) +} diff --git a/yaml/types/map.go b/yaml/types/map.go new file mode 100644 index 000000000..bd88ac000 --- /dev/null +++ b/yaml/types/map.go @@ -0,0 +1,38 @@ +package types + +import "strings" + +// MapEqualSlice is a custom Yaml type that can hold a map or slice of strings +// in key=value format. +type MapEqualSlice struct { + parts map[string]string +} + +// UnmarshalYAML implements custom Yaml unmarshaling. +func (s *MapEqualSlice) UnmarshalYAML(unmarshal func(interface{}) error) error { + s.parts = map[string]string{} + err := unmarshal(&s.parts) + if err == nil { + return nil + } + + var slice []string + err = unmarshal(&slice) + if err != nil { + return err + } + for _, v := range slice { + parts := strings.SplitN(v, "=", 2) + if len(parts) == 2 { + key := parts[0] + val := parts[1] + s.parts[key] = val + } + } + return nil +} + +// Map returns the Yaml information as a map. +func (s *MapEqualSlice) Map() map[string]string { + return s.parts +} diff --git a/yaml/types/map_test.go b/yaml/types/map_test.go new file mode 100644 index 000000000..7ed1e1c50 --- /dev/null +++ b/yaml/types/map_test.go @@ -0,0 +1,44 @@ +package types + +import ( + "testing" + + "github.com/franela/goblin" + "gopkg.in/yaml.v2" +) + +func TestMapEqualSlice(t *testing.T) { + g := goblin.Goblin(t) + + g.Describe("Yaml map equal slice", func() { + + g.It("should unmarshal a map", func() { + in := []byte("foo: bar") + out := MapEqualSlice{} + err := yaml.Unmarshal(in, &out) + if err != nil { + g.Fail(err) + } + g.Assert(len(out.Map())).Equal(1) + g.Assert(out.Map()["foo"]).Equal("bar") + }) + + g.It("should unmarshal a map equal slice", func() { + in := []byte("[ foo=bar ]") + out := MapEqualSlice{} + err := yaml.Unmarshal(in, &out) + if err != nil { + g.Fail(err) + } + g.Assert(len(out.parts)).Equal(1) + g.Assert(out.parts["foo"]).Equal("bar") + }) + + g.It("should throw error when invalid map equal slice", func() { + in := []byte("foo") // string value should fail parse + out := MapEqualSlice{} + err := yaml.Unmarshal(in, &out) + g.Assert(err != nil).IsTrue("expects error") + }) + }) +} diff --git a/yaml/types.go b/yaml/types/slice.go similarity index 52% rename from yaml/types.go rename to yaml/types/slice.go index 9c1eefa56..8174c87d6 100644 --- a/yaml/types.go +++ b/yaml/types/slice.go @@ -1,11 +1,12 @@ -package yaml +package types -// stringOrSlice represents a string or an array of strings. -type stringOrSlice struct { +// StringOrSlice is a custom Yaml type that can hold a string or slice of strings. +type StringOrSlice struct { parts []string } -func (s *stringOrSlice) UnmarshalYAML(unmarshal func(interface{}) error) error { +// UnmarshalYAML implements custom Yaml unmarshaling. +func (s *StringOrSlice) UnmarshalYAML(unmarshal func(interface{}) error) error { var sliceType []string err := unmarshal(&sliceType) if err == nil { @@ -23,6 +24,7 @@ func (s *stringOrSlice) UnmarshalYAML(unmarshal func(interface{}) error) error { return err } -func (s stringOrSlice) Slice() []string { +// Slice returns the slice of strings. +func (s StringOrSlice) Slice() []string { return s.parts } diff --git a/yaml/types_test.go b/yaml/types/slice_test.go similarity index 74% rename from yaml/types_test.go rename to yaml/types/slice_test.go index 8d095223f..dcf9a25f6 100644 --- a/yaml/types_test.go +++ b/yaml/types/slice_test.go @@ -1,4 +1,4 @@ -package yaml +package types import ( "testing" @@ -7,26 +7,26 @@ import ( "gopkg.in/yaml.v2" ) -func TestTypes(t *testing.T) { +func TestStringSlice(t *testing.T) { g := goblin.Goblin(t) - g.Describe("Yaml types", func() { + g.Describe("Yaml string slice", func() { g.Describe("given a yaml file", func() { g.It("should unmarshal a string", func() { in := []byte("foo") - out := stringOrSlice{} + out := StringOrSlice{} err := yaml.Unmarshal(in, &out) if err != nil { g.Fail(err) } - g.Assert(len(out.parts)).Equal(1) - g.Assert(out.parts[0]).Equal("foo") + g.Assert(len(out.Slice())).Equal(1) + g.Assert(out.Slice()[0]).Equal("foo") }) g.It("should unmarshal a string slice", func() { in := []byte("[ foo ]") - out := stringOrSlice{} + out := StringOrSlice{} err := yaml.Unmarshal(in, &out) if err != nil { g.Fail(err) @@ -37,7 +37,7 @@ func TestTypes(t *testing.T) { g.It("should throw error when invalid string slice", func() { in := []byte("{ }") // string value should fail parse - out := stringOrSlice{} + out := StringOrSlice{} err := yaml.Unmarshal(in, &out) g.Assert(err != nil).IsTrue("expects error") }) diff --git a/yaml/volume.go b/yaml/volume.go new file mode 100644 index 000000000..20d297bf9 --- /dev/null +++ b/yaml/volume.go @@ -0,0 +1,51 @@ +package yaml + +import ( + "fmt" + + "gopkg.in/yaml.v2" +) + +// Volume defines a Docker volume. +type Volume struct { + Name string + Driver string + DriverOpts map[string]string `yaml:"driver_opts"` + External bool +} + +// volumeList is an intermediate type used for decoding a slice of volumes +// in a format compatible with docker-compose.yml +type volumeList struct { + volumes []*Volume +} + +// UnmarshalYAML implements custom Yaml unmarshaling. +func (v *volumeList) UnmarshalYAML(unmarshal func(interface{}) error) error { + slice := yaml.MapSlice{} + err := unmarshal(&slice) + if err != nil { + return err + } + + for _, s := range slice { + vv := Volume{} + out, merr := yaml.Marshal(s.Value) + if merr != nil { + return merr + } + + err = yaml.Unmarshal(out, &vv) + if err != nil { + return err + } + if vv.Name == "" { + vv.Name = fmt.Sprintf("%v", s.Key) + } + if vv.Driver == "" { + vv.Driver = "local" + } + v.volumes = append(v.volumes, &vv) + } + return err +} diff --git a/yaml/volume_test.go b/yaml/volume_test.go new file mode 100644 index 000000000..ebeaa9ae1 --- /dev/null +++ b/yaml/volume_test.go @@ -0,0 +1,51 @@ +package yaml + +import ( + "testing" + + "github.com/franela/goblin" + "gopkg.in/yaml.v2" +) + +func TestVolumes(t *testing.T) { + g := goblin.Goblin(t) + + g.Describe("Volumes", func() { + g.Describe("given a yaml file", func() { + + g.It("should unmarshal", func() { + in := []byte("foo: { driver: blockbridge }") + out := volumeList{} + err := yaml.Unmarshal(in, &out) + if err != nil { + g.Fail(err) + } + g.Assert(len(out.volumes)).Equal(1) + g.Assert(out.volumes[0].Name).Equal("foo") + g.Assert(out.volumes[0].Driver).Equal("blockbridge") + }) + + g.It("should unmarshal named", func() { + in := []byte("foo: { name: bar }") + out := volumeList{} + err := yaml.Unmarshal(in, &out) + if err != nil { + g.Fail(err) + } + g.Assert(len(out.volumes)).Equal(1) + g.Assert(out.volumes[0].Name).Equal("bar") + }) + + g.It("should unmarshal and use default driver", func() { + in := []byte("foo: { name: bar }") + out := volumeList{} + err := yaml.Unmarshal(in, &out) + if err != nil { + g.Fail(err) + } + g.Assert(len(out.volumes)).Equal(1) + g.Assert(out.volumes[0].Driver).Equal("local") + }) + }) + }) +}