simplified build execution

yaml/container.go

@@ -23,6 +23,7 @@ type Container struct {
 	Pull           bool
 	AuthConfig     Auth
 	Detached       bool
+	Disabled       bool
 	Privileged     bool
 	WorkingDir     string
 	Environment    map[string]string

yaml/interpreter/convert.go

@@ -0,0 +1 @@
+package interpreter

yaml/interpreter/internal/README

@@ -0,0 +1 @@
+This is an internal copy of the Docker stdcopy package that removes the logrus debug logging. The original package is found at https://github.com/docker/docker/tree/master/pkg/stdcopy

yaml/interpreter/internal/stdcopy.go

@@ -0,0 +1,167 @@
+package internal
+
+import (
+	"encoding/binary"
+	"errors"
+	"fmt"
+	"io"
+)
+
+// StdType is the type of standard stream
+// a writer can multiplex to.
+type StdType byte
+
+const (
+	// Stdin represents standard input stream type.
+	Stdin StdType = iota
+	// Stdout represents standard output stream type.
+	Stdout
+	// Stderr represents standard error steam type.
+	Stderr
+
+	stdWriterPrefixLen = 8
+	stdWriterFdIndex   = 0
+	stdWriterSizeIndex = 4
+
+	startingBufLen = 32*1024 + stdWriterPrefixLen + 1
+)
+
+// stdWriter is wrapper of io.Writer with extra customized info.
+type stdWriter struct {
+	io.Writer
+	prefix byte
+}
+
+// Write sends the buffer to the underneath writer.
+// It insert the prefix header before the buffer,
+// so stdcopy.StdCopy knows where to multiplex the output.
+// It makes stdWriter to implement io.Writer.
+func (w *stdWriter) Write(buf []byte) (n int, err error) {
+	if w == nil || w.Writer == nil {
+		return 0, errors.New("Writer not instantiated")
+	}
+	if buf == nil {
+		return 0, nil
+	}
+
+	header := [stdWriterPrefixLen]byte{stdWriterFdIndex: w.prefix}
+	binary.BigEndian.PutUint32(header[stdWriterSizeIndex:], uint32(len(buf)))
+
+	line := append(header[:], buf...)
+
+	n, err = w.Writer.Write(line)
+	n -= stdWriterPrefixLen
+
+	if n < 0 {
+		n = 0
+	}
+	return
+}
+
+// NewStdWriter instantiates a new Writer.
+// Everything written to it will be encapsulated using a custom format,
+// and written to the underlying `w` stream.
+// This allows multiple write streams (e.g. stdout and stderr) to be muxed into a single connection.
+// `t` indicates the id of the stream to encapsulate.
+// It can be stdcopy.Stdin, stdcopy.Stdout, stdcopy.Stderr.
+func NewStdWriter(w io.Writer, t StdType) io.Writer {
+	return &stdWriter{
+		Writer: w,
+		prefix: byte(t),
+	}
+}
+
+// StdCopy is a modified version of io.Copy.
+//
+// StdCopy will demultiplex `src`, assuming that it contains two streams,
+// previously multiplexed together using a StdWriter instance.
+// As it reads from `src`, StdCopy will write to `dstout` and `dsterr`.
+//
+// StdCopy will read until it hits EOF on `src`. It will then return a nil error.
+// In other words: if `err` is non nil, it indicates a real underlying error.
+//
+// `written` will hold the total number of bytes written to `dstout` and `dsterr`.
+func StdCopy(dstout, dsterr io.Writer, src io.Reader) (written int64, err error) {
+	var (
+		buf       = make([]byte, startingBufLen)
+		bufLen    = len(buf)
+		nr, nw    int
+		er, ew    error
+		out       io.Writer
+		frameSize int
+	)
+
+	for {
+		// Make sure we have at least a full header
+		for nr < stdWriterPrefixLen {
+			var nr2 int
+			nr2, er = src.Read(buf[nr:])
+			nr += nr2
+			if er == io.EOF {
+				if nr < stdWriterPrefixLen {
+					return written, nil
+				}
+				break
+			}
+			if er != nil {
+				return 0, er
+			}
+		}
+
+		// Check the first byte to know where to write
+		switch StdType(buf[stdWriterFdIndex]) {
+		case Stdin:
+			fallthrough
+		case Stdout:
+			// Write on stdout
+			out = dstout
+		case Stderr:
+			// Write on stderr
+			out = dsterr
+		default:
+			return 0, fmt.Errorf("Unrecognized input header: %d", buf[stdWriterFdIndex])
+		}
+
+		// Retrieve the size of the frame
+		frameSize = int(binary.BigEndian.Uint32(buf[stdWriterSizeIndex : stdWriterSizeIndex+4]))
+
+		// Check if the buffer is big enough to read the frame.
+		// Extend it if necessary.
+		if frameSize+stdWriterPrefixLen > bufLen {
+			buf = append(buf, make([]byte, frameSize+stdWriterPrefixLen-bufLen+1)...)
+			bufLen = len(buf)
+		}
+
+		// While the amount of bytes read is less than the size of the frame + header, we keep reading
+		for nr < frameSize+stdWriterPrefixLen {
+			var nr2 int
+			nr2, er = src.Read(buf[nr:])
+			nr += nr2
+			if er == io.EOF {
+				if nr < frameSize+stdWriterPrefixLen {
+					return written, nil
+				}
+				break
+			}
+			if er != nil {
+				return 0, er
+			}
+		}
+
+		// Write the retrieved frame (without header)
+		nw, ew = out.Write(buf[stdWriterPrefixLen : frameSize+stdWriterPrefixLen])
+		if ew != nil {
+			return 0, ew
+		}
+		// If the frame has not been fully written: error
+		if nw != frameSize {
+			return 0, io.ErrShortWrite
+		}
+		written += int64(nw)
+
+		// Move the rest of the buffer to the beginning
+		copy(buf, buf[frameSize+stdWriterPrefixLen:])
+		// Move the index
+		nr -= frameSize + stdWriterPrefixLen
+	}
+}

yaml/interpreter/internal/stdcopy_test.go

@@ -0,0 +1,260 @@
+package internal
+
+import (
+	"bytes"
+	"errors"
+	"io"
+	"io/ioutil"
+	"strings"
+	"testing"
+)
+
+func TestNewStdWriter(t *testing.T) {
+	writer := NewStdWriter(ioutil.Discard, Stdout)
+	if writer == nil {
+		t.Fatalf("NewStdWriter with an invalid StdType should not return nil.")
+	}
+}
+
+func TestWriteWithUnitializedStdWriter(t *testing.T) {
+	writer := stdWriter{
+		Writer: nil,
+		prefix: byte(Stdout),
+	}
+	n, err := writer.Write([]byte("Something here"))
+	if n != 0 || err == nil {
+		t.Fatalf("Should fail when given an uncomplete or uninitialized StdWriter")
+	}
+}
+
+func TestWriteWithNilBytes(t *testing.T) {
+	writer := NewStdWriter(ioutil.Discard, Stdout)
+	n, err := writer.Write(nil)
+	if err != nil {
+		t.Fatalf("Shouldn't have fail when given no data")
+	}
+	if n > 0 {
+		t.Fatalf("Write should have written 0 byte, but has written %d", n)
+	}
+}
+
+func TestWrite(t *testing.T) {
+	writer := NewStdWriter(ioutil.Discard, Stdout)
+	data := []byte("Test StdWrite.Write")
+	n, err := writer.Write(data)
+	if err != nil {
+		t.Fatalf("Error while writing with StdWrite")
+	}
+	if n != len(data) {
+		t.Fatalf("Write should have written %d byte but wrote %d.", len(data), n)
+	}
+}
+
+type errWriter struct {
+	n   int
+	err error
+}
+
+func (f *errWriter) Write(buf []byte) (int, error) {
+	return f.n, f.err
+}
+
+func TestWriteWithWriterError(t *testing.T) {
+	expectedError := errors.New("expected")
+	expectedReturnedBytes := 10
+	writer := NewStdWriter(&errWriter{
+		n:   stdWriterPrefixLen + expectedReturnedBytes,
+		err: expectedError}, Stdout)
+	data := []byte("This won't get written, sigh")
+	n, err := writer.Write(data)
+	if err != expectedError {
+		t.Fatalf("Didn't get expected error.")
+	}
+	if n != expectedReturnedBytes {
+		t.Fatalf("Didn't get expected written bytes %d, got %d.",
+			expectedReturnedBytes, n)
+	}
+}
+
+func TestWriteDoesNotReturnNegativeWrittenBytes(t *testing.T) {
+	writer := NewStdWriter(&errWriter{n: -1}, Stdout)
+	data := []byte("This won't get written, sigh")
+	actual, _ := writer.Write(data)
+	if actual != 0 {
+		t.Fatalf("Expected returned written bytes equal to 0, got %d", actual)
+	}
+}
+
+func getSrcBuffer(stdOutBytes, stdErrBytes []byte) (buffer *bytes.Buffer, err error) {
+	buffer = new(bytes.Buffer)
+	dstOut := NewStdWriter(buffer, Stdout)
+	_, err = dstOut.Write(stdOutBytes)
+	if err != nil {
+		return
+	}
+	dstErr := NewStdWriter(buffer, Stderr)
+	_, err = dstErr.Write(stdErrBytes)
+	return
+}
+
+func TestStdCopyWriteAndRead(t *testing.T) {
+	stdOutBytes := []byte(strings.Repeat("o", startingBufLen))
+	stdErrBytes := []byte(strings.Repeat("e", startingBufLen))
+	buffer, err := getSrcBuffer(stdOutBytes, stdErrBytes)
+	if err != nil {
+		t.Fatal(err)
+	}
+	written, err := StdCopy(ioutil.Discard, ioutil.Discard, buffer)
+	if err != nil {
+		t.Fatal(err)
+	}
+	expectedTotalWritten := len(stdOutBytes) + len(stdErrBytes)
+	if written != int64(expectedTotalWritten) {
+		t.Fatalf("Expected to have total of %d bytes written, got %d", expectedTotalWritten, written)
+	}
+}
+
+type customReader struct {
+	n            int
+	err          error
+	totalCalls   int
+	correctCalls int
+	src          *bytes.Buffer
+}
+
+func (f *customReader) Read(buf []byte) (int, error) {
+	f.totalCalls++
+	if f.totalCalls <= f.correctCalls {
+		return f.src.Read(buf)
+	}
+	return f.n, f.err
+}
+
+func TestStdCopyReturnsErrorReadingHeader(t *testing.T) {
+	expectedError := errors.New("error")
+	reader := &customReader{
+		err: expectedError}
+	written, err := StdCopy(ioutil.Discard, ioutil.Discard, reader)
+	if written != 0 {
+		t.Fatalf("Expected 0 bytes read, got %d", written)
+	}
+	if err != expectedError {
+		t.Fatalf("Didn't get expected error")
+	}
+}
+
+func TestStdCopyReturnsErrorReadingFrame(t *testing.T) {
+	expectedError := errors.New("error")
+	stdOutBytes := []byte(strings.Repeat("o", startingBufLen))
+	stdErrBytes := []byte(strings.Repeat("e", startingBufLen))
+	buffer, err := getSrcBuffer(stdOutBytes, stdErrBytes)
+	if err != nil {
+		t.Fatal(err)
+	}
+	reader := &customReader{
+		correctCalls: 1,
+		n:            stdWriterPrefixLen + 1,
+		err:          expectedError,
+		src:          buffer}
+	written, err := StdCopy(ioutil.Discard, ioutil.Discard, reader)
+	if written != 0 {
+		t.Fatalf("Expected 0 bytes read, got %d", written)
+	}
+	if err != expectedError {
+		t.Fatalf("Didn't get expected error")
+	}
+}
+
+func TestStdCopyDetectsCorruptedFrame(t *testing.T) {
+	stdOutBytes := []byte(strings.Repeat("o", startingBufLen))
+	stdErrBytes := []byte(strings.Repeat("e", startingBufLen))
+	buffer, err := getSrcBuffer(stdOutBytes, stdErrBytes)
+	if err != nil {
+		t.Fatal(err)
+	}
+	reader := &customReader{
+		correctCalls: 1,
+		n:            stdWriterPrefixLen + 1,
+		err:          io.EOF,
+		src:          buffer}
+	written, err := StdCopy(ioutil.Discard, ioutil.Discard, reader)
+	if written != startingBufLen {
+		t.Fatalf("Expected %d bytes read, got %d", startingBufLen, written)
+	}
+	if err != nil {
+		t.Fatal("Didn't get nil error")
+	}
+}
+
+func TestStdCopyWithInvalidInputHeader(t *testing.T) {
+	dstOut := NewStdWriter(ioutil.Discard, Stdout)
+	dstErr := NewStdWriter(ioutil.Discard, Stderr)
+	src := strings.NewReader("Invalid input")
+	_, err := StdCopy(dstOut, dstErr, src)
+	if err == nil {
+		t.Fatal("StdCopy with invalid input header should fail.")
+	}
+}
+
+func TestStdCopyWithCorruptedPrefix(t *testing.T) {
+	data := []byte{0x01, 0x02, 0x03}
+	src := bytes.NewReader(data)
+	written, err := StdCopy(nil, nil, src)
+	if err != nil {
+		t.Fatalf("StdCopy should not return an error with corrupted prefix.")
+	}
+	if written != 0 {
+		t.Fatalf("StdCopy should have written 0, but has written %d", written)
+	}
+}
+
+func TestStdCopyReturnsWriteErrors(t *testing.T) {
+	stdOutBytes := []byte(strings.Repeat("o", startingBufLen))
+	stdErrBytes := []byte(strings.Repeat("e", startingBufLen))
+	buffer, err := getSrcBuffer(stdOutBytes, stdErrBytes)
+	if err != nil {
+		t.Fatal(err)
+	}
+	expectedError := errors.New("expected")
+
+	dstOut := &errWriter{err: expectedError}
+
+	written, err := StdCopy(dstOut, ioutil.Discard, buffer)
+	if written != 0 {
+		t.Fatalf("StdCopy should have written 0, but has written %d", written)
+	}
+	if err != expectedError {
+		t.Fatalf("Didn't get expected error, got %v", err)
+	}
+}
+
+func TestStdCopyDetectsNotFullyWrittenFrames(t *testing.T) {
+	stdOutBytes := []byte(strings.Repeat("o", startingBufLen))
+	stdErrBytes := []byte(strings.Repeat("e", startingBufLen))
+	buffer, err := getSrcBuffer(stdOutBytes, stdErrBytes)
+	if err != nil {
+		t.Fatal(err)
+	}
+	dstOut := &errWriter{n: startingBufLen - 10}
+
+	written, err := StdCopy(dstOut, ioutil.Discard, buffer)
+	if written != 0 {
+		t.Fatalf("StdCopy should have return 0 written bytes, but returned %d", written)
+	}
+	if err != io.ErrShortWrite {
+		t.Fatalf("Didn't get expected io.ErrShortWrite error")
+	}
+}
+
+func BenchmarkWrite(b *testing.B) {
+	w := NewStdWriter(ioutil.Discard, Stdout)
+	data := []byte("Test line for testing stdwriter performance\n")
+	data = bytes.Repeat(data, 100)
+	b.SetBytes(int64(len(data)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		if _, err := w.Write(data); err != nil {
+			b.Fatal(err)
+		}
+	}
+}

yaml/interpreter/pipeline.go

@@ -5,8 +5,10 @@ import (
 	"fmt"
 	"io"
 	"strings"
+	"time"
 
 	"github.com/drone/drone/yaml"
+	"github.com/drone/drone/yaml/interpreter/internal"
 
 	"github.com/samalba/dockerclient"
 )
@@ -22,10 +24,12 @@ type Pipeline struct {
 	conf *yaml.Config
 	head *element
 	tail *element
+	pipe chan (*Line)
 	next chan (error)
 	done chan (error)
 	err  error
 
+	ambassador string
 	containers []string
 	volumes    []string
 	networks   []string
@@ -34,20 +38,24 @@ type Pipeline struct {
 }
 
 // Load loads the pipeline from the Yaml configuration file.
-func Load(conf *yaml.Config) *Pipeline {
+func Load(conf *yaml.Config, client dockerclient.Client) *Pipeline {
 	pipeline := Pipeline{
-		conf: conf,
-		next: make(chan error),
-		done: make(chan error),
+		client: client,
+		pipe:   make(chan *Line, 500), // buffer 500 lines of logs
+		next:   make(chan error),
+		done:   make(chan error),
 	}
 
 	var containers []*yaml.Container
 	containers = append(containers, conf.Services...)
 	containers = append(containers, conf.Pipeline...)
 
-	for i, c := range containers {
+	for _, c := range containers {
+		if c.Disabled {
+			continue
+		}
 		next := &element{Container: c}
-		if i == 0 {
+		if pipeline.head == nil {
 			pipeline.head = next
 			pipeline.tail = next
 		} else {
@@ -80,11 +88,13 @@ func (p *Pipeline) Next() <-chan error {
 
 // Exec executes the current step.
 func (p *Pipeline) Exec() {
-	err := p.exec(p.head.Container)
-	if err != nil {
-		p.err = err
-	}
-	p.step()
+	go func() {
+		err := p.exec(p.head.Container)
+		if err != nil {
+			p.err = err
+		}
+		p.step()
+	}()
 }
 
 // Skip skips the current step.
@@ -92,6 +102,11 @@ func (p *Pipeline) Skip() {
 	p.step()
 }
 
+// Pipe returns the build output pipe.
+func (p *Pipeline) Pipe() <-chan *Line {
+	return p.pipe
+}
+
 // Head returns the head item in the list.
 func (p *Pipeline) Head() *yaml.Container {
 	return p.head.Container
@@ -104,8 +119,9 @@ func (p *Pipeline) Tail() *yaml.Container {
 
 // Stop stops the pipeline.
 func (p *Pipeline) Stop() {
-	p.close(ErrTerm)
-	return
+	go func() {
+		p.done <- ErrTerm
+	}()
 }
 
 // Setup prepares the build pipeline environment.
@@ -126,26 +142,29 @@ func (p *Pipeline) Teardown() {
 	for _, id := range p.volumes {
 		p.client.RemoveVolume(id)
 	}
+	close(p.next)
+	close(p.done)
+	close(p.pipe)
 }
 
 // step steps through the pipeline to head.next
 func (p *Pipeline) step() {
 	if p.head == p.tail {
-		p.close(nil)
-		return
+		go func() {
+			p.done <- nil
+		}()
+	} else {
+		go func() {
+			p.head = p.head.next
+			p.next <- nil
+		}()
 	}
-	go func() {
-		p.head = p.head.next
-		p.next <- nil
-	}()
 }
 
 // close closes open channels and signals the pipeline is done.
 func (p *Pipeline) close(err error) {
 	go func() {
-		p.done <- nil
-		close(p.next)
-		close(p.done)
+		p.done <- err
 	}()
 }
 
@@ -156,7 +175,7 @@ func (p *Pipeline) exec(c *yaml.Container) error {
 	// check for the image and pull if not exists or if configured to always
 	// pull the latest version.
 	_, err := p.client.InspectImage(c.Image)
-	if err == nil || c.Pull {
+	if err != nil || c.Pull {
 		err = p.client.PullImage(c.Image, auth)
 		if err != nil {
 			return err
@@ -184,15 +203,15 @@ func (p *Pipeline) exec(c *yaml.Container) error {
 		defer rc.Close()
 
 		num := 0
-		// now := time.Now().UTC()
+		now := time.Now().UTC()
 		scanner := bufio.NewScanner(rc)
 		for scanner.Scan() {
-			// r.pipe.lines <- &Line{
-			// 	Proc: c.Name,
-			// 	Time: int64(time.Since(now).Seconds()),
-			// 	Pos:  num,
-			// 	Out:  scanner.Text(),
-			// }
+			p.pipe <- &Line{
+				Proc: c.Name,
+				Time: int64(time.Since(now).Seconds()),
+				Pos:  num,
+				Out:  scanner.Text(),
+			}
 			num++
 		}
 	}()
@@ -243,7 +262,7 @@ func toLogs(client dockerclient.Client, id string) (io.ReadCloser, error) {
 			defer rc.Close()
 
 			// use Docker StdCopy
-			// internal.StdCopy(pipew, pipew, rc)
+			internal.StdCopy(pipew, pipew, rc)
 
 			// check to see if the container is still running. If not,  we can safely
 			// exit and assume there are no more logs left to stream.

yaml/interpreter/pipeline_test.go

@@ -14,14 +14,19 @@ func TestInterpreter(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	pipeline := Load(conf)
-
+	pipeline := Load(conf, nil)
+	pipeline.pipe <- &Line{Out: "foo"}
+	pipeline.pipe <- &Line{Out: "bar"}
+	pipeline.pipe <- &Line{Out: "baz"}
 	for {
 		select {
 		case <-pipeline.Done():
 			fmt.Println("GOT DONE")
 			return
 
+		case line := <-pipeline.Pipe():
+			fmt.Println(line.String())
+
 		case <-pipeline.Next():
 			pipeline.Exec()
 		}

yaml/transform/clone.go

@@ -0,0 +1,21 @@
+package transform
+
+import "github.com/drone/drone/yaml"
+
+const clone = "clone"
+
+// Clone transforms the Yaml to include a clone step.
+func Clone(c *yaml.Config, plugin string) error {
+	for _, p := range c.Pipeline {
+		if p.Name == clone {
+			return nil
+		}
+	}
+
+	s := &yaml.Container{
+		Image: plugin,
+		Name:  clone,
+	}
+	c.Pipeline = append([]*yaml.Container{s}, c.Pipeline...)
+	return nil
+}

yaml/transform/command.go

@@ -0,0 +1,82 @@
+package transform
+
+import (
+	"bytes"
+	"encoding/base64"
+	"fmt"
+	"strings"
+
+	"github.com/drone/drone/yaml"
+)
+
+// CommandTransform transforms the custom shell commands in the Yaml pipeline
+// into a container ENTRYPOINT and and CMD for execution.
+func CommandTransform(c *yaml.Config) error {
+	for _, p := range c.Pipeline {
+
+		if len(p.Commands) == 0 {
+			continue
+		}
+
+		p.Entrypoint = []string{
+			"/bin/sh", "-c",
+		}
+		p.Command = []string{
+			"echo $DRONE_SCRIPT | base64 -d | /bin/sh -e",
+		}
+		if p.Environment == nil {
+			p.Environment = map[string]string{}
+		}
+		p.Environment["HOME"] = "/root"
+		p.Environment["SHELL"] = "/bin/sh"
+		p.Environment["DRONE_SCRIPT"] = toScript(
+			p.Commands,
+		)
+	}
+	return nil
+}
+
+func toScript(commands []string) string {
+	var buf bytes.Buffer
+	for _, command := range commands {
+		escaped := fmt.Sprintf("%q", command)
+		escaped = strings.Replace(command, "$", `$\`, -1)
+		buf.WriteString(fmt.Sprintf(
+			traceScript,
+			escaped,
+			command,
+		))
+	}
+
+	script := fmt.Sprintf(
+		setupScript,
+		buf.String(),
+	)
+
+	return base64.StdEncoding.EncodeToString([]byte(script))
+}
+
+// setupScript is a helper script this is added to the build to ensure
+// a minimum set of environment variables are set correctly.
+const setupScript = `
+if [ -n "$DRONE_NETRC_MACHINE" ]; then
+cat <<EOF > $HOME/.netrc
+machine $DRONE_NETRC_MACHINE
+login $DRONE_NETRC_USERNAME
+password $DRONE_NETRC_PASSWORD
+EOF
+fi
+
+unset DRONE_NETRC_USERNAME
+unset DRONE_NETRC_PASSWORD
+unset DRONE_SCRIPT
+
+%s
+`
+
+// traceScript is a helper script that is added to the build script
+// to trace a command.
+const traceScript = `
+echo + %s
+%s
+`

yaml/transform/environ.go

@@ -0,0 +1,20 @@
+package transform
+
+import "github.com/drone/drone/yaml"
+
+// Environ transforms the steps in the Yaml pipeline to include runtime
+// environment variables.
+func Environ(c *yaml.Config, envs map[string]string) error {
+	for _, p := range c.Pipeline {
+		if p.Environment == nil {
+			p.Environment = map[string]string{}
+		}
+		for k, v := range envs {
+			if v == "" {
+				continue
+			}
+			p.Environment[k] = v
+		}
+	}
+	return nil
+}

yaml/transform/identifier.go

@@ -0,0 +1,30 @@
+package transform
+
+import (
+	"encoding/base64"
+	"fmt"
+
+	"github.com/drone/drone/yaml"
+
+	"github.com/gorilla/securecookie"
+)
+
+// Identifier transforms the container steps in the Yaml and assigns a unique
+// container identifier.
+func Identifier(c *yaml.Config) error {
+
+	// creates a random prefix for the build
+	rand := base64.RawURLEncoding.EncodeToString(
+		securecookie.GenerateRandomKey(8),
+	)
+
+	for i, step := range c.Services {
+		step.ID = fmt.Sprintf("drone_%s_%d", rand, i)
+	}
+
+	for i, step := range c.Pipeline {
+		step.ID = fmt.Sprintf("drone_%s_%d", rand, i+len(c.Services))
+	}
+
+	return nil
+}

yaml/transform/image.go

@@ -0,0 +1,63 @@
+package transform
+
+import (
+	"path/filepath"
+	"strings"
+
+	"github.com/drone/drone/yaml"
+)
+
+func ImagePull(conf *yaml.Config, pull bool) error {
+	for _, plugin := range conf.Pipeline {
+		if len(plugin.Commands) == 0 || len(plugin.Vargs) == 0 {
+			continue
+		}
+		plugin.Pull = pull
+	}
+	return nil
+}
+
+func ImageTag(conf *yaml.Config) error {
+	for _, image := range conf.Pipeline {
+		if !strings.Contains(image.Image, ":") {
+			image.Image = image.Image + ":latest"
+		}
+	}
+	for _, image := range conf.Services {
+		if !strings.Contains(image.Image, ":") {
+			image.Image = image.Image + ":latest"
+		}
+	}
+	return nil
+}
+
+func ImageName(conf *yaml.Config) error {
+	for _, image := range conf.Pipeline {
+		image.Image = strings.Replace(image.Image, "_", "-", -1)
+	}
+	return nil
+}
+
+func ImageNamespace(conf *yaml.Config, namespace string) error {
+	for _, image := range conf.Pipeline {
+		if strings.Contains(image.Image, "/") {
+			continue
+		}
+		if len(image.Vargs) == 0 {
+			continue
+		}
+		image.Image = filepath.Join(namespace, image.Image)
+	}
+	return nil
+}
+
+func ImageEscalate(conf *yaml.Config, patterns []string) error {
+	for _, c := range conf.Pipeline {
+		for _, pattern := range patterns {
+			if ok, _ := filepath.Match(pattern, c.Image); ok {
+				c.Privileged = true
+			}
+		}
+	}
+	return nil
+}

yaml/transform/plugin.go

@@ -0,0 +1,80 @@
+package transform
+
+import "github.com/drone/drone/yaml"
+
+// PluginDisable disables plugins. This is intended for use when executing the
+// pipeline locally on your own computer.
+func PluginDisable(conf *yaml.Config, disabled bool) {
+	for _, container := range conf.Pipeline {
+		if len(container.Vargs) != 0 || container.Name == "clone" {
+			container.Disabled = disabled
+		}
+	}
+}
+
+//
+// import (
+// 	"fmt"
+// 	"reflect"
+// 	"strconv"
+// 	"strings"
+//
+// 	"github.com/drone/drone/yaml"
+// 	"github.com/libcd/libyaml/parse"
+//
+// 	json "github.com/ghodss/yaml"
+// 	"gopkg.in/yaml.v2"
+// )
+//
+// func
+//
+// // argsToEnv uses reflection to convert a map[string]interface to a list
+// // of environment variables.
+// func argsToEnv(from map[string]interface{}, to map[string]string) error {
+//
+// 	for k, v := range from {
+// 		t := reflect.TypeOf(v)
+// 		vv := reflect.ValueOf(v)
+//
+// 		k = "PLUGIN_" + strings.ToUpper(k)
+//
+// 		switch t.Kind() {
+// 		case reflect.Bool:
+// 			to[k] = strconv.FormatBool(vv.Bool())
+//
+// 		case reflect.String:
+// 			to[k] = vv.String()
+//
+// 		case reflect.Int, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Int8:
+// 			to[k] = fmt.Sprintf("%v", vv.Int())
+//
+// 		case reflect.Float32, reflect.Float64:
+// 			to[k] = fmt.Sprintf("%v", vv.Float())
+//
+// 		case reflect.Map:
+// 			yml, _ := yaml.Marshal(vv.Interface())
+// 			out, _ := json.YAMLToJSON(yml)
+// 			to[k] = string(out)
+//
+// 		case reflect.Slice:
+// 			out, err := yaml.Marshal(vv.Interface())
+// 			if err != nil {
+// 				return err
+// 			}
+//
+// 			in := []string{}
+// 			err := yaml.Unmarshal(out, &in)
+// 			if err == nil {
+// 				to[k] = strings.Join(in, ",")
+// 			} else {
+// 				out, err = json.YAMLToJSON(out)
+// 				if err != nil {
+// 					return err
+// 				}
+// 				to[k] = string(out)
+// 			}
+// 		}
+// 	}
+//
+// 	return nil
+// }

yaml/transform/pod.go

@@ -0,0 +1,60 @@
+package transform
+
+import (
+	"encoding/base64"
+	"fmt"
+
+	"github.com/drone/drone/yaml"
+
+	"github.com/gorilla/securecookie"
+)
+
+// Pod transforms the containers in the Yaml to use Pod networking, where every
+// container shares the localhost connection.
+func Pod(c *yaml.Config) error {
+
+	rand := base64.RawURLEncoding.EncodeToString(
+		securecookie.GenerateRandomKey(8),
+	)
+
+	ambassador := &yaml.Container{
+		ID:         fmt.Sprintf("drone_ambassador_%s", rand),
+		Name:       "ambassador",
+		Image:      "busybox:latest",
+		Detached:   true,
+		Entrypoint: []string{"/bin/sleep"},
+		Command:    []string{"86400"},
+		Volumes:    []string{c.Workspace.Path, c.Workspace.Base},
+	}
+	network := fmt.Sprintf("container:%s", ambassador.ID)
+
+	var containers []*yaml.Container
+	containers = append(containers, c.Pipeline...)
+	containers = append(containers, c.Services...)
+
+	for _, container := range containers {
+		container.VolumesFrom = append(container.VolumesFrom, ambassador.ID)
+		if container.Network == "" {
+			container.Network = network
+		}
+	}
+
+	c.Services = append([]*yaml.Container{ambassador}, c.Services...)
+	return nil
+}
+
+// func (v *podOp) VisitContainer(node *parse.ContainerNode) error {
+// 	if node.Container.Network == "" {
+// 		parent := fmt.Sprintf("container:%s", v.name)
+// 		node.Container.Network = parent
+// 	}
+// 	node.Container.VolumesFrom = append(node.Container.VolumesFrom, v.name)
+// 	return nil
+// }
+//
+// func (v *podOp) VisitRoot(node *parse.RootNode) error {
+//
+//
+// 	node.Pod = service
+// 	return nil
+// }

yaml/transform/secret.go

@@ -0,0 +1,31 @@
+package transform
+
+import (
+	"github.com/drone/drone/model"
+	"github.com/drone/drone/yaml"
+)
+
+func Secret(c *yaml.Config, event string, secrets []*model.Secret) error {
+
+	for _, p := range c.Pipeline {
+		for _, secret := range secrets {
+
+			switch secret.Name {
+			case "REGISTRY_USERNAME":
+				p.AuthConfig.Username = secret.Value
+			case "REGISTRY_PASSWORD":
+				p.AuthConfig.Password = secret.Value
+			case "REGISTRY_EMAIL":
+				p.AuthConfig.Email = secret.Value
+			default:
+				if p.Environment == nil {
+					p.Environment = map[string]string{}
+				}
+				p.Environment[secret.Name] = secret.Value
+			}
+
+		}
+	}
+
+	return nil
+}

yaml/transform/transform.go

@@ -0,0 +1,6 @@
+package transform
+
+import "github.com/drone/drone/yaml"
+
+// TransformFunc defines an operation for transforming the Yaml file.
+type TransformFunc func(*yaml.Config) error

yaml/transform/validate.go

@@ -0,0 +1,74 @@
+package transform
+
+import (
+	"fmt"
+
+	"github.com/drone/drone/yaml"
+)
+
+func Check(c *yaml.Config, trusted bool) error {
+	var images []*yaml.Container
+	images = append(images, c.Pipeline...)
+	images = append(images, c.Services...)
+
+	for _, image := range images {
+		if err := CheckEntrypoint(image); err != nil {
+			return err
+		}
+		if trusted {
+			continue
+		}
+		if err := CheckTrusted(image); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// validate the plugin command and entrypoint and return an error
+// the user attempts to set or override these values.
+func CheckEntrypoint(c *yaml.Container) error {
+	if len(c.Vargs) == 0 {
+		return nil
+	}
+	if len(c.Entrypoint) != 0 {
+		return fmt.Errorf("Cannot set plugin Entrypoint")
+	}
+	if len(c.Command) != 0 {
+		return fmt.Errorf("Cannot set plugin Command")
+	}
+	return nil
+}
+
+// validate the container configuration and return an error if restricted
+// configurations are used.
+func CheckTrusted(c *yaml.Container) error {
+	if c.Privileged {
+		return fmt.Errorf("Insufficient privileges to use privileged mode")
+	}
+	if len(c.DNS) != 0 {
+		return fmt.Errorf("Insufficient privileges to use custom dns")
+	}
+	if len(c.DNSSearch) != 0 {
+		return fmt.Errorf("Insufficient privileges to use dns_search")
+	}
+	if len(c.Devices) != 0 {
+		return fmt.Errorf("Insufficient privileges to use devices")
+	}
+	if len(c.ExtraHosts) != 0 {
+		return fmt.Errorf("Insufficient privileges to use extra_hosts")
+	}
+	if len(c.Network) != 0 {
+		return fmt.Errorf("Insufficient privileges to override the network")
+	}
+	if c.OomKillDisable {
+		return fmt.Errorf("Insufficient privileges to disable oom_kill")
+	}
+	if len(c.Volumes) != 0 {
+		return fmt.Errorf("Insufficient privileges to use volumes")
+	}
+	if len(c.VolumesFrom) != 0 {
+		return fmt.Errorf("Insufficient privileges to use volumes_from")
+	}
+	return nil
+}

yaml/transform/volume.go

@@ -0,0 +1,20 @@
+package transform
+
+import "github.com/drone/drone/yaml"
+
+func ImageVolume(conf *yaml.Config, volumes []string) error {
+
+	if len(volumes) == 0 {
+		return nil
+	}
+
+	var containers []*yaml.Container
+	containers = append(containers, conf.Pipeline...)
+	containers = append(containers, conf.Services...)
+
+	for _, container := range containers {
+		container.Volumes = append(container.Volumes, volumes...)
+	}
+
+	return nil
+}

yaml/transform/workspace.go

@@ -0,0 +1,32 @@
+package transform
+
+import (
+	"path/filepath"
+
+	"github.com/drone/drone/yaml"
+)
+
+// WorkspaceTransform transforms ...
+func WorkspaceTransform(c *yaml.Config, base, path string) error {
+	if c.Workspace == nil {
+		c.Workspace = &yaml.Workspace{}
+	}
+
+	if c.Workspace.Base == "" {
+		c.Workspace.Base = base
+	}
+	if c.Workspace.Path == "" {
+		c.Workspace.Path = path
+	}
+	if !filepath.IsAbs(c.Workspace.Path) {
+		c.Workspace.Path = filepath.Join(
+			c.Workspace.Base,
+			c.Workspace.Path,
+		)
+	}
+
+	for _, p := range c.Pipeline {
+		p.WorkingDir = c.Workspace.Path
+	}
+	return nil
+}