diff --git a/cmd/drone-build/Dockerfile b/cmd/drone-build/Dockerfile new file mode 100644 index 000000000..3715317f6 --- /dev/null +++ b/cmd/drone-build/Dockerfile @@ -0,0 +1,18 @@ +# Docker image for Drone's git-clone plugin +# +# docker build --rm=true -t drone/drone-build . + +FROM library/golang:1.4 + +# copy the local package files to the container's workspace. +#ADD . /go/src/github.com/drone/drone-build/ + +# build the program inside the container. +#RUN go get github.com/drone/drone-build/... && \ +# go install github.com/drone/drone-build + + +ADD drone-build /go/bin/ + +# run the git-clone plugin when the container starts +ENTRYPOINT ["/go/bin/drone-build"] diff --git a/cmd/drone-build/client.go b/cmd/drone-build/client.go new file mode 100644 index 000000000..0fde96179 --- /dev/null +++ b/cmd/drone-build/client.go @@ -0,0 +1,203 @@ +package main + +import ( + "errors" + "os" + + "github.com/samalba/dockerclient" +) + +var ( + ErrTimeout = errors.New("Timeout") + ErrLogging = errors.New("Logs not available") +) + +var ( + // options to fetch the stdout and stderr logs + logOpts = &dockerclient.LogOptions{ + Stdout: true, + Stderr: true, + } + + // options to fetch the stdout and stderr logs + // by tailing the output. + logOptsTail = &dockerclient.LogOptions{ + Follow: true, + Stdout: true, + Stderr: true, + } +) + +// client is a wrapper around the default Docker client +// that tracks all created containers ensures some default +// configurations are in place. +type client struct { + dockerclient.Client + info *dockerclient.ContainerInfo + names []string // names of created containers +} + +func newClient(docker dockerclient.Client) (*client, error) { + + // creates an ambassador container + conf := &dockerclient.ContainerConfig{} + conf.HostConfig = dockerclient.HostConfig{} + conf.Entrypoint = []string{"/bin/sleep"} + conf.Cmd = []string{"86400"} + conf.Image = "busybox" + conf.Volumes = map[string]struct{}{} + conf.Volumes["/drone"] = struct{}{} + info, err := daemon(docker, conf, false) + if err != nil { + return nil, err + } + + return &client{Client: docker, info: info}, nil +} + +// CreateContainer creates a container and internally +// caches its container id. +func (c *client) CreateContainer(conf *dockerclient.ContainerConfig, name string) (string, error) { + conf.Env = append(conf.Env, "affinity:container=="+c.info.Id) + id, err := c.Client.CreateContainer(conf, name) + if err == nil { + c.names = append(c.names, id) + } + return id, err +} + +// StartContainer starts a container and links to an +// ambassador container sharing the build machiens volume. +func (c *client) StartContainer(id string, conf *dockerclient.HostConfig) error { + conf.VolumesFrom = append(conf.VolumesFrom, c.info.Id) + if len(conf.NetworkMode) == 0 { + conf.NetworkMode = "container:" + c.info.Id + } + return c.Client.StartContainer(id, conf) +} + +// Destroy will terminate and destroy all containers that +// were created by this client. +func (c *client) Destroy() error { + for _, id := range c.names { + c.Client.KillContainer(id, "9") + c.Client.RemoveContainer(id, true, true) + } + c.Client.KillContainer(c.info.Id, "9") + return c.Client.RemoveContainer(c.info.Id, true, true) +} + +func run(client dockerclient.Client, conf *dockerclient.ContainerConfig, pull bool) (*dockerclient.ContainerInfo, error) { + // force-pull the image if specified. + // TEMPORARY while we are in beta mode we should always re-pull drone plugins + if pull { //|| strings.HasPrefix(conf.Image, "plugins/") { + client.PullImage(conf.Image, nil) + } + + // attempts to create the contianer + id, err := client.CreateContainer(conf, "") + if err != nil { + // and pull the image and re-create if that fails + client.PullImage(conf.Image, nil) + id, err = client.CreateContainer(conf, "") + // make sure the container is removed in + // the event of a creation error. + if len(id) != 0 { + client.RemoveContainer(id, true, true) + } + if err != nil { + return nil, err + } + } + + // ensures the container is always stopped + // and ready to be removed. + defer func() { + client.StopContainer(id, 5) + client.KillContainer(id, "9") + }() + + // fetches the container information. + info, err := client.InspectContainer(id) + if err != nil { + client.RemoveContainer(id, true, true) + return nil, err + } + + // channel listening for errors while the + // container is running async. + errc := make(chan error, 1) + infoc := make(chan *dockerclient.ContainerInfo, 1) + go func() { + + // starts the container + err := client.StartContainer(id, &conf.HostConfig) + if err != nil { + errc <- err + return + } + + // blocks and waits for the container to finish + // by streaming the logs (to /dev/null). Ideally + // we could use the `wait` function instead + rc, err := client.ContainerLogs(id, logOptsTail) + if err != nil { + errc <- err + return + } + defer rc.Close() + StdCopy(os.Stdout, os.Stdout, rc) + + // fetches the container information + info, err := client.InspectContainer(id) + if err != nil { + errc <- err + return + } + infoc <- info + }() + + select { + case info := <-infoc: + return info, nil + case err := <-errc: + return info, err + // TODO checkout net.Context and cancel + // case <-time.After(timeout): + // return info, ErrTimeout + } +} + +func daemon(client dockerclient.Client, conf *dockerclient.ContainerConfig, pull bool) (*dockerclient.ContainerInfo, error) { + // force-pull the image + if pull { + client.PullImage(conf.Image, nil) + } + + // attempts to create the contianer + id, err := client.CreateContainer(conf, "") + if err != nil { + // and pull the image and re-create if that fails + client.PullImage(conf.Image, nil) + id, err = client.CreateContainer(conf, "") + // make sure the container is removed in + // the event of a creation error. + if len(id) != 0 { + client.RemoveContainer(id, true, true) + } + if err != nil { + return nil, err + } + } + + // fetches the container information + info, err := client.InspectContainer(id) + if err != nil { + client.RemoveContainer(id, true, true) + return nil, err + } + + // starts the container + err = client.StartContainer(id, &conf.HostConfig) + return info, err +} diff --git a/cmd/drone-build/copy.go b/cmd/drone-build/copy.go new file mode 100644 index 000000000..7764f582c --- /dev/null +++ b/cmd/drone-build/copy.go @@ -0,0 +1,124 @@ +package main + +import ( + "encoding/binary" + "errors" + "io" +) + +const ( + StdWriterPrefixLen = 8 + StdWriterFdIndex = 0 + StdWriterSizeIndex = 4 +) + +type StdType [StdWriterPrefixLen]byte + +var ( + Stdin StdType = StdType{0: 0} + Stdout StdType = StdType{0: 1} + Stderr StdType = StdType{0: 2} +) + +type StdWriter struct { + io.Writer + prefix StdType + sizeBuf []byte +} + +var ErrInvalidStdHeader = errors.New("Unrecognized input header") + +// StdCopy is a modified version of io.Copy. +// +// StdCopy will demultiplex `src`, assuming that it contains two streams, +// previously multiplexed together using a StdWriter instance. +// As it reads from `src`, StdCopy will write to `dstout` and `dsterr`. +// +// StdCopy will read until it hits EOF on `src`. It will then return a nil error. +// In other words: if `err` is non nil, it indicates a real underlying error. +// +// `written` will hold the total number of bytes written to `dstout` and `dsterr`. +func StdCopy(dstout, dsterr io.Writer, src io.Reader) (written int64, err error) { + var ( + buf = make([]byte, 32*1024+StdWriterPrefixLen+1) + bufLen = len(buf) + nr, nw int + er, ew error + out io.Writer + frameSize int + ) + + for { + // Make sure we have at least a full header + for nr < StdWriterPrefixLen { + var nr2 int + nr2, er = src.Read(buf[nr:]) + nr += nr2 + if er == io.EOF { + if nr < StdWriterPrefixLen { + return written, nil + } + break + } + if er != nil { + return 0, er + } + } + + // Check the first byte to know where to write + switch buf[StdWriterFdIndex] { + case 0: + fallthrough + case 1: + // Write on stdout + out = dstout + case 2: + // Write on stderr + out = dsterr + default: + return 0, ErrInvalidStdHeader + } + + // Retrieve the size of the frame + frameSize = int(binary.BigEndian.Uint32(buf[StdWriterSizeIndex : StdWriterSizeIndex+4])) + + // Check if the buffer is big enough to read the frame. + // Extend it if necessary. + if frameSize+StdWriterPrefixLen > bufLen { + buf = append(buf, make([]byte, frameSize+StdWriterPrefixLen-bufLen+1)...) + bufLen = len(buf) + } + + // While the amount of bytes read is less than the size of the frame + header, we keep reading + for nr < frameSize+StdWriterPrefixLen { + var nr2 int + nr2, er = src.Read(buf[nr:]) + nr += nr2 + if er == io.EOF { + if nr < frameSize+StdWriterPrefixLen { + return written, nil + } + break + } + if er != nil { + return 0, er + } + } + + // Write the retrieved frame (without header) + nw, ew = out.Write(buf[StdWriterPrefixLen : frameSize+StdWriterPrefixLen]) + if ew != nil { + return 0, ew + } + // If the frame has not been fully written: error + if nw != frameSize { + return 0, io.ErrShortWrite + } + written += int64(nw) + + // Move the rest of the buffer to the beginning + copy(buf, buf[frameSize+StdWriterPrefixLen:]) + // Move the index + nr -= frameSize + StdWriterPrefixLen + } +} diff --git a/cmd/drone-build/main.go b/cmd/drone-build/main.go new file mode 100644 index 000000000..7368ef125 --- /dev/null +++ b/cmd/drone-build/main.go @@ -0,0 +1,156 @@ +package main + +import ( + "bytes" + "encoding/json" + "flag" + "fmt" + "net/url" + "os" + "path/filepath" + "strings" + + "github.com/drone/drone/common" + "github.com/samalba/dockerclient" +) + +var ( + clone = flag.Bool("clone", false, "") + build = flag.Bool("build", false, "") + publish = flag.Bool("publish", false, "") + deploy = flag.Bool("deploy", false, "") + notify = flag.Bool("notify", false, "") +) + +func main() { + flag.Parse() + + ctx, err := parseContext() + if err != nil { + fmt.Println("Error launching build container.", err) + os.Exit(1) + return + } + createClone(ctx) + + // creates the Docker client, connecting to the + // linked Docker daemon + docker, err := dockerclient.NewDockerClient("unix:///var/run/docker.sock", nil) + if err != nil { + fmt.Println("Error connecting to build server.", err) + os.Exit(1) + return + } + + // creates a wrapper Docker client that uses an ambassador + // container to create a pod-like environment. + client, err := newClient(docker) + if err != nil { + fmt.Println("Error starting build server pod", err) + os.Exit(1) + } + ctx.client = client + defer client.Destroy() + + // performs some initial parsing and pre-processing steps + // prior to executing our build tasks. + err = setup(ctx) + if err != nil { + fmt.Println("Error processing .drone.yml file.", err) + client.Destroy() + os.Exit(1) + } + + var execs []execFunc + if *clone { + execs = append(execs, execClone) + } + if *build { + execs = append(execs, execSetup) + execs = append(execs, execCompose) + execs = append(execs, execBuild) + } + if *publish { + execs = append(execs, execPublish) + } + if *deploy { + execs = append(execs, execDeploy) + } + + // Loop through and execute each step. + for i, exec_ := range execs { + code, err := exec_(ctx) + if err != nil { + fmt.Printf("00%d Error executing build\n", i+1) + fmt.Println(err) + code = 255 + } + if code != 0 { + ctx.Build.ExitCode = code + break + } + } + + // Optionally execute notification steps. + if *notify { + execNotify(ctx) + } + + client.Destroy() + os.Exit(ctx.Build.ExitCode) +} + +func createClone(c *Context) error { + c.Clone = &common.Clone{ + Netrc: c.Netrc, + Keypair: c.Keys, + Remote: c.Repo.Clone, + Origin: c.Repo.Clone, + } + + c.Clone.Origin = c.Repo.Clone + c.Clone.Remote = c.Repo.Clone + c.Clone.Sha = c.Commit.Sha + c.Clone.Ref = c.Commit.Ref + c.Clone.Branch = c.Commit.Branch + // TODO move this to the main app (github package) + if strings.HasPrefix(c.Clone.Branch, "refs/heads/") { + c.Clone.Branch = c.Clone.Branch[11:] + } + + // TODO we should also pass the SourceSha, SourceBranch, etc + // to the clone object for merge requests from bitbucket, gitlab, et al + // if len(c.Commit.PullRequest) != 0 { + // } + + url_, err := url.Parse(c.Repo.Link) + if err != nil { + return err + } + c.Clone.Dir = filepath.Join("/drone/src", url_.Host, c.Repo.FullName) + + // attempt to extract the clone path. i'm not a huge fan of + // this, by the way, but for now we'll keep it. + // TODO consider moving this to a transform? + pathv, ok := c.Conf.Clone.Config["path"] + if ok { + path, ok := pathv.(string) + if ok { + c.Clone.Dir = filepath.Join("/drone/src", path) + } + } + return nil +} + +func parseContext() (*Context, error) { + c := &Context{} + for i, arg := range os.Args { + if arg == "--" && len(os.Args) != i+1 { + buf := bytes.NewBufferString(os.Args[i+1]) + err := json.NewDecoder(buf).Decode(c) + return c, err + } + } + err := json.NewDecoder(os.Stdin).Decode(c) + return c, err +} diff --git a/cmd/drone-build/run.go b/cmd/drone-build/run.go new file mode 100644 index 000000000..868773d09 --- /dev/null +++ b/cmd/drone-build/run.go @@ -0,0 +1,142 @@ +package main + +import ( + "encoding/base64" + "fmt" + + "github.com/drone/drone/common" + "github.com/drone/drone/parser" + "github.com/drone/drone/parser/inject" + "github.com/samalba/dockerclient" +) + +type Context struct { + // Links *common.Link + Clone *common.Clone `json:"clone"` + Repo *common.Repo `json:"repo"` + Commit *common.Commit `json:"commit"` + Build *common.Build `json:"build"` + Keys *common.Keypair `json:"keys"` + Netrc *common.Netrc `json:"netrc"` + Yaml []byte `json:"yaml"` + Conf *common.Config `json:"-"` + infos []*dockerclient.ContainerInfo + client dockerclient.Client +} + +func setup(c *Context) error { + var err error + + // inject the matrix parameters into the yaml + injected := inject.Inject(string(c.Yaml), c.Build.Environment) + c.Conf, err = parser.ParseSingle(injected, parser.DefaultOpts) + if err != nil { + return err + } + + // and append the matrix parameters as environment + // variables for the build + for k, v := range c.Build.Environment { + env := k + "=" + v + c.Conf.Build.Environment = append(c.Conf.Build.Environment, env) + } + + // and append drone, jenkins, travis and other + // environment variables that may be of use. + for k, v := range toEnv(c) { + env := k + "=" + v + c.Conf.Build.Environment = append(c.Conf.Build.Environment, env) + } + + return nil +} + +type execFunc func(c *Context) (int, error) + +func execClone(c *Context) (int, error) { + conf := toContainerConfig(c.Conf.Clone) + conf.Cmd = toCommand(c, c.Conf.Clone) + info, err := run(c.client, conf, c.Conf.Clone.Pull) + if err != nil { + return 255, err + } + return info.State.ExitCode, nil +} + +func execBuild(c *Context) (int, error) { + conf := toContainerConfig(c.Conf.Build) + conf.Entrypoint = []string{"/bin/bash", "-e"} + conf.Cmd = []string{"/drone/bin/build.sh"} + info, err := run(c.client, conf, c.Conf.Build.Pull) + if err != nil { + return 255, err + } + return info.State.ExitCode, nil +} + +func execSetup(c *Context) (int, error) { + conf := toContainerConfig(c.Conf.Setup) + conf.Cmd = toCommand(c, c.Conf.Setup) + info, err := run(c.client, conf, c.Conf.Setup.Pull) + if err != nil { + return 255, err + } + return info.State.ExitCode, nil +} + +func execDeploy(c *Context) (int, error) { + return runSteps(c, c.Conf.Deploy) +} + +func execPublish(c *Context) (int, error) { + return runSteps(c, c.Conf.Publish) +} + +func execNotify(c *Context) (int, error) { + return runSteps(c, c.Conf.Notify) +} + +func execCompose(c *Context) (int, error) { + for _, step := range c.Conf.Compose { + conf := toContainerConfig(step) + _, err := daemon(c.client, conf, step.Pull) + if err != nil { + return 0, err + } + } + return 0, nil +} + +func trace(s string) string { + cmd := fmt.Sprintf("$ %s\n", s) + encoded := base64.StdEncoding.EncodeToString([]byte(cmd)) + return fmt.Sprintf("echo %s | base64 --decode\n", encoded) +} + +func newline(s string) string { + return fmt.Sprintf("%s\n", s) +} + +func runSteps(c *Context, steps map[string]*common.Step) (int, error) { + for _, step := range steps { + + // verify the step matches the branch + // and other specifications + if step.Condition == nil || + !step.Condition.MatchOwner(c.Repo.Owner) || + !step.Condition.MatchBranch(c.Clone.Branch) || + !step.Condition.MatchMatrix(c.Build.Environment) { + continue + } + + conf := toContainerConfig(step) + conf.Cmd = toCommand(c, step) + info, err := run(c.client, conf, step.Pull) + if err != nil { + return 255, err + } else if info.State.ExitCode != 0 { + return info.State.ExitCode, nil + } + } + return 0, nil +} diff --git a/cmd/drone-build/util.go b/cmd/drone-build/util.go new file mode 100644 index 000000000..459427959 --- /dev/null +++ b/cmd/drone-build/util.go @@ -0,0 +1,92 @@ +package main + +import ( + "encoding/json" + "strconv" + "strings" + + "github.com/drone/drone/common" + "github.com/samalba/dockerclient" +) + +// helper function that converts the build step to +// a containerConfig for use with the dockerclient +func toContainerConfig(step *common.Step) *dockerclient.ContainerConfig { + config := &dockerclient.ContainerConfig{ + Image: step.Image, + Env: step.Environment, + Cmd: step.Command, + Entrypoint: step.Entrypoint, + WorkingDir: step.WorkingDir, + HostConfig: dockerclient.HostConfig{ + Privileged: step.Privileged, + NetworkMode: step.NetworkMode, + }, + } + + config.Volumes = map[string]struct{}{} + for _, path := range step.Volumes { + if strings.Index(path, ":") == -1 { + continue + } + parts := strings.Split(path, ":") + config.Volumes[parts[1]] = struct{}{} + config.HostConfig.Binds = append(config.HostConfig.Binds, path) + } + + return config +} + +// helper function to inject drone-specific environment +// variables into the container. +func toEnv(c *Context) map[string]string { + return map[string]string{ + "CI": "true", + "BUILD_DIR": c.Clone.Dir, + "BUILD_ID": strconv.Itoa(c.Commit.Sequence), + "BUILD_NUMBER": strconv.Itoa(c.Commit.Sequence), + "JOB_NAME": c.Repo.FullName, + "WORKSPACE": c.Clone.Dir, + "GIT_BRANCH": c.Clone.Branch, + "GIT_COMMIT": c.Clone.Sha, + + "DRONE": "true", + "DRONE_REPO": c.Repo.FullName, + "DRONE_BUILD": strconv.Itoa(c.Commit.Sequence), + "DRONE_BRANCH": c.Clone.Branch, + "DRONE_COMMIT": c.Clone.Sha, + "DRONE_DIR": c.Clone.Dir, + } +} + +// helper function to encode the build step to +// a json string. Primarily used for plugins, which +// expect a json encoded string in stdin or arg[1]. +func toCommand(c *Context, step *common.Step) []string { + p := payload{ + c.Repo, + c.Commit, + c.Build, + c.Clone, + step.Config, + } + return []string{p.Encode()} +} + +// payload represents the payload of a plugin +// that is serialized and sent to the plugin in JSON +// format via stdin or arg[1]. +type payload struct { + Repo *common.Repo `json:"repo"` + Commit *common.Commit `json:"commit"` + Build *common.Build `json:"build"` + Clone *common.Clone `json:"clone"` + + Config map[string]interface{} `json:"vargs"` +} + +// Encode encodes the payload in JSON format. +func (p *payload) Encode() string { + out, _ := json.Marshal(p) + return string(out) +}