diff --git a/engine/engine.go b/engine/engine.go index 29552f34a..f3d0c0c22 100644 --- a/engine/engine.go +++ b/engine/engine.go @@ -329,6 +329,11 @@ func (e *engine) runJob(c context.Context, r *Task, updater *updater, client doc info, builderr := docker.Wait(client, name) switch { + case info.State.Running: + // A build unblocked before actually being completed. + log.Errorf("incomplete build: %s", name) + r.Job.ExitCode = 1 + r.Job.Status = model.StatusError case info.State.ExitCode == 128: r.Job.ExitCode = info.State.ExitCode r.Job.Status = model.StatusKilled diff --git a/shared/docker/docker.go b/shared/docker/docker.go index 29ed10613..e08ec95e7 100644 --- a/shared/docker/docker.go +++ b/shared/docker/docker.go @@ -1,9 +1,9 @@ package docker import ( - "io" - "io/ioutil" + "errors" + log "github.com/Sirupsen/logrus" "github.com/samalba/dockerclient" ) @@ -77,33 +77,21 @@ func Wait(client dockerclient.Client, name string) (*dockerclient.ContainerInfo, client.KillContainer(name, "9") }() - errc := make(chan error, 1) - infoc := make(chan *dockerclient.ContainerInfo, 1) - go func() { - - // blocks and waits for the container to finish - // by streaming the logs (to /dev/null). Ideally - // we could use the `wait` function instead - rc, err := client.ContainerLogs(name, LogOptsTail) - if err != nil { - errc <- err - return - } - io.Copy(ioutil.Discard, rc) - rc.Close() + for attempts := 0; attempts < 5; attempts++ { + done := client.Wait(name) + <-done info, err := client.InspectContainer(name) if err != nil { - errc <- err - return + return nil, err } - infoc <- info - }() - select { - case info := <-infoc: - return info, nil - case err := <-errc: - return nil, err + if !info.State.Running { + return info, nil + } + + log.Debugf("attempting to resume waiting after %d attempts.\n", attempts) } + + return nil, errors.New("reached maximum wait attempts") }