Improve agent backend loading and suppress expectable errors (#818)

* improve agent loop loading backend once

* supress container not found or stopped warnings

cmd/agent/agent.go

@@ -132,28 +132,31 @@ func loop(c *cli.Context) error {
 	for i := 0; i < parallel; i++ {
 		go func() {
 			defer wg.Done()
+
+			// new engine
+			engine, err := backend.FindEngine(c.String("backend-engine"))
+			if err != nil {
+				log.Error().Err(err).Msgf("cannot find backend engine '%s'", c.String("backend-engine"))
+				return
+			}
+
+			// load engine (e.g. init api client)
+			err = engine.Load()
+			if err != nil {
+				log.Error().Err(err).Msg("cannot load backend engine")
+				return
+			}
+
+			r := agent.NewRunner(client, filter, hostname, counter, &engine)
+
+			log.Debug().Msgf("loaded %s backend engine", engine.Name())
+
 			for {
 				if sigterm.IsSet() {
 					return
 				}
 
-				// new engine
+				log.Debug().Msg("polling new jobs")
-				engine, err := backend.FindEngine(c.String("backend-engine"))
-				if err != nil {
-					log.Error().Err(err).Msgf("cannot find backend engine '%s'", c.String("backend-engine"))
-					return
-				}
-
-				// load engine (e.g. init api client)
-				err = engine.Load()
-				if err != nil {
-					log.Error().Err(err).Msg("cannot load backend engine")
-					return
-				}
-
-				log.Debug().Msgf("loaded %s backend engine", engine.Name())
-
-				r := agent.NewRunner(client, filter, hostname, counter, &engine)
 				if err := r.Run(ctx); err != nil {
 					log.Error().Err(err).Msg("pipeline done with error")
 					return

pipeline/backend/backend.go

@@ -10,16 +10,15 @@ import (
 var engines map[string]types.Engine
 
 func init() {
+	loadedEngines := []types.Engine{
+		docker.New(),
+		// kubernetes.New(), // TODO: disabled for now as kubernetes backend has not been implemented yet
+	}
+
 	engines = make(map[string]types.Engine)
-
+	for _, engine := range loadedEngines {
-	// TODO: disabled for now as kubernetes backend has not been implemented yet
+		engines[engine.Name()] = engine
-	// kubernetes
+	}
-	// engine = kubernetes.New("", "", "")
-	// engines[engine.Name()] = engine
-
-	// docker
-	engine := docker.New()
-	engines[engine.Name()] = engine
 }
 
 func FindEngine(engineName string) (types.Engine, error) {

pipeline/backend/docker/docker.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"io"
 	"os"
+	"strings"
 
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/network"
@@ -192,10 +193,10 @@ func (e *docker) Tail(ctx context.Context, proc *backend.Step) (io.ReadCloser, e
 func (e *docker) Destroy(_ context.Context, conf *backend.Config) error {
 	for _, stage := range conf.Stages {
 		for _, step := range stage.Steps {
-			if err := e.client.ContainerKill(noContext, step.Name, "9"); err != nil {
+			if err := e.client.ContainerKill(noContext, step.Name, "9"); err != nil && !isErrContainerNotFoundOrNotRunning(err) {
 				log.Error().Err(err).Msgf("could not kill container '%s'", stage.Name)
 			}
-			if err := e.client.ContainerRemove(noContext, step.Name, removeOpts); err != nil {
+			if err := e.client.ContainerRemove(noContext, step.Name, removeOpts); err != nil && !isErrContainerNotFoundOrNotRunning(err) {
 				log.Error().Err(err).Msgf("could not remove container '%s'", stage.Name)
 			}
 		}
@@ -232,3 +233,10 @@ var (
 		Timestamps: false,
 	}
 )
+
+func isErrContainerNotFoundOrNotRunning(err error) bool {
+	// Error response from daemon: Cannot kill container: ...: No such container: ...
+	// Error response from daemon: Cannot kill container: ...: Container ... is not running"
+	// Error: No such container: ...
+	return err != nil && (strings.Contains(err.Error(), "No such container") || strings.Contains(err.Error(), "is not running"))
+}