Let the backend engine report the current platform (#2688)

if you run woodpecker-agent on windows and connect it to an docker
daemon, there could be two different platforms possible, as you can
switch from linux to windows mode and visa versa


---
*Sponsored by Kithara Software GmbH*

cli/exec/exec.go

@@ -216,7 +216,7 @@ func execWithAxis(c *cli.Context, file, repoPath string, axis matrix.Axis) error
 		return err
 	}
 
-	if err = engine.Load(backendCtx); err != nil {
+	if _, err = engine.Load(backendCtx); err != nil {
 		return err
 	}
 

cmd/agent/agent.go

@@ -22,7 +22,6 @@ import (
 	"fmt"
 	"net/http"
 	"os"
-	"runtime"
 	"strings"
 	"sync"
 	"time"
@@ -57,8 +56,6 @@ func run(c *cli.Context) error {
 		hostname, _ = os.Hostname()
 	}
 
-	platform := runtime.GOOS + "/" + runtime.GOARCH
-
 	counter.Polling = c.Int("max-workflows")
 	counter.Running = 0
 
@@ -155,7 +152,15 @@ func run(c *cli.Context) error {
 		return err
 	}
 
-	agentConfig.AgentID, err = client.RegisterAgent(ctx, platform, engine.Name(), version.String(), parallel)
+	// load engine (e.g. init api client)
+	engInfo, err := engine.Load(backendCtx)
+	if err != nil {
+		log.Error().Err(err).Msg("cannot load backend engine")
+		return err
+	}
+	log.Debug().Msgf("loaded %s backend engine", engine.Name())
+
+	agentConfig.AgentID, err = client.RegisterAgent(ctx, engInfo.Platform, engine.Name(), version.String(), parallel)
 	if err != nil {
 		return err
 	}
@@ -164,7 +169,7 @@ func run(c *cli.Context) error {
 
 	labels := map[string]string{
 		"hostname": hostname,
-		"platform": platform,
+		"platform": engInfo.Platform,
 		"backend":  engine.Name(),
 		"repo":     "*", // allow all repos by default
 	}
@@ -195,13 +200,6 @@ func run(c *cli.Context) error {
 		}
 	}()
 
-	// load engine (e.g. init api client)
-	if err := engine.Load(backendCtx); err != nil {
-		log.Error().Err(err).Msg("cannot load backend engine")
-		return err
-	}
-	log.Debug().Msgf("loaded %s backend engine", engine.Name())
-
 	for i := 0; i < parallel; i++ {
 		i := i
 		go func() {
@@ -226,7 +224,7 @@ func run(c *cli.Context) error {
 
 	log.Info().Msgf(
 		"Starting Woodpecker agent with version '%s' and backend '%s' using platform '%s' running up to %d pipelines in parallel",
-		version.String(), engine.Name(), platform, parallel)
+		version.String(), engine.Name(), engInfo.Platform, parallel)
 
 	wg.Wait()
 	return nil

pipeline/backend/common/script.go

@@ -16,12 +16,11 @@ package common
 
 import (
 	"encoding/base64"
-	"runtime"
 )
 
-func GenerateContainerConf(commands []string) (env map[string]string, entry, cmd []string) {
+func GenerateContainerConf(commands []string, goos string) (env map[string]string, entry, cmd []string) {
 	env = make(map[string]string)
-	if runtime.GOOS == "windows" {
+	if goos == "windows" {
 		env["CI_SCRIPT"] = base64.StdEncoding.EncodeToString([]byte(generateScriptWindows(commands)))
 		env["HOME"] = "c:\\root"
 		env["SHELL"] = "powershell.exe"

pipeline/backend/common/script_win_test.go

@@ -0,0 +1,58 @@
+// Copyright 2022 Woodpecker Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package common
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGenerateScriptWin(t *testing.T) {
+	testdata := []struct {
+		from []string
+		want string
+	}{
+		{
+			from: []string{"echo %PATH%", "go build", "go test"},
+			want: `
+$ErrorActionPreference = 'Stop';
+&cmd /c "mkdir c:\root";
+if ($Env:CI_NETRC_MACHINE) {
+$netrc=[string]::Format("{0}\_netrc",$Env:HOME);
+"machine $Env:CI_NETRC_MACHINE" >> $netrc;
+"login $Env:CI_NETRC_USERNAME" >> $netrc;
+"password $Env:CI_NETRC_PASSWORD" >> $netrc;
+};
+[Environment]::SetEnvironmentVariable("CI_NETRC_PASSWORD",$null);
+[Environment]::SetEnvironmentVariable("CI_SCRIPT",$null);
+
+Write-Output ('+ "echo %PATH%"');
+& echo %PATH%; if ($LASTEXITCODE -ne 0) {exit $LASTEXITCODE}
+
+Write-Output ('+ "go build"');
+& go build; if ($LASTEXITCODE -ne 0) {exit $LASTEXITCODE}
+
+Write-Output ('+ "go test"');
+& go test; if ($LASTEXITCODE -ne 0) {exit $LASTEXITCODE}
+
+`,
+		},
+	}
+	for _, test := range testdata {
+		script := generateScriptWindows(test.from)
+		assert.EqualValues(t, test.want, script, "Want encoded script for %s", test.from)
+	}
+}

pipeline/backend/docker/convert.go

@@ -27,7 +27,7 @@ import (
 )
 
 // returns a container configuration.
-func toConfig(step *types.Step) *container.Config {
+func (e *docker) toConfig(step *types.Step) *container.Config {
 	config := &container.Config{
 		Image:        step.Image,
 		Labels:       map[string]string{"wp_uuid": step.UUID},
@@ -37,7 +37,7 @@ func toConfig(step *types.Step) *container.Config {
 	}
 
 	if len(step.Commands) != 0 {
-		env, entry, cmd := common.GenerateContainerConf(step.Commands)
+		env, entry, cmd := common.GenerateContainerConf(step.Commands, e.info.OSType)
 		for k, v := range env {
 			step.Environment[k] = v
 		}
@@ -76,9 +76,6 @@ func toHostConfig(step *types.Step) *container.HostConfig {
 		Sysctls:    step.Sysctls,
 	}
 
-	// if len(step.VolumesFrom) != 0 {
-	// 	config.VolumesFrom = step.VolumesFrom
-	// }
 	if len(step.NetworkMode) != 0 {
 		config.NetworkMode = container.NetworkMode(step.NetworkMode)
 	}

pipeline/backend/docker/docker.go

@@ -20,7 +20,6 @@ import (
 	"net/http"
 	"os"
 	"path/filepath"
-	"runtime"
 	"strings"
 
 	"github.com/docker/docker/api/types"
@@ -43,6 +42,7 @@ type docker struct {
 	enableIPv6 bool
 	network    string
 	volumes    []string
+	info       types.Info
 }
 
 const (
@@ -94,10 +94,10 @@ func httpClientOfOpts(dockerCertPath string, verifyTLS bool) *http.Client {
 }
 
 // Load new client for Docker Engine using environment variables.
-func (e *docker) Load(ctx context.Context) error {
+func (e *docker) Load(ctx context.Context) (*backend.EngineInfo, error) {
 	c, ok := ctx.Value(backend.CliContext).(*cli.Context)
 	if !ok {
-		return backend.ErrNoCliContextFound
+		return nil, backend.ErrNoCliContextFound
 	}
 
 	var dockerClientOpts []client.Opt
@@ -115,10 +115,15 @@ func (e *docker) Load(ctx context.Context) error {
 
 	cl, err := client.NewClientWithOpts(dockerClientOpts...)
 	if err != nil {
-		return err
+		return nil, err
 	}
 	e.client = cl
 
+	e.info, err = cl.Info(ctx)
+	if err != nil {
+		return nil, err
+	}
+
 	e.enableIPv6 = c.Bool("backend-docker-ipv6")
 	e.network = c.String("backend-docker-network")
 
@@ -137,7 +142,9 @@ func (e *docker) Load(ctx context.Context) error {
 		e.volumes = append(e.volumes, strings.Join(parts, ":"))
 	}
 
-	return nil
+	return &backend.EngineInfo{
+		Platform: e.info.OSType + "/" + normalizeArchType(e.info.Architecture),
+	}, nil
 }
 
 func (e *docker) SetupWorkflow(_ context.Context, conf *backend.Config, taskUUID string) error {
@@ -154,7 +161,7 @@ func (e *docker) SetupWorkflow(_ context.Context, conf *backend.Config, taskUUID
 	}
 
 	networkDriver := networkDriverBridge
-	if runtime.GOOS == "windows" {
+	if e.info.OSType == "windows" {
 		networkDriver = networkDriverNAT
 	}
 	for _, n := range conf.Networks {
@@ -172,7 +179,7 @@ func (e *docker) SetupWorkflow(_ context.Context, conf *backend.Config, taskUUID
 func (e *docker) StartStep(ctx context.Context, step *backend.Step, taskUUID string) error {
 	log.Trace().Str("taskUUID", taskUUID).Msgf("start step %s", step.Name)
 
-	config := toConfig(step)
+	config := e.toConfig(step)
 	hostConfig := toHostConfig(step)
 	containerName := toContainerName(step)
 
@@ -262,9 +269,6 @@ func (e *docker) WaitStep(ctx context.Context, step *backend.Step, taskUUID stri
 	if err != nil {
 		return nil, err
 	}
-	// if info.State.Running {
-	// TODO
-	// }
 
 	return &backend.State{
 		Exited:    true,
@@ -360,3 +364,15 @@ func isErrContainerNotFoundOrNotRunning(err error) bool {
 	// Error: No such container: ...
 	return err != nil && (strings.Contains(err.Error(), "No such container") || strings.Contains(err.Error(), "is not running"))
 }
+
+// normalizeArchType converts the arch type reported by docker info into
+// the runtime.GOARCH format
+// TODO: find out if we we need to convert other arch types too
+func normalizeArchType(s string) string {
+	switch s {
+	case "x86_64":
+		return "amd64"
+	default:
+		return s
+	}
+}

pipeline/backend/kubernetes/kubernetes.go

@@ -19,6 +19,7 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"runtime"
 	"strings"
 	"time"
 
@@ -46,6 +47,7 @@ type kube struct {
 	ctx    context.Context
 	client kubernetes.Interface
 	config *Config
+	goos   string
 }
 
 type Config struct {
@@ -104,10 +106,10 @@ func (e *kube) IsAvailable(context.Context) bool {
 	return len(host) > 0
 }
 
-func (e *kube) Load(context.Context) error {
+func (e *kube) Load(context.Context) (*types.EngineInfo, error) {
 	config, err := configFromCliContext(e.ctx)
 	if err != nil {
-		return err
+		return nil, err
 	}
 	e.config = config
 
@@ -120,12 +122,16 @@ func (e *kube) Load(context.Context) error {
 	}
 
 	if err != nil {
-		return err
+		return nil, err
 	}
 
 	e.client = kubeClient
 
-	return nil
+	// TODO(2693): use info resp of kubeClient to define platform var
+	e.goos = runtime.GOOS
+	return &types.EngineInfo{
+		Platform: runtime.GOOS + "/" + runtime.GOARCH,
+	}, nil
 }
 
 // Setup the pipeline environment.
@@ -183,7 +189,7 @@ func (e *kube) SetupWorkflow(ctx context.Context, conf *types.Config, taskUUID s
 
 // Start the pipeline step.
 func (e *kube) StartStep(ctx context.Context, step *types.Step, taskUUID string) error {
-	pod, err := Pod(e.config.Namespace, step, e.config.PodLabels, e.config.PodAnnotations)
+	pod, err := Pod(e.config.Namespace, step, e.config.PodLabels, e.config.PodAnnotations, e.goos)
 	if err != nil {
 		return err
 	}

pipeline/backend/kubernetes/pod.go

@@ -28,7 +28,7 @@ import (
 	"github.com/woodpecker-ci/woodpecker/pipeline/backend/types"
 )
 
-func Pod(namespace string, step *types.Step, labels, annotations map[string]string) (*v1.Pod, error) {
+func Pod(namespace string, step *types.Step, labels, annotations map[string]string, goos string) (*v1.Pod, error) {
 	var (
 		vols       []v1.Volume
 		volMounts  []v1.VolumeMount
@@ -66,7 +66,7 @@ func Pod(namespace string, step *types.Step, labels, annotations map[string]stri
 	}
 
 	if len(step.Commands) != 0 {
-		scriptEnv, entry, cmds := common.GenerateContainerConf(step.Commands)
+		scriptEnv, entry, cmds := common.GenerateContainerConf(step.Commands, goos)
 		for k, v := range scriptEnv {
 			step.Environment[k] = v
 		}

pipeline/backend/local/local.go

@@ -60,10 +60,12 @@ func (e *local) IsAvailable(context.Context) bool {
 	return true
 }
 
-func (e *local) Load(context.Context) error {
+func (e *local) Load(context.Context) (*types.EngineInfo, error) {
 	e.loadClone()
 
-	return nil
+	return &types.EngineInfo{
+		Platform: runtime.GOOS + "/" + runtime.GOARCH,
+	}, nil
 }
 
 // SetupWorkflow the pipeline environment.

pipeline/backend/types/engine.go

@@ -29,7 +29,7 @@ type Engine interface {
 	IsAvailable(ctx context.Context) bool
 
 	// Load loads the backend engine.
-	Load(ctx context.Context) error
+	Load(ctx context.Context) (*EngineInfo, error)
 
 	// SetupWorkflow sets up the workflow environment.
 	SetupWorkflow(ctx context.Context, conf *Config, taskUUID string) error
@@ -50,3 +50,8 @@ type Engine interface {
 	// DestroyWorkflow destroys the workflow environment.
 	DestroyWorkflow(ctx context.Context, conf *Config, taskUUID string) error
 }
+
+// EngineInfo represents the reported information of a loaded engine
+type EngineInfo struct {
+	Platform string
+}