From f829c07f3a89cebcf7b97fcdd8fd3b111ac3e2bb Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Tue, 26 Nov 2024 11:48:48 +0100 Subject: [PATCH] Docker Backend: fully support windows container (#4381) --- pipeline/backend/common/script.go | 4 +- pipeline/backend/docker/convert.go | 2 + pipeline/backend/docker/convert_test.go | 87 ++++++++++++++++++++- pipeline/backend/docker/convert_win.go | 82 +++++++++++++++++++ pipeline/backend/docker/convert_win_test.go | 44 +++++++++++ 5 files changed, 215 insertions(+), 4 deletions(-) create mode 100644 pipeline/backend/docker/convert_win.go create mode 100644 pipeline/backend/docker/convert_win_test.go diff --git a/pipeline/backend/common/script.go b/pipeline/backend/common/script.go index c46f78367..e7c4a2b98 100644 --- a/pipeline/backend/common/script.go +++ b/pipeline/backend/common/script.go @@ -18,9 +18,9 @@ import ( "encoding/base64" ) -func GenerateContainerConf(commands []string, goos, workDir string) (env map[string]string, entry []string) { +func GenerateContainerConf(commands []string, osType, workDir string) (env map[string]string, entry []string) { env = make(map[string]string) - if goos == "windows" { + if osType == "windows" { env["CI_SCRIPT"] = base64.StdEncoding.EncodeToString([]byte(generateScriptWindows(commands, workDir))) env["SHELL"] = "powershell.exe" // cspell:disable-next-line diff --git a/pipeline/backend/docker/convert.go b/pipeline/backend/docker/convert.go index eadca2800..9b4799da5 100644 --- a/pipeline/backend/docker/convert.go +++ b/pipeline/backend/docker/convert.go @@ -32,6 +32,8 @@ const minVolumeComponents = 2 // returns a container configuration. func (e *docker) toConfig(step *types.Step) *container.Config { + e.windowsPathPatch(step) + config := &container.Config{ Image: step.Image, Labels: map[string]string{ diff --git a/pipeline/backend/docker/convert_test.go b/pipeline/backend/docker/convert_test.go index c307f6959..f5ec5d970 100644 --- a/pipeline/backend/docker/convert_test.go +++ b/pipeline/backend/docker/convert_test.go @@ -15,8 +15,10 @@ package docker import ( + "encoding/base64" "reflect" "sort" + "strings" "testing" "github.com/docker/docker/api/types/container" @@ -166,7 +168,7 @@ func TestEncodeAuthToBase64(t *testing.T) { } func TestToConfigSmall(t *testing.T) { - engine := docker{info: system.Info{OSType: "linux/riscv64"}} + engine := docker{info: system.Info{OSType: "linux", Architecture: "riscv64"}} conf := engine.toConfig(&backend.Step{ Name: "test", @@ -193,7 +195,7 @@ func TestToConfigSmall(t *testing.T) { func TestToConfigFull(t *testing.T) { engine := docker{ - info: system.Info{OSType: "linux/riscv64"}, + info: system.Info{OSType: "linux", Architecture: "riscv64"}, config: config{ enableIPv6: true, resourceLimit: resourceLimit{ @@ -255,3 +257,84 @@ func TestToConfigFull(t *testing.T) { }, }, conf) } + +func TestToWindowsConfig(t *testing.T) { + engine := docker{ + info: system.Info{OSType: "windows", Architecture: "x86_64"}, + config: config{ + enableIPv6: true, + }, + } + + conf := engine.toConfig(&backend.Step{ + Name: "test", + UUID: "23434553", + Type: backend.StepTypeCommands, + Image: "golang:1.2.3", + WorkingDir: "/src/abc", + WorkspaceBase: "/src", + Environment: map[string]string{ + "TAGS": "sqlite", + "CI_WORKSPACE": "/src", + }, + Commands: []string{"go test", "go vet ./..."}, + ExtraHosts: []backend.HostAlias{{Name: "t", IP: "1.2.3.4"}}, + Volumes: []string{"wp_default_abc:/src", "/cache:/cache/some/more", "test:/test"}, + Networks: []backend.Conn{{Name: "extra-net", Aliases: []string{"extra.net"}}}, + DNS: []string{"9.9.9.9", "8.8.8.8"}, + Failure: "fail", + AuthConfig: backend.Auth{Username: "user", Password: "123456"}, + NetworkMode: "nat", + Ports: []backend.Port{{Number: 21}, {Number: 22}}, + }) + + assert.NotNil(t, conf) + sort.Strings(conf.Env) + assert.EqualValues(t, &container.Config{ + Image: "golang:1.2.3", + WorkingDir: "C:/src", + AttachStdout: true, + AttachStderr: true, + Entrypoint: []string{"powershell", "-noprofile", "-noninteractive", "-command", "[System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($Env:CI_SCRIPT)) | iex"}, + Labels: map[string]string{ + "wp_step": "test", + "wp_uuid": "23434553", + }, + Env: []string{ + "CI_SCRIPT=CiRFcnJvckFjdGlvblByZWZlcmVuY2UgPSAnU3RvcCc7CmlmICgtbm90IChUZXN0LVBhdGggIkM6L3NyYy9hYmMiKSkgeyBOZXctSXRlbSAtUGF0aCAiQzovc3JjL2FiYyIgLUl0ZW1UeXBlIERpcmVjdG9yeSAtRm9yY2UgfTsKaWYgKC1ub3QgW0Vudmlyb25tZW50XTo6R2V0RW52aXJvbm1lbnRWYXJpYWJsZSgnSE9NRScpKSB7IFtFbnZpcm9ubWVudF06OlNldEVudmlyb25tZW50VmFyaWFibGUoJ0hPTUUnLCAnYzpccm9vdCcpIH07CmlmICgtbm90IChUZXN0LVBhdGggIiRlbnY6SE9NRSIpKSB7IE5ldy1JdGVtIC1QYXRoICIkZW52OkhPTUUiIC1JdGVtVHlwZSBEaXJlY3RvcnkgLUZvcmNlIH07CmlmICgkRW52OkNJX05FVFJDX01BQ0hJTkUpIHsKJG5ldHJjPVtzdHJpbmddOjpGb3JtYXQoInswfVxfbmV0cmMiLCRFbnY6SE9NRSk7CiJtYWNoaW5lICRFbnY6Q0lfTkVUUkNfTUFDSElORSIgPj4gJG5ldHJjOwoibG9naW4gJEVudjpDSV9ORVRSQ19VU0VSTkFNRSIgPj4gJG5ldHJjOwoicGFzc3dvcmQgJEVudjpDSV9ORVRSQ19QQVNTV09SRCIgPj4gJG5ldHJjOwp9OwpbRW52aXJvbm1lbnRdOjpTZXRFbnZpcm9ubWVudFZhcmlhYmxlKCJDSV9ORVRSQ19QQVNTV09SRCIsJG51bGwpOwpbRW52aXJvbm1lbnRdOjpTZXRFbnZpcm9ubWVudFZhcmlhYmxlKCJDSV9TQ1JJUFQiLCRudWxsKTsKY2QgIkM6L3NyYy9hYmMiOwoKV3JpdGUtT3V0cHV0ICgnKyAiZ28gdGVzdCInKTsKJiBnbyB0ZXN0OyBpZiAoJExBU1RFWElUQ09ERSAtbmUgMCkge2V4aXQgJExBU1RFWElUQ09ERX0KCldyaXRlLU91dHB1dCAoJysgImdvIHZldCAuLy4uLiInKTsKJiBnbyB2ZXQgLi8uLi47IGlmICgkTEFTVEVYSVRDT0RFIC1uZSAwKSB7ZXhpdCAkTEFTVEVYSVRDT0RFfQo=", + "CI_WORKSPACE=C:/src", + "SHELL=powershell.exe", + "TAGS=sqlite", + }, + Volumes: map[string]struct{}{ + "C:/cache/some/more": {}, + "C:/src": {}, + "C:/test": {}, + }, + }, conf) + + ciScript, err := base64.StdEncoding.DecodeString(strings.TrimPrefix(conf.Env[0], "CI_SCRIPT=")) + if assert.NoError(t, err) { + assert.EqualValues(t, ` +$ErrorActionPreference = 'Stop'; +if (-not (Test-Path "C:/src/abc")) { New-Item -Path "C:/src/abc" -ItemType Directory -Force }; +if (-not [Environment]::GetEnvironmentVariable('HOME')) { [Environment]::SetEnvironmentVariable('HOME', 'c:\root') }; +if (-not (Test-Path "$env:HOME")) { New-Item -Path "$env:HOME" -ItemType Directory -Force }; +if ($Env:CI_NETRC_MACHINE) { +$netrc=[string]::Format("{0}\_netrc",$Env:HOME); +"machine $Env:CI_NETRC_MACHINE" >> $netrc; +"login $Env:CI_NETRC_USERNAME" >> $netrc; +"password $Env:CI_NETRC_PASSWORD" >> $netrc; +}; +[Environment]::SetEnvironmentVariable("CI_NETRC_PASSWORD",$null); +[Environment]::SetEnvironmentVariable("CI_SCRIPT",$null); +cd "C:/src/abc"; + +Write-Output ('+ "go test"'); +& go test; if ($LASTEXITCODE -ne 0) {exit $LASTEXITCODE} + +Write-Output ('+ "go vet ./..."'); +& go vet ./...; if ($LASTEXITCODE -ne 0) {exit $LASTEXITCODE} +`, string(ciScript)) + } +} diff --git a/pipeline/backend/docker/convert_win.go b/pipeline/backend/docker/convert_win.go new file mode 100644 index 000000000..37afef8e3 --- /dev/null +++ b/pipeline/backend/docker/convert_win.go @@ -0,0 +1,82 @@ +// Copyright 2024 Woodpecker Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package docker + +import ( + "path/filepath" + "regexp" + "strings" + + "go.woodpecker-ci.org/woodpecker/v2/pipeline/backend/types" +) + +const ( + osTypeWindows = "windows" + defaultWindowsDriverLetter = "C:" +) + +var MustNotAddWindowsLetterPattern = regexp.MustCompile(`^(?:` + + // Drive letter followed by colon and optional backslash (C: or C:\) + `[a-zA-Z]:(?:\\|$)|` + + + // Device path starting with \\ or // followed by .\ or ./ (\\.\ or //./ or \\./ or //.\ ) + `(?:\\\\|//)\.(?:\\|/).*|` + + + // UNC path starting with \\ or // followed by non-dot (\server or //server) + `(?:\\\\|//)[^.]|` + + + // Relative path starting with .\ or ./ (.\path or ./path) + `\.(?:\\|/)` + + `)`) + +func (e *docker) windowsPathPatch(step *types.Step) { + // only patch if target is windows + if strings.ToLower(e.info.OSType) != osTypeWindows { + return + } + + // patch volumes to have an letter if not already set + for i, vol := range step.Volumes { + volParts, err := splitVolumeParts(vol) + if err != nil || len(volParts) < 2 { + // ignore non valid volumes for now + continue + } + + // fix source destination + if strings.HasPrefix(volParts[0], "/") { + volParts[0] = filepath.Join(defaultWindowsDriverLetter, volParts[0]) + } + + // fix mount destination + if !MustNotAddWindowsLetterPattern.MatchString(volParts[1]) { + volParts[1] = filepath.Join(defaultWindowsDriverLetter, volParts[1]) + } + step.Volumes[i] = strings.Join(volParts, ":") + } + + // patch workspace + if !MustNotAddWindowsLetterPattern.MatchString(step.WorkspaceBase) { + step.WorkspaceBase = filepath.Join(defaultWindowsDriverLetter, step.WorkspaceBase) + } + if !MustNotAddWindowsLetterPattern.MatchString(step.WorkingDir) { + step.WorkingDir = filepath.Join(defaultWindowsDriverLetter, step.WorkingDir) + } + if ciWorkspace, ok := step.Environment["CI_WORKSPACE"]; ok { + if !MustNotAddWindowsLetterPattern.MatchString(ciWorkspace) { + step.Environment["CI_WORKSPACE"] = filepath.Join(defaultWindowsDriverLetter, ciWorkspace) + } + } +} diff --git a/pipeline/backend/docker/convert_win_test.go b/pipeline/backend/docker/convert_win_test.go new file mode 100644 index 000000000..bfc71b148 --- /dev/null +++ b/pipeline/backend/docker/convert_win_test.go @@ -0,0 +1,44 @@ +// Copyright 2024 Woodpecker Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package docker + +import "testing" + +func TestMustNotAddWindowsLetterPattern(t *testing.T) { + tests := map[string]bool{ + `C:\Users`: true, + `D:\Data`: true, + `\\.\PhysicalDrive0`: true, + `//./COM1`: true, + `E:`: true, + `\\server\share`: true, // UNC path + `.\relative\path`: true, // Relative path + `./path`: true, // Relative with forward slash + `//server/share`: true, // UNC with forward slashes + `not/a/windows/path`: false, + ``: false, + `/usr/local`: false, + `COM1`: false, + `\\.`: false, // Incomplete device path + `//`: false, + } + + for testCase, expected := range tests { + result := MustNotAddWindowsLetterPattern.MatchString(testCase) + if result != expected { + t.Errorf("Test case %q: expected %v but got %v", testCase, expected, result) + } + } +}