Enhance local backend (#2017)

make local backend able to clone from private

---------
*Sponsored by Kithara Software GmbH*
Co-authored-by: Bruno BELANYI <bruno@belanyi.fr>

pipeline/backend/local/clone.go

@@ -0,0 +1,185 @@
+// Copyright 2023 Woodpecker Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package local
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"strings"
+
+	"github.com/rs/zerolog/log"
+	"github.com/woodpecker-ci/woodpecker/pipeline/backend/types"
+	"github.com/woodpecker-ci/woodpecker/shared/constant"
+)
+
+// checkGitCloneCap check if we have the git binary on hand
+func checkGitCloneCap() error {
+	_, err := exec.LookPath("git")
+	return err
+}
+
+// loadClone on backend start determine if there is a global plugin-git binary
+func (e *local) loadClone() {
+	binary, err := exec.LookPath("plugin-git")
+	if err != nil || binary == "" {
+		// could not found global git plugin, just ignore it
+		return
+	}
+	e.pluginGitBinary = binary
+}
+
+// setupClone prepare the clone environment before exec
+func (e *local) setupClone(state *workflowState) error {
+	if e.pluginGitBinary != "" {
+		state.pluginGitBinary = e.pluginGitBinary
+		return nil
+	}
+
+	log.Info().Msg("no global 'plugin-git' installed, try to download for current workflow")
+	state.pluginGitBinary = filepath.Join(state.homeDir, "plugin-git")
+	if runtime.GOOS == "windows" {
+		state.pluginGitBinary += ".exe"
+	}
+	return downloadLatestGitPluginBinary(state.pluginGitBinary)
+}
+
+// execClone executes a clone-step locally
+func (e *local) execClone(ctx context.Context, step *types.Step, state *workflowState, env []string) error {
+	if err := e.setupClone(state); err != nil {
+		return fmt.Errorf("setup clone step failed: %w", err)
+	}
+
+	if err := checkGitCloneCap(); err != nil {
+		return fmt.Errorf("check for git clone capabilities failed: %w", err)
+	}
+
+	if step.Image != constant.DefaultCloneImage {
+		// TODO: write message into log
+		log.Warn().Msgf("clone step image '%s' does not match default git clone image. We ignore it assume git.", step.Image)
+	}
+
+	rmCmd, err := writeNetRC(step, state)
+	if err != nil {
+		return err
+	}
+
+	env = append(env, "CI_WORKSPACE="+state.workspaceDir)
+
+	// Prepare command
+	var cmd *exec.Cmd
+	if runtime.GOOS == "windows" {
+		pwsh, err := exec.LookPath("powershell.exe")
+		if err != nil {
+			return err
+		}
+		cmd = exec.CommandContext(ctx, pwsh, "-Command", fmt.Sprintf("%s ; $code=$? ; %s ; if (!$code) {[Environment]::Exit(1)}", state.pluginGitBinary, rmCmd))
+	} else {
+		cmd = exec.CommandContext(ctx, "/bin/sh", "-c", fmt.Sprintf("%s ; $code=$? ; %s ; exit $code", state.pluginGitBinary, rmCmd))
+	}
+	cmd.Env = env
+	cmd.Dir = state.workspaceDir
+
+	// Get output and redirect Stderr to Stdout
+	e.output, _ = cmd.StdoutPipe()
+	cmd.Stderr = cmd.Stdout
+
+	state.stepCMDs[step.Name] = cmd
+
+	return cmd.Start()
+}
+
+// writeNetRC write a netrc file into the home dir of a given workflow state
+func writeNetRC(step *types.Step, state *workflowState) (string, error) {
+	if step.Environment["CI_NETRC_MACHINE"] == "" {
+		return "", nil
+	}
+
+	file := filepath.Join(state.homeDir, ".netrc")
+	rmCmd := fmt.Sprintf("rm \"%s\"", file)
+	if runtime.GOOS == "windows" {
+		file = filepath.Join(state.homeDir, "_netrc")
+		rmCmd = fmt.Sprintf("del \"%s\"", file)
+	}
+
+	return rmCmd, os.WriteFile(file, []byte(fmt.Sprintf(
+		netrcFile,
+		step.Environment["CI_NETRC_MACHINE"],
+		step.Environment["CI_NETRC_USERNAME"],
+		step.Environment["CI_NETRC_PASSWORD"],
+	)), 0o600)
+}
+
+// downloadLatestGitPluginBinary download the latest plugin-git binary based on runtime OS and Arch
+// and saves it to dest
+func downloadLatestGitPluginBinary(dest string) error {
+	type asset struct {
+		Name               string
+		BrowserDownloadURL string `json:"browser_download_url"`
+	}
+
+	type release struct {
+		Assets []asset
+	}
+
+	// get latest release
+	req, _ := http.NewRequest(http.MethodGet, "https://api.github.com/repos/woodpecker-ci/plugin-git/releases/latest", nil)
+	req.Header.Set("Accept", "application/vnd.github+json")
+	req.Header.Set("X-GitHub-Api-Version", "2022-11-28")
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return fmt.Errorf("could not get latest release: %w", err)
+	}
+	raw, _ := io.ReadAll(resp.Body)
+	_ = resp.Body.Close()
+	var rel release
+	if err := json.Unmarshal(raw, &rel); err != nil {
+		return fmt.Errorf("could not unmarshal github response: %w", err)
+	}
+
+	for _, at := range rel.Assets {
+		if strings.Contains(at.Name, runtime.GOOS) && strings.Contains(at.Name, runtime.GOARCH) {
+			resp2, err := http.Get(at.BrowserDownloadURL)
+			if err != nil {
+				return fmt.Errorf("could not download plugin-git: %w", err)
+			}
+			defer resp2.Body.Close()
+
+			file, err := os.Create(dest)
+			if err != nil {
+				return fmt.Errorf("could not create plugin-git: %w", err)
+			}
+			defer file.Close()
+
+			if _, err := io.Copy(file, resp2.Body); err != nil {
+				return fmt.Errorf("could not download plugin-git: %w", err)
+			}
+			if err := os.Chmod(dest, 0o755); err != nil {
+				return err
+			}
+
+			// download successful
+			return nil
+		}
+	}
+
+	return fmt.Errorf("could not download plugin-git, binary for this os/arch not found")
+}

pipeline/backend/local/const.go

@@ -0,0 +1,38 @@
+// Copyright 2023 Woodpecker Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package local
+
+import "errors"
+
+// notAllowedEnvVarOverwrites are all env vars that can not be overwritten by step config
+var notAllowedEnvVarOverwrites = []string{
+	"CI_NETRC_MACHINE",
+	"CI_NETRC_USERNAME",
+	"CI_NETRC_PASSWORD",
+	"CI_SCRIPT",
+	"HOME",
+	"SHELL",
+}
+
+var (
+	ErrUnsupportedStepType   = errors.New("unsupported step type")
+	ErrWorkflowStateNotFound = errors.New("workflow state not found")
+)
+
+const netrcFile = `
+machine %s
+login %s
+password %s
+`

pipeline/backend/local/local.go

@@ -23,42 +23,32 @@ import (
 	"os/exec"
 	"path/filepath"
 	"strings"
+	"sync"
 
 	"github.com/alessio/shellescape"
 	"github.com/rs/zerolog/log"
 	"golang.org/x/exp/slices"
 
 	"github.com/woodpecker-ci/woodpecker/pipeline/backend/types"
-	"github.com/woodpecker-ci/woodpecker/shared/constant"
 )
 
-// notAllowedEnvVarOverwrites are all env vars that can not be overwritten by step config
-var notAllowedEnvVarOverwrites = []string{
-	"CI_NETRC_MACHINE",
-	"CI_NETRC_USERNAME",
-	"CI_NETRC_PASSWORD",
-	"CI_SCRIPT",
-	"HOME",
-	"SHELL",
-}
-
 type workflowState struct {
-	stepCMDs     map[string]*exec.Cmd
+	stepCMDs        map[string]*exec.Cmd
-	baseDir      string
+	baseDir         string
-	homeDir      string
+	homeDir         string
-	workspaceDir string
+	workspaceDir    string
+	pluginGitBinary string
 }
 
 type local struct {
-	workflows map[string]*workflowState
+	workflows       sync.Map
-	output    io.ReadCloser
+	output          io.ReadCloser
+	pluginGitBinary string
 }
 
 // New returns a new local Engine.
 func New() types.Engine {
-	return &local{
+	return &local{}
-		workflows: make(map[string]*workflowState),
-	}
 }
 
 func (e *local) Name() string {
@@ -70,13 +60,13 @@ func (e *local) IsAvailable(context.Context) bool {
 }
 
 func (e *local) Load(context.Context) error {
-	// TODO: download plugin-git binary if not exist
+	e.loadClone()
 
 	return nil
 }
 
 // SetupWorkflow the pipeline environment.
-func (e *local) SetupWorkflow(_ context.Context, conf *types.Config, taskUUID string) error {
+func (e *local) SetupWorkflow(_ context.Context, _ *types.Config, taskUUID string) error {
 	log.Trace().Str("taskUUID", taskUUID).Msg("create workflow environment")
 
 	baseDir, err := os.MkdirTemp("", "woodpecker-local-*")
@@ -99,14 +89,7 @@ func (e *local) SetupWorkflow(_ context.Context, conf *types.Config, taskUUID st
 		return err
 	}
 
-	// TODO: copy plugin-git binary to homeDir and set PATH
+	e.saveState(taskUUID, state)
-
-	workflowID, err := e.getWorkflowIDFromConfig(conf)
-	if err != nil {
-		return err
-	}
-
-	e.workflows[workflowID] = state
 
 	return nil
 }
@@ -115,7 +98,7 @@ func (e *local) SetupWorkflow(_ context.Context, conf *types.Config, taskUUID st
 func (e *local) StartStep(ctx context.Context, step *types.Step, taskUUID string) error {
 	log.Trace().Str("taskUUID", taskUUID).Msgf("start step %s", step.Name)
 
-	state, err := e.getWorkflowStateFromStep(step)
+	state, err := e.getState(taskUUID)
 	if err != nil {
 		return err
 	}
@@ -132,30 +115,27 @@ func (e *local) StartStep(ctx context.Context, step *types.Step, taskUUID string
 	// Set HOME
 	env = append(env, "HOME="+state.homeDir)
 
-	var command []string
+	switch step.Type {
-	if step.Image == constant.DefaultCloneImage {
+	case types.StepTypeClone:
-		// Default clone step
+		return e.execClone(ctx, step, state, env)
-		// TODO: use tmp HOME and insert netrc and delete it after clone
+	case types.StepTypeCommands:
-		env = append(env, "CI_WORKSPACE="+state.workspaceDir)
+		return e.execCommands(ctx, step, state, env)
-		command = append(command, "plugin-git")
+	default:
-	} else {
+		return ErrUnsupportedStepType
-		// Use "image name" as run command
-		command = append(command, step.Image)
-		command = append(command, "-c")
-
-		// TODO: use commands directly
-		script := ""
-		for _, cmd := range step.Commands {
-			script += fmt.Sprintf("echo + %s\n%s\n\n", shellescape.Quote(cmd), cmd)
-		}
-		script = strings.TrimSpace(script)
-
-		// Deleting the initial lines removes netrc support but adds compatibility for more shells like fish
-		command = append(command, script)
 	}
+}
+
+func (e *local) execCommands(ctx context.Context, step *types.Step, state *workflowState, env []string) error {
+	// TODO: use commands directly
+	script := ""
+	for _, cmd := range step.Commands {
+		script += fmt.Sprintf("echo + %s\n%s\n", strings.TrimSpace(shellescape.Quote(cmd)), cmd)
+	}
+	script = strings.TrimSpace(script)
 
 	// Prepare command
-	cmd := exec.CommandContext(ctx, command[0], command[1:]...)
+	// Use "image name" as run command (indicate shell)
+	cmd := exec.CommandContext(ctx, step.Image, "-c", script)
 	cmd.Env = env
 	cmd.Dir = state.workspaceDir
 
@@ -173,7 +153,7 @@ func (e *local) StartStep(ctx context.Context, step *types.Step, taskUUID string
 func (e *local) WaitStep(_ context.Context, step *types.Step, taskUUID string) (*types.State, error) {
 	log.Trace().Str("taskUUID", taskUUID).Msgf("wait for step %s", step.Name)
 
-	state, err := e.getWorkflowStateFromStep(step)
+	state, err := e.getState(taskUUID)
 	if err != nil {
 		return nil, err
 	}
@@ -206,10 +186,10 @@ func (e *local) TailStep(_ context.Context, step *types.Step, taskUUID string) (
 }
 
 // DestroyWorkflow the pipeline environment.
-func (e *local) DestroyWorkflow(_ context.Context, conf *types.Config, taskUUID string) error {
+func (e *local) DestroyWorkflow(_ context.Context, _ *types.Config, taskUUID string) error {
 	log.Trace().Str("taskUUID", taskUUID).Msgf("delete workflow environment")
 
-	state, err := e.getWorkflowStateFromConfig(conf)
+	state, err := e.getState(taskUUID)
 	if err != nil {
 		return err
 	}
@@ -219,69 +199,23 @@ func (e *local) DestroyWorkflow(_ context.Context, conf *types.Config, taskUUID
 		return err
 	}
 
-	workflowID, err := e.getWorkflowIDFromConfig(conf)
+	e.deleteState(taskUUID)
-	if err != nil {
-		return err
-	}
-
-	delete(e.workflows, workflowID)
 
 	return err
 }
 
-func (e *local) getWorkflowIDFromStep(step *types.Step) (string, error) {
+func (e *local) getState(taskUUID string) (*workflowState, error) {
-	sep := "_step_"
+	state, ok := e.workflows.Load(taskUUID)
-	if strings.Contains(step.Name, sep) {
-		prefix := strings.Split(step.Name, sep)
-		if len(prefix) == 2 {
-			return prefix[0], nil
-		}
-	}
-
-	sep = "_clone"
-	if strings.Contains(step.Name, sep) {
-		prefix := strings.Split(step.Name, sep)
-		if len(prefix) == 2 {
-			return prefix[0], nil
-		}
-	}
-
-	return "", fmt.Errorf("invalid step name (%s) %s", sep, step.Name)
-}
-
-func (e *local) getWorkflowIDFromConfig(c *types.Config) (string, error) {
-	if len(c.Volumes) < 1 {
-		return "", fmt.Errorf("no volumes found in config")
-	}
-
-	prefix := strings.Replace(c.Volumes[0].Name, "_default", "", 1)
-	return prefix, nil
-}
-
-func (e *local) getWorkflowStateFromConfig(c *types.Config) (*workflowState, error) {
-	workflowID, err := e.getWorkflowIDFromConfig(c)
-	if err != nil {
-		return nil, err
-	}
-
-	state, ok := e.workflows[workflowID]
 	if !ok {
-		return nil, fmt.Errorf("workflow %s not found", workflowID)
+		return nil, ErrWorkflowStateNotFound
 	}
-
+	return state.(*workflowState), nil
-	return state, nil
 }
 
-func (e *local) getWorkflowStateFromStep(step *types.Step) (*workflowState, error) {
+func (e *local) saveState(taskUUID string, state *workflowState) {
-	workflowID, err := e.getWorkflowIDFromStep(step)
+	e.workflows.Store(taskUUID, state)
-	if err != nil {
+}
-		return nil, err
+
-	}
+func (e *local) deleteState(taskUUID string) {
-
+	e.workflows.Delete(taskUUID)
-	state, ok := e.workflows[workflowID]
-	if !ok {
-		return nil, fmt.Errorf("workflow %s not found", workflowID)
-	}
-
-	return state, nil
 }

pipeline/frontend/yaml/linter/linter_test.go

@@ -7,7 +7,8 @@ import (
 )
 
 func TestLint(t *testing.T) {
-	testdatas := []struct{ Title, Data string }{{Title: "map", Data: `
+	testdatas := []struct{ Title, Data string }{{
+		Title: "map", Data: `
 steps:
   build:
     image: docker
@@ -26,7 +27,9 @@ steps:
 services:
   redis:
     image: redis
-`}, {Title: "list", Data: `
+`,
+	}, {
+		Title: "list", Data: `
 steps:
   - name: build
     image: docker
@@ -42,7 +45,8 @@ steps:
     repo: foo/bar
     settings:
       foo: bar
-`}, {
+`,
+	}, {
 		Title: "merge maps", Data: `
 variables:
   step_template: &base-step