From 10b1cfcd3b39c6f504e45dc0a9026d5eecdfbd0d Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Mon, 7 Aug 2023 15:39:58 +0200 Subject: [PATCH] Enhance local backend (#2017) make local backend able to clone from private --------- *Sponsored by Kithara Software GmbH* Co-authored-by: Bruno BELANYI --- pipeline/backend/local/clone.go | 185 +++++++++++++++++++ pipeline/backend/local/const.go | 38 ++++ pipeline/backend/local/local.go | 158 +++++----------- pipeline/frontend/yaml/linter/linter_test.go | 10 +- 4 files changed, 276 insertions(+), 115 deletions(-) create mode 100644 pipeline/backend/local/clone.go create mode 100644 pipeline/backend/local/const.go diff --git a/pipeline/backend/local/clone.go b/pipeline/backend/local/clone.go new file mode 100644 index 000000000..e859eb898 --- /dev/null +++ b/pipeline/backend/local/clone.go @@ -0,0 +1,185 @@ +// Copyright 2023 Woodpecker Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package local + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + + "github.com/rs/zerolog/log" + "github.com/woodpecker-ci/woodpecker/pipeline/backend/types" + "github.com/woodpecker-ci/woodpecker/shared/constant" +) + +// checkGitCloneCap check if we have the git binary on hand +func checkGitCloneCap() error { + _, err := exec.LookPath("git") + return err +} + +// loadClone on backend start determine if there is a global plugin-git binary +func (e *local) loadClone() { + binary, err := exec.LookPath("plugin-git") + if err != nil || binary == "" { + // could not found global git plugin, just ignore it + return + } + e.pluginGitBinary = binary +} + +// setupClone prepare the clone environment before exec +func (e *local) setupClone(state *workflowState) error { + if e.pluginGitBinary != "" { + state.pluginGitBinary = e.pluginGitBinary + return nil + } + + log.Info().Msg("no global 'plugin-git' installed, try to download for current workflow") + state.pluginGitBinary = filepath.Join(state.homeDir, "plugin-git") + if runtime.GOOS == "windows" { + state.pluginGitBinary += ".exe" + } + return downloadLatestGitPluginBinary(state.pluginGitBinary) +} + +// execClone executes a clone-step locally +func (e *local) execClone(ctx context.Context, step *types.Step, state *workflowState, env []string) error { + if err := e.setupClone(state); err != nil { + return fmt.Errorf("setup clone step failed: %w", err) + } + + if err := checkGitCloneCap(); err != nil { + return fmt.Errorf("check for git clone capabilities failed: %w", err) + } + + if step.Image != constant.DefaultCloneImage { + // TODO: write message into log + log.Warn().Msgf("clone step image '%s' does not match default git clone image. We ignore it assume git.", step.Image) + } + + rmCmd, err := writeNetRC(step, state) + if err != nil { + return err + } + + env = append(env, "CI_WORKSPACE="+state.workspaceDir) + + // Prepare command + var cmd *exec.Cmd + if runtime.GOOS == "windows" { + pwsh, err := exec.LookPath("powershell.exe") + if err != nil { + return err + } + cmd = exec.CommandContext(ctx, pwsh, "-Command", fmt.Sprintf("%s ; $code=$? ; %s ; if (!$code) {[Environment]::Exit(1)}", state.pluginGitBinary, rmCmd)) + } else { + cmd = exec.CommandContext(ctx, "/bin/sh", "-c", fmt.Sprintf("%s ; $code=$? ; %s ; exit $code", state.pluginGitBinary, rmCmd)) + } + cmd.Env = env + cmd.Dir = state.workspaceDir + + // Get output and redirect Stderr to Stdout + e.output, _ = cmd.StdoutPipe() + cmd.Stderr = cmd.Stdout + + state.stepCMDs[step.Name] = cmd + + return cmd.Start() +} + +// writeNetRC write a netrc file into the home dir of a given workflow state +func writeNetRC(step *types.Step, state *workflowState) (string, error) { + if step.Environment["CI_NETRC_MACHINE"] == "" { + return "", nil + } + + file := filepath.Join(state.homeDir, ".netrc") + rmCmd := fmt.Sprintf("rm \"%s\"", file) + if runtime.GOOS == "windows" { + file = filepath.Join(state.homeDir, "_netrc") + rmCmd = fmt.Sprintf("del \"%s\"", file) + } + + return rmCmd, os.WriteFile(file, []byte(fmt.Sprintf( + netrcFile, + step.Environment["CI_NETRC_MACHINE"], + step.Environment["CI_NETRC_USERNAME"], + step.Environment["CI_NETRC_PASSWORD"], + )), 0o600) +} + +// downloadLatestGitPluginBinary download the latest plugin-git binary based on runtime OS and Arch +// and saves it to dest +func downloadLatestGitPluginBinary(dest string) error { + type asset struct { + Name string + BrowserDownloadURL string `json:"browser_download_url"` + } + + type release struct { + Assets []asset + } + + // get latest release + req, _ := http.NewRequest(http.MethodGet, "https://api.github.com/repos/woodpecker-ci/plugin-git/releases/latest", nil) + req.Header.Set("Accept", "application/vnd.github+json") + req.Header.Set("X-GitHub-Api-Version", "2022-11-28") + resp, err := http.DefaultClient.Do(req) + if err != nil { + return fmt.Errorf("could not get latest release: %w", err) + } + raw, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + var rel release + if err := json.Unmarshal(raw, &rel); err != nil { + return fmt.Errorf("could not unmarshal github response: %w", err) + } + + for _, at := range rel.Assets { + if strings.Contains(at.Name, runtime.GOOS) && strings.Contains(at.Name, runtime.GOARCH) { + resp2, err := http.Get(at.BrowserDownloadURL) + if err != nil { + return fmt.Errorf("could not download plugin-git: %w", err) + } + defer resp2.Body.Close() + + file, err := os.Create(dest) + if err != nil { + return fmt.Errorf("could not create plugin-git: %w", err) + } + defer file.Close() + + if _, err := io.Copy(file, resp2.Body); err != nil { + return fmt.Errorf("could not download plugin-git: %w", err) + } + if err := os.Chmod(dest, 0o755); err != nil { + return err + } + + // download successful + return nil + } + } + + return fmt.Errorf("could not download plugin-git, binary for this os/arch not found") +} diff --git a/pipeline/backend/local/const.go b/pipeline/backend/local/const.go new file mode 100644 index 000000000..cc4817309 --- /dev/null +++ b/pipeline/backend/local/const.go @@ -0,0 +1,38 @@ +// Copyright 2023 Woodpecker Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package local + +import "errors" + +// notAllowedEnvVarOverwrites are all env vars that can not be overwritten by step config +var notAllowedEnvVarOverwrites = []string{ + "CI_NETRC_MACHINE", + "CI_NETRC_USERNAME", + "CI_NETRC_PASSWORD", + "CI_SCRIPT", + "HOME", + "SHELL", +} + +var ( + ErrUnsupportedStepType = errors.New("unsupported step type") + ErrWorkflowStateNotFound = errors.New("workflow state not found") +) + +const netrcFile = ` +machine %s +login %s +password %s +` diff --git a/pipeline/backend/local/local.go b/pipeline/backend/local/local.go index 2405c19bb..88050bc5f 100644 --- a/pipeline/backend/local/local.go +++ b/pipeline/backend/local/local.go @@ -23,42 +23,32 @@ import ( "os/exec" "path/filepath" "strings" + "sync" "github.com/alessio/shellescape" "github.com/rs/zerolog/log" "golang.org/x/exp/slices" "github.com/woodpecker-ci/woodpecker/pipeline/backend/types" - "github.com/woodpecker-ci/woodpecker/shared/constant" ) -// notAllowedEnvVarOverwrites are all env vars that can not be overwritten by step config -var notAllowedEnvVarOverwrites = []string{ - "CI_NETRC_MACHINE", - "CI_NETRC_USERNAME", - "CI_NETRC_PASSWORD", - "CI_SCRIPT", - "HOME", - "SHELL", -} - type workflowState struct { - stepCMDs map[string]*exec.Cmd - baseDir string - homeDir string - workspaceDir string + stepCMDs map[string]*exec.Cmd + baseDir string + homeDir string + workspaceDir string + pluginGitBinary string } type local struct { - workflows map[string]*workflowState - output io.ReadCloser + workflows sync.Map + output io.ReadCloser + pluginGitBinary string } // New returns a new local Engine. func New() types.Engine { - return &local{ - workflows: make(map[string]*workflowState), - } + return &local{} } func (e *local) Name() string { @@ -70,13 +60,13 @@ func (e *local) IsAvailable(context.Context) bool { } func (e *local) Load(context.Context) error { - // TODO: download plugin-git binary if not exist + e.loadClone() return nil } // SetupWorkflow the pipeline environment. -func (e *local) SetupWorkflow(_ context.Context, conf *types.Config, taskUUID string) error { +func (e *local) SetupWorkflow(_ context.Context, _ *types.Config, taskUUID string) error { log.Trace().Str("taskUUID", taskUUID).Msg("create workflow environment") baseDir, err := os.MkdirTemp("", "woodpecker-local-*") @@ -99,14 +89,7 @@ func (e *local) SetupWorkflow(_ context.Context, conf *types.Config, taskUUID st return err } - // TODO: copy plugin-git binary to homeDir and set PATH - - workflowID, err := e.getWorkflowIDFromConfig(conf) - if err != nil { - return err - } - - e.workflows[workflowID] = state + e.saveState(taskUUID, state) return nil } @@ -115,7 +98,7 @@ func (e *local) SetupWorkflow(_ context.Context, conf *types.Config, taskUUID st func (e *local) StartStep(ctx context.Context, step *types.Step, taskUUID string) error { log.Trace().Str("taskUUID", taskUUID).Msgf("start step %s", step.Name) - state, err := e.getWorkflowStateFromStep(step) + state, err := e.getState(taskUUID) if err != nil { return err } @@ -132,30 +115,27 @@ func (e *local) StartStep(ctx context.Context, step *types.Step, taskUUID string // Set HOME env = append(env, "HOME="+state.homeDir) - var command []string - if step.Image == constant.DefaultCloneImage { - // Default clone step - // TODO: use tmp HOME and insert netrc and delete it after clone - env = append(env, "CI_WORKSPACE="+state.workspaceDir) - command = append(command, "plugin-git") - } else { - // Use "image name" as run command - command = append(command, step.Image) - command = append(command, "-c") - - // TODO: use commands directly - script := "" - for _, cmd := range step.Commands { - script += fmt.Sprintf("echo + %s\n%s\n\n", shellescape.Quote(cmd), cmd) - } - script = strings.TrimSpace(script) - - // Deleting the initial lines removes netrc support but adds compatibility for more shells like fish - command = append(command, script) + switch step.Type { + case types.StepTypeClone: + return e.execClone(ctx, step, state, env) + case types.StepTypeCommands: + return e.execCommands(ctx, step, state, env) + default: + return ErrUnsupportedStepType } +} + +func (e *local) execCommands(ctx context.Context, step *types.Step, state *workflowState, env []string) error { + // TODO: use commands directly + script := "" + for _, cmd := range step.Commands { + script += fmt.Sprintf("echo + %s\n%s\n", strings.TrimSpace(shellescape.Quote(cmd)), cmd) + } + script = strings.TrimSpace(script) // Prepare command - cmd := exec.CommandContext(ctx, command[0], command[1:]...) + // Use "image name" as run command (indicate shell) + cmd := exec.CommandContext(ctx, step.Image, "-c", script) cmd.Env = env cmd.Dir = state.workspaceDir @@ -173,7 +153,7 @@ func (e *local) StartStep(ctx context.Context, step *types.Step, taskUUID string func (e *local) WaitStep(_ context.Context, step *types.Step, taskUUID string) (*types.State, error) { log.Trace().Str("taskUUID", taskUUID).Msgf("wait for step %s", step.Name) - state, err := e.getWorkflowStateFromStep(step) + state, err := e.getState(taskUUID) if err != nil { return nil, err } @@ -206,10 +186,10 @@ func (e *local) TailStep(_ context.Context, step *types.Step, taskUUID string) ( } // DestroyWorkflow the pipeline environment. -func (e *local) DestroyWorkflow(_ context.Context, conf *types.Config, taskUUID string) error { +func (e *local) DestroyWorkflow(_ context.Context, _ *types.Config, taskUUID string) error { log.Trace().Str("taskUUID", taskUUID).Msgf("delete workflow environment") - state, err := e.getWorkflowStateFromConfig(conf) + state, err := e.getState(taskUUID) if err != nil { return err } @@ -219,69 +199,23 @@ func (e *local) DestroyWorkflow(_ context.Context, conf *types.Config, taskUUID return err } - workflowID, err := e.getWorkflowIDFromConfig(conf) - if err != nil { - return err - } - - delete(e.workflows, workflowID) + e.deleteState(taskUUID) return err } -func (e *local) getWorkflowIDFromStep(step *types.Step) (string, error) { - sep := "_step_" - if strings.Contains(step.Name, sep) { - prefix := strings.Split(step.Name, sep) - if len(prefix) == 2 { - return prefix[0], nil - } - } - - sep = "_clone" - if strings.Contains(step.Name, sep) { - prefix := strings.Split(step.Name, sep) - if len(prefix) == 2 { - return prefix[0], nil - } - } - - return "", fmt.Errorf("invalid step name (%s) %s", sep, step.Name) -} - -func (e *local) getWorkflowIDFromConfig(c *types.Config) (string, error) { - if len(c.Volumes) < 1 { - return "", fmt.Errorf("no volumes found in config") - } - - prefix := strings.Replace(c.Volumes[0].Name, "_default", "", 1) - return prefix, nil -} - -func (e *local) getWorkflowStateFromConfig(c *types.Config) (*workflowState, error) { - workflowID, err := e.getWorkflowIDFromConfig(c) - if err != nil { - return nil, err - } - - state, ok := e.workflows[workflowID] +func (e *local) getState(taskUUID string) (*workflowState, error) { + state, ok := e.workflows.Load(taskUUID) if !ok { - return nil, fmt.Errorf("workflow %s not found", workflowID) + return nil, ErrWorkflowStateNotFound } - - return state, nil + return state.(*workflowState), nil } -func (e *local) getWorkflowStateFromStep(step *types.Step) (*workflowState, error) { - workflowID, err := e.getWorkflowIDFromStep(step) - if err != nil { - return nil, err - } - - state, ok := e.workflows[workflowID] - if !ok { - return nil, fmt.Errorf("workflow %s not found", workflowID) - } - - return state, nil +func (e *local) saveState(taskUUID string, state *workflowState) { + e.workflows.Store(taskUUID, state) +} + +func (e *local) deleteState(taskUUID string) { + e.workflows.Delete(taskUUID) } diff --git a/pipeline/frontend/yaml/linter/linter_test.go b/pipeline/frontend/yaml/linter/linter_test.go index d752e8e29..11d0d9c2a 100644 --- a/pipeline/frontend/yaml/linter/linter_test.go +++ b/pipeline/frontend/yaml/linter/linter_test.go @@ -7,7 +7,8 @@ import ( ) func TestLint(t *testing.T) { - testdatas := []struct{ Title, Data string }{{Title: "map", Data: ` + testdatas := []struct{ Title, Data string }{{ + Title: "map", Data: ` steps: build: image: docker @@ -26,7 +27,9 @@ steps: services: redis: image: redis -`}, {Title: "list", Data: ` +`, + }, { + Title: "list", Data: ` steps: - name: build image: docker @@ -42,7 +45,8 @@ steps: repo: foo/bar settings: foo: bar -`}, { +`, + }, { Title: "merge maps", Data: ` variables: step_template: &base-step