mirror of
https://github.com/woodpecker-ci/woodpecker.git
synced 2024-11-22 09:51:01 +00:00
Enhance local backend (#2017)
make local backend able to clone from private --------- *Sponsored by Kithara Software GmbH* Co-authored-by: Bruno BELANYI <bruno@belanyi.fr>
This commit is contained in:
6543
GitHub
parent
0ecaa7074a
commit
10b1cfcd3b
4 changed files with 276 additions and 115 deletions
185
pipeline/backend/local/clone.go
Normal file
185
pipeline/backend/local/clone.go
Normal file
|
|
@ -0,0 +1,185 @@
|
|||
// Copyright 2023 Woodpecker Authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package local
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
|
||||
"github.com/rs/zerolog/log"
|
||||
"github.com/woodpecker-ci/woodpecker/pipeline/backend/types"
|
||||
"github.com/woodpecker-ci/woodpecker/shared/constant"
|
||||
)
|
||||
|
||||
// checkGitCloneCap check if we have the git binary on hand
|
||||
func checkGitCloneCap() error {
|
||||
_, err := exec.LookPath("git")
|
||||
return err
|
||||
}
|
||||
|
||||
// loadClone on backend start determine if there is a global plugin-git binary
|
||||
func (e *local) loadClone() {
|
||||
binary, err := exec.LookPath("plugin-git")
|
||||
if err != nil || binary == "" {
|
||||
// could not found global git plugin, just ignore it
|
||||
return
|
||||
}
|
||||
e.pluginGitBinary = binary
|
||||
}
|
||||
|
||||
// setupClone prepare the clone environment before exec
|
||||
func (e *local) setupClone(state *workflowState) error {
|
||||
if e.pluginGitBinary != "" {
|
||||
state.pluginGitBinary = e.pluginGitBinary
|
||||
return nil
|
||||
}
|
||||
|
||||
log.Info().Msg("no global 'plugin-git' installed, try to download for current workflow")
|
||||
state.pluginGitBinary = filepath.Join(state.homeDir, "plugin-git")
|
||||
if runtime.GOOS == "windows" {
|
||||
state.pluginGitBinary += ".exe"
|
||||
}
|
||||
return downloadLatestGitPluginBinary(state.pluginGitBinary)
|
||||
}
|
||||
|
||||
// execClone executes a clone-step locally
|
||||
func (e *local) execClone(ctx context.Context, step *types.Step, state *workflowState, env []string) error {
|
||||
if err := e.setupClone(state); err != nil {
|
||||
return fmt.Errorf("setup clone step failed: %w", err)
|
||||
}
|
||||
|
||||
if err := checkGitCloneCap(); err != nil {
|
||||
return fmt.Errorf("check for git clone capabilities failed: %w", err)
|
||||
}
|
||||
|
||||
if step.Image != constant.DefaultCloneImage {
|
||||
// TODO: write message into log
|
||||
log.Warn().Msgf("clone step image '%s' does not match default git clone image. We ignore it assume git.", step.Image)
|
||||
}
|
||||
|
||||
rmCmd, err := writeNetRC(step, state)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
env = append(env, "CI_WORKSPACE="+state.workspaceDir)
|
||||
|
||||
// Prepare command
|
||||
var cmd *exec.Cmd
|
||||
if runtime.GOOS == "windows" {
|
||||
pwsh, err := exec.LookPath("powershell.exe")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
cmd = exec.CommandContext(ctx, pwsh, "-Command", fmt.Sprintf("%s ; $code=$? ; %s ; if (!$code) {[Environment]::Exit(1)}", state.pluginGitBinary, rmCmd))
|
||||
} else {
|
||||
cmd = exec.CommandContext(ctx, "/bin/sh", "-c", fmt.Sprintf("%s ; $code=$? ; %s ; exit $code", state.pluginGitBinary, rmCmd))
|
||||
}
|
||||
cmd.Env = env
|
||||
cmd.Dir = state.workspaceDir
|
||||
|
||||
// Get output and redirect Stderr to Stdout
|
||||
e.output, _ = cmd.StdoutPipe()
|
||||
cmd.Stderr = cmd.Stdout
|
||||
|
||||
state.stepCMDs[step.Name] = cmd
|
||||
|
||||
return cmd.Start()
|
||||
}
|
||||
|
||||
// writeNetRC write a netrc file into the home dir of a given workflow state
|
||||
func writeNetRC(step *types.Step, state *workflowState) (string, error) {
|
||||
if step.Environment["CI_NETRC_MACHINE"] == "" {
|
||||
return "", nil
|
||||
}
|
||||
|
||||
file := filepath.Join(state.homeDir, ".netrc")
|
||||
rmCmd := fmt.Sprintf("rm \"%s\"", file)
|
||||
if runtime.GOOS == "windows" {
|
||||
file = filepath.Join(state.homeDir, "_netrc")
|
||||
rmCmd = fmt.Sprintf("del \"%s\"", file)
|
||||
}
|
||||
|
||||
return rmCmd, os.WriteFile(file, []byte(fmt.Sprintf(
|
||||
netrcFile,
|
||||
step.Environment["CI_NETRC_MACHINE"],
|
||||
step.Environment["CI_NETRC_USERNAME"],
|
||||
step.Environment["CI_NETRC_PASSWORD"],
|
||||
)), 0o600)
|
||||
}
|
||||
|
||||
// downloadLatestGitPluginBinary download the latest plugin-git binary based on runtime OS and Arch
|
||||
// and saves it to dest
|
||||
func downloadLatestGitPluginBinary(dest string) error {
|
||||
type asset struct {
|
||||
Name string
|
||||
BrowserDownloadURL string `json:"browser_download_url"`
|
||||
}
|
||||
|
||||
type release struct {
|
||||
Assets []asset
|
||||
}
|
||||
|
||||
// get latest release
|
||||
req, _ := http.NewRequest(http.MethodGet, "https://api.github.com/repos/woodpecker-ci/plugin-git/releases/latest", nil)
|
||||
req.Header.Set("Accept", "application/vnd.github+json")
|
||||
req.Header.Set("X-GitHub-Api-Version", "2022-11-28")
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not get latest release: %w", err)
|
||||
}
|
||||
raw, _ := io.ReadAll(resp.Body)
|
||||
_ = resp.Body.Close()
|
||||
var rel release
|
||||
if err := json.Unmarshal(raw, &rel); err != nil {
|
||||
return fmt.Errorf("could not unmarshal github response: %w", err)
|
||||
}
|
||||
|
||||
for _, at := range rel.Assets {
|
||||
if strings.Contains(at.Name, runtime.GOOS) && strings.Contains(at.Name, runtime.GOARCH) {
|
||||
resp2, err := http.Get(at.BrowserDownloadURL)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not download plugin-git: %w", err)
|
||||
}
|
||||
defer resp2.Body.Close()
|
||||
|
||||
file, err := os.Create(dest)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not create plugin-git: %w", err)
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
if _, err := io.Copy(file, resp2.Body); err != nil {
|
||||
return fmt.Errorf("could not download plugin-git: %w", err)
|
||||
}
|
||||
if err := os.Chmod(dest, 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// download successful
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
return fmt.Errorf("could not download plugin-git, binary for this os/arch not found")
|
||||
}
|
||||
38
pipeline/backend/local/const.go
Normal file
38
pipeline/backend/local/const.go
Normal file
|
|
@ -0,0 +1,38 @@
|
|||
// Copyright 2023 Woodpecker Authors
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package local
|
||||
|
||||
import "errors"
|
||||
|
||||
// notAllowedEnvVarOverwrites are all env vars that can not be overwritten by step config
|
||||
var notAllowedEnvVarOverwrites = []string{
|
||||
"CI_NETRC_MACHINE",
|
||||
"CI_NETRC_USERNAME",
|
||||
"CI_NETRC_PASSWORD",
|
||||
"CI_SCRIPT",
|
||||
"HOME",
|
||||
"SHELL",
|
||||
}
|
||||
|
||||
var (
|
||||
ErrUnsupportedStepType = errors.New("unsupported step type")
|
||||
ErrWorkflowStateNotFound = errors.New("workflow state not found")
|
||||
)
|
||||
|
||||
const netrcFile = `
|
||||
machine %s
|
||||
login %s
|
||||
password %s
|
||||
`
|
||||
|
|
@ -23,42 +23,32 @@ import (
|
|||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/alessio/shellescape"
|
||||
"github.com/rs/zerolog/log"
|
||||
"golang.org/x/exp/slices"
|
||||
|
||||
"github.com/woodpecker-ci/woodpecker/pipeline/backend/types"
|
||||
"github.com/woodpecker-ci/woodpecker/shared/constant"
|
||||
)
|
||||
|
||||
// notAllowedEnvVarOverwrites are all env vars that can not be overwritten by step config
|
||||
var notAllowedEnvVarOverwrites = []string{
|
||||
"CI_NETRC_MACHINE",
|
||||
"CI_NETRC_USERNAME",
|
||||
"CI_NETRC_PASSWORD",
|
||||
"CI_SCRIPT",
|
||||
"HOME",
|
||||
"SHELL",
|
||||
}
|
||||
|
||||
type workflowState struct {
|
||||
stepCMDs map[string]*exec.Cmd
|
||||
baseDir string
|
||||
homeDir string
|
||||
workspaceDir string
|
||||
stepCMDs map[string]*exec.Cmd
|
||||
baseDir string
|
||||
homeDir string
|
||||
workspaceDir string
|
||||
pluginGitBinary string
|
||||
}
|
||||
|
||||
type local struct {
|
||||
workflows map[string]*workflowState
|
||||
output io.ReadCloser
|
||||
workflows sync.Map
|
||||
output io.ReadCloser
|
||||
pluginGitBinary string
|
||||
}
|
||||
|
||||
// New returns a new local Engine.
|
||||
func New() types.Engine {
|
||||
return &local{
|
||||
workflows: make(map[string]*workflowState),
|
||||
}
|
||||
return &local{}
|
||||
}
|
||||
|
||||
func (e *local) Name() string {
|
||||
|
|
@ -70,13 +60,13 @@ func (e *local) IsAvailable(context.Context) bool {
|
|||
}
|
||||
|
||||
func (e *local) Load(context.Context) error {
|
||||
// TODO: download plugin-git binary if not exist
|
||||
e.loadClone()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// SetupWorkflow the pipeline environment.
|
||||
func (e *local) SetupWorkflow(_ context.Context, conf *types.Config, taskUUID string) error {
|
||||
func (e *local) SetupWorkflow(_ context.Context, _ *types.Config, taskUUID string) error {
|
||||
log.Trace().Str("taskUUID", taskUUID).Msg("create workflow environment")
|
||||
|
||||
baseDir, err := os.MkdirTemp("", "woodpecker-local-*")
|
||||
|
|
@ -99,14 +89,7 @@ func (e *local) SetupWorkflow(_ context.Context, conf *types.Config, taskUUID st
|
|||
return err
|
||||
}
|
||||
|
||||
// TODO: copy plugin-git binary to homeDir and set PATH
|
||||
|
||||
workflowID, err := e.getWorkflowIDFromConfig(conf)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
e.workflows[workflowID] = state
|
||||
e.saveState(taskUUID, state)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
|
@ -115,7 +98,7 @@ func (e *local) SetupWorkflow(_ context.Context, conf *types.Config, taskUUID st
|
|||
func (e *local) StartStep(ctx context.Context, step *types.Step, taskUUID string) error {
|
||||
log.Trace().Str("taskUUID", taskUUID).Msgf("start step %s", step.Name)
|
||||
|
||||
state, err := e.getWorkflowStateFromStep(step)
|
||||
state, err := e.getState(taskUUID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
@ -132,30 +115,27 @@ func (e *local) StartStep(ctx context.Context, step *types.Step, taskUUID string
|
|||
// Set HOME
|
||||
env = append(env, "HOME="+state.homeDir)
|
||||
|
||||
var command []string
|
||||
if step.Image == constant.DefaultCloneImage {
|
||||
// Default clone step
|
||||
// TODO: use tmp HOME and insert netrc and delete it after clone
|
||||
env = append(env, "CI_WORKSPACE="+state.workspaceDir)
|
||||
command = append(command, "plugin-git")
|
||||
} else {
|
||||
// Use "image name" as run command
|
||||
command = append(command, step.Image)
|
||||
command = append(command, "-c")
|
||||
|
||||
// TODO: use commands directly
|
||||
script := ""
|
||||
for _, cmd := range step.Commands {
|
||||
script += fmt.Sprintf("echo + %s\n%s\n\n", shellescape.Quote(cmd), cmd)
|
||||
}
|
||||
script = strings.TrimSpace(script)
|
||||
|
||||
// Deleting the initial lines removes netrc support but adds compatibility for more shells like fish
|
||||
command = append(command, script)
|
||||
switch step.Type {
|
||||
case types.StepTypeClone:
|
||||
return e.execClone(ctx, step, state, env)
|
||||
case types.StepTypeCommands:
|
||||
return e.execCommands(ctx, step, state, env)
|
||||
default:
|
||||
return ErrUnsupportedStepType
|
||||
}
|
||||
}
|
||||
|
||||
func (e *local) execCommands(ctx context.Context, step *types.Step, state *workflowState, env []string) error {
|
||||
// TODO: use commands directly
|
||||
script := ""
|
||||
for _, cmd := range step.Commands {
|
||||
script += fmt.Sprintf("echo + %s\n%s\n", strings.TrimSpace(shellescape.Quote(cmd)), cmd)
|
||||
}
|
||||
script = strings.TrimSpace(script)
|
||||
|
||||
// Prepare command
|
||||
cmd := exec.CommandContext(ctx, command[0], command[1:]...)
|
||||
// Use "image name" as run command (indicate shell)
|
||||
cmd := exec.CommandContext(ctx, step.Image, "-c", script)
|
||||
cmd.Env = env
|
||||
cmd.Dir = state.workspaceDir
|
||||
|
||||
|
|
@ -173,7 +153,7 @@ func (e *local) StartStep(ctx context.Context, step *types.Step, taskUUID string
|
|||
func (e *local) WaitStep(_ context.Context, step *types.Step, taskUUID string) (*types.State, error) {
|
||||
log.Trace().Str("taskUUID", taskUUID).Msgf("wait for step %s", step.Name)
|
||||
|
||||
state, err := e.getWorkflowStateFromStep(step)
|
||||
state, err := e.getState(taskUUID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -206,10 +186,10 @@ func (e *local) TailStep(_ context.Context, step *types.Step, taskUUID string) (
|
|||
}
|
||||
|
||||
// DestroyWorkflow the pipeline environment.
|
||||
func (e *local) DestroyWorkflow(_ context.Context, conf *types.Config, taskUUID string) error {
|
||||
func (e *local) DestroyWorkflow(_ context.Context, _ *types.Config, taskUUID string) error {
|
||||
log.Trace().Str("taskUUID", taskUUID).Msgf("delete workflow environment")
|
||||
|
||||
state, err := e.getWorkflowStateFromConfig(conf)
|
||||
state, err := e.getState(taskUUID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
@ -219,69 +199,23 @@ func (e *local) DestroyWorkflow(_ context.Context, conf *types.Config, taskUUID
|
|||
return err
|
||||
}
|
||||
|
||||
workflowID, err := e.getWorkflowIDFromConfig(conf)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
delete(e.workflows, workflowID)
|
||||
e.deleteState(taskUUID)
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
func (e *local) getWorkflowIDFromStep(step *types.Step) (string, error) {
|
||||
sep := "_step_"
|
||||
if strings.Contains(step.Name, sep) {
|
||||
prefix := strings.Split(step.Name, sep)
|
||||
if len(prefix) == 2 {
|
||||
return prefix[0], nil
|
||||
}
|
||||
}
|
||||
|
||||
sep = "_clone"
|
||||
if strings.Contains(step.Name, sep) {
|
||||
prefix := strings.Split(step.Name, sep)
|
||||
if len(prefix) == 2 {
|
||||
return prefix[0], nil
|
||||
}
|
||||
}
|
||||
|
||||
return "", fmt.Errorf("invalid step name (%s) %s", sep, step.Name)
|
||||
}
|
||||
|
||||
func (e *local) getWorkflowIDFromConfig(c *types.Config) (string, error) {
|
||||
if len(c.Volumes) < 1 {
|
||||
return "", fmt.Errorf("no volumes found in config")
|
||||
}
|
||||
|
||||
prefix := strings.Replace(c.Volumes[0].Name, "_default", "", 1)
|
||||
return prefix, nil
|
||||
}
|
||||
|
||||
func (e *local) getWorkflowStateFromConfig(c *types.Config) (*workflowState, error) {
|
||||
workflowID, err := e.getWorkflowIDFromConfig(c)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
state, ok := e.workflows[workflowID]
|
||||
func (e *local) getState(taskUUID string) (*workflowState, error) {
|
||||
state, ok := e.workflows.Load(taskUUID)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("workflow %s not found", workflowID)
|
||||
return nil, ErrWorkflowStateNotFound
|
||||
}
|
||||
|
||||
return state, nil
|
||||
return state.(*workflowState), nil
|
||||
}
|
||||
|
||||
func (e *local) getWorkflowStateFromStep(step *types.Step) (*workflowState, error) {
|
||||
workflowID, err := e.getWorkflowIDFromStep(step)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
state, ok := e.workflows[workflowID]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("workflow %s not found", workflowID)
|
||||
}
|
||||
|
||||
return state, nil
|
||||
func (e *local) saveState(taskUUID string, state *workflowState) {
|
||||
e.workflows.Store(taskUUID, state)
|
||||
}
|
||||
|
||||
func (e *local) deleteState(taskUUID string) {
|
||||
e.workflows.Delete(taskUUID)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,7 +7,8 @@ import (
|
|||
)
|
||||
|
||||
func TestLint(t *testing.T) {
|
||||
testdatas := []struct{ Title, Data string }{{Title: "map", Data: `
|
||||
testdatas := []struct{ Title, Data string }{{
|
||||
Title: "map", Data: `
|
||||
steps:
|
||||
build:
|
||||
image: docker
|
||||
|
|
@ -26,7 +27,9 @@ steps:
|
|||
services:
|
||||
redis:
|
||||
image: redis
|
||||
`}, {Title: "list", Data: `
|
||||
`,
|
||||
}, {
|
||||
Title: "list", Data: `
|
||||
steps:
|
||||
- name: build
|
||||
image: docker
|
||||
|
|
@ -42,7 +45,8 @@ steps:
|
|||
repo: foo/bar
|
||||
settings:
|
||||
foo: bar
|
||||
`}, {
|
||||
`,
|
||||
}, {
|
||||
Title: "merge maps", Data: `
|
||||
variables:
|
||||
step_template: &base-step
|
||||
|
|
|
|||
Loading…
Reference in a new issue