mirrors/woodpecker
1
0
Fork 0
mirror of https://github.com/woodpecker-ci/woodpecker.git synced 2024-10-23 10:33:58 +00:00
Code Issues Projects Releases Wiki Activity

add files

Browse source
This commit is contained in:
pat-s 2024-10-23 09:50:35 +02:00
parent 1b5ee05307
commit 2ff9183059
No known key found for this signature in database
GPG key ID: 3C6318841EF78925
17 changed files with 162 additions and 135 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
agent/runner.go
View file

@ -23,10 +23,11 @@ import (
"time" "time"
"github.com/rs/zerolog/log" "github.com/rs/zerolog/log"
"google.golang.org/grpc/metadata" grpc_metadata "google.golang.org/grpc/metadata"
"go.woodpecker-ci.org/woodpecker/v2/pipeline" "go.woodpecker-ci.org/woodpecker/v2/pipeline"
backend "go.woodpecker-ci.org/woodpecker/v2/pipeline/backend/types" backend "go.woodpecker-ci.org/woodpecker/v2/pipeline/backend/types"
"go.woodpecker-ci.org/woodpecker/v2/pipeline/frontend/metadata"
"go.woodpecker-ci.org/woodpecker/v2/pipeline/rpc" "go.woodpecker-ci.org/woodpecker/v2/pipeline/rpc"
"go.woodpecker-ci.org/woodpecker/v2/shared/constant" "go.woodpecker-ci.org/woodpecker/v2/shared/constant"
"go.woodpecker-ci.org/woodpecker/v2/shared/utils" "go.woodpecker-ci.org/woodpecker/v2/shared/utils"
@ -50,11 +51,11 @@ func NewRunner(workEngine rpc.Peer, f rpc.Filter, h string, state *State, backen
} }
} }
func (r *Runner) Run(runnerCtx, shutdownCtx context.Context) error { //nolint:contextcheck func (r *Runner) Run(runnerCtx, shutdownCtx context.Context, metadata *metadata.Metadata) error { //nolint:contextcheck
log.Debug().Msg("request next execution") log.Debug().Msg("request next execution")
meta, _ := metadata.FromOutgoingContext(runnerCtx) meta, _ := grpc_metadata.FromOutgoingContext(runnerCtx)
ctxMeta := metadata.NewOutgoingContext(context.Background(), meta) ctxMeta := grpc_metadata.NewOutgoingContext(context.Background(), meta)
// get the next workflow from the queue // get the next workflow from the queue
workflow, err := r.client.Next(runnerCtx, r.filter) workflow, err := r.client.Next(runnerCtx, r.filter)
@ -150,7 +151,7 @@ func (r *Runner) Run(runnerCtx, shutdownCtx context.Context) error { //nolint:co
"repo": repoName, "repo": repoName,
"pipeline_number": pipelineNumber, "pipeline_number": pipelineNumber,
}), }),
).Run(runnerCtx) ).Run(runnerCtx, metadata)
state.Finished = time.Now().Unix() state.Finished = time.Now().Unix()

2
cli/exec/exec.go
View file

@ -282,7 +282,7 @@ func execWithAxis(ctx context.Context, c *cli.Command, file, repoPath string, ax
pipeline.WithDescription(map[string]string{ pipeline.WithDescription(map[string]string{
"CLI": "exec", "CLI": "exec",
}), }),
).Run(ctx) ).Run(ctx, metadata)
} }
// convertPathForWindows converts a path to use slash separators // convertPathForWindows converts a path to use slash separators

13
cmd/agent/core/agent.go
View file

@ -35,13 +35,14 @@ import (
grpc_credentials "google.golang.org/grpc/credentials" grpc_credentials "google.golang.org/grpc/credentials"
"google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/credentials/insecure"
"google.golang.org/grpc/keepalive" "google.golang.org/grpc/keepalive"
"google.golang.org/grpc/metadata" grpc_metadata "google.golang.org/grpc/metadata"
"google.golang.org/grpc/status" "google.golang.org/grpc/status"
"go.woodpecker-ci.org/woodpecker/v2/agent" "go.woodpecker-ci.org/woodpecker/v2/agent"
agent_rpc "go.woodpecker-ci.org/woodpecker/v2/agent/rpc" agent_rpc "go.woodpecker-ci.org/woodpecker/v2/agent/rpc"
"go.woodpecker-ci.org/woodpecker/v2/pipeline/backend" "go.woodpecker-ci.org/woodpecker/v2/pipeline/backend"
"go.woodpecker-ci.org/woodpecker/v2/pipeline/backend/types" "go.woodpecker-ci.org/woodpecker/v2/pipeline/backend/types"
"go.woodpecker-ci.org/woodpecker/v2/pipeline/frontend/metadata"
"go.woodpecker-ci.org/woodpecker/v2/pipeline/rpc" "go.woodpecker-ci.org/woodpecker/v2/pipeline/rpc"
"go.woodpecker-ci.org/woodpecker/v2/shared/logger" "go.woodpecker-ci.org/woodpecker/v2/shared/logger"
"go.woodpecker-ci.org/woodpecker/v2/shared/utils" "go.woodpecker-ci.org/woodpecker/v2/shared/utils"
@ -63,7 +64,7 @@ var (
shutdownCtx = context.Background() shutdownCtx = context.Background()
) )
func run(ctx context.Context, c *cli.Command, backends []types.Backend) error { func run(ctx context.Context, c *cli.Command, backends []types.Backend, metadata *metadata.Metadata) error {
agentCtx, ctxCancel := context.WithCancelCause(ctx) agentCtx, ctxCancel := context.WithCancelCause(ctx)
stopAgentFunc = func(err error) { stopAgentFunc = func(err error) {
msg := "shutdown of whole agent" msg := "shutdown of whole agent"
@ -160,7 +161,7 @@ func run(ctx context.Context, c *cli.Command, backends []types.Backend) error {
client := agent_rpc.NewGrpcClient(ctx, conn) client := agent_rpc.NewGrpcClient(ctx, conn)
agentConfigPersisted := atomic.Bool{} agentConfigPersisted := atomic.Bool{}
grpcCtx := metadata.NewOutgoingContext(grpcClientCtx, metadata.Pairs("hostname", hostname)) grpcCtx := grpc_metadata.NewOutgoingContext(grpcClientCtx, grpc_metadata.Pairs("hostname", hostname))
// check if grpc server version is compatible with agent // check if grpc server version is compatible with agent
grpcServerVersion, err := client.Version(grpcCtx) //nolint:contextcheck grpcServerVersion, err := client.Version(grpcCtx) //nolint:contextcheck
@ -290,7 +291,7 @@ func run(ctx context.Context, c *cli.Command, backends []types.Backend) error {
} }
log.Debug().Msg("polling new steps") log.Debug().Msg("polling new steps")
if err := runner.Run(agentCtx, shutdownCtx); err != nil { if err := runner.Run(agentCtx, shutdownCtx, metadata); err != nil {
log.Error().Err(err).Msg("runner done with error") log.Error().Err(err).Msg("runner done with error")
return err return err
} }
@ -305,7 +306,7 @@ func run(ctx context.Context, c *cli.Command, backends []types.Backend) error {
return serviceWaitingGroup.Wait() return serviceWaitingGroup.Wait()
} }
func runWithRetry(backendEngines []types.Backend) func(ctx context.Context, c *cli.Command) error { func runWithRetry(backendEngines []types.Backend, metadata *metadata.Metadata) func(ctx context.Context, c *cli.Command) error {
return func(ctx context.Context, c *cli.Command) error { return func(ctx context.Context, c *cli.Command) error {
if err := logger.SetupGlobalLogger(ctx, c, true); err != nil { if err := logger.SetupGlobalLogger(ctx, c, true); err != nil {
return err return err
@ -317,7 +318,7 @@ func runWithRetry(backendEngines []types.Backend) func(ctx context.Context, c *c
retryDelay := c.Duration("connect-retry-delay") retryDelay := c.Duration("connect-retry-delay")
var err error var err error
for i := 0; i < retryCount; i++ { for i := 0; i < retryCount; i++ {
if err = run(ctx, c, backendEngines); status.Code(err) == codes.Unavailable { if err = run(ctx, c, backendEngines, metadata); status.Code(err) == codes.Unavailable {
log.Warn().Err(err).Msg(fmt.Sprintf("cannot connect to server, retrying in %v", retryDelay)) log.Warn().Err(err).Msg(fmt.Sprintf("cannot connect to server, retrying in %v", retryDelay))
time.Sleep(retryDelay) time.Sleep(retryDelay)
} else { } else {

5
cmd/agent/core/run.go
View file

@ -24,17 +24,18 @@ import (
"github.com/urfave/cli/v3" "github.com/urfave/cli/v3"
backend "go.woodpecker-ci.org/woodpecker/v2/pipeline/backend/types" backend "go.woodpecker-ci.org/woodpecker/v2/pipeline/backend/types"
"go.woodpecker-ci.org/woodpecker/v2/pipeline/frontend/metadata"
"go.woodpecker-ci.org/woodpecker/v2/shared/logger" "go.woodpecker-ci.org/woodpecker/v2/shared/logger"
"go.woodpecker-ci.org/woodpecker/v2/shared/utils" "go.woodpecker-ci.org/woodpecker/v2/shared/utils"
"go.woodpecker-ci.org/woodpecker/v2/version" "go.woodpecker-ci.org/woodpecker/v2/version"
) )
func RunAgent(ctx context.Context, backends []backend.Backend) { func RunAgent(ctx context.Context, backends []backend.Backend, metadata *metadata.Metadata) {
app := &cli.Command{} app := &cli.Command{}
app.Name = "woodpecker-agent" app.Name = "woodpecker-agent"
app.Version = version.String() app.Version = version.String()
app.Usage = "woodpecker agent" app.Usage = "woodpecker agent"
app.Action = runWithRetry(backends) app.Action = runWithRetry(backends, metadata)
app.Commands = []*cli.Command{ app.Commands = []*cli.Command{
{ {
Name: "ping", Name: "ping",

5
cmd/agent/main.go
View file

@ -24,6 +24,7 @@ import (
"go.woodpecker-ci.org/woodpecker/v2/pipeline/backend/kubernetes" "go.woodpecker-ci.org/woodpecker/v2/pipeline/backend/kubernetes"
"go.woodpecker-ci.org/woodpecker/v2/pipeline/backend/local" "go.woodpecker-ci.org/woodpecker/v2/pipeline/backend/local"
backendTypes "go.woodpecker-ci.org/woodpecker/v2/pipeline/backend/types" backendTypes "go.woodpecker-ci.org/woodpecker/v2/pipeline/backend/types"
"go.woodpecker-ci.org/woodpecker/v2/pipeline/frontend/metadata"
"go.woodpecker-ci.org/woodpecker/v2/shared/utils" "go.woodpecker-ci.org/woodpecker/v2/shared/utils"
) )
@ -33,9 +34,11 @@ var backends = []backendTypes.Backend{
local.New(), local.New(),
} }
var metadataList = &metadata.Metadata{}
func main() { func main() {
ctx := utils.WithContextSigtermCallback(context.Background(), func() { ctx := utils.WithContextSigtermCallback(context.Background(), func() {
log.Info().Msg("termination signal is received, shutting down agent") log.Info().Msg("termination signal is received, shutting down agent")
}) })
core.RunAgent(ctx, backends) core.RunAgent(ctx, backends, metadataList)
} }

11
pipeline/backend/docker/docker.go
View file

@ -36,6 +36,7 @@ import (
"github.com/urfave/cli/v3" "github.com/urfave/cli/v3"
backend "go.woodpecker-ci.org/woodpecker/v2/pipeline/backend/types" backend "go.woodpecker-ci.org/woodpecker/v2/pipeline/backend/types"
"go.woodpecker-ci.org/woodpecker/v2/pipeline/frontend/metadata"
"go.woodpecker-ci.org/woodpecker/v2/shared/utils" "go.woodpecker-ci.org/woodpecker/v2/shared/utils"
) )
@ -169,7 +170,7 @@ func (e *docker) SetupWorkflow(ctx context.Context, conf *backend.Config, taskUU
return nil return nil
} }
func (e *docker) StartStep(ctx context.Context, step *backend.Step, taskUUID string) error { func (e *docker) StartStep(ctx context.Context, step *backend.Step, taskUUID string, metadata *metadata.Metadata) error {
log.Trace().Str("taskUUID", taskUUID).Msgf("start step %s", step.Name) log.Trace().Str("taskUUID", taskUUID).Msgf("start step %s", step.Name)
config := e.toConfig(step) config := e.toConfig(step)
@ -247,7 +248,7 @@ func (e *docker) StartStep(ctx context.Context, step *backend.Step, taskUUID str
return e.client.ContainerStart(ctx, containerName, container.StartOptions{}) return e.client.ContainerStart(ctx, containerName, container.StartOptions{})
} }
func (e *docker) WaitStep(ctx context.Context, step *backend.Step, taskUUID string) (*backend.State, error) { func (e *docker) WaitStep(ctx context.Context, step *backend.Step, taskUUID string, metadata *metadata.Metadata) (*backend.State, error) {
log.Trace().Str("taskUUID", taskUUID).Msgf("wait for step %s", step.Name) log.Trace().Str("taskUUID", taskUUID).Msgf("wait for step %s", step.Name)
containerName := toContainerName(step) containerName := toContainerName(step)
@ -270,7 +271,7 @@ func (e *docker) WaitStep(ctx context.Context, step *backend.Step, taskUUID stri
}, nil }, nil
} }
func (e *docker) TailStep(ctx context.Context, step *backend.Step, taskUUID string) (io.ReadCloser, error) { func (e *docker) TailStep(ctx context.Context, step *backend.Step, taskUUID string, metadata *metadata.Metadata) (io.ReadCloser, error) {
log.Trace().Str("taskUUID", taskUUID).Msgf("tail logs of step %s", step.Name) log.Trace().Str("taskUUID", taskUUID).Msgf("tail logs of step %s", step.Name)
logs, err := e.client.ContainerLogs(ctx, toContainerName(step), container.LogsOptions{ logs, err := e.client.ContainerLogs(ctx, toContainerName(step), container.LogsOptions{
@ -294,7 +295,7 @@ func (e *docker) TailStep(ctx context.Context, step *backend.Step, taskUUID stri
return rc, nil return rc, nil
} }
func (e *docker) DestroyStep(ctx context.Context, step *backend.Step, taskUUID string) error { func (e *docker) DestroyStep(ctx context.Context, step *backend.Step, taskUUID string, metadata *metadata.Metadata) error {
log.Trace().Str("taskUUID", taskUUID).Msgf("stop step %s", step.Name) log.Trace().Str("taskUUID", taskUUID).Msgf("stop step %s", step.Name)
containerName := toContainerName(step) containerName := toContainerName(step)
@ -310,7 +311,7 @@ func (e *docker) DestroyStep(ctx context.Context, step *backend.Step, taskUUID s
return nil return nil
} }
func (e *docker) DestroyWorkflow(ctx context.Context, conf *backend.Config, taskUUID string) error { func (e *docker) DestroyWorkflow(ctx context.Context, conf *backend.Config, taskUUID string, metadata *metadata.Metadata) error {
log.Trace().Str("taskUUID", taskUUID).Msgf("delete workflow environment") log.Trace().Str("taskUUID", taskUUID).Msgf("delete workflow environment")
for _, stage := range conf.Stages { for _, stage := range conf.Stages {

11
pipeline/backend/dummy/dummy.go
View file

@ -30,6 +30,7 @@ import (
"github.com/urfave/cli/v3" "github.com/urfave/cli/v3"
backend "go.woodpecker-ci.org/woodpecker/v2/pipeline/backend/types" backend "go.woodpecker-ci.org/woodpecker/v2/pipeline/backend/types"
"go.woodpecker-ci.org/woodpecker/v2/pipeline/frontend/metadata"
) )
type dummy struct { type dummy struct {
@ -87,7 +88,7 @@ func (e *dummy) SetupWorkflow(_ context.Context, _ *backend.Config, taskUUID str
return nil return nil
} }
func (e *dummy) StartStep(_ context.Context, step *backend.Step, taskUUID string) error { func (e *dummy) StartStep(_ context.Context, step *backend.Step, taskUUID string, metadata *metadata.Metadata) error {
log.Trace().Str("taskUUID", taskUUID).Msgf("start step %s", step.Name) log.Trace().Str("taskUUID", taskUUID).Msgf("start step %s", step.Name)
// internal state checks // internal state checks
@ -114,7 +115,7 @@ func (e *dummy) StartStep(_ context.Context, step *backend.Step, taskUUID string
return nil return nil
} }
func (e *dummy) WaitStep(ctx context.Context, step *backend.Step, taskUUID string) (*backend.State, error) { func (e *dummy) WaitStep(ctx context.Context, step *backend.Step, taskUUID string, metadata *metadata.Metadata) (*backend.State, error) {
log.Trace().Str("taskUUID", taskUUID).Msgf("wait for step %s", step.Name) log.Trace().Str("taskUUID", taskUUID).Msgf("wait for step %s", step.Name)
_, exist := e.kv.Load("task_" + taskUUID) _, exist := e.kv.Load("task_" + taskUUID)
@ -172,7 +173,7 @@ func (e *dummy) WaitStep(ctx context.Context, step *backend.Step, taskUUID strin
}, nil }, nil
} }
func (e *dummy) TailStep(_ context.Context, step *backend.Step, taskUUID string) (io.ReadCloser, error) { func (e *dummy) TailStep(_ context.Context, step *backend.Step, taskUUID string, metadata *metadata.Metadata) (io.ReadCloser, error) {
log.Trace().Str("taskUUID", taskUUID).Msgf("tail logs of step %s", step.Name) log.Trace().Str("taskUUID", taskUUID).Msgf("tail logs of step %s", step.Name)
_, exist := e.kv.Load("task_" + taskUUID) _, exist := e.kv.Load("task_" + taskUUID)
@ -196,7 +197,7 @@ func (e *dummy) TailStep(_ context.Context, step *backend.Step, taskUUID string)
return io.NopCloser(strings.NewReader(dummyExecStepOutput(step))), nil return io.NopCloser(strings.NewReader(dummyExecStepOutput(step))), nil
} }
func (e *dummy) DestroyStep(_ context.Context, step *backend.Step, taskUUID string) error { func (e *dummy) DestroyStep(_ context.Context, step *backend.Step, taskUUID string, metadata *metadata.Metadata) error {
log.Trace().Str("taskUUID", taskUUID).Msgf("stop step %s", step.Name) log.Trace().Str("taskUUID", taskUUID).Msgf("stop step %s", step.Name)
_, exist := e.kv.Load("task_" + taskUUID) _, exist := e.kv.Load("task_" + taskUUID)
@ -217,7 +218,7 @@ func (e *dummy) DestroyStep(_ context.Context, step *backend.Step, taskUUID stri
return nil return nil
} }
func (e *dummy) DestroyWorkflow(_ context.Context, _ *backend.Config, taskUUID string) error { func (e *dummy) DestroyWorkflow(_ context.Context, _ *backend.Config, taskUUID string, metadata *metadata.Metadata) error {
log.Trace().Str("taskUUID", taskUUID).Msgf("delete workflow environment") log.Trace().Str("taskUUID", taskUUID).Msgf("delete workflow environment")
_, exist := e.kv.Load("task_" + taskUUID) _, exist := e.kv.Load("task_" + taskUUID)

54
pipeline/backend/dummy/dummy_test.go
View file

@ -23,6 +23,7 @@ import (
"go.woodpecker-ci.org/woodpecker/v2/pipeline/backend/dummy" "go.woodpecker-ci.org/woodpecker/v2/pipeline/backend/dummy"
"go.woodpecker-ci.org/woodpecker/v2/pipeline/backend/types" "go.woodpecker-ci.org/woodpecker/v2/pipeline/backend/types"
"go.woodpecker-ci.org/woodpecker/v2/pipeline/frontend/metadata"
) )
func TestSmalPipelineDummyRun(t *testing.T) { func TestSmalPipelineDummyRun(t *testing.T) {
@ -39,17 +40,18 @@ func TestSmalPipelineDummyRun(t *testing.T) {
t.Run("expect fail of step func with non setup workflow", func(t *testing.T) { t.Run("expect fail of step func with non setup workflow", func(t *testing.T) {
step := &types.Step{Name: "step1", UUID: "SID_1"} step := &types.Step{Name: "step1", UUID: "SID_1"}
nonExistWorkflowID := "WID_NONE" nonExistWorkflowID := "WID_NONE"
metadata := &metadata.Metadata{}
err := dummyEngine.StartStep(ctx, step, nonExistWorkflowID) err := dummyEngine.StartStep(ctx, step, nonExistWorkflowID, metadata)
assert.Error(t, err) assert.Error(t, err)
_, err = dummyEngine.TailStep(ctx, step, nonExistWorkflowID) _, err = dummyEngine.TailStep(ctx, step, nonExistWorkflowID, metadata)
assert.Error(t, err) assert.Error(t, err)
_, err = dummyEngine.WaitStep(ctx, step, nonExistWorkflowID) _, err = dummyEngine.WaitStep(ctx, step, nonExistWorkflowID, metadata)
assert.Error(t, err) assert.Error(t, err)
err = dummyEngine.DestroyStep(ctx, step, nonExistWorkflowID) err = dummyEngine.DestroyStep(ctx, step, nonExistWorkflowID, metadata)
assert.Error(t, err) assert.Error(t, err)
}) })
@ -61,13 +63,14 @@ func TestSmalPipelineDummyRun(t *testing.T) {
Environment: map[string]string{}, Environment: map[string]string{},
Commands: []string{"echo ja", "echo nein"}, Commands: []string{"echo ja", "echo nein"},
} }
metadata := &metadata.Metadata{}
workflowUUID := "WID_1" workflowUUID := "WID_1"
assert.NoError(t, dummyEngine.SetupWorkflow(ctx, nil, workflowUUID)) assert.NoError(t, dummyEngine.SetupWorkflow(ctx, nil, workflowUUID))
assert.NoError(t, dummyEngine.StartStep(ctx, step, workflowUUID)) assert.NoError(t, dummyEngine.StartStep(ctx, step, workflowUUID, metadata))
reader, err := dummyEngine.TailStep(ctx, step, workflowUUID) reader, err := dummyEngine.TailStep(ctx, step, workflowUUID, metadata)
assert.NoError(t, err) assert.NoError(t, err)
log, err := io.ReadAll(reader) log, err := io.ReadAll(reader)
assert.NoError(t, err) assert.NoError(t, err)
@ -81,14 +84,14 @@ echo nein
------------------ ------------------
`, string(log)) `, string(log))
state, err := dummyEngine.WaitStep(ctx, step, workflowUUID) state, err := dummyEngine.WaitStep(ctx, step, workflowUUID, metadata)
assert.NoError(t, err) assert.NoError(t, err)
assert.NoError(t, state.Error) assert.NoError(t, state.Error)
assert.EqualValues(t, 0, state.ExitCode) assert.EqualValues(t, 0, state.ExitCode)
assert.NoError(t, dummyEngine.DestroyStep(ctx, step, workflowUUID)) assert.NoError(t, dummyEngine.DestroyStep(ctx, step, workflowUUID, metadata))
assert.NoError(t, dummyEngine.DestroyWorkflow(ctx, nil, workflowUUID)) assert.NoError(t, dummyEngine.DestroyWorkflow(ctx, nil, workflowUUID, metadata))
}) })
t.Run("step exec error", func(t *testing.T) { t.Run("step exec error", func(t *testing.T) {
@ -98,23 +101,24 @@ echo nein
Type: types.StepTypePlugin, Type: types.StepTypePlugin,
Environment: map[string]string{dummy.EnvKeyStepType: "plugin", dummy.EnvKeyStepExitCode: "1"}, Environment: map[string]string{dummy.EnvKeyStepType: "plugin", dummy.EnvKeyStepExitCode: "1"},
} }
metadata := &metadata.Metadata{}
workflowUUID := "WID_1" workflowUUID := "WID_1"
assert.NoError(t, dummyEngine.SetupWorkflow(ctx, nil, workflowUUID)) assert.NoError(t, dummyEngine.SetupWorkflow(ctx, nil, workflowUUID))
assert.NoError(t, dummyEngine.StartStep(ctx, step, workflowUUID)) assert.NoError(t, dummyEngine.StartStep(ctx, step, workflowUUID, metadata))
_, err := dummyEngine.TailStep(ctx, step, workflowUUID) _, err := dummyEngine.TailStep(ctx, step, workflowUUID, metadata)
assert.NoError(t, err) assert.NoError(t, err)
state, err := dummyEngine.WaitStep(ctx, step, workflowUUID) state, err := dummyEngine.WaitStep(ctx, step, workflowUUID, metadata)
assert.NoError(t, err) assert.NoError(t, err)
assert.NoError(t, state.Error) assert.NoError(t, state.Error)
assert.EqualValues(t, 1, state.ExitCode) assert.EqualValues(t, 1, state.ExitCode)
assert.NoError(t, dummyEngine.DestroyStep(ctx, step, workflowUUID)) assert.NoError(t, dummyEngine.DestroyStep(ctx, step, workflowUUID, metadata))
assert.NoError(t, dummyEngine.DestroyWorkflow(ctx, nil, workflowUUID)) assert.NoError(t, dummyEngine.DestroyWorkflow(ctx, nil, workflowUUID, metadata))
}) })
t.Run("step tail error", func(t *testing.T) { t.Run("step tail error", func(t *testing.T) {
@ -123,21 +127,22 @@ echo nein
UUID: "SID_2", UUID: "SID_2",
Environment: map[string]string{dummy.EnvKeyStepTailFail: "true"}, Environment: map[string]string{dummy.EnvKeyStepTailFail: "true"},
} }
metadata := &metadata.Metadata{}
workflowUUID := "WID_1" workflowUUID := "WID_1"
assert.NoError(t, dummyEngine.SetupWorkflow(ctx, nil, workflowUUID)) assert.NoError(t, dummyEngine.SetupWorkflow(ctx, nil, workflowUUID))
assert.NoError(t, dummyEngine.StartStep(ctx, step, workflowUUID)) assert.NoError(t, dummyEngine.StartStep(ctx, step, workflowUUID, metadata))
_, err := dummyEngine.TailStep(ctx, step, workflowUUID) _, err := dummyEngine.TailStep(ctx, step, workflowUUID, metadata)
assert.Error(t, err) assert.Error(t, err)
_, err = dummyEngine.WaitStep(ctx, step, workflowUUID) _, err = dummyEngine.WaitStep(ctx, step, workflowUUID, metadata)
assert.NoError(t, err) assert.NoError(t, err)
assert.NoError(t, dummyEngine.DestroyStep(ctx, step, workflowUUID)) assert.NoError(t, dummyEngine.DestroyStep(ctx, step, workflowUUID, metadata))
assert.NoError(t, dummyEngine.DestroyWorkflow(ctx, nil, workflowUUID)) assert.NoError(t, dummyEngine.DestroyWorkflow(ctx, nil, workflowUUID, metadata))
}) })
t.Run("step start fail", func(t *testing.T) { t.Run("step start fail", func(t *testing.T) {
@ -147,22 +152,23 @@ echo nein
Type: types.StepTypeService, Type: types.StepTypeService,
Environment: map[string]string{dummy.EnvKeyStepType: "service", dummy.EnvKeyStepStartFail: "true"}, Environment: map[string]string{dummy.EnvKeyStepType: "service", dummy.EnvKeyStepStartFail: "true"},
} }
metadata := &metadata.Metadata{}
workflowUUID := "WID_1" workflowUUID := "WID_1"
assert.NoError(t, dummyEngine.SetupWorkflow(ctx, nil, workflowUUID)) assert.NoError(t, dummyEngine.SetupWorkflow(ctx, nil, workflowUUID))
assert.Error(t, dummyEngine.StartStep(ctx, step, workflowUUID)) assert.Error(t, dummyEngine.StartStep(ctx, step, workflowUUID, metadata))
_, err := dummyEngine.TailStep(ctx, step, workflowUUID) _, err := dummyEngine.TailStep(ctx, step, workflowUUID, metadata)
assert.Error(t, err) assert.Error(t, err)
state, err := dummyEngine.WaitStep(ctx, step, workflowUUID) state, err := dummyEngine.WaitStep(ctx, step, workflowUUID, metadata)
assert.Error(t, err) assert.Error(t, err)
assert.Error(t, state.Error) assert.Error(t, state.Error)
assert.EqualValues(t, 0, state.ExitCode) assert.EqualValues(t, 0, state.ExitCode)
assert.Error(t, dummyEngine.DestroyStep(ctx, step, workflowUUID)) assert.Error(t, dummyEngine.DestroyStep(ctx, step, workflowUUID, metadata))
assert.NoError(t, dummyEngine.DestroyWorkflow(ctx, nil, workflowUUID)) assert.NoError(t, dummyEngine.DestroyWorkflow(ctx, nil, workflowUUID, metadata))
}) })
} }

25
pipeline/backend/kubernetes/kubernetes.go
View file

@ -38,6 +38,7 @@ import (
"k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/cache"
"go.woodpecker-ci.org/woodpecker/v2/pipeline/backend/types" "go.woodpecker-ci.org/woodpecker/v2/pipeline/backend/types"
"go.woodpecker-ci.org/woodpecker/v2/pipeline/frontend/metadata"
) )
const ( const (
@ -220,28 +221,28 @@ func (e *kube) SetupWorkflow(ctx context.Context, conf *types.Config, taskUUID s
} }
// StartStep starts the pipeline step. // StartStep starts the pipeline step.
func (e *kube) StartStep(ctx context.Context, step *types.Step, taskUUID string) error { func (e *kube) StartStep(ctx context.Context, step *types.Step, taskUUID string, metadata *metadata.Metadata) error {
options, err := parseBackendOptions(step) options, err := parseBackendOptions(step)
if err != nil { if err != nil {
log.Error().Err(err).Msg("could not parse backend options") log.Error().Err(err).Msg("could not parse backend options")
} }
if needsRegistrySecret(step) { if needsRegistrySecret(step) {
err = startRegistrySecret(ctx, e, step) err = startRegistrySecret(ctx, e, step, metadata)
if err != nil { if err != nil {
return err return err
} }
} }
log.Trace().Str("taskUUID", taskUUID).Msgf("starting step: %s", step.Name) log.Trace().Str("taskUUID", taskUUID).Msgf("starting step: %s", step.Name)
_, err = startPod(ctx, e, step, options) _, err = startPod(ctx, e, step, options, metadata)
return err return err
} }
// WaitStep waits for the pipeline step to complete and returns // WaitStep waits for the pipeline step to complete and returns
// the completion results. // the completion results.
func (e *kube) WaitStep(ctx context.Context, step *types.Step, taskUUID string) (*types.State, error) { func (e *kube) WaitStep(ctx context.Context, step *types.Step, taskUUID string, metadata *metadata.Metadata) (*types.State, error) {
podName, err := stepToPodName(step) podName, err := stepToPodName(step, metadata)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -316,8 +317,8 @@ func (e *kube) WaitStep(ctx context.Context, step *types.Step, taskUUID string)
} }
// TailStep tails the pipeline step logs. // TailStep tails the pipeline step logs.
func (e *kube) TailStep(ctx context.Context, step *types.Step, taskUUID string) (io.ReadCloser, error) { func (e *kube) TailStep(ctx context.Context, step *types.Step, taskUUID string, metadata *metadata.Metadata) (io.ReadCloser, error) {
podName, err := stepToPodName(step) podName, err := stepToPodName(step, metadata)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -389,17 +390,17 @@ func (e *kube) TailStep(ctx context.Context, step *types.Step, taskUUID string)
return rc, nil return rc, nil
} }
func (e *kube) DestroyStep(ctx context.Context, step *types.Step, taskUUID string) error { func (e *kube) DestroyStep(ctx context.Context, step *types.Step, taskUUID string, metadata *metadata.Metadata) error {
var errs []error var errs []error
log.Trace().Str("taskUUID", taskUUID).Msgf("Stopping step: %s", step.Name) log.Trace().Str("taskUUID", taskUUID).Msgf("Stopping step: %s", step.Name)
if needsRegistrySecret(step) { if needsRegistrySecret(step) {
err := stopRegistrySecret(ctx, e, step, defaultDeleteOptions) err := stopRegistrySecret(ctx, e, step, defaultDeleteOptions, metadata)
if err != nil { if err != nil {
errs = append(errs, err) errs = append(errs, err)
} }
} }
err := stopPod(ctx, e, step, defaultDeleteOptions) err := stopPod(ctx, e, step, defaultDeleteOptions, metadata)
if err != nil { if err != nil {
errs = append(errs, err) errs = append(errs, err)
} }
@ -407,12 +408,12 @@ func (e *kube) DestroyStep(ctx context.Context, step *types.Step, taskUUID strin
} }
// DestroyWorkflow destroys the pipeline environment. // DestroyWorkflow destroys the pipeline environment.
func (e *kube) DestroyWorkflow(ctx context.Context, conf *types.Config, taskUUID string) error { func (e *kube) DestroyWorkflow(ctx context.Context, conf *types.Config, taskUUID string, metadata *metadata.Metadata) error {
log.Trace().Str("taskUUID", taskUUID).Msg("deleting Kubernetes primitives") log.Trace().Str("taskUUID", taskUUID).Msg("deleting Kubernetes primitives")
for _, stage := range conf.Stages { for _, stage := range conf.Stages {
for _, step := range stage.Steps { for _, step := range stage.Steps {
err := stopPod(ctx, e, step, defaultDeleteOptions) err := stopPod(ctx, e, step, defaultDeleteOptions, metadata)
if err != nil { if err != nil {
return err return err
} }

27
pipeline/backend/kubernetes/pod.go
View file

@ -28,6 +28,7 @@ import (
"go.woodpecker-ci.org/woodpecker/v2/pipeline/backend/common" "go.woodpecker-ci.org/woodpecker/v2/pipeline/backend/common"
"go.woodpecker-ci.org/woodpecker/v2/pipeline/backend/types" "go.woodpecker-ci.org/woodpecker/v2/pipeline/backend/types"
"go.woodpecker-ci.org/woodpecker/v2/pipeline/frontend/metadata"
) )
const ( const (
@ -35,7 +36,7 @@ const (
podPrefix = "wp-" podPrefix = "wp-"
) )
func mkPod(step *types.Step, config *config, podName, goos string, options BackendOptions) (*v1.Pod, error) { func mkPod(step *types.Step, config *config, podName, goos string, options BackendOptions, metadata *metadata.Metadata) (*v1.Pod, error) {
var err error var err error
nsp := newNativeSecretsProcessor(config, options.Secrets) nsp := newNativeSecretsProcessor(config, options.Secrets)
@ -49,7 +50,7 @@ func mkPod(step *types.Step, config *config, podName, goos string, options Backe
return nil, err return nil, err
} }
spec, err := podSpec(step, config, options, nsp) spec, err := podSpec(step, config, options, nsp, metadata)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -68,15 +69,15 @@ func mkPod(step *types.Step, config *config, podName, goos string, options Backe
return pod, nil return pod, nil
} }
func stepToPodName(step *types.Step) (name string, err error) { func stepToPodName(step *types.Step, metadata *metadata.Metadata) (name string, err error) {
if step.Type == types.StepTypeService { if step.Type == types.StepTypeService {
return serviceName(step) return serviceName(step)
} }
return podName(step) return podName(step, metadata)
} }
func podName(step *types.Step) (string, error) { func podName(step *types.Step, metadata *metadata.Metadata) (string, error) {
return dnsName(podPrefix + step.UUID) return dnsName(strings.ReplaceAll(podPrefix+metadata.Repo.Owner+"-"+metadata.Repo.Name+"-"+metadata.Workflow.Name+"-"+step.Name+"-"+step.UUID[:5], "_", "-"))
} }
func podMeta(step *types.Step, config *config, options BackendOptions, podName string) (meta_v1.ObjectMeta, error) { func podMeta(step *types.Step, config *config, options BackendOptions, podName string) (meta_v1.ObjectMeta, error) {
@ -145,7 +146,7 @@ func podAnnotations(config *config, options BackendOptions) map[string]string {
return annotations return annotations
} }
func podSpec(step *types.Step, config *config, options BackendOptions, nsp nativeSecretsProcessor) (v1.PodSpec, error) { func podSpec(step *types.Step, config *config, options BackendOptions, nsp nativeSecretsProcessor, metadata *metadata.Metadata) (v1.PodSpec, error) {
var err error var err error
spec := v1.PodSpec{ spec := v1.PodSpec{
RestartPolicy: v1.RestartPolicyNever, RestartPolicy: v1.RestartPolicyNever,
@ -165,7 +166,7 @@ func podSpec(step *types.Step, config *config, options BackendOptions, nsp nativ
spec.ImagePullSecrets = secretsReferences(config.ImagePullSecretNames) spec.ImagePullSecrets = secretsReferences(config.ImagePullSecretNames)
if needsRegistrySecret(step) { if needsRegistrySecret(step) {
log.Trace().Msgf("using an image pull secret from registries") log.Trace().Msgf("using an image pull secret from registries")
name, err := registrySecretName(step) name, err := registrySecretName(step, metadata)
if err != nil { if err != nil {
return spec, err return spec, err
} }
@ -502,13 +503,13 @@ func mapToEnvVars(m map[string]string) []v1.EnvVar {
return ev return ev
} }
func startPod(ctx context.Context, engine *kube, step *types.Step, options BackendOptions) (*v1.Pod, error) { func startPod(ctx context.Context, engine *kube, step *types.Step, options BackendOptions, metadata *metadata.Metadata) (*v1.Pod, error) {
podName, err := stepToPodName(step) podName, err := stepToPodName(step, metadata)
if err != nil { if err != nil {
return nil, err return nil, err
} }
engineConfig := engine.getConfig() engineConfig := engine.getConfig()
pod, err := mkPod(step, engineConfig, podName, engine.goos, options) pod, err := mkPod(step, engineConfig, podName, engine.goos, options, metadata)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -517,8 +518,8 @@ func startPod(ctx context.Context, engine *kube, step *types.Step, options Backe
return engine.client.CoreV1().Pods(engineConfig.Namespace).Create(ctx, pod, meta_v1.CreateOptions{}) return engine.client.CoreV1().Pods(engineConfig.Namespace).Create(ctx, pod, meta_v1.CreateOptions{})
} }
func stopPod(ctx context.Context, engine *kube, step *types.Step, deleteOpts meta_v1.DeleteOptions) error { func stopPod(ctx context.Context, engine *kube, step *types.Step, deleteOpts meta_v1.DeleteOptions, metadata *metadata.Metadata) error {
podName, err := stepToPodName(step) podName, err := stepToPodName(step, metadata)
if err != nil { if err != nil {
return err return err
} }

45
pipeline/backend/kubernetes/pod_test.go
View file

@ -23,34 +23,35 @@ import (
v1 "k8s.io/api/core/v1" v1 "k8s.io/api/core/v1"
"go.woodpecker-ci.org/woodpecker/v2/pipeline/backend/types" "go.woodpecker-ci.org/woodpecker/v2/pipeline/backend/types"
"go.woodpecker-ci.org/woodpecker/v2/pipeline/frontend/metadata"
) )
func TestPodName(t *testing.T) { func TestPodName(t *testing.T) {
name, err := podName(&types.Step{UUID: "01he8bebctabr3kgk0qj36d2me-0"}) name, err := podName(&types.Step{UUID: "01he8bebctabr3kgk0qj36d2me-0", Name: "stepName"}, &metadata.Metadata{Workflow: metadata.Workflow{Name: "my-workflow"}, Repo: metadata.Repo{Owner: "owner", Name: "repo_name"}})
assert.NoError(t, err) assert.NoError(t, err)
assert.Equal(t, "wp-01he8bebctabr3kgk0qj36d2me-0", name) assert.Equal(t, "wp-owner-repo-name-my-workflow-stepname-01he8", name)
_, err = podName(&types.Step{UUID: "01he8bebctabr3kgk0qj36d2me\\0a"}) _, err = podName(&types.Step{UUID: "01he8bebctabr3kgk0qj36d2me\\0a", Name: ".."}, &metadata.Metadata{Workflow: metadata.Workflow{Name: "my-workflow"}, Repo: metadata.Repo{Owner: "owner", Name: "repo_name!"}})
assert.ErrorIs(t, err, ErrDNSPatternInvalid) assert.ErrorIs(t, err, ErrDNSPatternInvalid)
_, err = podName(&types.Step{UUID: "01he8bebctabr3kgk0qj36d2me-0-services-0..woodpecker-runtime.svc.cluster.local"}) _, err = podName(&types.Step{UUID: "01he8bebctabr3kgk0qj36d2me-0-services-0..woodpecker-runtime.svc.cluster.local"}, &metadata.Metadata{Workflow: metadata.Workflow{Name: "my-workflow"}, Repo: metadata.Repo{Owner: "owner", Name: "repo_name.."}})
assert.ErrorIs(t, err, ErrDNSPatternInvalid) assert.ErrorIs(t, err, ErrDNSPatternInvalid)
} }
func TestStepToPodName(t *testing.T) { func TestStepToPodName(t *testing.T) {
name, err := stepToPodName(&types.Step{UUID: "01he8bebctabr3kg", Name: "clone", Type: types.StepTypeClone}) name, err := stepToPodName(&types.Step{UUID: "01he8bebctabr3kg", Name: "clone", Type: types.StepTypeClone}, &metadata.Metadata{Workflow: metadata.Workflow{Name: "my-workflow"}, Repo: metadata.Repo{Owner: "owner", Name: "repo_name"}})
assert.NoError(t, err) assert.NoError(t, err)
assert.EqualValues(t, "wp-01he8bebctabr3kg", name) assert.EqualValues(t, "wp-owner-repo-name-my-workflow-clone-01he8", name)
name, err = stepToPodName(&types.Step{UUID: "01he8bebctabr3kg", Name: "cache", Type: types.StepTypeCache}) name, err = stepToPodName(&types.Step{UUID: "01he8bebctabr3kg", Name: "cache", Type: types.StepTypeCache}, &metadata.Metadata{Workflow: metadata.Workflow{Name: "my-workflow"}, Repo: metadata.Repo{Owner: "owner", Name: "repo_name"}})
assert.NoError(t, err) assert.NoError(t, err)
assert.EqualValues(t, "wp-01he8bebctabr3kg", name) assert.EqualValues(t, "wp-owner-repo-name-my-workflow-cache-01he8", name)
name, err = stepToPodName(&types.Step{UUID: "01he8bebctabr3kg", Name: "release", Type: types.StepTypePlugin}) name, err = stepToPodName(&types.Step{UUID: "01he8bebctabr3kg", Name: "release", Type: types.StepTypePlugin}, &metadata.Metadata{Workflow: metadata.Workflow{Name: "my-workflow"}, Repo: metadata.Repo{Owner: "owner", Name: "repo_name"}})
assert.NoError(t, err) assert.NoError(t, err)
assert.EqualValues(t, "wp-01he8bebctabr3kg", name) assert.EqualValues(t, "wp-owner-repo-name-my-workflow-release-01he8", name)
name, err = stepToPodName(&types.Step{UUID: "01he8bebctabr3kg", Name: "prepare-env", Type: types.StepTypeCommands}) name, err = stepToPodName(&types.Step{UUID: "01he8bebctabr3kg", Name: "prepare-env", Type: types.StepTypeCommands}, &metadata.Metadata{Workflow: metadata.Workflow{Name: "my-workflow"}, Repo: metadata.Repo{Owner: "owner", Name: "repo_name"}})
assert.NoError(t, err) assert.NoError(t, err)
assert.EqualValues(t, "wp-01he8bebctabr3kg", name) assert.EqualValues(t, "wp-owner-repo-name-my-workflow-prepare-env-01he8", name)
name, err = stepToPodName(&types.Step{UUID: "01he8bebctabr3kg", Name: "postgres", Type: types.StepTypeService}) name, err = stepToPodName(&types.Step{UUID: "01he8bebctabr3kg", Name: "postgres", Type: types.StepTypeService}, &metadata.Metadata{Workflow: metadata.Workflow{Name: "workflow-postgres"}, Repo: metadata.Repo{Owner: "owner", Name: "repo_name"}})
assert.NoError(t, err) assert.NoError(t, err)
assert.EqualValues(t, "wp-svc-01he8bebctabr3kg-postgres", name) assert.EqualValues(t, "wp-svc-01he8bebctabr3kg-postgres", name)
} }
@ -138,7 +139,7 @@ func TestTinyPod(t *testing.T) {
Environment: map[string]string{"CI": "woodpecker"}, Environment: map[string]string{"CI": "woodpecker"},
}, &config{ }, &config{
Namespace: "woodpecker", Namespace: "woodpecker",
}, "wp-01he8bebctabr3kgk0qj36d2me-0", "linux/amd64", BackendOptions{}) }, "wp-01he8bebctabr3kgk0qj36d2me-0", "linux/amd64", BackendOptions{}, &metadata.Metadata{Workflow: metadata.Workflow{Name: "my-workflow"}})
assert.NoError(t, err) assert.NoError(t, err)
podJSON, err := json.Marshal(pod) podJSON, err := json.Marshal(pod)
@ -152,7 +153,7 @@ func TestFullPod(t *testing.T) {
const expected = ` const expected = `
{ {
"metadata": { "metadata": {
"name": "wp-01he8bebctabr3kgk0qj36d2me-0", "name": "wp-owner-repo-name-my-workflow-go-test-01he8",
"namespace": "woodpecker", "namespace": "woodpecker",
"creationTimestamp": null, "creationTimestamp": null,
"labels": { "labels": {
@ -176,7 +177,7 @@ func TestFullPod(t *testing.T) {
], ],
"containers": [ "containers": [
{ {
"name": "wp-01he8bebctabr3kgk0qj36d2me-0", "name": "wp-owner-repo-name-my-workflow-go-test-01he8",
"image": "meltwater/drone-cache", "image": "meltwater/drone-cache",
"command": [ "command": [
"/bin/sh", "/bin/sh",
@ -266,7 +267,7 @@ func TestFullPod(t *testing.T) {
"name": "another-pull-secret" "name": "another-pull-secret"
}, },
{ {
"name": "wp-01he8bebctabr3kgk0qj36d2me-0" "name": "wp-owner-repo-name-my-workflow-go-test-01he8"
} }
], ],
"tolerations": [ "tolerations": [
@ -345,7 +346,7 @@ func TestFullPod(t *testing.T) {
PodAnnotationsAllowFromStep: true, PodAnnotationsAllowFromStep: true,
PodNodeSelector: map[string]string{"topology.kubernetes.io/region": "eu-central-1"}, PodNodeSelector: map[string]string{"topology.kubernetes.io/region": "eu-central-1"},
SecurityContext: SecurityContextConfig{RunAsNonRoot: false}, SecurityContext: SecurityContextConfig{RunAsNonRoot: false},
}, "wp-01he8bebctabr3kgk0qj36d2me-0", "linux/amd64", BackendOptions{ }, "wp-owner-repo-name-my-workflow-go-test-01he8", "linux/amd64", BackendOptions{
Labels: map[string]string{"part-of": "woodpecker-ci"}, Labels: map[string]string{"part-of": "woodpecker-ci"},
Annotations: map[string]string{"kubernetes.io/limit-ranger": "LimitRanger plugin set: cpu, memory request and limit for container"}, Annotations: map[string]string{"kubernetes.io/limit-ranger": "LimitRanger plugin set: cpu, memory request and limit for container"},
NodeSelector: map[string]string{"storage": "ssd"}, NodeSelector: map[string]string{"storage": "ssd"},
@ -357,7 +358,7 @@ func TestFullPod(t *testing.T) {
Limits: map[string]string{"memory": "256Mi", "cpu": "2"}, Limits: map[string]string{"memory": "256Mi", "cpu": "2"},
}, },
SecurityContext: &secCtx, SecurityContext: &secCtx,
}) }, &metadata.Metadata{Workflow: metadata.Workflow{Name: "my-workflow"}, Repo: metadata.Repo{Owner: "owner", Name: "repo_name"}})
assert.NoError(t, err) assert.NoError(t, err)
podJSON, err := json.Marshal(pod) podJSON, err := json.Marshal(pod)
@ -378,7 +379,7 @@ func TestPodPrivilege(t *testing.T) {
SecurityContext: SecurityContextConfig{RunAsNonRoot: globalRunAsRoot}, SecurityContext: SecurityContextConfig{RunAsNonRoot: globalRunAsRoot},
}, "wp-01he8bebctabr3kgk0qj36d2me-0", "linux/amd64", BackendOptions{ }, "wp-01he8bebctabr3kgk0qj36d2me-0", "linux/amd64", BackendOptions{
SecurityContext: &secCtx, SecurityContext: &secCtx,
}) }, &metadata.Metadata{Workflow: metadata.Workflow{Name: "my-workflow"}})
} }
// securty context is requesting user and group 101 (non-root) // securty context is requesting user and group 101 (non-root)
@ -471,7 +472,7 @@ func TestScratchPod(t *testing.T) {
Entrypoint: []string{"/usr/bin/curl", "-v", "google.com"}, Entrypoint: []string{"/usr/bin/curl", "-v", "google.com"},
}, &config{ }, &config{
Namespace: "woodpecker", Namespace: "woodpecker",
}, "wp-01he8bebctabr3kgk0qj36d2me-0", "linux/amd64", BackendOptions{}) }, "wp-01he8bebctabr3kgk0qj36d2me-0", "linux/amd64", BackendOptions{}, &metadata.Metadata{Workflow: metadata.Workflow{Name: "my-workflow"}})
assert.NoError(t, err) assert.NoError(t, err)
podJSON, err := json.Marshal(pod) podJSON, err := json.Marshal(pod)
@ -590,7 +591,7 @@ func TestSecrets(t *testing.T) {
Target: SecretTarget{File: "~/.docker/config.json"}, Target: SecretTarget{File: "~/.docker/config.json"},
}, },
}, },
}) }, &metadata.Metadata{Workflow: metadata.Workflow{Name: "my-workflow"}})
assert.NoError(t, err) assert.NoError(t, err)
podJSON, err := json.Marshal(pod) podJSON, err := json.Marshal(pod)

17
pipeline/backend/kubernetes/secrets.go
View file

@ -29,6 +29,7 @@ import (
meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"go.woodpecker-ci.org/woodpecker/v2/pipeline/backend/types" "go.woodpecker-ci.org/woodpecker/v2/pipeline/backend/types"
"go.woodpecker-ci.org/woodpecker/v2/pipeline/frontend/metadata"
"go.woodpecker-ci.org/woodpecker/v2/pipeline/frontend/yaml/utils" "go.woodpecker-ci.org/woodpecker/v2/pipeline/frontend/yaml/utils"
) )
@ -204,8 +205,8 @@ func needsRegistrySecret(step *types.Step) bool {
return step.AuthConfig.Username != "" && step.AuthConfig.Password != "" return step.AuthConfig.Username != "" && step.AuthConfig.Password != ""
} }
func mkRegistrySecret(step *types.Step, config *config) (*v1.Secret, error) { func mkRegistrySecret(step *types.Step, config *config, metadata *metadata.Metadata) (*v1.Secret, error) {
name, err := registrySecretName(step) name, err := registrySecretName(step, metadata)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -247,8 +248,8 @@ func mkRegistrySecret(step *types.Step, config *config) (*v1.Secret, error) {
}, nil }, nil
} }
func registrySecretName(step *types.Step) (string, error) { func registrySecretName(step *types.Step, metadata *metadata.Metadata) (string, error) {
return podName(step) return podName(step, metadata)
} }
func registrySecretLabels(step *types.Step) (map[string]string, error) { func registrySecretLabels(step *types.Step) (map[string]string, error) {
@ -266,8 +267,8 @@ func registrySecretLabels(step *types.Step) (map[string]string, error) {
return labels, nil return labels, nil
} }
func startRegistrySecret(ctx context.Context, engine *kube, step *types.Step) error { func startRegistrySecret(ctx context.Context, engine *kube, step *types.Step, metadata *metadata.Metadata) error {
secret, err := mkRegistrySecret(step, engine.config) secret, err := mkRegistrySecret(step, engine.config, metadata)
if err != nil { if err != nil {
return err return err
} }
@ -279,8 +280,8 @@ func startRegistrySecret(ctx context.Context, engine *kube, step *types.Step) er
return nil return nil
} }
func stopRegistrySecret(ctx context.Context, engine *kube, step *types.Step, deleteOpts meta_v1.DeleteOptions) error { func stopRegistrySecret(ctx context.Context, engine *kube, step *types.Step, deleteOpts meta_v1.DeleteOptions, metadata *metadata.Metadata) error {
name, err := registrySecretName(step) name, err := registrySecretName(step, metadata)
if err != nil { if err != nil {
return err return err
} }

9
pipeline/backend/kubernetes/secrets_test.go
View file

@ -23,6 +23,7 @@ import (
v1 "k8s.io/api/core/v1" v1 "k8s.io/api/core/v1"
"go.woodpecker-ci.org/woodpecker/v2/pipeline/backend/types" "go.woodpecker-ci.org/woodpecker/v2/pipeline/backend/types"
"go.woodpecker-ci.org/woodpecker/v2/pipeline/frontend/metadata"
) )
func TestNativeSecretsEnabled(t *testing.T) { func TestNativeSecretsEnabled(t *testing.T) {
@ -208,7 +209,7 @@ func TestUsernameAndPasswordNeedsSecret(t *testing.T) {
func TestRegistrySecret(t *testing.T) { func TestRegistrySecret(t *testing.T) {
const expected = `{ const expected = `{
"metadata": { "metadata": {
"name": "wp-01he8bebctabr3kgk0qj36d2me-0", "name": "wp-owner-repo-name-my-workflow-go-test-01he8",
"namespace": "woodpecker", "namespace": "woodpecker",
"creationTimestamp": null, "creationTimestamp": null,
"labels": { "labels": {
@ -231,7 +232,11 @@ func TestRegistrySecret(t *testing.T) {
}, },
}, &config{ }, &config{
Namespace: "woodpecker", Namespace: "woodpecker",
}) },
&metadata.Metadata{
Workflow: metadata.Workflow{Name: "my-workflow"},
Repo: metadata.Repo{Owner: "owner", Name: "repo_name"},
})
assert.NoError(t, err) assert.NoError(t, err)
secretJSON, err := json.Marshal(secret) secretJSON, err := json.Marshal(secret)

11
pipeline/backend/local/local.go
View file

@ -32,6 +32,7 @@ import (
"golang.org/x/text/transform" "golang.org/x/text/transform"
"go.woodpecker-ci.org/woodpecker/v2/pipeline/backend/types" "go.woodpecker-ci.org/woodpecker/v2/pipeline/backend/types"
"go.woodpecker-ci.org/woodpecker/v2/pipeline/frontend/metadata"
) )
type workflowState struct { type workflowState struct {
@ -119,7 +120,7 @@ func (e *local) SetupWorkflow(_ context.Context, _ *types.Config, taskUUID strin
} }
// StartStep the pipeline step. // StartStep the pipeline step.
func (e *local) StartStep(ctx context.Context, step *types.Step, taskUUID string) error { func (e *local) StartStep(ctx context.Context, step *types.Step, taskUUID string, metadata *metadata.Metadata) error {
log.Trace().Str("taskUUID", taskUUID).Msgf("start step %s", step.Name) log.Trace().Str("taskUUID", taskUUID).Msgf("start step %s", step.Name)
state, err := e.getState(taskUUID) state, err := e.getState(taskUUID)
@ -204,7 +205,7 @@ func (e *local) execPlugin(ctx context.Context, step *types.Step, state *workflo
// WaitStep for the pipeline step to complete and returns // WaitStep for the pipeline step to complete and returns
// the completion results. // the completion results.
func (e *local) WaitStep(_ context.Context, step *types.Step, taskUUID string) (*types.State, error) { func (e *local) WaitStep(_ context.Context, step *types.Step, taskUUID string, metadata *metadata.Metadata) (*types.State, error) {
log.Trace().Str("taskUUID", taskUUID).Msgf("wait for step %s", step.Name) log.Trace().Str("taskUUID", taskUUID).Msgf("wait for step %s", step.Name)
state, err := e.getState(taskUUID) state, err := e.getState(taskUUID)
@ -234,18 +235,18 @@ func (e *local) WaitStep(_ context.Context, step *types.Step, taskUUID string) (
} }
// TailStep the pipeline step logs. // TailStep the pipeline step logs.
func (e *local) TailStep(_ context.Context, step *types.Step, taskUUID string) (io.ReadCloser, error) { func (e *local) TailStep(_ context.Context, step *types.Step, taskUUID string, metadata *metadata.Metadata) (io.ReadCloser, error) {
log.Trace().Str("taskUUID", taskUUID).Msgf("tail logs of step %s", step.Name) log.Trace().Str("taskUUID", taskUUID).Msgf("tail logs of step %s", step.Name)
return e.output, nil return e.output, nil
} }
func (e *local) DestroyStep(_ context.Context, _ *types.Step, _ string) error { func (e *local) DestroyStep(_ context.Context, _ *types.Step, _ string, _ *metadata.Metadata) error {
// WaitStep already waits for the command to finish, so there is nothing to do here. // WaitStep already waits for the command to finish, so there is nothing to do here.
return nil return nil
} }
// DestroyWorkflow the pipeline environment. // DestroyWorkflow the pipeline environment.
func (e *local) DestroyWorkflow(_ context.Context, _ *types.Config, taskUUID string) error { func (e *local) DestroyWorkflow(_ context.Context, _ *types.Config, taskUUID string, metadata *metadata.Metadata) error {
log.Trace().Str("taskUUID", taskUUID).Msg("delete workflow environment") log.Trace().Str("taskUUID", taskUUID).Msg("delete workflow environment")
state, err := e.getState(taskUUID) state, err := e.getState(taskUUID)

11
pipeline/backend/types/backend.go
View file

@ -19,6 +19,7 @@ import (
"io" "io"
"github.com/urfave/cli/v3" "github.com/urfave/cli/v3"
"go.woodpecker-ci.org/woodpecker/v2/pipeline/frontend/metadata"
) )
// Backend defines a container orchestration backend and is used // Backend defines a container orchestration backend and is used
@ -40,20 +41,20 @@ type Backend interface {
SetupWorkflow(ctx context.Context, conf *Config, taskUUID string) error SetupWorkflow(ctx context.Context, conf *Config, taskUUID string) error
// StartStep starts the workflow step. // StartStep starts the workflow step.
StartStep(ctx context.Context, step *Step, taskUUID string) error StartStep(ctx context.Context, step *Step, taskUUID string, metadata *metadata.Metadata) error
// WaitStep waits for the workflow step to complete and returns // WaitStep waits for the workflow step to complete and returns
// the completion results. // the completion results.
WaitStep(ctx context.Context, step *Step, taskUUID string) (*State, error) WaitStep(ctx context.Context, step *Step, taskUUID string, metadata *metadata.Metadata) (*State, error)
// TailStep tails the workflow step logs. // TailStep tails the workflow step logs.
TailStep(ctx context.Context, step *Step, taskUUID string) (io.ReadCloser, error) TailStep(ctx context.Context, step *Step, taskUUID string, metadata *metadata.Metadata) (io.ReadCloser, error)
// DestroyStep destroys the workflow step. // DestroyStep destroys the workflow step.
DestroyStep(ctx context.Context, step *Step, taskUUID string) error DestroyStep(ctx context.Context, step *Step, taskUUID string, metadata *metadata.Metadata) error
// DestroyWorkflow destroys the workflow environment. // DestroyWorkflow destroys the workflow environment.
DestroyWorkflow(ctx context.Context, conf *Config, taskUUID string) error DestroyWorkflow(ctx context.Context, conf *Config, taskUUID string, metadata *metadata.Metadata) error
} }
// BackendInfo represents the reported information of a loaded backend. // BackendInfo represents the reported information of a loaded backend.

17
pipeline/frontend/metadata/types.go
View file

@ -17,14 +17,15 @@ package metadata
type ( type (
// Metadata defines runtime m. // Metadata defines runtime m.
Metadata struct { Metadata struct {
ID string `json:"id,omitempty"` ID string `json:"id,omitempty"`
Repo Repo `json:"repo,omitempty"` Repo Repo `json:"repo,omitempty"`
Curr Pipeline `json:"curr,omitempty"` Curr Pipeline `json:"curr,omitempty"`
Prev Pipeline `json:"prev,omitempty"` Prev Pipeline `json:"prev,omitempty"`
Workflow Workflow `json:"workflow,omitempty"` Workflow Workflow `json:"workflow,omitempty"`
Step Step `json:"step,omitempty"` Step Step `json:"step,omitempty"`
Sys System `json:"sys,omitempty"` Sys System `json:"sys,omitempty"`
Forge Forge `json:"forge,omitempty"` Forge Forge `json:"forge,omitempty"`
FailureIgnore string `json:"failure_ignore,omitempty"`
} }
// Repo defines runtime metadata for a repository. // Repo defines runtime metadata for a repository.

23
pipeline/pipeline.go
View file

@ -28,6 +28,7 @@ import (
backend "go.woodpecker-ci.org/woodpecker/v2/pipeline/backend/types" backend "go.woodpecker-ci.org/woodpecker/v2/pipeline/backend/types"
"go.woodpecker-ci.org/woodpecker/v2/pipeline/frontend/metadata" "go.woodpecker-ci.org/woodpecker/v2/pipeline/frontend/metadata"
meta "go.woodpecker-ci.org/woodpecker/v2/pipeline/frontend/metadata"
) )
// TODO: move runtime into "runtime" subpackage // TODO: move runtime into "runtime" subpackage
@ -89,7 +90,7 @@ func (r *Runtime) MakeLogger() zerolog.Logger {
} }
// Run starts the execution of a workflow and waits for it to complete. // Run starts the execution of a workflow and waits for it to complete.
func (r *Runtime) Run(runnerCtx context.Context) error { func (r *Runtime) Run(runnerCtx context.Context, metadata *metadata.Metadata) error {
logger := r.MakeLogger() logger := r.MakeLogger()
logger.Debug().Msgf("executing %d stages, in order of:", len(r.spec.Stages)) logger.Debug().Msgf("executing %d stages, in order of:", len(r.spec.Stages))
for stagePos, stage := range r.spec.Stages { for stagePos, stage := range r.spec.Stages {
@ -109,7 +110,7 @@ func (r *Runtime) Run(runnerCtx context.Context) error {
if ctx.Err() != nil { if ctx.Err() != nil {
ctx = GetShutdownCtx() ctx = GetShutdownCtx()
} }
if err := r.engine.DestroyWorkflow(ctx, r.spec, r.taskUUID); err != nil { if err := r.engine.DestroyWorkflow(ctx, r.spec, r.taskUUID, metadata); err != nil {
logger.Error().Err(err).Msg("could not destroy engine") logger.Error().Err(err).Msg("could not destroy engine")
} }
}() }()
@ -123,7 +124,7 @@ func (r *Runtime) Run(runnerCtx context.Context) error {
select { select {
case <-r.ctx.Done(): case <-r.ctx.Done():
return ErrCancel return ErrCancel
case err := <-r.execAll(stage.Steps): case err := <-r.execAll(stage.Steps, metadata):
if err != nil { if err != nil {
r.err = err r.err = err
} }
@ -163,7 +164,7 @@ func (r *Runtime) traceStep(processState *backend.State, err error, step *backen
} }
// Executes a set of parallel steps. // Executes a set of parallel steps.
func (r *Runtime) execAll(steps []*backend.Step) <-chan error { func (r *Runtime) execAll(steps []*backend.Step, metadata *metadata.Metadata) <-chan error {
var g errgroup.Group var g errgroup.Group
done := make(chan error) done := make(chan error)
logger := r.MakeLogger() logger := r.MakeLogger()
@ -200,13 +201,13 @@ func (r *Runtime) execAll(steps []*backend.Step) <-chan error {
} }
// add compatibility for drone-ci plugins // add compatibility for drone-ci plugins
metadata.SetDroneEnviron(step.Environment) meta.SetDroneEnviron(step.Environment)
logger.Debug(). logger.Debug().
Str("step", step.Name). Str("step", step.Name).
Msg("executing") Msg("executing")
processState, err := r.exec(step) processState, err := r.exec(step, metadata)
logger.Debug(). logger.Debug().
Str("step", step.Name). Str("step", step.Name).
@ -229,14 +230,14 @@ func (r *Runtime) execAll(steps []*backend.Step) <-chan error {
} }
// Executes the step and returns the state and error. // Executes the step and returns the state and error.
func (r *Runtime) exec(step *backend.Step) (*backend.State, error) { func (r *Runtime) exec(step *backend.Step, metadata *metadata.Metadata) (*backend.State, error) {
if err := r.engine.StartStep(r.ctx, step, r.taskUUID); err != nil { if err := r.engine.StartStep(r.ctx, step, r.taskUUID, metadata); err != nil {
return nil, err return nil, err
} }
var wg sync.WaitGroup var wg sync.WaitGroup
if r.logger != nil { if r.logger != nil {
rc, err := r.engine.TailStep(r.ctx, step, r.taskUUID) rc, err := r.engine.TailStep(r.ctx, step, r.taskUUID, metadata)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -261,7 +262,7 @@ func (r *Runtime) exec(step *backend.Step) (*backend.State, error) {
// We wait until all data was logged. (Needed for some backends like local as WaitStep kills the log stream) // We wait until all data was logged. (Needed for some backends like local as WaitStep kills the log stream)
wg.Wait() wg.Wait()
waitState, err := r.engine.WaitStep(r.ctx, step, r.taskUUID) waitState, err := r.engine.WaitStep(r.ctx, step, r.taskUUID, metadata)
if err != nil { if err != nil {
if errors.Is(err, context.Canceled) { if errors.Is(err, context.Canceled) {
return waitState, ErrCancel return waitState, ErrCancel
@ -269,7 +270,7 @@ func (r *Runtime) exec(step *backend.Step) (*backend.State, error) {
return nil, err return nil, err
} }
if err := r.engine.DestroyStep(r.ctx, step, r.taskUUID); err != nil { if err := r.engine.DestroyStep(r.ctx, step, r.taskUUID, metadata); err != nil {
return nil, err return nil, err
} }