From 5d3300824da49de7e68e5f809eb86fe1d494c300 Mon Sep 17 00:00:00 2001 From: Anbraten <6918444+anbraten@users.noreply.github.com> Date: Wed, 1 Jan 2025 16:21:57 +0100 Subject: [PATCH] Fix workflow volume and network config (#4650) --- pipeline/backend/docker/docker.go | 46 +++++++++++++------ pipeline/backend/kubernetes/kubernetes.go | 16 ++++--- pipeline/backend/types/config.go | 8 ++-- pipeline/frontend/yaml/compiler/compiler.go | 8 ++-- .../frontend/yaml/compiler/compiler_test.go | 46 +++++++++---------- 5 files changed, 72 insertions(+), 52 deletions(-) diff --git a/pipeline/backend/docker/docker.go b/pipeline/backend/docker/docker.go index eacc59d5a..88c43fee3 100644 --- a/pipeline/backend/docker/docker.go +++ b/pipeline/backend/docker/docker.go @@ -144,23 +144,32 @@ func (e *docker) Load(ctx context.Context) (*backend.BackendInfo, error) { func (e *docker) SetupWorkflow(ctx context.Context, conf *backend.Config, taskUUID string) error { log.Trace().Str("taskUUID", taskUUID).Msg("create workflow environment") - _, err := e.client.VolumeCreate(ctx, volume.CreateOptions{ - Name: conf.Volume.Name, - Driver: volumeDriver, - }) - if err != nil { - return err + for _, vol := range conf.Volumes { + _, err := e.client.VolumeCreate(ctx, volume.CreateOptions{ + Name: vol.Name, + Driver: volumeDriver, + }) + if err != nil { + return err + } } networkDriver := networkDriverBridge if e.info.OSType == "windows" { networkDriver = networkDriverNAT } - _, err = e.client.NetworkCreate(ctx, conf.Network.Name, network.CreateOptions{ - Driver: networkDriver, - EnableIPv6: &e.config.enableIPv6, - }) - return err + + for _, n := range conf.Networks { + _, err := e.client.NetworkCreate(ctx, n.Name, network.CreateOptions{ + Driver: networkDriver, + EnableIPv6: &e.config.enableIPv6, + }) + if err != nil { + return err + } + } + + return nil } func (e *docker) StartStep(ctx context.Context, step *backend.Step, taskUUID string) error { @@ -323,12 +332,19 @@ func (e *docker) DestroyWorkflow(ctx context.Context, conf *backend.Config, task } } } - if err := e.client.VolumeRemove(ctx, conf.Volume.Name, true); err != nil { - log.Error().Err(err).Msgf("could not remove volume '%s'", conf.Volume.Name) + + for _, v := range conf.Volumes { + if err := e.client.VolumeRemove(ctx, v.Name, true); err != nil { + log.Error().Err(err).Msgf("could not remove volume '%s'", v.Name) + } } - if err := e.client.NetworkRemove(ctx, conf.Network.Name); err != nil { - log.Error().Err(err).Msgf("could not remove network '%s'", conf.Network.Name) + + for _, n := range conf.Networks { + if err := e.client.NetworkRemove(ctx, n.Name); err != nil { + log.Error().Err(err).Msgf("could not remove network '%s'", n.Name) + } } + return nil } diff --git a/pipeline/backend/kubernetes/kubernetes.go b/pipeline/backend/kubernetes/kubernetes.go index 2ddf17852..7aa11893e 100644 --- a/pipeline/backend/kubernetes/kubernetes.go +++ b/pipeline/backend/kubernetes/kubernetes.go @@ -191,9 +191,11 @@ func (e *kube) getConfig() *config { func (e *kube) SetupWorkflow(ctx context.Context, conf *types.Config, taskUUID string) error { log.Trace().Str("taskUUID", taskUUID).Msgf("Setting up Kubernetes primitives") - _, err := startVolume(ctx, e, conf.Volume.Name) - if err != nil { - return err + for _, vol := range conf.Volumes { + _, err := startVolume(ctx, e, vol.Name) + if err != nil { + return err + } } var extraHosts []types.HostAlias @@ -425,9 +427,11 @@ func (e *kube) DestroyWorkflow(ctx context.Context, conf *types.Config, taskUUID } } - err := stopVolume(ctx, e, conf.Volume.Name, defaultDeleteOptions) - if err != nil { - return err + for _, vol := range conf.Volumes { + err := stopVolume(ctx, e, vol.Name, defaultDeleteOptions) + if err != nil { + return err + } } return nil diff --git a/pipeline/backend/types/config.go b/pipeline/backend/types/config.go index 158c75b62..8434f736c 100644 --- a/pipeline/backend/types/config.go +++ b/pipeline/backend/types/config.go @@ -16,10 +16,10 @@ package types // Config defines the runtime configuration of a workflow. type Config struct { - Stages []*Stage `json:"pipeline"` // workflow stages - Network *Network `json:"network"` // network definitions - Volume *Volume `json:"volume"` // volume definition - Secrets []*Secret `json:"secrets"` // secret definitions + Stages []*Stage `json:"pipeline"` // workflow stages + Networks []*Network `json:"network"` // network definitions + Volumes []*Volume `json:"volume"` // volume definitions + Secrets []*Secret `json:"secrets"` // secret definitions } // CliCommand is the context key to pass cli context to backends if needed. diff --git a/pipeline/frontend/yaml/compiler/compiler.go b/pipeline/frontend/yaml/compiler/compiler.go index 76ccf3c97..858b94abf 100644 --- a/pipeline/frontend/yaml/compiler/compiler.go +++ b/pipeline/frontend/yaml/compiler/compiler.go @@ -129,14 +129,14 @@ func (c *Compiler) Compile(conf *yaml_types.Workflow) (*backend_types.Config, er } // create a default volume - config.Volume = &backend_types.Volume{ + config.Volumes = append(config.Volumes, &backend_types.Volume{ Name: fmt.Sprintf("%s_default", c.prefix), - } + }) // create a default network - config.Network = &backend_types.Network{ + config.Networks = append(config.Networks, &backend_types.Network{ Name: fmt.Sprintf("%s_default", c.prefix), - } + }) // create secrets for mask for _, sec := range c.secrets { diff --git a/pipeline/frontend/yaml/compiler/compiler_test.go b/pipeline/frontend/yaml/compiler/compiler_test.go index f8179e7c3..fc4c68fa9 100644 --- a/pipeline/frontend/yaml/compiler/compiler_test.go +++ b/pipeline/frontend/yaml/compiler/compiler_test.go @@ -81,12 +81,12 @@ func TestCompilerCompile(t *testing.T) { WithWorkspaceFromURL("/test", repoURL), ) - defaultNetwork := &backend_types.Network{ + defaultNetworks := []*backend_types.Network{{ Name: "test_default", - } - defaultVolume := &backend_types.Volume{ + }} + defaultVolumes := []*backend_types.Volume{{ Name: "test_default", - } + }} defaultCloneStage := &backend_types.Stage{ Steps: []*backend_types.Step{{ @@ -95,7 +95,7 @@ func TestCompilerCompile(t *testing.T) { Image: constant.DefaultClonePlugin, OnSuccess: true, Failure: "fail", - Volumes: []string{defaultVolume.Name + ":/woodpecker"}, + Volumes: []string{defaultVolumes[0].Name + ":/woodpecker"}, WorkingDir: "/woodpecker/src/github.com/octocat/hello-world", WorkspaceBase: "/woodpecker", Networks: []backend_types.Conn{{Name: "test_default", Aliases: []string{"clone"}}}, @@ -113,17 +113,17 @@ func TestCompilerCompile(t *testing.T) { name: "empty workflow, no clone", fronConf: &yaml_types.Workflow{SkipClone: true}, backConf: &backend_types.Config{ - Network: defaultNetwork, - Volume: defaultVolume, + Networks: defaultNetworks, + Volumes: defaultVolumes, }, }, { name: "empty workflow, default clone", fronConf: &yaml_types.Workflow{}, backConf: &backend_types.Config{ - Network: defaultNetwork, - Volume: defaultVolume, - Stages: []*backend_types.Stage{defaultCloneStage}, + Networks: defaultNetworks, + Volumes: defaultVolumes, + Stages: []*backend_types.Stage{defaultCloneStage}, }, }, { @@ -133,8 +133,8 @@ func TestCompilerCompile(t *testing.T) { Image: "dummy_img", }}}}, backConf: &backend_types.Config{ - Network: defaultNetwork, - Volume: defaultVolume, + Networks: defaultNetworks, + Volumes: defaultVolumes, Stages: []*backend_types.Stage{defaultCloneStage, { Steps: []*backend_types.Step{{ Name: "dummy", @@ -142,7 +142,7 @@ func TestCompilerCompile(t *testing.T) { Image: "dummy_img", OnSuccess: true, Failure: "fail", - Volumes: []string{defaultVolume.Name + ":/woodpecker"}, + Volumes: []string{defaultVolumes[0].Name + ":/woodpecker"}, WorkingDir: "/woodpecker/src/github.com/octocat/hello-world", WorkspaceBase: "/woodpecker", Networks: []backend_types.Conn{{Name: "test_default", Aliases: []string{"dummy"}}}, @@ -167,8 +167,8 @@ func TestCompilerCompile(t *testing.T) { Commands: []string{"echo 2"}, }}}}, backConf: &backend_types.Config{ - Network: defaultNetwork, - Volume: defaultVolume, + Networks: defaultNetworks, + Volumes: defaultVolumes, Stages: []*backend_types.Stage{ defaultCloneStage, { Steps: []*backend_types.Step{{ @@ -178,7 +178,7 @@ func TestCompilerCompile(t *testing.T) { Commands: []string{"env"}, OnSuccess: true, Failure: "fail", - Volumes: []string{defaultVolume.Name + ":/test"}, + Volumes: []string{defaultVolumes[0].Name + ":/test"}, WorkingDir: "/test/src/github.com/octocat/hello-world", WorkspaceBase: "/test", Networks: []backend_types.Conn{{Name: "test_default", Aliases: []string{"echo env"}}}, @@ -192,7 +192,7 @@ func TestCompilerCompile(t *testing.T) { Commands: []string{"echo 1"}, OnSuccess: true, Failure: "fail", - Volumes: []string{defaultVolume.Name + ":/test"}, + Volumes: []string{defaultVolumes[0].Name + ":/test"}, WorkingDir: "/test/src/github.com/octocat/hello-world", WorkspaceBase: "/test", Networks: []backend_types.Conn{{Name: "test_default", Aliases: []string{"parallel echo 1"}}}, @@ -206,7 +206,7 @@ func TestCompilerCompile(t *testing.T) { Commands: []string{"echo 2"}, OnSuccess: true, Failure: "fail", - Volumes: []string{defaultVolume.Name + ":/test"}, + Volumes: []string{defaultVolumes[0].Name + ":/test"}, WorkingDir: "/test/src/github.com/octocat/hello-world", WorkspaceBase: "/test", Networks: []backend_types.Conn{{Name: "test_default", Aliases: []string{"parallel echo 2"}}}, @@ -233,8 +233,8 @@ func TestCompilerCompile(t *testing.T) { Commands: []string{"echo 2"}, }}}}, backConf: &backend_types.Config{ - Network: defaultNetwork, - Volume: defaultVolume, + Networks: defaultNetworks, + Volumes: defaultVolumes, Stages: []*backend_types.Stage{defaultCloneStage, { Steps: []*backend_types.Step{{ Name: "echo env", @@ -243,7 +243,7 @@ func TestCompilerCompile(t *testing.T) { Commands: []string{"env"}, OnSuccess: true, Failure: "fail", - Volumes: []string{defaultVolume.Name + ":/test"}, + Volumes: []string{defaultVolumes[0].Name + ":/test"}, WorkingDir: "/test/src/github.com/octocat/hello-world", WorkspaceBase: "/test", Networks: []backend_types.Conn{{Name: "test_default", Aliases: []string{"echo env"}}}, @@ -255,7 +255,7 @@ func TestCompilerCompile(t *testing.T) { Commands: []string{"echo 2"}, OnSuccess: true, Failure: "fail", - Volumes: []string{defaultVolume.Name + ":/test"}, + Volumes: []string{defaultVolumes[0].Name + ":/test"}, WorkingDir: "/test/src/github.com/octocat/hello-world", WorkspaceBase: "/test", Networks: []backend_types.Conn{{Name: "test_default", Aliases: []string{"echo 2"}}}, @@ -269,7 +269,7 @@ func TestCompilerCompile(t *testing.T) { Commands: []string{"echo 1"}, OnSuccess: true, Failure: "fail", - Volumes: []string{defaultVolume.Name + ":/test"}, + Volumes: []string{defaultVolumes[0].Name + ":/test"}, WorkingDir: "/test/src/github.com/octocat/hello-world", WorkspaceBase: "/test", Networks: []backend_types.Conn{{Name: "test_default", Aliases: []string{"echo 1"}}},