Remove workflow-level volumes and networks (#4636)

pipeline/backend/docker/docker.go

@@ -144,30 +144,23 @@ func (e *docker) Load(ctx context.Context) (*backend.BackendInfo, error) {
 func (e *docker) SetupWorkflow(ctx context.Context, conf *backend.Config, taskUUID string) error {
 	log.Trace().Str("taskUUID", taskUUID).Msg("create workflow environment")
 
-	for _, vol := range conf.Volumes {
+	_, err := e.client.VolumeCreate(ctx, volume.CreateOptions{
-		_, err := e.client.VolumeCreate(ctx, volume.CreateOptions{
+		Name:   conf.Volume.Name,
-			Name:   vol.Name,
+		Driver: volumeDriver,
-			Driver: volumeDriver,
+	})
-		})
+	if err != nil {
-		if err != nil {
+		return err
-			return err
-		}
 	}
 
 	networkDriver := networkDriverBridge
 	if e.info.OSType == "windows" {
 		networkDriver = networkDriverNAT
 	}
-	for _, n := range conf.Networks {
+	_, err = e.client.NetworkCreate(ctx, conf.Network.Name, network.CreateOptions{
-		_, err := e.client.NetworkCreate(ctx, n.Name, network.CreateOptions{
+		Driver:     networkDriver,
-			Driver:     networkDriver,
+		EnableIPv6: &e.config.enableIPv6,
-			EnableIPv6: &e.config.enableIPv6,
+	})
-		})
+	return err
-		if err != nil {
-			return err
-		}
-	}
-	return nil
 }
 
 func (e *docker) StartStep(ctx context.Context, step *backend.Step, taskUUID string) error {
@@ -330,15 +323,11 @@ func (e *docker) DestroyWorkflow(ctx context.Context, conf *backend.Config, task
 			}
 		}
 	}
-	for _, v := range conf.Volumes {
+	if err := e.client.VolumeRemove(ctx, conf.Volume.Name, true); err != nil {
-		if err := e.client.VolumeRemove(ctx, v.Name, true); err != nil {
+		log.Error().Err(err).Msgf("could not remove volume '%s'", conf.Volume.Name)
-			log.Error().Err(err).Msgf("could not remove volume '%s'", v.Name)
-		}
 	}
-	for _, n := range conf.Networks {
+	if err := e.client.NetworkRemove(ctx, conf.Network.Name); err != nil {
-		if err := e.client.NetworkRemove(ctx, n.Name); err != nil {
+		log.Error().Err(err).Msgf("could not remove network '%s'", conf.Network.Name)
-			log.Error().Err(err).Msgf("could not remove network '%s'", n.Name)
-		}
 	}
 	return nil
 }

pipeline/backend/kubernetes/kubernetes.go

@@ -191,11 +191,9 @@ func (e *kube) getConfig() *config {
 func (e *kube) SetupWorkflow(ctx context.Context, conf *types.Config, taskUUID string) error {
 	log.Trace().Str("taskUUID", taskUUID).Msgf("Setting up Kubernetes primitives")
 
-	for _, vol := range conf.Volumes {
+	_, err := startVolume(ctx, e, conf.Volume.Name)
-		_, err := startVolume(ctx, e, vol.Name)
+	if err != nil {
-		if err != nil {
+		return err
-			return err
-		}
 	}
 
 	var extraHosts []types.HostAlias
@@ -427,11 +425,9 @@ func (e *kube) DestroyWorkflow(ctx context.Context, conf *types.Config, taskUUID
 		}
 	}
 
-	for _, vol := range conf.Volumes {
+	err := stopVolume(ctx, e, conf.Volume.Name, defaultDeleteOptions)
-		err := stopVolume(ctx, e, vol.Name, defaultDeleteOptions)
+	if err != nil {
-		if err != nil {
+		return err
-			return err
-		}
 	}
 
 	return nil

pipeline/backend/types/config.go

@@ -16,10 +16,10 @@ package types
 
 // Config defines the runtime configuration of a workflow.
 type Config struct {
-	Stages   []*Stage   `json:"pipeline"` // workflow stages
+	Stages  []*Stage  `json:"pipeline"` // workflow stages
-	Networks []*Network `json:"networks"` // network definitions
+	Network *Network  `json:"network"`  // network definitions
-	Volumes  []*Volume  `json:"volumes"`  // volume definitions
+	Volume  *Volume   `json:"volume"`   // volume definition
-	Secrets  []*Secret  `json:"secrets"`  // secret definitions
+	Secrets []*Secret `json:"secrets"`  // secret definitions
 }
 
 // CliCommand is the context key to pass cli context to backends if needed.

pipeline/frontend/yaml/compiler/compiler.go

@@ -129,14 +129,14 @@ func (c *Compiler) Compile(conf *yaml_types.Workflow) (*backend_types.Config, er
 	}
 
 	// create a default volume
-	config.Volumes = append(config.Volumes, &backend_types.Volume{
+	config.Volume = &backend_types.Volume{
 		Name: fmt.Sprintf("%s_default", c.prefix),
-	})
+	}
 
 	// create a default network
-	config.Networks = append(config.Networks, &backend_types.Network{
+	config.Network = &backend_types.Network{
 		Name: fmt.Sprintf("%s_default", c.prefix),
-	})
+	}
 
 	// create secrets for mask
 	for _, sec := range c.secrets {

pipeline/frontend/yaml/compiler/compiler_test.go

@@ -81,12 +81,12 @@ func TestCompilerCompile(t *testing.T) {
 		WithWorkspaceFromURL("/test", repoURL),
 	)
 
-	defaultNetworks := []*backend_types.Network{{
+	defaultNetwork := &backend_types.Network{
 		Name: "test_default",
-	}}
+	}
-	defaultVolumes := []*backend_types.Volume{{
+	defaultVolume := &backend_types.Volume{
 		Name: "test_default",
-	}}
+	}
 
 	defaultCloneStage := &backend_types.Stage{
 		Steps: []*backend_types.Step{{
@@ -95,7 +95,7 @@ func TestCompilerCompile(t *testing.T) {
 			Image:         constant.DefaultClonePlugin,
 			OnSuccess:     true,
 			Failure:       "fail",
-			Volumes:       []string{defaultVolumes[0].Name + ":/woodpecker"},
+			Volumes:       []string{defaultVolume.Name + ":/woodpecker"},
 			WorkingDir:    "/woodpecker/src/github.com/octocat/hello-world",
 			WorkspaceBase: "/woodpecker",
 			Networks:      []backend_types.Conn{{Name: "test_default", Aliases: []string{"clone"}}},
@@ -113,17 +113,17 @@ func TestCompilerCompile(t *testing.T) {
 			name:     "empty workflow, no clone",
 			fronConf: &yaml_types.Workflow{SkipClone: true},
 			backConf: &backend_types.Config{
-				Networks: defaultNetworks,
+				Network: defaultNetwork,
-				Volumes:  defaultVolumes,
+				Volume:  defaultVolume,
 			},
 		},
 		{
 			name:     "empty workflow, default clone",
 			fronConf: &yaml_types.Workflow{},
 			backConf: &backend_types.Config{
-				Networks: defaultNetworks,
+				Network: defaultNetwork,
-				Volumes:  defaultVolumes,
+				Volume:  defaultVolume,
-				Stages:   []*backend_types.Stage{defaultCloneStage},
+				Stages:  []*backend_types.Stage{defaultCloneStage},
 			},
 		},
 		{
@@ -133,8 +133,8 @@ func TestCompilerCompile(t *testing.T) {
 				Image: "dummy_img",
 			}}}},
 			backConf: &backend_types.Config{
-				Networks: defaultNetworks,
+				Network: defaultNetwork,
-				Volumes:  defaultVolumes,
+				Volume:  defaultVolume,
 				Stages: []*backend_types.Stage{defaultCloneStage, {
 					Steps: []*backend_types.Step{{
 						Name:          "dummy",
@@ -142,7 +142,7 @@ func TestCompilerCompile(t *testing.T) {
 						Image:         "dummy_img",
 						OnSuccess:     true,
 						Failure:       "fail",
-						Volumes:       []string{defaultVolumes[0].Name + ":/woodpecker"},
+						Volumes:       []string{defaultVolume.Name + ":/woodpecker"},
 						WorkingDir:    "/woodpecker/src/github.com/octocat/hello-world",
 						WorkspaceBase: "/woodpecker",
 						Networks:      []backend_types.Conn{{Name: "test_default", Aliases: []string{"dummy"}}},
@@ -167,8 +167,8 @@ func TestCompilerCompile(t *testing.T) {
 				Commands: []string{"echo 2"},
 			}}}},
 			backConf: &backend_types.Config{
-				Networks: defaultNetworks,
+				Network: defaultNetwork,
-				Volumes:  defaultVolumes,
+				Volume:  defaultVolume,
 				Stages: []*backend_types.Stage{
 					defaultCloneStage, {
 						Steps: []*backend_types.Step{{
@@ -178,7 +178,7 @@ func TestCompilerCompile(t *testing.T) {
 							Commands:      []string{"env"},
 							OnSuccess:     true,
 							Failure:       "fail",
-							Volumes:       []string{defaultVolumes[0].Name + ":/test"},
+							Volumes:       []string{defaultVolume.Name + ":/test"},
 							WorkingDir:    "/test/src/github.com/octocat/hello-world",
 							WorkspaceBase: "/test",
 							Networks:      []backend_types.Conn{{Name: "test_default", Aliases: []string{"echo env"}}},
@@ -192,7 +192,7 @@ func TestCompilerCompile(t *testing.T) {
 							Commands:      []string{"echo 1"},
 							OnSuccess:     true,
 							Failure:       "fail",
-							Volumes:       []string{defaultVolumes[0].Name + ":/test"},
+							Volumes:       []string{defaultVolume.Name + ":/test"},
 							WorkingDir:    "/test/src/github.com/octocat/hello-world",
 							WorkspaceBase: "/test",
 							Networks:      []backend_types.Conn{{Name: "test_default", Aliases: []string{"parallel echo 1"}}},
@@ -206,7 +206,7 @@ func TestCompilerCompile(t *testing.T) {
 							Commands:      []string{"echo 2"},
 							OnSuccess:     true,
 							Failure:       "fail",
-							Volumes:       []string{defaultVolumes[0].Name + ":/test"},
+							Volumes:       []string{defaultVolume.Name + ":/test"},
 							WorkingDir:    "/test/src/github.com/octocat/hello-world",
 							WorkspaceBase: "/test",
 							Networks:      []backend_types.Conn{{Name: "test_default", Aliases: []string{"parallel echo 2"}}},
@@ -233,8 +233,8 @@ func TestCompilerCompile(t *testing.T) {
 				Commands: []string{"echo 2"},
 			}}}},
 			backConf: &backend_types.Config{
-				Networks: defaultNetworks,
+				Network: defaultNetwork,
-				Volumes:  defaultVolumes,
+				Volume:  defaultVolume,
 				Stages: []*backend_types.Stage{defaultCloneStage, {
 					Steps: []*backend_types.Step{{
 						Name:          "echo env",
@@ -243,7 +243,7 @@ func TestCompilerCompile(t *testing.T) {
 						Commands:      []string{"env"},
 						OnSuccess:     true,
 						Failure:       "fail",
-						Volumes:       []string{defaultVolumes[0].Name + ":/test"},
+						Volumes:       []string{defaultVolume.Name + ":/test"},
 						WorkingDir:    "/test/src/github.com/octocat/hello-world",
 						WorkspaceBase: "/test",
 						Networks:      []backend_types.Conn{{Name: "test_default", Aliases: []string{"echo env"}}},
@@ -255,7 +255,7 @@ func TestCompilerCompile(t *testing.T) {
 						Commands:      []string{"echo 2"},
 						OnSuccess:     true,
 						Failure:       "fail",
-						Volumes:       []string{defaultVolumes[0].Name + ":/test"},
+						Volumes:       []string{defaultVolume.Name + ":/test"},
 						WorkingDir:    "/test/src/github.com/octocat/hello-world",
 						WorkspaceBase: "/test",
 						Networks:      []backend_types.Conn{{Name: "test_default", Aliases: []string{"echo 2"}}},
@@ -269,7 +269,7 @@ func TestCompilerCompile(t *testing.T) {
 						Commands:      []string{"echo 1"},
 						OnSuccess:     true,
 						Failure:       "fail",
-						Volumes:       []string{defaultVolumes[0].Name + ":/test"},
+						Volumes:       []string{defaultVolume.Name + ":/test"},
 						WorkingDir:    "/test/src/github.com/octocat/hello-world",
 						WorkspaceBase: "/test",
 						Networks:      []backend_types.Conn{{Name: "test_default", Aliases: []string{"echo 1"}}},

pipeline/frontend/yaml/parse_test.go

@@ -32,10 +32,6 @@ func TestParse(t *testing.T) {
 
 		assert.Equal(t, "/go", out.Workspace.Base)
 		assert.Equal(t, "src/github.com/octocat/hello-world", out.Workspace.Path)
-		assert.Equal(t, "custom", out.Volumes.WorkflowVolumes[0].Name)
-		assert.Equal(t, "blockbridge", out.Volumes.WorkflowVolumes[0].Driver)
-		assert.Equal(t, "custom", out.Networks.WorkflowNetworks[0].Name)
-		assert.Equal(t, "overlay", out.Networks.WorkflowNetworks[0].Driver)
 		assert.Equal(t, "database", out.Services.ContainerList[0].Name)
 		assert.Equal(t, "mysql", out.Services.ContainerList[0].Image)
 		assert.Equal(t, "test", out.Steps.ContainerList[0].Name)
@@ -201,12 +197,6 @@ steps:
 services:
   database:
     image: mysql
-networks:
-  custom:
-    driver: overlay
-volumes:
-  custom:
-    driver: blockbridge
 labels:
   com.example.type: "build"
   com.example.team: "frontend"

pipeline/frontend/yaml/types/workflow.go

@@ -30,10 +30,6 @@ type (
 		DependsOn []string          `yaml:"depends_on,omitempty"`
 		RunsOn    []string          `yaml:"runs_on,omitempty"`
 		SkipClone bool              `yaml:"skip_clone"`
-
-		// Undocumented
-		Networks WorkflowNetworks `yaml:"networks,omitempty"`
-		Volumes  WorkflowVolumes  `yaml:"volumes,omitempty"`
 	}
 
 	// Workspace defines a pipeline workspace.

pipeline/frontend/yaml/types/workflow_network.go

@@ -1,52 +0,0 @@
-// Copyright 2023 Woodpecker Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package types
-
-import (
-	"fmt"
-
-	"gopkg.in/yaml.v3"
-)
-
-type (
-	// WorkflowNetworks defines a collection of networks.
-	WorkflowNetworks struct {
-		WorkflowNetworks []*WorkflowNetwork
-	}
-
-	// WorkflowNetwork defines a container network.
-	WorkflowNetwork struct {
-		Name       string            `yaml:"name,omitempty"`
-		Driver     string            `yaml:"driver,omitempty"`
-		DriverOpts map[string]string `yaml:"driver_opts,omitempty"`
-	}
-)
-
-// UnmarshalYAML implements the Unmarshaler interface.
-func (n *WorkflowNetworks) UnmarshalYAML(value *yaml.Node) error {
-	networks := map[string]WorkflowNetwork{}
-	err := value.Decode(&networks)
-
-	for key, nn := range networks {
-		if nn.Name == "" {
-			nn.Name = fmt.Sprintf("%v", key)
-		}
-		if nn.Driver == "" {
-			nn.Driver = "bridge"
-		}
-		n.WorkflowNetworks = append(n.WorkflowNetworks, &nn)
-	}
-	return err
-}

pipeline/frontend/yaml/types/workflow_network_test.go

@@ -1,111 +0,0 @@
-// Copyright 2023 Woodpecker Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package types
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-	"gopkg.in/yaml.v3"
-)
-
-func TestUnmarshalNetwork(t *testing.T) {
-	testdata := []struct {
-		from string
-		want WorkflowNetwork
-	}{
-		{
-			from: "{ name: foo, driver: bar }",
-			want: WorkflowNetwork{
-				Name:   "foo",
-				Driver: "bar",
-			},
-		},
-		{
-			from: "{ name: foo, driver: bar, driver_opts: { baz: qux } }",
-			want: WorkflowNetwork{
-				Name:   "foo",
-				Driver: "bar",
-				DriverOpts: map[string]string{
-					"baz": "qux",
-				},
-			},
-		},
-	}
-
-	for _, test := range testdata {
-		in := []byte(test.from)
-		got := WorkflowNetwork{}
-		err := yaml.Unmarshal(in, &got)
-		assert.NoError(t, err)
-		assert.EqualValues(t, test.want, got, "problem parsing network %q", test.from)
-	}
-}
-
-func TestUnmarshalWorkflowNetworks(t *testing.T) {
-	testdata := []struct {
-		from string
-		want []*WorkflowNetwork
-	}{
-		{
-			from: "foo: { driver: bar }",
-			want: []*WorkflowNetwork{
-				{
-					Name:   "foo",
-					Driver: "bar",
-				},
-			},
-		},
-		{
-			from: "foo: { name: baz }",
-			want: []*WorkflowNetwork{
-				{
-					Name:   "baz",
-					Driver: "bridge",
-				},
-			},
-		},
-		{
-			from: "foo: { name: baz, driver: bar }",
-			want: []*WorkflowNetwork{
-				{
-					Name:   "baz",
-					Driver: "bar",
-				},
-			},
-		},
-	}
-
-	for _, test := range testdata {
-		in := []byte(test.from)
-		got := WorkflowNetworks{}
-		err := yaml.Unmarshal(in, &got)
-		assert.NoError(t, err)
-		assert.EqualValues(t, test.want, got.WorkflowNetworks, "problem parsing network %q", test.from)
-	}
-}
-
-func TestUnmarshalNetworkErr(t *testing.T) {
-	testdata := []string{
-		"foo: { name: [ foo, bar] }",
-		"- foo",
-	}
-
-	for _, test := range testdata {
-		in := []byte(test)
-		err := yaml.Unmarshal(in, new(WorkflowNetworks))
-		assert.Error(t, err, "wanted error for networks %q", test)
-	}
-}

pipeline/frontend/yaml/types/workflow_volume.go

@@ -1,57 +0,0 @@
-// Copyright 2023 Woodpecker Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package types
-
-import (
-	"fmt"
-
-	"gopkg.in/yaml.v3"
-)
-
-type (
-	// WorkflowVolumes defines a collection of volumes.
-	WorkflowVolumes struct {
-		WorkflowVolumes []*WorkflowVolume
-	}
-
-	// WorkflowVolume defines a container volume.
-	WorkflowVolume struct {
-		Name       string            `yaml:"name,omitempty"`
-		Driver     string            `yaml:"driver,omitempty"`
-		DriverOpts map[string]string `yaml:"driver_opts,omitempty"`
-	}
-)
-
-// UnmarshalYAML implements the Unmarshaler interface.
-func (v *WorkflowVolumes) UnmarshalYAML(value *yaml.Node) error {
-	y, _ := yaml.Marshal(value)
-
-	volumes := map[string]WorkflowVolume{}
-	err := yaml.Unmarshal(y, &volumes)
-	if err != nil {
-		return err
-	}
-
-	for key, vv := range volumes {
-		if vv.Name == "" {
-			vv.Name = fmt.Sprintf("%v", key)
-		}
-		if vv.Driver == "" {
-			vv.Driver = "local"
-		}
-		v.WorkflowVolumes = append(v.WorkflowVolumes, &vv)
-	}
-	return err
-}

pipeline/frontend/yaml/types/workflow_volume_test.go

@@ -1,111 +0,0 @@
-// Copyright 2023 Woodpecker Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package types
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-	"gopkg.in/yaml.v3"
-)
-
-func TestUnmarshalVolume(t *testing.T) {
-	testdata := []struct {
-		from string
-		want WorkflowVolume
-	}{
-		{
-			from: "{ name: foo, driver: bar }",
-			want: WorkflowVolume{
-				Name:   "foo",
-				Driver: "bar",
-			},
-		},
-		{
-			from: "{ name: foo, driver: bar, driver_opts: { baz: qux } }",
-			want: WorkflowVolume{
-				Name:   "foo",
-				Driver: "bar",
-				DriverOpts: map[string]string{
-					"baz": "qux",
-				},
-			},
-		},
-	}
-
-	for _, test := range testdata {
-		in := []byte(test.from)
-		got := WorkflowVolume{}
-		err := yaml.Unmarshal(in, &got)
-		assert.NoError(t, err)
-		assert.EqualValues(t, test.want, got, "problem parsing volume %q", test.from)
-	}
-}
-
-func TestUnmarshalWorkflowVolumes(t *testing.T) {
-	testdata := []struct {
-		from string
-		want []*WorkflowVolume
-	}{
-		{
-			from: "foo: { driver: bar }",
-			want: []*WorkflowVolume{
-				{
-					Name:   "foo",
-					Driver: "bar",
-				},
-			},
-		},
-		{
-			from: "foo: { name: baz }",
-			want: []*WorkflowVolume{
-				{
-					Name:   "baz",
-					Driver: "local",
-				},
-			},
-		},
-		{
-			from: "foo: { name: baz, driver: bar }",
-			want: []*WorkflowVolume{
-				{
-					Name:   "baz",
-					Driver: "bar",
-				},
-			},
-		},
-	}
-
-	for _, test := range testdata {
-		in := []byte(test.from)
-		got := WorkflowVolumes{}
-		err := yaml.Unmarshal(in, &got)
-		assert.NoError(t, err)
-		assert.EqualValues(t, test.want, got.WorkflowVolumes, "problem parsing volumes %q", test.from)
-	}
-}
-
-func TestUnmarshalVolumesErr(t *testing.T) {
-	testdata := []string{
-		"foo: { name: [ foo, bar] }",
-		"- foo",
-	}
-
-	for _, test := range testdata {
-		in := []byte(test)
-		err := yaml.Unmarshal(in, new(WorkflowVolumes))
-		assert.Error(t, err, "wanted error for volumes %q", test)
-	}
-}