mirrors/woodpecker
1
0
Fork 0
mirror of https://github.com/woodpecker-ci/woodpecker.git synced 2025-01-10 17:45:36 +00:00
Code Issues Projects Releases Wiki Activity

Remove workflow-level volumes and networks (#4636)

Browse source
This commit is contained in:
qwerty287 2024-12-30 11:34:59 +02:00 committed by GitHub
parent afa6dee30b
commit 428ba689cb
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
11 changed files with 52 additions and 412 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

41
pipeline/backend/docker/docker.go
View file

@ -144,30 +144,23 @@ func (e *docker) Load(ctx context.Context) (*backend.BackendInfo, error) {
func (e *docker) SetupWorkflow(ctx context.Context, conf *backend.Config, taskUUID string) error {
log.Trace().Str("taskUUID", taskUUID).Msg("create workflow environment")
for _, vol := range conf.Volumes {
_, err := e.client.VolumeCreate(ctx, volume.CreateOptions{
Name: vol.Name,
Driver: volumeDriver,
})
if err != nil {
return err
}
_, err := e.client.VolumeCreate(ctx, volume.CreateOptions{
Name: conf.Volume.Name,
Driver: volumeDriver,
})
if err != nil {
return err
}
networkDriver := networkDriverBridge
if e.info.OSType == "windows" {
networkDriver = networkDriverNAT
}
for _, n := range conf.Networks {
_, err := e.client.NetworkCreate(ctx, n.Name, network.CreateOptions{
Driver: networkDriver,
EnableIPv6: &e.config.enableIPv6,
})
if err != nil {
return err
}
}
return nil
_, err = e.client.NetworkCreate(ctx, conf.Network.Name, network.CreateOptions{
Driver: networkDriver,
EnableIPv6: &e.config.enableIPv6,
})
return err
}
func (e *docker) StartStep(ctx context.Context, step *backend.Step, taskUUID string) error {
@ -330,15 +323,11 @@ func (e *docker) DestroyWorkflow(ctx context.Context, conf *backend.Config, task
}
}
}
for _, v := range conf.Volumes {
if err := e.client.VolumeRemove(ctx, v.Name, true); err != nil {
log.Error().Err(err).Msgf("could not remove volume '%s'", v.Name)
}
if err := e.client.VolumeRemove(ctx, conf.Volume.Name, true); err != nil {
log.Error().Err(err).Msgf("could not remove volume '%s'", conf.Volume.Name)
}
for _, n := range conf.Networks {
if err := e.client.NetworkRemove(ctx, n.Name); err != nil {
log.Error().Err(err).Msgf("could not remove network '%s'", n.Name)
}
if err := e.client.NetworkRemove(ctx, conf.Network.Name); err != nil {
log.Error().Err(err).Msgf("could not remove network '%s'", conf.Network.Name)
}
return nil
}

16
pipeline/backend/kubernetes/kubernetes.go
View file

@ -191,11 +191,9 @@ func (e *kube) getConfig() *config {
func (e *kube) SetupWorkflow(ctx context.Context, conf *types.Config, taskUUID string) error {
log.Trace().Str("taskUUID", taskUUID).Msgf("Setting up Kubernetes primitives")
for _, vol := range conf.Volumes {
_, err := startVolume(ctx, e, vol.Name)
if err != nil {
return err
}
_, err := startVolume(ctx, e, conf.Volume.Name)
if err != nil {
return err
}
var extraHosts []types.HostAlias
@ -427,11 +425,9 @@ func (e *kube) DestroyWorkflow(ctx context.Context, conf *types.Config, taskUUID
}
}
for _, vol := range conf.Volumes {
err := stopVolume(ctx, e, vol.Name, defaultDeleteOptions)
if err != nil {
return err
}
err := stopVolume(ctx, e, conf.Volume.Name, defaultDeleteOptions)
if err != nil {
return err
}
return nil

8
pipeline/backend/types/config.go
View file

@ -16,10 +16,10 @@ package types
// Config defines the runtime configuration of a workflow.
type Config struct {
Stages []*Stage `json:"pipeline"` // workflow stages
Networks []*Network `json:"networks"` // network definitions
Volumes []*Volume `json:"volumes"` // volume definitions
Secrets []*Secret `json:"secrets"` // secret definitions
Stages []*Stage `json:"pipeline"` // workflow stages
Network *Network `json:"network"` // network definitions
Volume *Volume `json:"volume"` // volume definition
Secrets []*Secret `json:"secrets"` // secret definitions
}
// CliCommand is the context key to pass cli context to backends if needed.

8
pipeline/frontend/yaml/compiler/compiler.go
View file

@ -129,14 +129,14 @@ func (c *Compiler) Compile(conf *yaml_types.Workflow) (*backend_types.Config, er
}
// create a default volume
config.Volumes = append(config.Volumes, &backend_types.Volume{
config.Volume = &backend_types.Volume{
Name: fmt.Sprintf("%s_default", c.prefix),
})
}
// create a default network
config.Networks = append(config.Networks, &backend_types.Network{
config.Network = &backend_types.Network{
Name: fmt.Sprintf("%s_default", c.prefix),
})
}
// create secrets for mask
for _, sec := range c.secrets {

46
pipeline/frontend/yaml/compiler/compiler_test.go
View file

@ -81,12 +81,12 @@ func TestCompilerCompile(t *testing.T) {
WithWorkspaceFromURL("/test", repoURL),
)
defaultNetworks := []*backend_types.Network{{
defaultNetwork := &backend_types.Network{
Name: "test_default",
}}
defaultVolumes := []*backend_types.Volume{{
}
defaultVolume := &backend_types.Volume{
Name: "test_default",
}}
}
defaultCloneStage := &backend_types.Stage{
Steps: []*backend_types.Step{{
@ -95,7 +95,7 @@ func TestCompilerCompile(t *testing.T) {
Image: constant.DefaultClonePlugin,
OnSuccess: true,
Failure: "fail",
Volumes: []string{defaultVolumes[0].Name + ":/woodpecker"},
Volumes: []string{defaultVolume.Name + ":/woodpecker"},
WorkingDir: "/woodpecker/src/github.com/octocat/hello-world",
WorkspaceBase: "/woodpecker",
Networks: []backend_types.Conn{{Name: "test_default", Aliases: []string{"clone"}}},
@ -113,17 +113,17 @@ func TestCompilerCompile(t *testing.T) {
name: "empty workflow, no clone",
fronConf: &yaml_types.Workflow{SkipClone: true},
backConf: &backend_types.Config{
Networks: defaultNetworks,
Volumes: defaultVolumes,
Network: defaultNetwork,
Volume: defaultVolume,
},
},
{
name: "empty workflow, default clone",
fronConf: &yaml_types.Workflow{},
backConf: &backend_types.Config{
Networks: defaultNetworks,
Volumes: defaultVolumes,
Stages: []*backend_types.Stage{defaultCloneStage},
Network: defaultNetwork,
Volume: defaultVolume,
Stages: []*backend_types.Stage{defaultCloneStage},
},
},
{
@ -133,8 +133,8 @@ func TestCompilerCompile(t *testing.T) {
Image: "dummy_img",
}}}},
backConf: &backend_types.Config{
Networks: defaultNetworks,
Volumes: defaultVolumes,
Network: defaultNetwork,
Volume: defaultVolume,
Stages: []*backend_types.Stage{defaultCloneStage, {
Steps: []*backend_types.Step{{
Name: "dummy",
@ -142,7 +142,7 @@ func TestCompilerCompile(t *testing.T) {
Image: "dummy_img",
OnSuccess: true,
Failure: "fail",
Volumes: []string{defaultVolumes[0].Name + ":/woodpecker"},
Volumes: []string{defaultVolume.Name + ":/woodpecker"},
WorkingDir: "/woodpecker/src/github.com/octocat/hello-world",
WorkspaceBase: "/woodpecker",
Networks: []backend_types.Conn{{Name: "test_default", Aliases: []string{"dummy"}}},
@ -167,8 +167,8 @@ func TestCompilerCompile(t *testing.T) {
Commands: []string{"echo 2"},
}}}},
backConf: &backend_types.Config{
Networks: defaultNetworks,
Volumes: defaultVolumes,
Network: defaultNetwork,
Volume: defaultVolume,
Stages: []*backend_types.Stage{
defaultCloneStage, {
Steps: []*backend_types.Step{{
@ -178,7 +178,7 @@ func TestCompilerCompile(t *testing.T) {
Commands: []string{"env"},
OnSuccess: true,
Failure: "fail",
Volumes: []string{defaultVolumes[0].Name + ":/test"},
Volumes: []string{defaultVolume.Name + ":/test"},
WorkingDir: "/test/src/github.com/octocat/hello-world",
WorkspaceBase: "/test",
Networks: []backend_types.Conn{{Name: "test_default", Aliases: []string{"echo env"}}},
@ -192,7 +192,7 @@ func TestCompilerCompile(t *testing.T) {
Commands: []string{"echo 1"},
OnSuccess: true,
Failure: "fail",
Volumes: []string{defaultVolumes[0].Name + ":/test"},
Volumes: []string{defaultVolume.Name + ":/test"},
WorkingDir: "/test/src/github.com/octocat/hello-world",
WorkspaceBase: "/test",
Networks: []backend_types.Conn{{Name: "test_default", Aliases: []string{"parallel echo 1"}}},
@ -206,7 +206,7 @@ func TestCompilerCompile(t *testing.T) {
Commands: []string{"echo 2"},
OnSuccess: true,
Failure: "fail",
Volumes: []string{defaultVolumes[0].Name + ":/test"},
Volumes: []string{defaultVolume.Name + ":/test"},
WorkingDir: "/test/src/github.com/octocat/hello-world",
WorkspaceBase: "/test",
Networks: []backend_types.Conn{{Name: "test_default", Aliases: []string{"parallel echo 2"}}},
@ -233,8 +233,8 @@ func TestCompilerCompile(t *testing.T) {
Commands: []string{"echo 2"},
}}}},
backConf: &backend_types.Config{
Networks: defaultNetworks,
Volumes: defaultVolumes,
Network: defaultNetwork,
Volume: defaultVolume,
Stages: []*backend_types.Stage{defaultCloneStage, {
Steps: []*backend_types.Step{{
Name: "echo env",
@ -243,7 +243,7 @@ func TestCompilerCompile(t *testing.T) {
Commands: []string{"env"},
OnSuccess: true,
Failure: "fail",
Volumes: []string{defaultVolumes[0].Name + ":/test"},
Volumes: []string{defaultVolume.Name + ":/test"},
WorkingDir: "/test/src/github.com/octocat/hello-world",
WorkspaceBase: "/test",
Networks: []backend_types.Conn{{Name: "test_default", Aliases: []string{"echo env"}}},
@ -255,7 +255,7 @@ func TestCompilerCompile(t *testing.T) {
Commands: []string{"echo 2"},
OnSuccess: true,
Failure: "fail",
Volumes: []string{defaultVolumes[0].Name + ":/test"},
Volumes: []string{defaultVolume.Name + ":/test"},
WorkingDir: "/test/src/github.com/octocat/hello-world",
WorkspaceBase: "/test",
Networks: []backend_types.Conn{{Name: "test_default", Aliases: []string{"echo 2"}}},
@ -269,7 +269,7 @@ func TestCompilerCompile(t *testing.T) {
Commands: []string{"echo 1"},
OnSuccess: true,
Failure: "fail",
Volumes: []string{defaultVolumes[0].Name + ":/test"},
Volumes: []string{defaultVolume.Name + ":/test"},
WorkingDir: "/test/src/github.com/octocat/hello-world",
WorkspaceBase: "/test",
Networks: []backend_types.Conn{{Name: "test_default", Aliases: []string{"echo 1"}}},

10
pipeline/frontend/yaml/parse_test.go
View file

@ -32,10 +32,6 @@ func TestParse(t *testing.T) {
assert.Equal(t, "/go", out.Workspace.Base)
assert.Equal(t, "src/github.com/octocat/hello-world", out.Workspace.Path)
assert.Equal(t, "custom", out.Volumes.WorkflowVolumes[0].Name)
assert.Equal(t, "blockbridge", out.Volumes.WorkflowVolumes[0].Driver)
assert.Equal(t, "custom", out.Networks.WorkflowNetworks[0].Name)
assert.Equal(t, "overlay", out.Networks.WorkflowNetworks[0].Driver)
assert.Equal(t, "database", out.Services.ContainerList[0].Name)
assert.Equal(t, "mysql", out.Services.ContainerList[0].Image)
assert.Equal(t, "test", out.Steps.ContainerList[0].Name)
@ -201,12 +197,6 @@ steps:
services:
database:
image: mysql
networks:
custom:
driver: overlay
volumes:
custom:
driver: blockbridge
labels:
com.example.type: "build"
com.example.team: "frontend"

4
pipeline/frontend/yaml/types/workflow.go
View file

@ -30,10 +30,6 @@ type (
DependsOn []string `yaml:"depends_on,omitempty"`
RunsOn []string `yaml:"runs_on,omitempty"`
SkipClone bool `yaml:"skip_clone"`
// Undocumented
Networks WorkflowNetworks `yaml:"networks,omitempty"`
Volumes WorkflowVolumes `yaml:"volumes,omitempty"`
}
// Workspace defines a pipeline workspace.

52
pipeline/frontend/yaml/types/workflow_network.go
View file

@ -1,52 +0,0 @@
// Copyright 2023 Woodpecker Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package types
import (
"fmt"
"gopkg.in/yaml.v3"
)
type (
// WorkflowNetworks defines a collection of networks.
WorkflowNetworks struct {
WorkflowNetworks []*WorkflowNetwork
}
// WorkflowNetwork defines a container network.
WorkflowNetwork struct {
Name string `yaml:"name,omitempty"`
Driver string `yaml:"driver,omitempty"`
DriverOpts map[string]string `yaml:"driver_opts,omitempty"`
}
)
// UnmarshalYAML implements the Unmarshaler interface.
func (n *WorkflowNetworks) UnmarshalYAML(value *yaml.Node) error {
networks := map[string]WorkflowNetwork{}
err := value.Decode(&networks)
for key, nn := range networks {
if nn.Name == "" {
nn.Name = fmt.Sprintf("%v", key)
}
if nn.Driver == "" {
nn.Driver = "bridge"
}
n.WorkflowNetworks = append(n.WorkflowNetworks, &nn)
}
return err
}

111
pipeline/frontend/yaml/types/workflow_network_test.go
View file

@ -1,111 +0,0 @@
// Copyright 2023 Woodpecker Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package types
import (
"testing"
"github.com/stretchr/testify/assert"
"gopkg.in/yaml.v3"
)
func TestUnmarshalNetwork(t *testing.T) {
testdata := []struct {
from string
want WorkflowNetwork
}{
{
from: "{ name: foo, driver: bar }",
want: WorkflowNetwork{
Name: "foo",
Driver: "bar",
},
},
{
from: "{ name: foo, driver: bar, driver_opts: { baz: qux } }",
want: WorkflowNetwork{
Name: "foo",
Driver: "bar",
DriverOpts: map[string]string{
"baz": "qux",
},
},
},
}
for _, test := range testdata {
in := []byte(test.from)
got := WorkflowNetwork{}
err := yaml.Unmarshal(in, &got)
assert.NoError(t, err)
assert.EqualValues(t, test.want, got, "problem parsing network %q", test.from)
}
}
func TestUnmarshalWorkflowNetworks(t *testing.T) {
testdata := []struct {
from string
want []*WorkflowNetwork
}{
{
from: "foo: { driver: bar }",
want: []*WorkflowNetwork{
{
Name: "foo",
Driver: "bar",
},
},
},
{
from: "foo: { name: baz }",
want: []*WorkflowNetwork{
{
Name: "baz",
Driver: "bridge",
},
},
},
{
from: "foo: { name: baz, driver: bar }",
want: []*WorkflowNetwork{
{
Name: "baz",
Driver: "bar",
},
},
},
}
for _, test := range testdata {
in := []byte(test.from)
got := WorkflowNetworks{}
err := yaml.Unmarshal(in, &got)
assert.NoError(t, err)
assert.EqualValues(t, test.want, got.WorkflowNetworks, "problem parsing network %q", test.from)
}
}
func TestUnmarshalNetworkErr(t *testing.T) {
testdata := []string{
"foo: { name: [ foo, bar] }",
"- foo",
}
for _, test := range testdata {
in := []byte(test)
err := yaml.Unmarshal(in, new(WorkflowNetworks))
assert.Error(t, err, "wanted error for networks %q", test)
}
}

57
pipeline/frontend/yaml/types/workflow_volume.go
View file

@ -1,57 +0,0 @@
// Copyright 2023 Woodpecker Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package types
import (
"fmt"
"gopkg.in/yaml.v3"
)
type (
// WorkflowVolumes defines a collection of volumes.
WorkflowVolumes struct {
WorkflowVolumes []*WorkflowVolume
}
// WorkflowVolume defines a container volume.
WorkflowVolume struct {
Name string `yaml:"name,omitempty"`
Driver string `yaml:"driver,omitempty"`
DriverOpts map[string]string `yaml:"driver_opts,omitempty"`
}
)
// UnmarshalYAML implements the Unmarshaler interface.
func (v *WorkflowVolumes) UnmarshalYAML(value *yaml.Node) error {
y, _ := yaml.Marshal(value)
volumes := map[string]WorkflowVolume{}
err := yaml.Unmarshal(y, &volumes)
if err != nil {
return err
}
for key, vv := range volumes {
if vv.Name == "" {
vv.Name = fmt.Sprintf("%v", key)
}
if vv.Driver == "" {
vv.Driver = "local"
}
v.WorkflowVolumes = append(v.WorkflowVolumes, &vv)
}
return err
}

111
pipeline/frontend/yaml/types/workflow_volume_test.go
View file

@ -1,111 +0,0 @@
// Copyright 2023 Woodpecker Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package types
import (
"testing"
"github.com/stretchr/testify/assert"
"gopkg.in/yaml.v3"
)
func TestUnmarshalVolume(t *testing.T) {
testdata := []struct {
from string
want WorkflowVolume
}{
{
from: "{ name: foo, driver: bar }",
want: WorkflowVolume{
Name: "foo",
Driver: "bar",
},
},
{
from: "{ name: foo, driver: bar, driver_opts: { baz: qux } }",
want: WorkflowVolume{
Name: "foo",
Driver: "bar",
DriverOpts: map[string]string{
"baz": "qux",
},
},
},
}
for _, test := range testdata {
in := []byte(test.from)
got := WorkflowVolume{}
err := yaml.Unmarshal(in, &got)
assert.NoError(t, err)
assert.EqualValues(t, test.want, got, "problem parsing volume %q", test.from)
}
}
func TestUnmarshalWorkflowVolumes(t *testing.T) {
testdata := []struct {
from string
want []*WorkflowVolume
}{
{
from: "foo: { driver: bar }",
want: []*WorkflowVolume{
{
Name: "foo",
Driver: "bar",
},
},
},
{
from: "foo: { name: baz }",
want: []*WorkflowVolume{
{
Name: "baz",
Driver: "local",
},
},
},
{
from: "foo: { name: baz, driver: bar }",
want: []*WorkflowVolume{
{
Name: "baz",
Driver: "bar",
},
},
},
}
for _, test := range testdata {
in := []byte(test.from)
got := WorkflowVolumes{}
err := yaml.Unmarshal(in, &got)
assert.NoError(t, err)
assert.EqualValues(t, test.want, got.WorkflowVolumes, "problem parsing volumes %q", test.from)
}
}
func TestUnmarshalVolumesErr(t *testing.T) {
testdata := []string{
"foo: { name: [ foo, bar] }",
"- foo",
}
for _, test := range testdata {
in := []byte(test)
err := yaml.Unmarshal(in, new(WorkflowVolumes))
assert.Error(t, err, "wanted error for volumes %q", test)
}
}