Move constrain to only have a single command in backend to run to dedicated backends (#1032)

at the moment we compile a script that we can pipe in as single command
this is because of the constrains the docker backend gives us.

so we move it into the docker backend and eventually get rid of it altogether
This commit is contained in:
6543 2022-10-31 00:26:49 +01:00 committed by GitHub
parent e61f97f8ac
commit b15ca52a63
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
19 changed files with 157 additions and 259 deletions

View file

@ -0,0 +1,43 @@
// Copyright 2022 Woodpecker Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package common
import "runtime"
func GenerateContainerConf(commands []string) (env map[string]string, entry, cmd []string) {
env = make(map[string]string)
if runtime.GOOS == "windows" {
env["CI_SCRIPT"] = generateScriptWindows(commands)
env["HOME"] = "c:\\root"
env["SHELL"] = "powershell.exe"
entry = []string{"powershell", "-noprofile", "-noninteractive", "-command"}
cmd = []string{"[System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($Env:CI_SCRIPT)) | iex"}
} else {
env["CI_SCRIPT"] = generateScriptPosix(commands)
env["HOME"] = "/root"
env["SHELL"] = "/bin/sh"
entry = []string{"/bin/sh", "-c"}
cmd = []string{"echo $CI_SCRIPT | base64 -d | /bin/sh -e"}
}
return env, entry, cmd
}
func GenerateScript(commands []string) string {
if runtime.GOOS == "windows" {
return generateScriptWindows(commands)
}
return generateScriptPosix(commands)
}

View file

@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package compiler
package common
import (
"bytes"

View file

@ -1,4 +1,18 @@
package compiler
// Copyright 2022 Woodpecker Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package common
import (
"encoding/base64"

View file

@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package compiler
package common
import (
"bytes"

View file

@ -1,3 +1,17 @@
// Copyright 2022 Woodpecker Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package docker
import (
@ -8,6 +22,7 @@ import (
"github.com/docker/docker/api/types/container"
"github.com/woodpecker-ci/woodpecker/pipeline/backend/common"
"github.com/woodpecker-ci/woodpecker/pipeline/backend/types"
)
@ -20,15 +35,19 @@ func toConfig(step *types.Step) *container.Config {
AttachStdout: true,
AttachStderr: true,
}
if len(step.Commands) != 0 {
env, entry, cmd := common.GenerateContainerConf(step.Commands)
for k, v := range env {
step.Environment[k] = v
}
config.Entrypoint = entry
config.Cmd = cmd
}
if len(step.Environment) != 0 {
config.Env = toEnv(step.Environment)
}
if len(step.Command) != 0 {
config.Cmd = step.Command
}
if len(step.Entrypoint) != 0 {
config.Entrypoint = step.Entrypoint
}
if len(step.Volumes) != 0 {
config.Volumes = toVol(step.Volumes)
}

View file

@ -1,3 +1,17 @@
// Copyright 2022 Woodpecker Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package docker
import (

View file

@ -1,3 +1,17 @@
// Copyright 2022 Woodpecker Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package docker
import (

View file

@ -1,42 +0,0 @@
package docker
// import (
// )
//
// // Pool manages a pool of Docker clients.
// type Pool struct {
// queue chan (backend.Engine)
// }
//
// // NewPool returns a Pool.
// func NewPool(engines ...backend.Engine) *Pool {
// return &Pool{
// queue: make(chan backend.Engine, len(engines)),
// }
// }
//
// // Reserve requests the next available Docker client in the pool.
// func (p *Pool) Reserve(c context.Context) backend.Engine {
// select {
// case <-c.Done():
// case docker := <-p.queue:
// return docker
// }
// return nil
// }
//
// // Release releases the Docker client back to the pool.
// func (p *Pool) Release(docker backend.Engine) {
// p.queue <- docker
// }
// pool := docker.Pool(
// docker.FromEnvironmentMust(),
// docker.FromEnvironmentMust(),
// docker.FromEnvironmentMust(),
// docker.FromEnvironmentMust(),
// )
//
// client := pool.Reserve()
// defer pool.Release(client)

View file

@ -3,6 +3,7 @@ package kubernetes
import (
"strings"
"github.com/woodpecker-ci/woodpecker/pipeline/backend/common"
"github.com/woodpecker-ci/woodpecker/pipeline/backend/types"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
@ -10,8 +11,13 @@ import (
)
func Pod(namespace string, step *types.Step) *v1.Pod {
var vols []v1.Volume
var volMounts []v1.VolumeMount
var (
vols []v1.Volume
volMounts []v1.VolumeMount
entrypoint []string
args []string
)
if step.WorkingDir != "" {
for _, vol := range step.Volumes {
vols = append(vols, v1.Volume{
@ -36,13 +42,13 @@ func Pod(namespace string, step *types.Step) *v1.Pod {
pullPolicy = v1.PullAlways
}
command := step.Entrypoint
args := step.Command
envs := mapToEnvVars(step.Environment)
if _, hasScript := step.Environment["CI_SCRIPT"]; !strings.HasSuffix(step.Name, "_clone") && hasScript {
command = []string{"/bin/sh", "-c"}
args = []string{"echo $CI_SCRIPT | base64 -d | /bin/sh -e"}
if len(step.Commands) != 0 {
scriptEnv, entry, cmds := common.GenerateContainerConf(step.Commands)
for k, v := range scriptEnv {
step.Environment[k] = v
}
entrypoint = entry
args = cmds
}
hostAliases := []v1.HostAlias{}
@ -97,10 +103,10 @@ func Pod(namespace string, step *types.Step) *v1.Pod {
Name: podName(step),
Image: step.Image,
ImagePullPolicy: pullPolicy,
Command: command,
Command: entrypoint,
Args: args,
WorkingDir: step.WorkingDir,
Env: envs,
Env: mapToEnvVars(step.Environment),
VolumeMounts: volMounts,
Resources: resources,
SecurityContext: &v1.SecurityContext{

View file

@ -16,12 +16,12 @@ package local
import (
"context"
"encoding/base64"
"io"
"os"
"os/exec"
"strings"
"github.com/woodpecker-ci/woodpecker/pipeline/backend/common"
"github.com/woodpecker-ci/woodpecker/pipeline/backend/types"
"github.com/woodpecker-ci/woodpecker/shared/constant"
)
@ -62,32 +62,32 @@ func (e *local) Setup(ctx context.Context, config *types.Config) error {
// Exec the pipeline step.
func (e *local) Exec(ctx context.Context, step *types.Step) error {
// Get environment variables
Env := os.Environ()
env := os.Environ()
for a, b := range step.Environment {
if a != "HOME" && a != "SHELL" { // Don't override $HOME and $SHELL
Env = append(Env, a+"="+b)
env = append(env, a+"="+b)
}
}
Command := []string{}
command := []string{}
if step.Image == constant.DefaultCloneImage {
// Default clone step
Env = append(Env, "CI_WORKSPACE="+e.workingdir+"/"+step.Environment["CI_REPO"])
Command = append(Command, "plugin-git")
env = append(env, "CI_WORKSPACE="+e.workingdir+"/"+step.Environment["CI_REPO"])
command = append(command, "plugin-git")
} else {
// Use "image name" as run command
Command = append(Command, step.Image)
Command = append(Command, "-c")
command = append(command, step.Image)
command = append(command, "-c")
// Decode script and delete initial lines
// TODO: use commands directly
script := common.GenerateScript(step.Commands)
// Deleting the initial lines removes netrc support but adds compatibility for more shells like fish
Script, _ := base64.RawStdEncoding.DecodeString(step.Environment["CI_SCRIPT"])
Command = append(Command, string(Script)[strings.Index(string(Script), "\n\n")+2:])
command = append(command, string(script)[strings.Index(string(script), "\n\n")+2:])
}
// Prepare command
e.cmd = exec.CommandContext(ctx, Command[0], Command[1:]...)
e.cmd.Env = Env
e.cmd = exec.CommandContext(ctx, command[0], command[1:]...)
e.cmd.Env = env
// Prepare working directory
if step.Image == constant.DefaultCloneImage {

View file

@ -2,7 +2,6 @@ package ssh
import (
"context"
"encoding/base64"
"fmt"
"io"
"os"
@ -10,6 +9,7 @@ import (
"github.com/melbahja/goph"
"github.com/woodpecker-ci/woodpecker/pipeline/backend/common"
"github.com/woodpecker-ci/woodpecker/pipeline/backend/types"
"github.com/woodpecker-ci/woodpecker/shared/constant"
)
@ -91,31 +91,31 @@ func (e *ssh) Setup(ctx context.Context, config *types.Config) error {
// Exec the pipeline step.
func (e *ssh) Exec(ctx context.Context, step *types.Step) error {
// Get environment variables
Command := []string{}
command := []string{}
for a, b := range step.Environment {
if a != "HOME" && a != "SHELL" { // Don't override $HOME and $SHELL
Command = append(Command, a+"="+b)
command = append(command, a+"="+b)
}
}
if step.Image == constant.DefaultCloneImage {
// Default clone step
Command = append(Command, "CI_WORKSPACE="+e.workingdir+"/"+step.Environment["CI_REPO"])
Command = append(Command, "plugin-git")
command = append(command, "CI_WORKSPACE="+e.workingdir+"/"+step.Environment["CI_REPO"])
command = append(command, "plugin-git")
} else {
// Use "image name" as run command
Command = append(Command, step.Image)
Command = append(Command, "-c")
command = append(command, step.Image)
command = append(command, "-c")
// Decode script and delete initial lines
// TODO: use commands directly
script := common.GenerateScript(step.Commands)
// Deleting the initial lines removes netrc support but adds compatibility for more shells like fish
Script, _ := base64.RawStdEncoding.DecodeString(step.Environment["CI_SCRIPT"])
Command = append(Command, "cd "+e.workingdir+"/"+step.Environment["CI_REPO"]+" && "+string(Script)[strings.Index(string(Script), "\n\n")+2:])
command = append(command, "cd "+e.workingdir+"/"+step.Environment["CI_REPO"]+" && "+string(script)[strings.Index(string(script), "\n\n")+2:])
}
// Prepare command
var err error
e.cmd, err = e.client.CommandContext(ctx, "/bin/env", Command...)
e.cmd, err = e.client.CommandContext(ctx, "/bin/env", command...)
if err != nil {
return err
}

View file

@ -12,7 +12,7 @@ type Step struct {
Environment map[string]string `json:"environment,omitempty"`
Labels map[string]string `json:"labels,omitempty"`
Entrypoint []string `json:"entrypoint,omitempty"`
Command []string `json:"command,omitempty"`
Commands []string `json:"commands,omitempty"`
ExtraHosts []string `json:"extra_hosts,omitempty"`
Volumes []string `json:"volumes,omitempty"`
Tmpfs []string `json:"tmpfs,omitempty"`

View file

@ -20,8 +20,6 @@ func (c *Compiler) createProcess(name string, container *yaml.Container, section
workspace = fmt.Sprintf("%s_default:%s", c.prefix, c.base)
privileged = container.Privileged
entrypoint = container.Entrypoint
command = container.Command
networkMode = container.NetworkMode
ipcMode = container.IpcMode
// network = container.Network
@ -85,26 +83,8 @@ func (c *Compiler) createProcess(name string, container *yaml.Container, section
}
}
if len(container.Commands) != 0 {
if c.metadata.Sys.Platform == "windows/amd64" {
entrypoint = []string{"powershell", "-noprofile", "-noninteractive", "-command"}
command = []string{"[System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($Env:CI_SCRIPT)) | iex"}
environment["CI_SCRIPT"] = generateScriptWindows(container.Commands)
environment["HOME"] = "c:\\root"
environment["SHELL"] = "powershell.exe"
} else {
entrypoint = []string{"/bin/sh", "-c"}
command = []string{"echo $CI_SCRIPT | base64 -d | /bin/sh -e"}
environment["CI_SCRIPT"] = generateScriptPosix(container.Commands)
environment["HOME"] = "/root"
environment["SHELL"] = "/bin/sh"
}
}
if matchImage(container.Image, c.escalated...) {
privileged = true
entrypoint = []string{}
command = []string{}
}
authConfig := backend.Auth{
@ -169,8 +149,7 @@ func (c *Compiler) createProcess(name string, container *yaml.Container, section
WorkingDir: workingdir,
Environment: environment,
Labels: container.Labels,
Entrypoint: entrypoint,
Command: command,
Commands: container.Commands,
ExtraHosts: container.ExtraHosts,
Volumes: volumes,
Tmpfs: container.Tmpfs,

View file

@ -27,7 +27,6 @@ type (
AuthConfig AuthConfig `yaml:"auth_config,omitempty"`
CapAdd []string `yaml:"cap_add,omitempty"`
CapDrop []string `yaml:"cap_drop,omitempty"`
Command types.Command `yaml:"command,omitempty"`
Commands types.Stringorslice `yaml:"commands,omitempty"`
CPUQuota types.StringorInt `yaml:"cpu_quota,omitempty"`
CPUSet string `yaml:"cpuset,omitempty"`
@ -38,7 +37,6 @@ type (
DNS types.Stringorslice `yaml:"dns,omitempty"`
DNSSearch types.Stringorslice `yaml:"dns_search,omitempty"`
Directory string `yaml:"directory,omitempty"`
Entrypoint types.Command `yaml:"entrypoint,omitempty"`
Environment types.SliceorMap `yaml:"environment,omitempty"`
ExtraHosts []string `yaml:"extra_hosts,omitempty"`
Group string `yaml:"group,omitempty"`
@ -113,5 +111,5 @@ func (c *Containers) UnmarshalYAML(value *yaml.Node) error {
}
func (c *Container) IsPlugin() bool {
return len(c.Commands) == 0 && len(c.Command) == 0
return len(c.Commands) == 0
}

View file

@ -18,7 +18,6 @@ auth_config:
password: password
cap_add: [ ALL ]
cap_drop: [ NET_ADMIN, SYS_ADMIN ]
command: bundle exec thin -p 3000
commands:
- go build
- go test
@ -31,7 +30,6 @@ devices:
directory: example/
dns: 8.8.8.8
dns_search: example.com
entrypoint: /code/entrypoint.sh
environment:
- RACK_ENV=development
- SHOW=true
@ -76,7 +74,6 @@ func TestUnmarshalContainer(t *testing.T) {
},
CapAdd: []string{"ALL"},
CapDrop: []string{"NET_ADMIN", "SYS_ADMIN"},
Command: types.Command{"bundle exec thin -p 3000"},
Commands: types.Stringorslice{"go build", "go test"},
CPUQuota: types.StringorInt(11),
CPUSet: "1,2",
@ -86,7 +83,6 @@ func TestUnmarshalContainer(t *testing.T) {
Directory: "example/",
DNS: types.Stringorslice{"8.8.8.8"},
DNSSearch: types.Stringorslice{"example.com"},
Entrypoint: types.Command{"/code/entrypoint.sh"},
Environment: types.SliceorMap{"RACK_ENV": "development", "SHOW": "true"},
ExtraHosts: []string{"somehost:162.242.195.82", "otherhost:50.31.209.229"},
Image: "golang:latest",

View file

@ -53,11 +53,6 @@ func (l *Linter) lint(containers []*yaml.Container, block uint8) error {
return err
}
}
if block != blockServices && !container.Detached {
if err := l.lintEntrypoint(container); err != nil {
return err
}
}
if err := l.lintCommands(container); err != nil {
return err
}
@ -83,22 +78,6 @@ func (l *Linter) lintCommands(c *yaml.Container) error {
}
return fmt.Errorf("Cannot configure both commands and custom attributes %v", keys)
}
if len(c.Entrypoint) != 0 {
return fmt.Errorf("Cannot configure both commands and entrypoint attributes")
}
if len(c.Command) != 0 {
return fmt.Errorf("Cannot configure both commands and command attributes")
}
return nil
}
func (l *Linter) lintEntrypoint(c *yaml.Container) error {
if len(c.Entrypoint) != 0 {
return fmt.Errorf("Cannot override container entrypoint")
}
if len(c.Command) != 0 {
return fmt.Errorf("Cannot override container command")
}
return nil
}

View file

@ -26,8 +26,6 @@ pipeline:
services:
redis:
image: redis
entrypoint: [ /bin/redis-server ]
command: [ -v ]
`}, {Title: "list", Data: `
pipeline:
- name: build
@ -117,24 +115,6 @@ func TestLintErrors(t *testing.T) {
from: "pipeline: { build: { image: golang, sysctls: [ net.core.somaxconn=1024 ] } }",
want: "Insufficient privileges to use sysctls",
},
// cannot override entypoint, command for script steps
{
from: "pipeline: { build: { image: golang, commands: [ 'go build' ], entrypoint: [ '/bin/bash' ] } }",
want: "Cannot override container entrypoint",
},
{
from: "pipeline: { build: { image: golang, commands: [ 'go build' ], command: [ '/bin/bash' ] } }",
want: "Cannot override container command",
},
// cannot override entypoint, command for plugin steps
{
from: "pipeline: { publish: { image: plugins/docker, repo: foo/bar, entrypoint: [ '/bin/bash' ] } }",
want: "Cannot override container entrypoint",
},
{
from: "pipeline: { publish: { image: plugins/docker, repo: foo/bar, command: [ '/bin/bash' ] } }",
want: "Cannot override container command",
},
}
for _, test := range testdata {

View file

@ -1,37 +0,0 @@
package types
import (
"errors"
"fmt"
"github.com/docker/docker/api/types/strslice"
)
// Command represents a docker command, can be a string or an array of strings.
type Command strslice.StrSlice
// UnmarshalYAML implements the Unmarshaler interface.
func (s *Command) UnmarshalYAML(unmarshal func(interface{}) error) error {
var stringType string
if err := unmarshal(&stringType); err == nil {
*s = []string{stringType}
return nil
}
var sliceType []interface{}
if err := unmarshal(&sliceType); err == nil {
parts, err := toStrings(sliceType)
if err != nil {
return err
}
*s = parts
return nil
}
var interfaceType interface{}
if err := unmarshal(&interfaceType); err == nil {
fmt.Println(interfaceType)
}
return errors.New("Failed to unmarshal Command")
}

View file

@ -1,65 +0,0 @@
package types
import (
"strings"
"testing"
"gopkg.in/yaml.v3"
"github.com/stretchr/testify/assert"
)
type StructCommand struct {
Entrypoint Command `yaml:"entrypoint,flow,omitempty"`
Command Command `yaml:"command,flow,omitempty"`
}
func TestUnmarshalCommand(t *testing.T) {
s := &StructCommand{}
err := yaml.Unmarshal([]byte(`command: bash`), s)
assert.Nil(t, err)
assert.Equal(t, Command{"bash"}, s.Command)
assert.Nil(t, s.Entrypoint)
bytes, err := yaml.Marshal(s)
assert.Nil(t, err)
s2 := &StructCommand{}
err = yaml.Unmarshal(bytes, s2)
assert.Nil(t, err)
assert.Equal(t, Command{"bash"}, s2.Command)
assert.Nil(t, s2.Entrypoint)
s3 := &StructCommand{}
err = yaml.Unmarshal([]byte(`command:
- echo AAA; echo "wow"
- sleep 3s`), s3)
assert.Nil(t, err)
assert.Equal(t, Command{`echo AAA; echo "wow"`, `sleep 3s`}, s3.Command)
s4 := &StructCommand{}
err = yaml.Unmarshal([]byte(`command: echo AAA; echo "wow"`), s4)
assert.Nil(t, err)
assert.Equal(t, Command{`echo AAA; echo "wow"`}, s4.Command)
}
var sampleEmptyCommand = `{}`
func TestUnmarshalEmptyCommand(t *testing.T) {
s := &StructCommand{}
err := yaml.Unmarshal([]byte(sampleEmptyCommand), s)
assert.Nil(t, err)
assert.Nil(t, s.Command)
bytes, err := yaml.Marshal(s)
assert.Nil(t, err)
assert.Equal(t, "{}", strings.TrimSpace(string(bytes)))
s2 := &StructCommand{}
err = yaml.Unmarshal(bytes, s2)
assert.Nil(t, err)
assert.Nil(t, s2.Command)
}