Make sure we dont have hidden options for backend and pipeline compiler (#2123)

move options based on **os.Getenv** into flags

---------
*Sponsored by Kithara Software GmbH*

cli/exec/exec.go

@@ -30,6 +30,10 @@ import (
 	"github.com/woodpecker-ci/woodpecker/cli/common"
 	"github.com/woodpecker-ci/woodpecker/pipeline"
 	"github.com/woodpecker-ci/woodpecker/pipeline/backend"
+	"github.com/woodpecker-ci/woodpecker/pipeline/backend/docker"
+	"github.com/woodpecker-ci/woodpecker/pipeline/backend/kubernetes"
+	"github.com/woodpecker-ci/woodpecker/pipeline/backend/local"
+	"github.com/woodpecker-ci/woodpecker/pipeline/backend/ssh"
 	backendTypes "github.com/woodpecker-ci/woodpecker/pipeline/backend/types"
 	"github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml"
 	"github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml/compiler"
@@ -45,7 +49,7 @@ var Command = &cli.Command{
 	Usage:     "execute a local pipeline",
 	ArgsUsage: "[path/to/.woodpecker.yaml]",
 	Action:    run,
-	Flags:     append(common.GlobalFlags, flags...),
+	Flags:     utils.MergeSlices(common.GlobalFlags, flags, docker.Flags, ssh.Flags, kubernetes.Flags, local.Flags),
 }
 
 func run(c *cli.Context) error {
@@ -184,7 +188,11 @@ func execWithAxis(c *cli.Context, file, repoPath string, axis matrix.Axis) error
 		compiler.WithPrefix(
 			c.String("prefix"),
 		),
-		compiler.WithProxy(),
+		compiler.WithProxy(compiler.ProxyOptions{
+			NoProxy:    c.String("backend-no-proxy"),
+			HTTPProxy:  c.String("backend-http-proxy"),
+			HTTPSProxy: c.String("backend-https-proxy"),
+		}),
 		compiler.WithLocal(
 			c.Bool("local"),
 		),

cli/exec/flags.go

@@ -64,6 +64,25 @@ var flags = []cli.Flag{
 		Value:   "auto-detect",
 	},
 
+	//
+	// backend options for pipeline compiler
+	//
+	&cli.StringFlag{
+		EnvVars: []string{"WOODPECKER_BACKEND_NO_PROXY", "NO_PROXY", "no_proxy"},
+		Usage:   "if set, pass the environment variable down as \"NO_PROXY\" to steps",
+		Name:    "backend-no-proxy",
+	},
+	&cli.StringFlag{
+		EnvVars: []string{"WOODPECKER_BACKEND_HTTP_PROXY", "HTTP_PROXY", "http_proxy"},
+		Usage:   "if set, pass the environment variable down as \"NO_PROXY\" to steps",
+		Name:    "backend-http-proxy",
+	},
+	&cli.StringFlag{
+		EnvVars: []string{"WOODPECKER_BACKEND_HTTPS_PROXY", "HTTPS_PROXY", "https_proxy"},
+		Usage:   "if set, pass the environment variable down as \"HTTPS_PROXY\" to steps",
+		Name:    "backend-https-proxy",
+	},
+
 	//
 	// Please note the below flags should match the flags from
 	// pipeline/frontend/metadata.go and should be kept synchronized.
@@ -292,87 +311,4 @@ var flags = []cli.Flag{
 		EnvVars: []string{"CI_FORGE_URL"},
 		Name:    "forge-url",
 	},
-
-	// backend docker
-	&cli.BoolFlag{
-		EnvVars: []string{"WOODPECKER_BACKEND_DOCKER_ENABLE_IPV6"},
-		Name:    "backend-docker-ipv6",
-		Usage:   "backend docker enable IPV6",
-		Value:   false,
-	},
-	&cli.StringFlag{
-		EnvVars: []string{"WOODPECKER_BACKEND_DOCKER_NETWORK"},
-		Name:    "backend-docker-network",
-		Usage:   "backend docker network",
-	},
-	&cli.StringFlag{
-		EnvVars: []string{"WOODPECKER_BACKEND_DOCKER_VOLUMES"},
-		Name:    "backend-docker-volumes",
-		Usage:   "backend docker volumes (comma separated)",
-	},
-
-	// backend ssh
-	&cli.StringFlag{
-		EnvVars: []string{"WOODPECKER_BACKEND_SSH_ADDRESS"},
-		Name:    "backend-ssh-address",
-		Usage:   "backend ssh address",
-	},
-	&cli.StringFlag{
-		EnvVars: []string{"WOODPECKER_BACKEND_SSH_USER"},
-		Name:    "backend-ssh-user",
-		Usage:   "backend ssh user",
-	},
-	&cli.StringFlag{
-		EnvVars: []string{"WOODPECKER_BACKEND_SSH_KEY"},
-		Name:    "backend-ssh-key",
-		Usage:   "backend ssh key file",
-	},
-	&cli.StringFlag{
-		EnvVars: []string{"WOODPECKER_BACKEND_SSH_KEY_PASSWORD"},
-		Name:    "backend-ssh-key-password",
-		Usage:   "backend ssh key password",
-	},
-	&cli.StringFlag{
-		EnvVars: []string{"WOODPECKER_BACKEND_SSH_PASSWORD"},
-		Name:    "backend-ssh-password",
-		Usage:   "backend ssh password",
-	},
-
-	// backend k8s
-	&cli.StringFlag{
-		EnvVars: []string{"WOODPECKER_BACKEND_K8S_NAMESPACE"},
-		Name:    "backend-k8s-namespace",
-		Usage:   "backend k8s namespace",
-		Value:   "woodpecker",
-	},
-	&cli.StringFlag{
-		EnvVars: []string{"WOODPECKER_BACKEND_K8S_VOLUME_SIZE"},
-		Name:    "backend-k8s-volume-size",
-		Usage:   "backend k8s volume size (default 10G)",
-		Value:   "10G",
-	},
-	&cli.StringFlag{
-		EnvVars: []string{"WOODPECKER_BACKEND_K8S_STORAGE_CLASS"},
-		Name:    "backend-k8s-storage-class",
-		Usage:   "backend k8s storage class",
-		Value:   "",
-	},
-	&cli.BoolFlag{
-		EnvVars: []string{"WOODPECKER_BACKEND_K8S_STORAGE_RWX"},
-		Name:    "backend-k8s-storage-rwx",
-		Usage:   "backend k8s storage access mode, should ReadWriteMany (RWX) instead of ReadWriteOnce (RWO) be used? (default: true)",
-		Value:   true,
-	},
-	&cli.StringFlag{
-		EnvVars: []string{"WOODPECKER_BACKEND_K8S_POD_LABELS"},
-		Name:    "backend-k8s-pod-labels",
-		Usage:   "backend k8s additional worker pod labels",
-		Value:   "",
-	},
-	&cli.StringFlag{
-		EnvVars: []string{"WOODPECKER_BACKEND_K8S_POD_ANNOTATIONS"},
-		Name:    "backend-k8s-pod-annotations",
-		Usage:   "backend k8s additional worker pod annotations",
-		Value:   "",
-	},
 }

cmd/agent/flags.go

@@ -20,11 +20,9 @@ import (
 	"time"
 
 	"github.com/urfave/cli/v2"
-
-	"github.com/woodpecker-ci/woodpecker/cmd/common"
 )
 
-var flags = append([]cli.Flag{
+var flags = []cli.Flag{
 	&cli.StringFlag{
 		EnvVars: []string{"WOODPECKER_SERVER"},
 		Name:    "server",
@@ -99,99 +97,4 @@ var flags = append([]cli.Flag{
 		Usage:   "backend engine to run pipelines on",
 		Value:   "auto-detect",
 	},
-
-	// backend docker
-	&cli.BoolFlag{
-		EnvVars: []string{"WOODPECKER_BACKEND_DOCKER_ENABLE_IPV6"},
-		Name:    "backend-docker-ipv6",
-		Usage:   "backend docker enable IPV6",
-		Value:   false,
-	},
-	&cli.StringFlag{
-		EnvVars: []string{"WOODPECKER_BACKEND_DOCKER_NETWORK"},
-		Name:    "backend-docker-network",
-		Usage:   "backend docker network",
-	},
-	&cli.StringFlag{
-		EnvVars: []string{"WOODPECKER_BACKEND_DOCKER_VOLUMES"},
-		Name:    "backend-docker-volumes",
-		Usage:   "backend docker volumes (comma separated)",
-	},
-
-	// backend ssh
-	&cli.StringFlag{
-		EnvVars: []string{"WOODPECKER_BACKEND_SSH_ADDRESS"},
-		Name:    "backend-ssh-address",
-		Usage:   "backend ssh address",
-	},
-	&cli.StringFlag{
-		EnvVars: []string{"WOODPECKER_BACKEND_SSH_USER"},
-		Name:    "backend-ssh-user",
-		Usage:   "backend ssh user",
-	},
-	&cli.StringFlag{
-		EnvVars: []string{"WOODPECKER_BACKEND_SSH_KEY"},
-		Name:    "backend-ssh-key",
-		Usage:   "backend ssh key file",
-	},
-	&cli.StringFlag{
-		EnvVars: []string{"WOODPECKER_BACKEND_SSH_KEY_PASSWORD"},
-		Name:    "backend-ssh-key-password",
-		Usage:   "backend ssh key password",
-	},
-	&cli.StringFlag{
-		EnvVars: []string{"WOODPECKER_BACKEND_SSH_PASSWORD"},
-		Name:    "backend-ssh-password",
-		Usage:   "backend ssh password",
-	},
-
-	// backend k8s
-	&cli.StringFlag{
-		EnvVars: []string{"WOODPECKER_BACKEND_K8S_NAMESPACE"},
-		Name:    "backend-k8s-namespace",
-		Usage:   "backend k8s namespace",
-		Value:   "woodpecker",
-	},
-	&cli.StringFlag{
-		EnvVars: []string{"WOODPECKER_BACKEND_K8S_VOLUME_SIZE"},
-		Name:    "backend-k8s-volume-size",
-		Usage:   "backend k8s volume size (default 10G)",
-		Value:   "10G",
-	},
-	&cli.StringFlag{
-		EnvVars: []string{"WOODPECKER_BACKEND_K8S_STORAGE_CLASS"},
-		Name:    "backend-k8s-storage-class",
-		Usage:   "backend k8s storage class",
-		Value:   "",
-	},
-	&cli.BoolFlag{
-		EnvVars: []string{"WOODPECKER_BACKEND_K8S_STORAGE_RWX"},
-		Name:    "backend-k8s-storage-rwx",
-		Usage:   "backend k8s storage access mode, should ReadWriteMany (RWX) instead of ReadWriteOnce (RWO) be used? (default: true)",
-		Value:   true,
-	},
-	&cli.StringFlag{
-		EnvVars: []string{"WOODPECKER_BACKEND_K8S_POD_LABELS"},
-		Name:    "backend-k8s-pod-labels",
-		Usage:   "backend k8s additional worker pod labels",
-		Value:   "",
-	},
-	&cli.StringFlag{
-		EnvVars: []string{"WOODPECKER_BACKEND_K8S_POD_ANNOTATIONS"},
-		Name:    "backend-k8s-pod-annotations",
-		Usage:   "backend k8s additional worker pod annotations",
-		Value:   "",
-	},
-	&cli.IntFlag{
-		EnvVars: []string{"WOODPECKER_CONNECT_RETRY_COUNT"},
-		Name:    "connect-retry-count",
-		Usage:   "number of times to retry connecting to the server",
-		Value:   5,
-	},
-	&cli.DurationFlag{
-		EnvVars: []string{"WOODPECKER_CONNECT_RETRY_DELAY"},
-		Name:    "connect-retry-delay",
-		Usage:   "duration to wait before retrying to connect to the server",
-		Value:   time.Second * 2,
-	},
-}, common.GlobalLoggerFlags...)
+}

cmd/agent/main.go

@@ -21,6 +21,12 @@ import (
 	_ "github.com/joho/godotenv/autoload"
 	"github.com/urfave/cli/v2"
 
+	"github.com/woodpecker-ci/woodpecker/cmd/common"
+	"github.com/woodpecker-ci/woodpecker/pipeline/backend/docker"
+	"github.com/woodpecker-ci/woodpecker/pipeline/backend/kubernetes"
+	"github.com/woodpecker-ci/woodpecker/pipeline/backend/local"
+	"github.com/woodpecker-ci/woodpecker/pipeline/backend/ssh"
+	"github.com/woodpecker-ci/woodpecker/shared/utils"
 	"github.com/woodpecker-ci/woodpecker/version"
 )
 
@@ -37,7 +43,7 @@ func main() {
 			Action: pinger,
 		},
 	}
-	app.Flags = flags
+	app.Flags = utils.MergeSlices(flags, common.GlobalLoggerFlags, docker.Flags, ssh.Flags, kubernetes.Flags, local.Flags)
 
 	if err := app.Run(os.Args); err != nil {
 		fmt.Fprintln(os.Stderr, err)

cmd/server/flags.go

@@ -263,6 +263,24 @@ var flags = append([]cli.Flag{
 		Value:   true,
 	},
 	//
+	// backend options for pipeline compiler
+	//
+	&cli.StringFlag{
+		EnvVars: []string{"WOODPECKER_BACKEND_NO_PROXY", "NO_PROXY", "no_proxy"},
+		Usage:   "if set, pass the environment variable down as \"NO_PROXY\" to steps",
+		Name:    "backend-no-proxy",
+	},
+	&cli.StringFlag{
+		EnvVars: []string{"WOODPECKER_BACKEND_HTTP_PROXY", "HTTP_PROXY", "http_proxy"},
+		Usage:   "if set, pass the environment variable down as \"NO_PROXY\" to steps",
+		Name:    "backend-http-proxy",
+	},
+	&cli.StringFlag{
+		EnvVars: []string{"WOODPECKER_BACKEND_HTTPS_PROXY", "HTTPS_PROXY", "https_proxy"},
+		Usage:   "if set, pass the environment variable down as \"HTTPS_PROXY\" to steps",
+		Name:    "backend-https-proxy",
+	},
+	//
 	// resource limit parameters
 	//
 	&cli.DurationFlag{

cmd/server/server.go

@@ -318,6 +318,11 @@ func setupEvilGlobals(c *cli.Context, v store.Store, f forge.Forge) {
 	server.Config.Pipeline.Limits.CPUShares = c.Int64("limit-cpu-shares")
 	server.Config.Pipeline.Limits.CPUSet = c.String("limit-cpu-set")
 
+	// backend options for pipeline compiler
+	server.Config.Pipeline.Proxy.No = c.String("backend-no-proxy")
+	server.Config.Pipeline.Proxy.HTTP = c.String("backend-http-proxy")
+	server.Config.Pipeline.Proxy.HTTPS = c.String("backend-https-proxy")
+
 	// server configuration
 	server.Config.Server.Cert = c.String("server-cert")
 	server.Config.Server.Key = c.String("server-key")

pipeline/backend/docker/docker.go

@@ -17,13 +17,16 @@ package docker
 import (
 	"context"
 	"io"
+	"net/http"
 	"os"
+	"path/filepath"
 	"runtime"
 	"strings"
 
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/network"
 	"github.com/docker/docker/api/types/volume"
+	"github.com/docker/go-connections/tlsconfig"
 	"github.com/moby/moby/client"
 	"github.com/moby/moby/pkg/jsonmessage"
 	"github.com/moby/moby/pkg/stdcopy"
@@ -67,20 +70,56 @@ func (e *docker) IsAvailable(context.Context) bool {
 	return err == nil
 }
 
+func httpClientOfOpts(dockerCertPath string, verifyTLS bool) *http.Client {
+	if dockerCertPath == "" {
+		return nil
+	}
+
+	options := tlsconfig.Options{
+		CAFile:             filepath.Join(dockerCertPath, "ca.pem"),
+		CertFile:           filepath.Join(dockerCertPath, "cert.pem"),
+		KeyFile:            filepath.Join(dockerCertPath, "key.pem"),
+		InsecureSkipVerify: !verifyTLS,
+	}
+	tlsConf, err := tlsconfig.Client(options)
+	if err != nil {
+		log.Error().Err(err).Msg("could not create http client out of docker backend options")
+		return nil
+	}
+
+	return &http.Client{
+		Transport:     &http.Transport{TLSClientConfig: tlsConf},
+		CheckRedirect: client.CheckRedirect,
+	}
+}
+
 // Load new client for Docker Engine using environment variables.
 func (e *docker) Load(ctx context.Context) error {
-	cl, err := client.NewClientWithOpts(client.FromEnv)
+	c, ok := ctx.Value(backend.CliContext).(*cli.Context)
+	if !ok {
+		return backend.ErrNoCliContextFound
+	}
+
+	var dockerClientOpts []client.Opt
+	if httpClient := httpClientOfOpts(c.String("backend-docker-cert"), c.Bool("backend-docker-tls-verify")); httpClient != nil {
+		dockerClientOpts = append(dockerClientOpts, client.WithHTTPClient(httpClient))
+	}
+	if dockerHost := c.String("backend-docker-host"); dockerHost != "" {
+		dockerClientOpts = append(dockerClientOpts, client.WithHost(dockerHost))
+	}
+	if dockerAPIVersion := c.String("backend-docker-api-version"); dockerAPIVersion != "" {
+		dockerClientOpts = append(dockerClientOpts, client.WithVersion(dockerAPIVersion))
+	} else {
+		dockerClientOpts = append(dockerClientOpts, client.WithAPIVersionNegotiation())
+	}
+
+	cl, err := client.NewClientWithOpts(dockerClientOpts...)
 	if err != nil {
 		return err
 	}
 	e.client = cl
 
-	c, ok := ctx.Value(backend.CliContext).(*cli.Context)
-	if !ok {
-		return backend.ErrNoCliContextFound
-	}
 	e.enableIPv6 = c.Bool("backend-docker-ipv6")
-
 	e.network = c.String("backend-docker-network")
 
 	volumes := strings.Split(c.String("backend-docker-volumes"), ",")

pipeline/backend/docker/flags.go

@@ -0,0 +1,59 @@
+// Copyright 2023 Woodpecker Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package docker
+
+import (
+	"github.com/urfave/cli/v2"
+)
+
+var Flags = []cli.Flag{
+	&cli.StringFlag{
+		EnvVars: []string{"WOODPECKER_BACKEND_DOCKER_HOST", "DOCKER_HOST"},
+		Name:    "backend-docker-host",
+		Usage:   "path to docker socket or url to the docker server",
+	},
+	&cli.StringFlag{
+		EnvVars: []string{"WOODPECKER_BACKEND_DOCKER_API_VERSION", "DOCKER_API_VERSION"},
+		Name:    "backend-docker-api-version",
+		Usage:   "the version of the API to reach, leave empty for latest.",
+	},
+	&cli.StringFlag{
+		EnvVars: []string{"WOODPECKER_BACKEND_DOCKER_CERT_PATH", "DOCKER_CERT_PATH"},
+		Name:    "backend-docker-cert",
+		Usage:   "path to load the TLS certificates for connecting to docker server",
+	},
+	&cli.BoolFlag{
+		EnvVars: []string{"WOODPECKER_BACKEND_DOCKER_TLS_VERIFY", "DOCKER_TLS_VERIFY"},
+		Name:    "backend-docker-tls-verify",
+		Usage:   "enable or disable TLS verification for connecting to docker server",
+		Value:   true,
+	},
+	&cli.BoolFlag{
+		EnvVars: []string{"WOODPECKER_BACKEND_DOCKER_ENABLE_IPV6"},
+		Name:    "backend-docker-ipv6",
+		Usage:   "backend docker enable IPV6",
+		Value:   false,
+	},
+	&cli.StringFlag{
+		EnvVars: []string{"WOODPECKER_BACKEND_DOCKER_NETWORK"},
+		Name:    "backend-docker-network",
+		Usage:   "backend docker network",
+	},
+	&cli.StringFlag{
+		EnvVars: []string{"WOODPECKER_BACKEND_DOCKER_VOLUMES"},
+		Name:    "backend-docker-volumes",
+		Usage:   "backend docker volumes (comma separated)",
+	},
+}

pipeline/backend/kubernetes/flags.go

@@ -0,0 +1,72 @@
+// Copyright 2023 Woodpecker Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package kubernetes
+
+import (
+	"time"
+
+	"github.com/urfave/cli/v2"
+)
+
+var Flags = []cli.Flag{
+	&cli.StringFlag{
+		EnvVars: []string{"WOODPECKER_BACKEND_K8S_NAMESPACE"},
+		Name:    "backend-k8s-namespace",
+		Usage:   "backend k8s namespace",
+		Value:   "woodpecker",
+	},
+	&cli.StringFlag{
+		EnvVars: []string{"WOODPECKER_BACKEND_K8S_VOLUME_SIZE"},
+		Name:    "backend-k8s-volume-size",
+		Usage:   "backend k8s volume size (default 10G)",
+		Value:   "10G",
+	},
+	&cli.StringFlag{
+		EnvVars: []string{"WOODPECKER_BACKEND_K8S_STORAGE_CLASS"},
+		Name:    "backend-k8s-storage-class",
+		Usage:   "backend k8s storage class",
+		Value:   "",
+	},
+	&cli.BoolFlag{
+		EnvVars: []string{"WOODPECKER_BACKEND_K8S_STORAGE_RWX"},
+		Name:    "backend-k8s-storage-rwx",
+		Usage:   "backend k8s storage access mode, should ReadWriteMany (RWX) instead of ReadWriteOnce (RWO) be used? (default: true)",
+		Value:   true,
+	},
+	&cli.StringFlag{
+		EnvVars: []string{"WOODPECKER_BACKEND_K8S_POD_LABELS"},
+		Name:    "backend-k8s-pod-labels",
+		Usage:   "backend k8s additional worker pod labels",
+		Value:   "",
+	},
+	&cli.StringFlag{
+		EnvVars: []string{"WOODPECKER_BACKEND_K8S_POD_ANNOTATIONS"},
+		Name:    "backend-k8s-pod-annotations",
+		Usage:   "backend k8s additional worker pod annotations",
+		Value:   "",
+	},
+	&cli.IntFlag{
+		EnvVars: []string{"WOODPECKER_CONNECT_RETRY_COUNT"},
+		Name:    "connect-retry-count",
+		Usage:   "number of times to retry connecting to the server",
+		Value:   5,
+	},
+	&cli.DurationFlag{
+		EnvVars: []string{"WOODPECKER_CONNECT_RETRY_DELAY"},
+		Name:    "connect-retry-delay",
+		Usage:   "duration to wait before retrying to connect to the server",
+		Value:   time.Second * 2,
+	},
+}

pipeline/backend/kubernetes/kubernetes.go

@@ -1,3 +1,17 @@
+// Copyright 2022 Woodpecker Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package kubernetes
 
 import (

pipeline/backend/kubernetes/pod.go

@@ -1,3 +1,17 @@
+// Copyright 2022 Woodpecker Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package kubernetes
 
 import (

pipeline/backend/kubernetes/service.go

@@ -1,3 +1,17 @@
+// Copyright 2022 Woodpecker Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package kubernetes
 
 import (

pipeline/backend/kubernetes/service_test.go

@@ -1,3 +1,17 @@
+// Copyright 2022 Woodpecker Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package kubernetes
 
 import (

pipeline/backend/kubernetes/utils.go

@@ -1,3 +1,17 @@
+// Copyright 2022 Woodpecker Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package kubernetes
 
 import (
@@ -41,13 +55,13 @@ func isImagePullBackOffState(pod *v1.Pod) bool {
 
 // getClientOutOfCluster returns a k8s clientset to the request from outside of cluster
 func getClientOutOfCluster() (kubernetes.Interface, error) {
-	kubeconfigPath := os.Getenv("KUBECONFIG")
-	if kubeconfigPath == "" {
-		kubeconfigPath = os.Getenv("HOME") + "/.kube/config"
+	kubeConfigPath := os.Getenv("KUBECONFIG")
+	if kubeConfigPath == "" {
+		kubeConfigPath = os.Getenv("HOME") + "/.kube/config"
 	}
 
 	// use the current context in kubeconfig
-	config, err := clientcmd.BuildConfigFromFlags("", kubeconfigPath)
+	config, err := clientcmd.BuildConfigFromFlags("", kubeConfigPath)
 	if err != nil {
 		return nil, err
 	}

pipeline/backend/kubernetes/volume.go

@@ -1,3 +1,17 @@
+// Copyright 2022 Woodpecker Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package kubernetes
 
 import (

pipeline/backend/kubernetes/volume_test.go

@@ -1,3 +1,17 @@
+// Copyright 2022 Woodpecker Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package kubernetes
 
 import (

pipeline/backend/local/flags.go

@@ -0,0 +1,21 @@
+// Copyright 2023 Woodpecker Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package local
+
+import (
+	"github.com/urfave/cli/v2"
+)
+
+var Flags = []cli.Flag{}

pipeline/backend/ssh/flags.go

@@ -0,0 +1,47 @@
+// Copyright 2023 Woodpecker Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package ssh
+
+import (
+	"github.com/urfave/cli/v2"
+)
+
+var Flags = []cli.Flag{
+	&cli.StringFlag{
+		EnvVars: []string{"WOODPECKER_BACKEND_SSH_ADDRESS"},
+		Name:    "backend-ssh-address",
+		Usage:   "backend ssh address",
+	},
+	&cli.StringFlag{
+		EnvVars: []string{"WOODPECKER_BACKEND_SSH_USER"},
+		Name:    "backend-ssh-user",
+		Usage:   "backend ssh user",
+	},
+	&cli.StringFlag{
+		EnvVars: []string{"WOODPECKER_BACKEND_SSH_KEY"},
+		Name:    "backend-ssh-key",
+		Usage:   "backend ssh key file",
+	},
+	&cli.StringFlag{
+		EnvVars: []string{"WOODPECKER_BACKEND_SSH_KEY_PASSWORD"},
+		Name:    "backend-ssh-key-password",
+		Usage:   "backend ssh key password",
+	},
+	&cli.StringFlag{
+		EnvVars: []string{"WOODPECKER_BACKEND_SSH_PASSWORD"},
+		Name:    "backend-ssh-password",
+		Usage:   "backend ssh password",
+	},
+}

pipeline/backend/ssh/ssh.go

@@ -1,3 +1,17 @@
+// Copyright 2022 Woodpecker Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package ssh
 
 import (

pipeline/frontend/yaml/compiler/option.go

@@ -16,7 +16,6 @@ package compiler
 
 import (
 	"net/url"
-	"os"
 	"path/filepath"
 	"strings"
 
@@ -174,22 +173,6 @@ func WithS3Cacher(access, secret, region, bucket string) Option {
 	}
 }
 
-// WithProxy configures the compiler with HTTP_PROXY, HTTPS_PROXY,
-// and NO_PROXY environment variables added by default to every
-// container in the pipeline.
-func WithProxy() Option {
-	return WithEnviron(
-		map[string]string{
-			"no_proxy":    noProxy,
-			"NO_PROXY":    noProxy,
-			"http_proxy":  httpProxy,
-			"HTTP_PROXY":  httpProxy,
-			"HTTPS_PROXY": httpsProxy,
-			"https_proxy": httpsProxy,
-		},
-	)
-}
-
 // WithNetworks configures the compiler with additional networks
 // to be connected to pipeline containers
 func WithNetworks(networks ...string) Option {
@@ -233,38 +216,24 @@ func WithNetrcOnlyTrusted(only bool) Option {
 	}
 }
 
-// TODO(bradrydzewski) consider an alternate approach to
-// WithProxy where the proxy strings are passed directly
-// to the function as named parameters.
-
-// func WithProxy2(http, https, none string) Option {
-// 	return WithEnviron(
-// 		map[string]string{
-// 			"no_proxy":    none,
-// 			"NO_PROXY":    none,
-// 			"http_proxy":  http,
-// 			"HTTP_PROXY":  http,
-// 			"HTTPS_PROXY": https,
-// 			"https_proxy": https,
-// 		},
-// 	)
-// }
-
-var (
-	noProxy    = getenv("no_proxy")
-	httpProxy  = getenv("https_proxy")
-	httpsProxy = getenv("https_proxy")
-)
-
-// getenv returns the named environment variable.
-func getenv(name string) (value string) {
-	name = strings.ToUpper(name)
-	if value := os.Getenv(name); value != "" {
-		return value
-	}
-	name = strings.ToLower(name)
-	if value := os.Getenv(name); value != "" {
-		return value
-	}
-	return
+type ProxyOptions struct {
+	NoProxy    string
+	HTTPProxy  string
+	HTTPSProxy string
+}
+
+// WithProxy configures the compiler with HTTP_PROXY, HTTPS_PROXY,
+// and NO_PROXY environment variables added by default to every
+// container in the pipeline.
+func WithProxy(opt ProxyOptions) Option {
+	return WithEnviron(
+		map[string]string{
+			"no_proxy":    opt.NoProxy,
+			"NO_PROXY":    opt.NoProxy,
+			"http_proxy":  opt.HTTPProxy,
+			"HTTP_PROXY":  opt.HTTPProxy,
+			"HTTPS_PROXY": opt.HTTPSProxy,
+			"https_proxy": opt.HTTPSProxy,
+		},
+	)
 }

pipeline/frontend/yaml/compiler/option_test.go

@@ -1,7 +1,6 @@
 package compiler
 
 import (
-	"os"
 	"reflect"
 	"testing"
 
@@ -153,24 +152,10 @@ func TestWithNetrc(t *testing.T) {
 }
 
 func TestWithProxy(t *testing.T) {
-	// do not execute the test if the host machine sets http proxy
-	// environment variables to avoid interference with other tests.
-	if noProxy != "" || httpProxy != "" || httpsProxy != "" {
-		t.SkipNow()
-		return
-	}
-
 	// alter the default values
-	noProxy = "example.com"
-	httpProxy = "bar.com"
-	httpsProxy = "baz.com"
-
-	// reset the default values
-	defer func() {
-		noProxy = ""
-		httpProxy = ""
-		httpsProxy = ""
-	}()
+	noProxy := "example.com"
+	httpProxy := "bar.com"
+	httpsProxy := "baz.com"
 
 	testdata := map[string]string{
 		"no_proxy":    noProxy,
@@ -181,7 +166,11 @@ func TestWithProxy(t *testing.T) {
 		"HTTPS_PROXY": httpsProxy,
 	}
 	compiler := New(
-		WithProxy(),
+		WithProxy(ProxyOptions{
+			NoProxy:    noProxy,
+			HTTPProxy:  httpProxy,
+			HTTPSProxy: httpsProxy,
+		}),
 	)
 	for key, value := range testdata {
 		if compiler.env[key] != value {
@@ -207,26 +196,6 @@ func TestWithEnviron(t *testing.T) {
 	}
 }
 
-func TestGetenv(t *testing.T) {
-	defer func() {
-		os.Unsetenv("X_TEST_FOO")
-		os.Unsetenv("x_test_bar")
-		os.Unsetenv("x_test_baz")
-	}()
-	os.Setenv("X_TEST_FOO", "foo")
-	os.Setenv("x_test_bar", "bar")
-	os.Setenv("x_test_baz", "")
-	if getenv("x_test_foo") != "foo" {
-		t.Errorf("Expect X_TEST_FOO=foo")
-	}
-	if getenv("X_TEST_BAR") != "bar" {
-		t.Errorf("Expect x_test_bar=bar")
-	}
-	if getenv("x_test_baz") != "" {
-		t.Errorf("Expect x_test_bar=bar is empty")
-	}
-}
-
 func TestWithVolumeCacher(t *testing.T) {
 	compiler := New(
 		WithVolumeCacher("/cache"),

pipeline/stepBuilder.go

@@ -39,16 +39,17 @@ import (
 
 // StepBuilder Takes the hook data and the yaml and returns in internal data model
 type StepBuilder struct {
-	Repo  *model.Repo
-	Curr  *model.Pipeline
-	Last  *model.Pipeline
-	Netrc *model.Netrc
-	Secs  []*model.Secret
-	Regs  []*model.Registry
-	Link  string
-	Yamls []*forge_types.FileMeta
-	Envs  map[string]string
-	Forge metadata.ServerForge
+	Repo      *model.Repo
+	Curr      *model.Pipeline
+	Last      *model.Pipeline
+	Netrc     *model.Netrc
+	Secs      []*model.Secret
+	Regs      []*model.Registry
+	Link      string
+	Yamls     []*forge_types.FileMeta
+	Envs      map[string]string
+	Forge     metadata.ServerForge
+	ProxyOpts compiler.ProxyOptions
 }
 
 type Item struct {
@@ -281,7 +282,7 @@ func (b *StepBuilder) toInternalRepresentation(parsed *yaml_types.Workflow, envi
 				stepID,
 			),
 		),
-		compiler.WithProxy(),
+		compiler.WithProxy(b.ProxyOpts),
 		compiler.WithWorkspaceFromURL("/woodpecker", b.Repo.Link),
 		compiler.WithMetadata(metadata),
 		compiler.WithTrusted(b.Repo.IsTrusted),

server/config.go

@@ -91,5 +91,10 @@ var Config = struct {
 		Privileged                          []string
 		DefaultTimeout                      int64
 		MaxTimeout                          int64
+		Proxy                               struct {
+			No    string
+			HTTP  string
+			HTTPS string
+		}
 	}
 }{}

server/pipeline/items.go

@@ -22,6 +22,7 @@ import (
 	"github.com/rs/zerolog/log"
 
 	"github.com/woodpecker-ci/woodpecker/pipeline"
+	"github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml/compiler"
 	"github.com/woodpecker-ci/woodpecker/server"
 	forge_types "github.com/woodpecker-ci/woodpecker/server/forge/types"
 	"github.com/woodpecker-ci/woodpecker/server/model"
@@ -78,6 +79,11 @@ func createPipelineItems(c context.Context, store store.Store,
 		Link:  server.Config.Server.Host,
 		Yamls: yamls,
 		Forge: server.Config.Services.Forge,
+		ProxyOpts: compiler.ProxyOptions{
+			NoProxy:    server.Config.Pipeline.Proxy.No,
+			HTTPProxy:  server.Config.Pipeline.Proxy.HTTP,
+			HTTPSProxy: server.Config.Pipeline.Proxy.HTTPS,
+		},
 	}
 	pipelineItems, err := b.Build()
 	if err != nil {

shared/utils/paginate.go

@@ -1,3 +1,17 @@
+// Copyright 2023 Woodpecker Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package utils
 
 // Paginate iterates over a func call until it does not return new items and return it as list

shared/utils/paginate_test.go

@@ -1,3 +1,17 @@
+// Copyright 2023 Woodpecker Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package utils
 
 import (

shared/utils/slices.go

@@ -0,0 +1,33 @@
+// Copyright 2023 Woodpecker Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package utils
+
+// MergeSlices return a new slice that combines all values of input slices
+// TODO: once https://github.com/golang/go/pull/61817 got merged, we should switch to it
+func MergeSlices[T any](slices ...[]T) []T {
+	sl := 0
+	for i := range slices {
+		sl += len(slices[i])
+	}
+	result := make([]T, sl)
+	cp := 0
+	for _, s := range slices {
+		if sLen := len(s); sLen != 0 {
+			copy(result[cp:], s)
+			cp += sLen
+		}
+	}
+	return result
+}

shared/utils/slices_test.go

@@ -0,0 +1,29 @@
+// Copyright 2023 Woodpecker Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package utils
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestMergeSlices(t *testing.T) {
+	resultSS := MergeSlices([]string{}, []string{"a", "b"}, []string{"c"}, nil)
+	assert.EqualValues(t, []string{"a", "b", "c"}, resultSS)
+
+	resultIS := MergeSlices([]int{}, []int{1, 2}, []int{4}, nil)
+	assert.EqualValues(t, []int{1, 2, 4}, resultIS)
+}