Allow loading sensitive flags from files (#815)

With systems like docker swarm or docker compose it is usually a little awkward to manage secrets. 
There is no way to directly inject them into the environment config. So you often have to write your secrets directly into the compose file

There are hacky workarounds such as overriding the entry-point of the container and loading a script which then fetches secrets from /run/secrets and replaces the environment variables, but this becomes very difficult once we are using docker images built from "scratch" (which is a really great practice otherwise) as there is no shell or standard tooling available

This adds a *_FILE variant of their Environment config values to work around this issue.

Signed-off-by: Lukas Bachschwell <lukas@lbsfilm.at>

cmd/agent/flags.go

@@ -15,6 +15,7 @@
 package main
 
 import (
+	"os"
 	"time"
 
 	"github.com/urfave/cli/v2"
@@ -37,6 +38,7 @@ var flags = []cli.Flag{
 		EnvVars:  []string{"WOODPECKER_AGENT_SECRET"},
 		Name:     "grpc-password",
 		Usage:    "server-agent shared password",
+		FilePath: os.Getenv("WOODPECKER_AGENT_SECRET_FILE"),
 	},
 	&cli.BoolFlag{
 		EnvVars: []string{"WOODPECKER_GRPC_SECURE"},

cmd/server/flags.go

@@ -15,6 +15,7 @@
 package main
 
 import (
+	"os"
 	"time"
 
 	"github.com/urfave/cli/v2"
@@ -140,6 +141,7 @@ var flags = []cli.Flag{
 		EnvVars:  []string{"WOODPECKER_AGENT_SECRET"},
 		Name:     "agent-secret",
 		Usage:    "server-agent shared password",
+		FilePath: os.Getenv("WOODPECKER_AGENT_SECRET_FILE"),
 	},
 	&cli.DurationFlag{
 		EnvVars: []string{"WOODPECKER_KEEPALIVE_MIN_TIME"},
@@ -182,12 +184,14 @@ var flags = []cli.Flag{
 		Name:     "datasource",
 		Usage:    "database driver configuration string",
 		Value:    "woodpecker.sqlite",
+		FilePath: os.Getenv("WOODPECKER_DATABASE_DATASOURCE_FILE"),
 	},
 	&cli.StringFlag{
 		EnvVars:  []string{"WOODPECKER_PROMETHEUS_AUTH_TOKEN"},
 		Name:     "prometheus-auth-token",
 		Usage:    "token to secure prometheus metrics endpoint",
 		Value:    "",
+		FilePath: os.Getenv("WOODPECKER_PROMETHEUS_AUTH_TOKEN_FILE"),
 	},
 	&cli.StringFlag{
 		EnvVars: []string{"WOODPECKER_STATUS_CONTEXT", "WOODPECKER_GITHUB_CONTEXT", "WOODPECKER_GITEA_CONTEXT"},
@@ -246,11 +250,13 @@ var flags = []cli.Flag{
 		EnvVars:  []string{"WOODPECKER_GITHUB_CLIENT"},
 		Name:     "github-client",
 		Usage:    "github oauth2 client id",
+		FilePath: os.Getenv("WOODPECKER_GITHUB_CLIENT_FILE"),
 	},
 	&cli.StringFlag{
 		EnvVars:  []string{"WOODPECKER_GITHUB_SECRET"},
 		Name:     "github-secret",
 		Usage:    "github oauth2 client secret",
+		FilePath: os.Getenv("WOODPECKER_GITHUB_SECRET_FILE"),
 	},
 	&cli.BoolFlag{
 		EnvVars: []string{"WOODPECKER_GITHUB_MERGE_REF"},
@@ -281,11 +287,13 @@ var flags = []cli.Flag{
 		EnvVars:  []string{"WOODPECKER_GOGS_GIT_USERNAME"},
 		Name:     "gogs-git-username",
 		Usage:    "gogs service account username",
+		FilePath: os.Getenv("WOODPECKER_GOGS_GIT_USERNAME_FILE"),
 	},
 	&cli.StringFlag{
 		EnvVars:  []string{"WOODPECKER_GOGS_GIT_PASSWORD"},
 		Name:     "gogs-git-password",
 		Usage:    "gogs service account password",
+		FilePath: os.Getenv("WOODPECKER_GOGS_GIT_PASSWORD_FILE"),
 	},
 	&cli.BoolFlag{
 		EnvVars: []string{"WOODPECKER_GOGS_PRIVATE_MODE"},
@@ -315,11 +323,13 @@ var flags = []cli.Flag{
 		EnvVars:  []string{"WOODPECKER_GITEA_CLIENT"},
 		Name:     "gitea-client",
 		Usage:    "gitea oauth2 client id",
+		FilePath: os.Getenv("WOODPECKER_GITEA_CLIENT_FILE"),
 	},
 	&cli.StringFlag{
 		EnvVars:  []string{"WOODPECKER_GITEA_SECRET"},
 		Name:     "gitea-secret",
 		Usage:    "gitea oauth2 client secret",
+		FilePath: os.Getenv("WOODPECKER_GITEA_SECRET_FILE"),
 	},
 	&cli.BoolFlag{
 		EnvVars: []string{"WOODPECKER_GITEA_SKIP_VERIFY"},
@@ -338,11 +348,13 @@ var flags = []cli.Flag{
 		EnvVars:  []string{"WOODPECKER_BITBUCKET_CLIENT"},
 		Name:     "bitbucket-client",
 		Usage:    "bitbucket oauth2 client id",
+		FilePath: os.Getenv("WOODPECKER_BITBUCKET_CLIENT_FILE"),
 	},
 	&cli.StringFlag{
 		EnvVars:  []string{"WOODPECKER_BITBUCKET_SECRET"},
 		Name:     "bitbucket-secret",
 		Usage:    "bitbucket oauth2 client secret",
+		FilePath: os.Getenv("WOODPECKER_BITBUCKET_SECRET_FILE"),
 	},
 	//
 	// Gitlab
@@ -362,11 +374,13 @@ var flags = []cli.Flag{
 		EnvVars:  []string{"WOODPECKER_GITLAB_CLIENT"},
 		Name:     "gitlab-client",
 		Usage:    "gitlab oauth2 client id",
+		FilePath: os.Getenv("WOODPECKER_GITLAB_CLIENT_FILE"),
 	},
 	&cli.StringFlag{
 		EnvVars:  []string{"WOODPECKER_GITLAB_SECRET"},
 		Name:     "gitlab-secret",
 		Usage:    "gitlab oauth2 client secret",
+		FilePath: os.Getenv("WOODPECKER_GITLAB_SECRET_FILE"),
 	},
 	&cli.BoolFlag{
 		EnvVars: []string{"WOODPECKER_GITLAB_SKIP_VERIFY"},
@@ -390,6 +404,7 @@ var flags = []cli.Flag{
 		EnvVars:  []string{"WOODPECKER_STASH_CONSUMER_KEY"},
 		Name:     "stash-consumer-key",
 		Usage:    "stash oauth1 consumer key",
+		FilePath: os.Getenv("WOODPECKER_STASH_CONSUMER_KEY_FILE"),
 	},
 	&cli.StringFlag{
 		EnvVars: []string{"WOODPECKER_STASH_CONSUMER_RSA"},
@@ -405,11 +420,13 @@ var flags = []cli.Flag{
 		EnvVars:  []string{"WOODPECKER_STASH_GIT_USERNAME"},
 		Name:     "stash-git-username",
 		Usage:    "stash service account username",
+		FilePath: os.Getenv("WOODPECKER_STASH_GIT_USERNAME_FILE"),
 	},
 	&cli.StringFlag{
 		EnvVars:  []string{"WOODPECKER_STASH_GIT_PASSWORD"},
 		Name:     "stash-git-password",
 		Usage:    "stash service account password",
+		FilePath: os.Getenv("WOODPECKER_STASH_GIT_PASSWORD_FILE"),
 	},
 	&cli.BoolFlag{
 		EnvVars: []string{"WOODPECKER_STASH_SKIP_VERIFY"},
@@ -434,11 +451,13 @@ var flags = []cli.Flag{
 		EnvVars:  []string{"WOODPECKER_CODING_CLIENT"},
 		Name:     "coding-client",
 		Usage:    "coding oauth2 client id",
+		FilePath: os.Getenv("WOODPECKER_CODING_CLIENT_FILE"),
 	},
 	&cli.StringFlag{
 		EnvVars:  []string{"WOODPECKER_CODING_SECRET"},
 		Name:     "coding-secret",
 		Usage:    "coding oauth2 client secret",
+		FilePath: os.Getenv("WOODPECKER_CODING_SECRET_FILE"),
 	},
 	&cli.StringSliceFlag{
 		EnvVars: []string{"WOODPECKER_CODING_SCOPE"},
@@ -460,11 +479,13 @@ var flags = []cli.Flag{
 		EnvVars:  []string{"WOODPECKER_CODING_GIT_USERNAME"},
 		Name:     "coding-git-username",
 		Usage:    "coding machine user username",
+		FilePath: os.Getenv("WOODPECKER_CODING_GIT_USERNAME_FILE"),
 	},
 	&cli.StringFlag{
 		EnvVars:  []string{"WOODPECKER_CODING_GIT_PASSWORD"},
 		Name:     "coding-git-password",
 		Usage:    "coding machine user password",
+		FilePath: os.Getenv("WOODPECKER_CODING_GIT_PASSWORD_FILE"),
 	},
 	&cli.BoolFlag{
 		EnvVars: []string{"WOODPECKER_CODING_SKIP_VERIFY"},

docs/docs/30-administration/10-server-config.md

@@ -72,6 +72,39 @@ services:
 +     - WOODPECKER_DOCKER_CONFIG=/home/user/.docker/config.json
 ```
 
+## Handling sensitive data in docker-compose and docker-swarm
+
+To handle sensitive data in docker-compose or docker-swarm configurations there are several options:
+
+For docker-compose you can use a .env file next to your compose condfiguration to store the secrets outside of the compose file. While this seperates configuration from secrets it is still not very secure.
+
+Alternatively use docker-secrets. As it may be difficult to use docker secrets for environment variables woodpecker allows to read sensible data from files by profiding a `*_FILE` option of all sensible configuration variables. Woodpecker will try to read the value directly from this file. Keep in mind that when the original environment varibale gets specified at the same time it will override the value read from the file.
+
+```diff
+# docker-compose.yml
+version: '3'
+
+services:
+  woodpecker-server:
+    [...]
+    environment:
+      - [...]
++     - WOODPECKER_AGENT_SECRET_FILE=/run/secrets/woodpecker-agent-secret
++   secrets:
++     - woodpecker-agent-secret
++
++secrets:
++  woodpecker-agent-secret:
++    external: true
+```
+
+Store a value to a docker secret like this:
+`echo "my_agent_secret_key" | docker secret create woodpecker-agent-secret -`
+
+or generate a random one like this:
+
+`openssl rand -hex 32 | docker secret create woodpecker-agent-secret -`
+
 ## All server configuration options
 
 The following list describes all available server configuration options.
@@ -214,6 +247,11 @@ Example: `WOODPECKER_NETWORK=network1,network2`
 
 A shared secret used by server and agents to authenticate communication. A secret can be generated by `openssl rand -hex 32`.
 
+### `WOODPECKER_AGENT_SECRET_FILE`
+> Default: empty
+
+Read the value for `WOODPECKER_AGENT_SECRET` from the specified filepath
+
 ### `WOODPECKER_KEEPALIVE_MIN_TIME`
 > Default: empty
 
@@ -242,11 +280,21 @@ WOODPECKER_DATABASE_DATASOURCE=root:password@tcp(1.2.3.4:3306)/woodpecker?parseT
 WOODPECKER_DATABASE_DATASOURCE=postgres://root:password@1.2.3.4:5432/woodpecker?sslmode=disable
 ```
 
+### `WOODPECKER_DATABASE_DATASOURCE_FILE`
+> Default: empty
+
+Read the value for `WOODPECKER_DATABASE_DATASOURCE` from the specified filepath
+
 ### `WOODPECKER_PROMETHEUS_AUTH_TOKEN`
 > Default: empty
 
 Token to secure the Prometheus metrics endpoint.
 
+### `WOODPECKER_PROMETHEUS_AUTH_TOKEN_FILE`
+> Default: empty
+
+Read the value for `WOODPECKER_PROMETHEUS_AUTH_TOKEN` from the specified filepath
+
 ### `WOODPECKER_STATUS_CONTEXT`
 > Default: `ci/woodpecker`
 

docs/docs/30-administration/11-vcs/20-github.md

@@ -46,11 +46,21 @@ Configures the GitHub server address.
 
 Configures the GitHub OAuth client id. This is used to authorize access.
 
+### `WOODPECKER_GITHUB_CLIENT_FILE`
+> Default: empty
+
+Read the value for `WOODPECKER_GITHUB_CLIENT` from the specified filepath
+
 ### `WOODPECKER_GITHUB_SECRET`
 > Default: empty
 
 Configures the GitHub OAuth client secret. This is used to authorize access.
 
+### `WOODPECKER_GITHUB_SECRET_FILE`
+> Default: empty
+
+Read the value for `WOODPECKER_GITHUB_SECRET` from the specified filepath
+
 ### `WOODPECKER_GITHUB_MERGE_REF`
 > Default: `true`
 

docs/docs/30-administration/11-vcs/30-gitea.md

@@ -54,11 +54,21 @@ Configures the Gitea server address.
 
 Configures the Gitea OAuth client id. This is used to authorize access.
 
+### `WOODPECKER_GITEA_CLIENT_FILE`
+> Default: empty
+
+Read the value for `WOODPECKER_GITEA_CLIENT` from the specified filepath
+
 ### `WOODPECKER_GITEA_SECRET`
 > Default: empty
 
 Configures the Gitea OAuth client secret. This is used to authorize access.
 
+### `WOODPECKER_GITEA_SECRET_FILE`
+> Default: empty
+
+Read the value for `WOODPECKER_GITEA_SECRET` from the specified filepath
+
 ### `WOODPECKER_GITEA_SKIP_VERIFY`
 > Default: `false`
 

docs/docs/30-administration/11-vcs/40-gitlab.md

@@ -46,11 +46,21 @@ Configures the GitLab server address.
 
 Configures the GitLab OAuth client id. This is used to authorize access.
 
+### `WOODPECKER_GITLAB_CLIENT_FILE`
+> Default: empty
+
+Read the value for `WOODPECKER_GITLAB_CLIENT` from the specified filepath
+
 ### `WOODPECKER_GITLAB_SECRET`
 > Default: empty
 
 Configures the GitLab OAuth client secret. This is used to authorize access.
 
+### `WOODPECKER_GITLAB_SECRET_FILE`
+> Default: empty
+
+Read the value for `WOODPECKER_GITLAB_SECRET` from the specified filepath
+
 ### `WOODPECKER_GITLAB_SKIP_VERIFY`
 > Default: `false`
 

docs/docs/30-administration/11-vcs/50-bitbucket.md

@@ -53,11 +53,21 @@ Enables the Bitbucket driver.
 
 Configures the Bitbucket OAuth client id. This is used to authorize access.
 
+### `WOODPECKER_BITBUCKET_CLIENT_FILE`
+> Default: empty
+
+Read the value for `WOODPECKER_BITBUCKET_CLIENT` from the specified filepath
+
 ### `WOODPECKER_BITBUCKET_SECRET`
 > Default: empty
 
 Configures the Bitbucket OAuth client secret. This is used to authorize access.
 
+### `WOODPECKER_BITBUCKET_SECRET_FILE`
+> Default: empty
+
+Read the value for `WOODPECKER_BITBUCKET_SECRET` from the specified filepath
+
 ## Missing Features
 
 Merge requests are not currently supported. We are interested in patches to include this functionality.

docs/docs/30-administration/11-vcs/60-bitbucket_server.md

@@ -115,6 +115,11 @@ Configures the Bitbucket Server address.
 
 Configures your Bitbucket Server consumer key.
 
+### `WOODPECKER_STASH_CONSUMER_KEY_FILE`
+> Default: empty
+
+Read the value for `WOODPECKER_STASH_CONSUMER_KEY` from the specified filepath
+
 ### `WOODPECKER_STASH_CONSUMER_RSA`
 > Default: empty
 
@@ -130,11 +135,21 @@ Configures your Bitbucket Server private key.
 
 This username is used to authenticate and clone all private repositories.
 
+### `WOODPECKER_STASH_GIT_USERNAME_FILE`
+> Default: empty
+
+Read the value for `WOODPECKER_STASH_GIT_USERNAME` from the specified filepath
+
 ### `WOODPECKER_STASH_GIT_PASSWORD`
 > Default: empty
 
 The password is used to authenticate and clone all private repositories.
 
+### `WOODPECKER_STASH_GIT_PASSWORD_FILE`
+> Default: empty
+
+Read the value for `WOODPECKER_STASH_GIT_PASSWORD` from the specified filepath
+
 ### `WOODPECKER_STASH_SKIP_VERIFY`
 > Default: `false`
 

docs/docs/30-administration/11-vcs/70-gogs.md

@@ -19,11 +19,21 @@ Configures the Gogs server address.
 
 This username is used to authenticate and clone all private repositories.
 
+### `WOODPECKER_GOGS_GIT_USERNAME_FILE`
+> Default: empty
+
+Read the value for `WOODPECKER_GOGS_GIT_USERNAME` from the specified filepath
+
 ### `WOODPECKER_GOGS_GIT_PASSWORD`
 > Default: empty
 
 The password is used to authenticate and clone all private repositories.
 
+### `WOODPECKER_GOGS_GIT_PASSWORD_FILE`
+> Default: empty
+
+Read the value for `WOODPECKER_GOGS_GIT_PASSWORD` from the specified filepath
+
 ### `WOODPECKER_GOGS_PRIVATE_MODE`
 > Default: `false`
 

docs/docs/30-administration/11-vcs/80-coding.md

@@ -19,11 +19,21 @@ Configures the Coding server address.
 
 Configures the Coding OAuth client id. This is used to authorize access.
 
+### `WOODPECKER_CODING_CLIENT_FILE`
+> Default: empty
+
+Read the value for `WOODPECKER_CODING_CLIENT` from the specified filepath
+
 ### `WOODPECKER_CODING_SECRET`
 > Default: empty
 
 Configures the Coding OAuth client secret. This is used to authorize access.
 
+### `WOODPECKER_CODING_SECRET_FILE`
+> Default: empty
+
+Read the value for `WOODPECKER_CODING_SECRET` from the specified filepath
+
 ### `WOODPECKER_CODING_SCOPE`
 > Default: `user, project, project:depot`
 
@@ -39,11 +49,21 @@ TODO
 
 This username is used to authenticate and clone all private repositories.
 
+### `WOODPECKER_CODING_GIT_USERNAME_FILE`
+> Default: empty
+
+Read the value for `WOODPECKER_CODING_GIT_USERNAME` from the specified filepath
+
 ### `WOODPECKER_CODING_GIT_PASSWORD`
 > Default: empty
 
 The password is used to authenticate and clone all private repositories.
 
+### `WOODPECKER_CODING_GIT_PASSWORD_FILE`
+> Default: empty
+
+Read the value for `WOODPECKER_CODING_GIT_PASSWORD` from the specified filepath
+
 ### `WOODPECKER_CODING_SKIP_VERIFY`
 > Default: `false`
 

docs/docs/30-administration/15-agent-config.md

@@ -95,6 +95,11 @@ The gRPC username.
 
 A shared secret used by server and agents to authenticate communication. A secret can be generated by `openssl rand -hex 32`.
 
+### `WOODPECKER_AGENT_SECRET_FILE`
+> Default: empty
+
+Read the value for `WOODPECKER_AGENT_SECRET` from the specified filepath
+
 ### `WOODPECKER_LOG_LEVEL`
 > Default: empty