Fail on missing secrets (#2749)

Co-authored-by: pat-s <patrick.schratz@gmail.com>

docs/docusaurus.config.ts

@@ -1,5 +1,5 @@
 import { themes } from 'prism-react-renderer';
-import type {Config} from '@docusaurus/types';
+import type { Config } from '@docusaurus/types';
 import type * as Preset from '@docusaurus/preset-classic';
 import * as path from 'path';
 
@@ -14,147 +14,146 @@ const config: Config = {
   organizationName: 'woodpecker-ci',
   projectName: 'woodpecker-ci.github.io',
   trailingSlash: false,
-  themeConfig:
+  themeConfig: {
-    ({
+    navbar: {
-      navbar: {
+      title: 'Woodpecker',
-        title: 'Woodpecker',
+      logo: {
-        logo: {
+        alt: 'Woodpecker Logo',
-          alt: 'Woodpecker Logo',
+        src: 'img/logo.svg',
-          src: 'img/logo.svg',
+      },
+      items: [
+        {
+          type: 'doc',
+          docId: 'intro',
+          activeBaseRegex: 'docs/(?!migrations|awesome)',
+          position: 'left',
+          label: 'Docs',
         },
-        items: [
+        {
-          {
+          to: '/plugins',
-            type: 'doc',
+          position: 'left',
-            docId: 'intro',
+          label: 'Plugins',
-            activeBaseRegex: 'docs/(?!migrations|awesome)',
+        },
-            position: 'left',
+        {
-            label: 'Docs',
+          to: '/docs/next/migrations', // Always point to newest migration guide
-          },
+          activeBaseRegex: 'docs/(next/)?migrations',
-          {
+          position: 'left',
-            to: '/plugins',
+          label: 'Migrations',
-            position: 'left',
+        },
-            label: 'Plugins',
+        {
-          },
+          to: '/faq',
-          {
+          position: 'left',
-            to: '/docs/next/migrations', // Always point to newest migration guide
+          label: 'FAQ',
-            activeBaseRegex: 'docs/(next/)?migrations',
+        },
-            position: 'left',
+        {
-            label: 'Migrations',
+          to: '/docs/next/awesome', // Always point to newest awesome list
-          },
+          activeBaseRegex: 'docs/(next/)?awesome',
-          {
+          position: 'left',
-            to: '/faq',
+          label: 'Awesome',
-            position: 'left',
+        },
-            label: 'FAQ',
+        {
-          },
+          to: '/api',
-          {
+          position: 'left',
-            to: '/docs/next/awesome', // Always point to newest awesome list
+          label: 'API',
-            activeBaseRegex: 'docs/(next/)?awesome',
+        },
-            position: 'left',
+        { to: 'blog', label: 'Blog', position: 'left' },
-            label: 'Awesome',
+        {
-          },
+          type: 'docsVersionDropdown',
-          {
+          position: 'right',
-            to: '/api',
+        },
-            position: 'left',
+        {
-            label: 'API',
+          href: 'https://github.com/woodpecker-ci/woodpecker',
-          },
+          position: 'right',
-          { to: 'blog', label: 'Blog', position: 'left' },
+          className: 'header-github-link',
-          {
+          'aria-label': 'GitHub repository',
-            type: 'docsVersionDropdown',
+        },
-            position: 'right',
+        {
-          },
+          label: '🧡 Sponsor Us',
-          {
+          position: 'right',
-            href: 'https://github.com/woodpecker-ci/woodpecker',
+          href: 'https://opencollective.com/woodpecker-ci',
-            position: 'right',
+        },
-            className: 'header-github-link',
+      ],
-            'aria-label': 'GitHub repository',
+    },
-          },
+    footer: {
-          {
+      style: 'dark',
-            label: '🧡 Sponsor Us',
+      links: [
-            position: 'right',
+        {
-            href: 'https://opencollective.com/woodpecker-ci',
+          title: 'Docs',
-          },
+          items: [
-        ],
+            {
-      },
+              label: 'Introduction',
-      footer: {
+              to: '/docs/intro',
-        style: 'dark',
+            },
-        links: [
+            {
-          {
+              label: 'Usage',
-            title: 'Docs',
+              to: '/docs/usage/intro',
-            items: [
+            },
-              {
+            {
-                label: 'Introduction',
+              label: 'Server setup',
-                to: '/docs/intro',
+              to: '/docs/administration/setup',
-              },
+            },
-              {
+            {
-                label: 'Usage',
+              label: 'FAQ',
-                to: '/docs/usage/intro',
+              to: '/faq',
-              },
+            },
-              {
+          ],
-                label: 'Server setup',
+        },
-                to: '/docs/administration/setup',
+        {
-              },
+          title: 'Community',
-              {
+          items: [
-                label: 'FAQ',
+            {
-                to: '/faq',
+              label: 'Discord',
-              },
+              href: 'https://discord.gg/fcMQqSMXJy',
-            ],
+            },
-          },
+            {
-          {
+              label: 'Matrix',
-            title: 'Community',
+              href: 'https://matrix.to/#/#woodpecker:matrix.org',
-            items: [
+            },
-              {
+            {
-                label: 'Discord',
+              label: 'Mastodon',
-                href: 'https://discord.gg/fcMQqSMXJy',
+              href: 'https://floss.social/@WoodpeckerCI',
-              },
+            },
-              {
+          ],
-                label: 'Matrix',
+        },
-                href: 'https://matrix.to/#/#woodpecker:matrix.org',
+        {
-              },
+          title: 'More',
-              {
+          items: [
-                label: 'Mastodon',
+            {
-                href: 'https://floss.social/@WoodpeckerCI',
+              label: 'Translate',
-              },
+              href: 'https://translate.woodpecker-ci.org/engage/woodpecker-ci/',
-            ],
+            },
-          },
+            {
-          {
+              label: 'GitHub',
-            title: 'More',
+              href: 'https://github.com/woodpecker-ci/woodpecker',
-            items: [
+            },
-              {
+            {
-                label: 'Translate',
+              href: 'https://ci.woodpecker-ci.org/woodpecker-ci/woodpecker',
-                href: 'https://translate.woodpecker-ci.org/engage/woodpecker-ci/',
+              label: 'CI',
-              },
+            },
-              {
+          ],
-                label: 'GitHub',
+        },
-                href: 'https://github.com/woodpecker-ci/woodpecker',
+      ],
-              },
+      copyright: `Copyright © ${new Date().getFullYear()} Woodpecker CI. Built with Docusaurus.`,
-              {
+    },
-                href: 'https://ci.woodpecker-ci.org/woodpecker-ci/woodpecker',
+    prism: {
-                label: 'CI',
+      theme: themes.github,
-              },
+      darkTheme: themes.dracula,
-            ],
+      additionalLanguages: ['diff', 'json', 'docker', 'javascript', 'css', 'bash', 'nginx', 'apacheconf'],
-          },
+    },
-        ],
+    announcementBar: {
-        copyright: `Copyright © ${new Date().getFullYear()} Woodpecker CI. Built with Docusaurus.`,
+      id: 'github-star',
-      },
+      content: ` If you like Woodpecker-CI, <a href=https://github.com/woodpecker-ci/woodpecker rel="noopener noreferrer" target="_blank">give us a star on GitHub</a> ! ⭐️`,
-      prism: {
+      backgroundColor: 'var(--ifm-color-primary)',
-        theme: themes.github,
+      textColor: 'var(--ifm-color-gray-900)',
-        darkTheme: themes.dracula,
+    },
-        additionalLanguages: ['diff', 'json', 'docker', 'javascript', 'css', 'bash', 'nginx', 'apacheconf'],
+    tableOfContents: {
-      },
+      minHeadingLevel: 2,
-      announcementBar: {
+      maxHeadingLevel: 4,
-        id: 'github-star',
+    },
-        content: ` If you like Woodpecker-CI, <a href=https://github.com/woodpecker-ci/woodpecker rel="noopener noreferrer" target="_blank">give us a star on GitHub</a> ! ⭐️`,
+    colorMode: {
-        backgroundColor: 'var(--ifm-color-primary)',
+      respectPrefersColorScheme: true,
-        textColor: 'var(--ifm-color-gray-900)',
+    },
-      },
+  } satisfies Preset.ThemeConfig,
-      tableOfContents: {
-        minHeadingLevel: 2,
-        maxHeadingLevel: 4,
-      },
-      colorMode: {
-        respectPrefersColorScheme: true,
-      },
-    } satisfies Preset.ThemeConfig),
   plugins: [
     () => ({
       name: 'docusaurus-plugin-favicon',
@@ -206,7 +205,7 @@ const config: Config = {
   presets: [
     [
       '@docusaurus/preset-classic',
-      ({
+      {
         docs: {
           sidebarPath: require.resolve('./sidebars.js'),
           editUrl: 'https://github.com/woodpecker-ci/woodpecker/edit/main/docs/',
@@ -235,7 +234,7 @@ const config: Config = {
         theme: {
           customCss: require.resolve('./src/css/custom.css'),
         },
-      } satisfies Preset.Options),
+      } satisfies Preset.Options,
     ],
     [
       'redocusaurus',
@@ -265,8 +264,8 @@ const config: Config = {
     }),
   },
   markdown: {
-    format: 'detect'
+    format: 'detect',
-  }
+  },
 };
 
 export default config;

pipeline/frontend/yaml/compiler/compiler.go

@@ -162,7 +162,10 @@ func (c *Compiler) Compile(conf *yaml_types.Workflow) (*backend_types.Config, er
 			Environment: c.cloneEnv,
 		}
 		name := fmt.Sprintf("%s_clone", c.prefix)
-		step := c.createProcess(name, container, backend_types.StepTypeClone)
+		step, err := c.createProcess(name, container, backend_types.StepTypeClone)
+		if err != nil {
+			return nil, err
+		}
 
 		stage := new(backend_types.Stage)
 		stage.Name = name
@@ -183,7 +186,10 @@ func (c *Compiler) Compile(conf *yaml_types.Workflow) (*backend_types.Config, er
 			stage.Alias = container.Name
 
 			name := fmt.Sprintf("%s_clone_%d", c.prefix, i)
-			step := c.createProcess(name, container, backend_types.StepTypeClone)
+			step, err := c.createProcess(name, container, backend_types.StepTypeClone)
+			if err != nil {
+				return nil, err
+			}
 
 			// only inject netrc if it's a trusted repo or a trusted plugin
 			if !c.netrcOnlyTrusted || c.trustedPipeline || (container.IsPlugin() && container.IsTrustedCloneImage()) {
@@ -198,7 +204,10 @@ func (c *Compiler) Compile(conf *yaml_types.Workflow) (*backend_types.Config, er
 		}
 	}
 
-	c.setupCache(conf, config)
+	err := c.setupCache(conf, config)
+	if err != nil {
+		return nil, err
+	}
 
 	// add services steps
 	if len(conf.Services.ContainerList) != 0 {
@@ -214,7 +223,11 @@ func (c *Compiler) Compile(conf *yaml_types.Workflow) (*backend_types.Config, er
 			}
 
 			name := fmt.Sprintf("%s_%s_%d", c.prefix, nameServices, i)
-			step := c.createProcess(name, container, backend_types.StepTypeService)
+			step, err := c.createProcess(name, container, backend_types.StepTypeService)
+			if err != nil {
+				return nil, err
+			}
+
 			stage.Steps = append(stage.Steps, step)
 		}
 		config.Stages = append(config.Stages, stage)
@@ -249,7 +262,10 @@ func (c *Compiler) Compile(conf *yaml_types.Workflow) (*backend_types.Config, er
 		if container.IsPlugin() {
 			stepType = backend_types.StepTypePlugin
 		}
-		step := c.createProcess(name, container, stepType)
+		step, err := c.createProcess(name, container, stepType)
+		if err != nil {
+			return nil, err
+		}
 
 		// inject netrc if it's a trusted repo or a trusted clone-plugin
 		if c.trustedPipeline || (container.IsPlugin() && container.IsTrustedCloneImage()) {
@@ -261,19 +277,25 @@ func (c *Compiler) Compile(conf *yaml_types.Workflow) (*backend_types.Config, er
 		stage.Steps = append(stage.Steps, step)
 	}
 
-	c.setupCacheRebuild(conf, config)
+	err = c.setupCacheRebuild(conf, config)
+	if err != nil {
+		return nil, err
+	}
 
 	return config, nil
 }
 
-func (c *Compiler) setupCache(conf *yaml_types.Workflow, ir *backend_types.Config) {
+func (c *Compiler) setupCache(conf *yaml_types.Workflow, ir *backend_types.Config) error {
 	if c.local || len(conf.Cache) == 0 || c.cacher == nil {
-		return
+		return nil
 	}
 
 	container := c.cacher.Restore(path.Join(c.metadata.Repo.Owner, c.metadata.Repo.Name), c.metadata.Curr.Commit.Branch, conf.Cache)
 	name := fmt.Sprintf("%s_restore_cache", c.prefix)
-	step := c.createProcess(name, container, backend_types.StepTypeCache)
+	step, err := c.createProcess(name, container, backend_types.StepTypeCache)
+	if err != nil {
+		return err
+	}
 
 	stage := new(backend_types.Stage)
 	stage.Name = name
@@ -281,16 +303,21 @@ func (c *Compiler) setupCache(conf *yaml_types.Workflow, ir *backend_types.Confi
 	stage.Steps = append(stage.Steps, step)
 
 	ir.Stages = append(ir.Stages, stage)
+
+	return nil
 }
 
-func (c *Compiler) setupCacheRebuild(conf *yaml_types.Workflow, ir *backend_types.Config) {
+func (c *Compiler) setupCacheRebuild(conf *yaml_types.Workflow, ir *backend_types.Config) error {
 	if c.local || len(conf.Cache) == 0 || c.metadata.Curr.Event != metadata.EventPush || c.cacher == nil {
-		return
+		return nil
 	}
 	container := c.cacher.Rebuild(path.Join(c.metadata.Repo.Owner, c.metadata.Repo.Name), c.metadata.Curr.Commit.Branch, conf.Cache)
 
 	name := fmt.Sprintf("%s_rebuild_cache", c.prefix)
-	step := c.createProcess(name, container, backend_types.StepTypeCache)
+	step, err := c.createProcess(name, container, backend_types.StepTypeCache)
+	if err != nil {
+		return err
+	}
 
 	stage := new(backend_types.Stage)
 	stage.Name = name
@@ -298,4 +325,6 @@ func (c *Compiler) setupCacheRebuild(conf *yaml_types.Workflow, ir *backend_type
 	stage.Steps = append(stage.Steps, step)
 
 	ir.Stages = append(ir.Stages, stage)
+
+	return nil
 }

pipeline/frontend/yaml/compiler/compiler_test.go

@@ -94,10 +94,10 @@ func TestCompilerCompile(t *testing.T) {
 	}
 
 	tests := []struct {
-		name     string
+		name        string
-		fronConf *yaml_types.Workflow
+		fronConf    *yaml_types.Workflow
-		backConf *backend_types.Config
+		backConf    *backend_types.Config
-		expErr   bool
+		expectedErr string
 	}{{
 		name:     "empty workflow, no clone",
 		fronConf: &yaml_types.Workflow{SkipClone: true},
@@ -197,13 +197,24 @@ func TestCompilerCompile(t *testing.T) {
 				}},
 			}},
 		},
+	}, {
+		name: "workflow with missing secret",
+		fronConf: &yaml_types.Workflow{Steps: yaml_types.ContainerList{ContainerList: []*yaml_types.Container{{
+			Name:     "step",
+			Image:    "bash",
+			Commands: []string{"env"},
+			Secrets:  yaml_types.Secrets{Secrets: []*yaml_types.Secret{{Source: "missing", Target: "missing"}}},
+		}}}},
+		backConf:    nil,
+		expectedErr: "secret \"missing\" not found or not allowed to be used",
 	}}
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
 			backConf, err := compiler.Compile(test.fronConf)
-			if test.expErr {
+			if test.expectedErr != "" {
 				assert.Error(t, err)
+				assert.Equal(t, err.Error(), test.expectedErr)
 			} else {
 				// we ignore uuids in steps and only check if global env got set ...
 				for _, st := range backConf.Stages {

pipeline/frontend/yaml/compiler/convert.go

@@ -21,7 +21,6 @@ import (
 	"strings"
 
 	"github.com/google/uuid"
-	"github.com/rs/zerolog/log"
 
 	backend_types "github.com/woodpecker-ci/woodpecker/pipeline/backend/types"
 	"github.com/woodpecker-ci/woodpecker/pipeline/frontend/metadata"
@@ -30,7 +29,7 @@ import (
 	"github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml/utils"
 )
 
-func (c *Compiler) createProcess(name string, container *yaml_types.Container, stepType backend_types.StepType) *backend_types.Step {
+func (c *Compiler) createProcess(name string, container *yaml_types.Container, stepType backend_types.StepType) (*backend_types.Step, error) {
 	var (
 		uuid = uuid.New()
 
@@ -90,7 +89,7 @@ func (c *Compiler) createProcess(name string, container *yaml_types.Container, s
 		}
 
 		if err := settings.ParamsToEnv(container.Settings, environment, pluginSecrets.toStringMap()); err != nil {
-			log.Error().Err(err).Msg("paramsToEnv")
+			return nil, err
 		}
 	}
 
@@ -112,6 +111,8 @@ func (c *Compiler) createProcess(name string, container *yaml_types.Container, s
 		secret, ok := c.secrets[strings.ToLower(requested.Source)]
 		if ok && secret.Available(container) {
 			environment[strings.ToUpper(requested.Target)] = secret.Value
+		} else {
+			return nil, fmt.Errorf("secret %q not found or not allowed to be used", requested.Source)
 		}
 	}
 
@@ -213,7 +214,7 @@ func (c *Compiler) createProcess(name string, container *yaml_types.Container, s
 		IpcMode:        ipcMode,
 		Ports:          ports,
 		BackendOptions: backendOptions,
-	}
+	}, nil
 }
 
 func (c *Compiler) stepWorkdir(container *yaml_types.Container) string {