Add support sub-settings and secrets in sub-settings (#1221)

pipeline/frontend/yaml/compiler/compiler.go

@@ -25,6 +25,7 @@ const (
 	namePipeline = "pipeline"
 )
 
+// Registry represents registry credentials
 type Registry struct {
 	Hostname string
 	Username string
@@ -39,6 +40,16 @@ type Secret struct {
 	Match []string
 }
 
+type secretMap map[string]Secret
+
+func (sm secretMap) toStringMap() map[string]string {
+	m := make(map[string]string, len(sm))
+	for k, v := range sm {
+		m[k] = v.Value
+	}
+	return m
+}
+
 type ResourceLimit struct {
 	MemSwapLimit int64
 	MemLimit     int64
@@ -61,7 +72,7 @@ type Compiler struct {
 	path              string
 	metadata          frontend.Metadata
 	registries        []Registry
-	secrets           map[string]Secret
+	secrets           secretMap
 	cacher            Cacher
 	reslimit          ResourceLimit
 	defaultCloneImage string

pipeline/frontend/yaml/compiler/convert.go

@@ -9,6 +9,7 @@ import (
 
 	backend "github.com/woodpecker-ci/woodpecker/pipeline/backend/types"
 	"github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml"
+	"github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml/compiler/settings"
 )
 
 func (c *Compiler) createProcess(name string, container *yaml.Container, section string) *backend.Step {
@@ -71,7 +72,7 @@ func (c *Compiler) createProcess(name string, container *yaml.Container, section
 	}
 
 	if !detached {
-		if err := paramsToEnv(container.Settings, environment, c.secrets); err != nil {
+		if err := settings.ParamsToEnv(container.Settings, environment, c.secrets.toStringMap()); err != nil {
 			log.Error().Err(err).Msg("paramsToEnv")
 		}
 	}

pipeline/frontend/yaml/compiler/params.go

@@ -1,111 +0,0 @@
-package compiler
-
-import (
-	"fmt"
-	"reflect"
-	"strconv"
-	"strings"
-
-	"codeberg.org/6543/go-yaml2json"
-	"gopkg.in/yaml.v3"
-)
-
-// paramsToEnv uses reflection to convert a map[string]interface to a list
-// of environment variables.
-func paramsToEnv(from map[string]interface{}, to map[string]string, secrets map[string]Secret) (err error) {
-	if to == nil {
-		return fmt.Errorf("no map to write to")
-	}
-	for k, v := range from {
-		if v == nil || len(k) == 0 {
-			continue
-		}
-		to[sanitizeParamKey(k)], err = sanitizeParamValue(v, secrets)
-		if err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-func sanitizeParamKey(k string) string {
-	return "PLUGIN_" +
-		strings.ToUpper(
-			strings.ReplaceAll(k, ".", "_"),
-		)
-}
-
-func isComplex(t reflect.Kind) bool {
-	switch t {
-	case reflect.Bool,
-		reflect.String,
-		reflect.Int, reflect.Int16, reflect.Int32, reflect.Int64,
-		reflect.Float32, reflect.Float64:
-		return false
-	default:
-		return true
-	}
-}
-
-func sanitizeParamValue(v interface{}, secrets map[string]Secret) (string, error) {
-	t := reflect.TypeOf(v)
-	vv := reflect.ValueOf(v)
-
-	switch t.Kind() {
-	case reflect.Bool:
-		return strconv.FormatBool(vv.Bool()), nil
-
-	case reflect.String:
-		return vv.String(), nil
-
-	case reflect.Int, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Int8:
-		return fmt.Sprintf("%v", vv.Int()), nil
-
-	case reflect.Float32, reflect.Float64:
-		return fmt.Sprintf("%v", vv.Float()), nil
-
-	case reflect.Map:
-		if fromSecret, ok := v.(map[string]interface{}); ok {
-			if secretNameI, ok := fromSecret["from_secret"]; ok {
-				if secretName, ok := secretNameI.(string); ok {
-					if secret, ok := secrets[strings.ToLower(secretName)]; ok {
-						return secret.Value, nil
-					}
-					return "", fmt.Errorf("no secret found for %q", secretName)
-				}
-			}
-		}
-		ymlOut, _ := yaml.Marshal(vv.Interface())
-		out, _ := yaml2json.Convert(ymlOut)
-		return string(out), nil
-
-	case reflect.Slice, reflect.Array:
-		if vv.Len() == 0 {
-			return "", nil
-		}
-		if !isComplex(t.Elem().Kind()) || t.Elem().Kind() == reflect.Interface {
-			in := make([]string, vv.Len())
-			for i := 0; i < vv.Len(); i++ {
-				var err error
-				if in[i], err = sanitizeParamValue(vv.Index(i).Interface(), secrets); err != nil {
-					return "", err
-				}
-			}
-			return strings.Join(in, ","), nil
-		}
-
-		// it's complex use yml.ToJSON
-		fallthrough
-
-	default:
-		out, err := yaml.Marshal(vv.Interface())
-		if err != nil {
-			return "", err
-		}
-		out, err = yaml2json.Convert(out)
-		if err != nil {
-			return "", err
-		}
-		return string(out), nil
-	}
-}

pipeline/frontend/yaml/compiler/settings/params.go

@@ -0,0 +1,219 @@
+// Copyright 2022 Woodpecker Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package settings
+
+import (
+	"fmt"
+	"reflect"
+	"strconv"
+	"strings"
+
+	"codeberg.org/6543/go-yaml2json"
+	"gopkg.in/yaml.v3"
+)
+
+// ParamsToEnv uses reflection to convert a map[string]interface to a list
+// of environment variables.
+func ParamsToEnv(from map[string]interface{}, to, secrets map[string]string) (err error) {
+	if to == nil {
+		return fmt.Errorf("no map to write to")
+	}
+	for k, v := range from {
+		if v == nil || len(k) == 0 {
+			continue
+		}
+		to[sanitizeParamKey(k)], err = sanitizeParamValue(v, secrets)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// format the environment variable key
+func sanitizeParamKey(k string) string {
+	return "PLUGIN_" + strings.ToUpper(
+		strings.ReplaceAll(strings.ReplaceAll(k, ".", "_"), "-", "_"))
+}
+
+// indicate if a data type can be turned into string without encoding as json
+func isComplex(t reflect.Kind) bool {
+	switch t {
+	case reflect.Bool,
+		reflect.String,
+		reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64,
+		reflect.Float32, reflect.Float64:
+		return false
+	default:
+		return true
+	}
+}
+
+// sanitizeParamValue returns the value of a setting as string prepared to be injected as environment variable
+func sanitizeParamValue(v interface{}, secrets map[string]string) (string, error) {
+	t := reflect.TypeOf(v)
+	vv := reflect.ValueOf(v)
+
+	switch t.Kind() {
+	case reflect.Bool:
+		return strconv.FormatBool(vv.Bool()), nil
+
+	case reflect.String:
+		return vv.String(), nil
+
+	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
+		return fmt.Sprintf("%v", vv.Int()), nil
+
+	case reflect.Float32, reflect.Float64:
+		return fmt.Sprintf("%v", vv.Float()), nil
+
+	case reflect.Map:
+		switch v := v.(type) {
+		// gopkg.in/yaml.v3 only emits this map interface
+		case map[string]interface{}:
+			// check if it's a secret and return value if it's the case
+			value, isSecret, err := injectSecret(v, secrets)
+			if err != nil {
+				return "", err
+			} else if isSecret {
+				return value, nil
+			}
+		default:
+			return "", fmt.Errorf("could not handle: %#v", v)
+		}
+
+		return handleComplex(vv.Interface(), secrets)
+
+	case reflect.Slice, reflect.Array:
+		if vv.Len() == 0 {
+			return "", nil
+		}
+
+		// if it's an interface unwrap and element check happen for each iteration later
+		if t.Elem().Kind() == reflect.Interface ||
+			// else check directly if element is not complex
+			!isComplex(t.Elem().Kind()) {
+			containsComplex := false
+			in := make([]string, vv.Len())
+
+			for i := 0; i < vv.Len(); i++ {
+				v := vv.Index(i).Interface()
+
+				// ensure each element is not complex
+				if isComplex(reflect.TypeOf(v).Kind()) {
+					containsComplex = true
+					break
+				}
+
+				var err error
+				if in[i], err = sanitizeParamValue(v, secrets); err != nil {
+					return "", err
+				}
+			}
+
+			if !containsComplex {
+				return strings.Join(in, ","), nil
+			}
+		}
+	}
+
+	// handle all elements which are not primitives, string-maps containing secrets or arrays
+	return handleComplex(vv.Interface(), secrets)
+}
+
+// handleComplex uses yaml2json to get json strings as values for environment variables
+func handleComplex(v interface{}, secrets map[string]string) (string, error) {
+	v, err := injectSecretRecursive(v, secrets)
+	if err != nil {
+		return "", err
+	}
+
+	out, err := yaml.Marshal(v)
+	if err != nil {
+		return "", err
+	}
+	out, err = yaml2json.Convert(out)
+	if err != nil {
+		return "", err
+	}
+	return string(out), nil
+}
+
+// injectSecret probes if a map is a from_secret request.
+// If it's a from_secret request it either  returns the secret value or an error if the secret was not found
+// else it just indicates to progress normally using the provided map as is
+func injectSecret(v map[string]interface{}, secrets map[string]string) (string, bool, error) {
+	if secretNameI, ok := v["from_secret"]; ok {
+		if secretName, ok := secretNameI.(string); ok {
+			if secret, ok := secrets[strings.ToLower(secretName)]; ok {
+				return secret, true, nil
+			}
+			return "", false, fmt.Errorf("no secret found for %q", secretName)
+		}
+		return "", false, fmt.Errorf("from_secret has to be a string")
+	}
+	return "", false, nil
+}
+
+// injectSecretRecursive iterates over all types and if they contain elements
+// it iterates recursively over them too, using injectSecret internally
+func injectSecretRecursive(v interface{}, secrets map[string]string) (interface{}, error) {
+	t := reflect.TypeOf(v)
+
+	if !isComplex(t.Kind()) {
+		return v, nil
+	}
+
+	switch t.Kind() {
+	case reflect.Map:
+		switch v := v.(type) {
+		// gopkg.in/yaml.v3 only emits this map interface
+		case map[string]interface{}:
+			// handle secrets
+			value, isSecret, err := injectSecret(v, secrets)
+			if err != nil {
+				return nil, err
+			} else if isSecret {
+				return value, nil
+			}
+
+			for key, val := range v {
+				v[key], err = injectSecretRecursive(val, secrets)
+				if err != nil {
+					return nil, err
+				}
+			}
+			return v, nil
+		default:
+			return v, fmt.Errorf("could not handle: %#v", v)
+		}
+
+	case reflect.Array, reflect.Slice:
+		vv := reflect.ValueOf(v)
+		vl := make([]interface{}, vv.Len())
+
+		for i := 0; i < vv.Len(); i++ {
+			v, err := injectSecretRecursive(vv.Index(i).Interface(), secrets)
+			if err != nil {
+				return nil, err
+			}
+			vl[i] = v
+		}
+		return vl, nil
+
+	default:
+		return v, nil
+	}
+}

pipeline/frontend/yaml/compiler/settings/params_test.go

@@ -1,4 +1,18 @@
-package compiler
+// Copyright 2022 Woodpecker Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package settings
 
 import (
 	"testing"
@@ -15,7 +29,7 @@ func TestParamsToEnv(t *testing.T) {
 		"float":            1.2,
 		"bool":             true,
 		"slice":            []int{1, 2, 3},
-		"map":              map[string]string{"hello": "world"},
+		"map":              map[string]interface{}{"hello": "world"},
 		"complex":          []struct{ Name string }{{"Jack"}, {"Jill"}},
 		"complex2":         struct{ Name string }{"Jack"},
 		"from.address":     "noreply@example.com",
@@ -39,14 +53,20 @@ func TestParamsToEnv(t *testing.T) {
 		"PLUGIN_MY_SECRET":        "FooBar",
 		"PLUGIN_UPPERCASE_SECRET": "FooBar",
 	}
-	secrets := map[string]Secret{
+	secrets := map[string]string{
-		"secret_token": {Name: "secret_token", Value: "FooBar", Match: nil},
+		"secret_token": "FooBar",
 	}
 	got := map[string]string{}
-	assert.NoError(t, paramsToEnv(from, got, secrets))
+	assert.NoError(t, ParamsToEnv(from, got, secrets))
 	assert.EqualValues(t, want, got, "Problem converting plugin parameters to environment variables")
 }
 
+func TestSanitizeParamKey(t *testing.T) {
+	assert.EqualValues(t, "PLUGIN_DRY_RUN", sanitizeParamKey("dry-run"))
+	assert.EqualValues(t, "PLUGIN_DRY_RUN", sanitizeParamKey("dry_Run"))
+	assert.EqualValues(t, "PLUGIN_DRY_RUN", sanitizeParamKey("dry.run"))
+}
+
 func TestYAMLToParamsToEnv(t *testing.T) {
 	fromYAML := []byte(`skip: ~
 string: stringz
@@ -56,6 +76,19 @@ bool: true
 slice: [1, 2, 3]
 my_secret:
   from_secret: secret_token
+map:
+  key: "value"
+  entry2:
+    - "a"
+    - "b"
+    - 3
+  secret:
+    from_secret: secret_token
+list.map:
+  - registry: https://codeberg.org
+    username: "6543"
+    password:
+      from_secret: cb_password
 `)
 	var from map[string]interface{}
 	err := yaml.Unmarshal(fromYAML, &from)
@@ -68,12 +101,15 @@ my_secret:
 		"PLUGIN_BOOL":      "true",
 		"PLUGIN_SLICE":     "1,2,3",
 		"PLUGIN_MY_SECRET": "FooBar",
+		"PLUGIN_MAP":       `{"entry2":["a","b",3],"key":"value","secret":"FooBar"}`,
+		"PLUGIN_LIST_MAP":  `[{"password":"geheim","registry":"https://codeberg.org","username":"6543"}]`,
 	}
-	secrets := map[string]Secret{
+	secrets := map[string]string{
-		"secret_token": {Name: "secret_token", Value: "FooBar", Match: nil},
+		"secret_token": "FooBar",
+		"cb_password":  "geheim",
 	}
 	got := map[string]string{}
-	assert.NoError(t, paramsToEnv(from, got, secrets))
+	assert.NoError(t, ParamsToEnv(from, got, secrets))
 	assert.EqualValues(t, want, got, "Problem converting plugin parameters to environment variables")
 }
 
@@ -84,10 +120,10 @@ func TestYAMLToParamsToEnvError(t *testing.T) {
 	var from map[string]interface{}
 	err := yaml.Unmarshal(fromYAML, &from)
 	assert.NoError(t, err)
-	secrets := map[string]Secret{
+	secrets := map[string]string{
-		"secret_token": {Name: "secret_token", Value: "FooBar", Match: nil},
+		"secret_token": "FooBar",
 	}
-	assert.Error(t, paramsToEnv(from, make(map[string]string), secrets))
+	assert.Error(t, ParamsToEnv(from, make(map[string]string), secrets))
 }
 
 func stringsToInterface(val ...string) []interface{} {