From da997fa34a3621f751b25da9843d2636db5ec35e Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Mon, 3 Oct 2022 19:25:43 +0200 Subject: [PATCH] Add support sub-settings and secrets in sub-settings (#1221) --- pipeline/frontend/yaml/compiler/compiler.go | 13 +- pipeline/frontend/yaml/compiler/convert.go | 3 +- pipeline/frontend/yaml/compiler/params.go | 111 --------- .../frontend/yaml/compiler/settings/params.go | 219 ++++++++++++++++++ .../compiler/{ => settings}/params_test.go | 58 ++++- 5 files changed, 280 insertions(+), 124 deletions(-) delete mode 100644 pipeline/frontend/yaml/compiler/params.go create mode 100644 pipeline/frontend/yaml/compiler/settings/params.go rename pipeline/frontend/yaml/compiler/{ => settings}/params_test.go (59%) diff --git a/pipeline/frontend/yaml/compiler/compiler.go b/pipeline/frontend/yaml/compiler/compiler.go index 0885c8128..c6b6f3002 100644 --- a/pipeline/frontend/yaml/compiler/compiler.go +++ b/pipeline/frontend/yaml/compiler/compiler.go @@ -25,6 +25,7 @@ const ( namePipeline = "pipeline" ) +// Registry represents registry credentials type Registry struct { Hostname string Username string @@ -39,6 +40,16 @@ type Secret struct { Match []string } +type secretMap map[string]Secret + +func (sm secretMap) toStringMap() map[string]string { + m := make(map[string]string, len(sm)) + for k, v := range sm { + m[k] = v.Value + } + return m +} + type ResourceLimit struct { MemSwapLimit int64 MemLimit int64 @@ -61,7 +72,7 @@ type Compiler struct { path string metadata frontend.Metadata registries []Registry - secrets map[string]Secret + secrets secretMap cacher Cacher reslimit ResourceLimit defaultCloneImage string diff --git a/pipeline/frontend/yaml/compiler/convert.go b/pipeline/frontend/yaml/compiler/convert.go index f4478604f..343a7a748 100644 --- a/pipeline/frontend/yaml/compiler/convert.go +++ b/pipeline/frontend/yaml/compiler/convert.go @@ -9,6 +9,7 @@ import ( backend "github.com/woodpecker-ci/woodpecker/pipeline/backend/types" "github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml" + "github.com/woodpecker-ci/woodpecker/pipeline/frontend/yaml/compiler/settings" ) func (c *Compiler) createProcess(name string, container *yaml.Container, section string) *backend.Step { @@ -71,7 +72,7 @@ func (c *Compiler) createProcess(name string, container *yaml.Container, section } if !detached { - if err := paramsToEnv(container.Settings, environment, c.secrets); err != nil { + if err := settings.ParamsToEnv(container.Settings, environment, c.secrets.toStringMap()); err != nil { log.Error().Err(err).Msg("paramsToEnv") } } diff --git a/pipeline/frontend/yaml/compiler/params.go b/pipeline/frontend/yaml/compiler/params.go deleted file mode 100644 index e8276e5fe..000000000 --- a/pipeline/frontend/yaml/compiler/params.go +++ /dev/null @@ -1,111 +0,0 @@ -package compiler - -import ( - "fmt" - "reflect" - "strconv" - "strings" - - "codeberg.org/6543/go-yaml2json" - "gopkg.in/yaml.v3" -) - -// paramsToEnv uses reflection to convert a map[string]interface to a list -// of environment variables. -func paramsToEnv(from map[string]interface{}, to map[string]string, secrets map[string]Secret) (err error) { - if to == nil { - return fmt.Errorf("no map to write to") - } - for k, v := range from { - if v == nil || len(k) == 0 { - continue - } - to[sanitizeParamKey(k)], err = sanitizeParamValue(v, secrets) - if err != nil { - return err - } - } - return nil -} - -func sanitizeParamKey(k string) string { - return "PLUGIN_" + - strings.ToUpper( - strings.ReplaceAll(k, ".", "_"), - ) -} - -func isComplex(t reflect.Kind) bool { - switch t { - case reflect.Bool, - reflect.String, - reflect.Int, reflect.Int16, reflect.Int32, reflect.Int64, - reflect.Float32, reflect.Float64: - return false - default: - return true - } -} - -func sanitizeParamValue(v interface{}, secrets map[string]Secret) (string, error) { - t := reflect.TypeOf(v) - vv := reflect.ValueOf(v) - - switch t.Kind() { - case reflect.Bool: - return strconv.FormatBool(vv.Bool()), nil - - case reflect.String: - return vv.String(), nil - - case reflect.Int, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Int8: - return fmt.Sprintf("%v", vv.Int()), nil - - case reflect.Float32, reflect.Float64: - return fmt.Sprintf("%v", vv.Float()), nil - - case reflect.Map: - if fromSecret, ok := v.(map[string]interface{}); ok { - if secretNameI, ok := fromSecret["from_secret"]; ok { - if secretName, ok := secretNameI.(string); ok { - if secret, ok := secrets[strings.ToLower(secretName)]; ok { - return secret.Value, nil - } - return "", fmt.Errorf("no secret found for %q", secretName) - } - } - } - ymlOut, _ := yaml.Marshal(vv.Interface()) - out, _ := yaml2json.Convert(ymlOut) - return string(out), nil - - case reflect.Slice, reflect.Array: - if vv.Len() == 0 { - return "", nil - } - if !isComplex(t.Elem().Kind()) || t.Elem().Kind() == reflect.Interface { - in := make([]string, vv.Len()) - for i := 0; i < vv.Len(); i++ { - var err error - if in[i], err = sanitizeParamValue(vv.Index(i).Interface(), secrets); err != nil { - return "", err - } - } - return strings.Join(in, ","), nil - } - - // it's complex use yml.ToJSON - fallthrough - - default: - out, err := yaml.Marshal(vv.Interface()) - if err != nil { - return "", err - } - out, err = yaml2json.Convert(out) - if err != nil { - return "", err - } - return string(out), nil - } -} diff --git a/pipeline/frontend/yaml/compiler/settings/params.go b/pipeline/frontend/yaml/compiler/settings/params.go new file mode 100644 index 000000000..9bbeb5d49 --- /dev/null +++ b/pipeline/frontend/yaml/compiler/settings/params.go @@ -0,0 +1,219 @@ +// Copyright 2022 Woodpecker Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package settings + +import ( + "fmt" + "reflect" + "strconv" + "strings" + + "codeberg.org/6543/go-yaml2json" + "gopkg.in/yaml.v3" +) + +// ParamsToEnv uses reflection to convert a map[string]interface to a list +// of environment variables. +func ParamsToEnv(from map[string]interface{}, to, secrets map[string]string) (err error) { + if to == nil { + return fmt.Errorf("no map to write to") + } + for k, v := range from { + if v == nil || len(k) == 0 { + continue + } + to[sanitizeParamKey(k)], err = sanitizeParamValue(v, secrets) + if err != nil { + return err + } + } + return nil +} + +// format the environment variable key +func sanitizeParamKey(k string) string { + return "PLUGIN_" + strings.ToUpper( + strings.ReplaceAll(strings.ReplaceAll(k, ".", "_"), "-", "_")) +} + +// indicate if a data type can be turned into string without encoding as json +func isComplex(t reflect.Kind) bool { + switch t { + case reflect.Bool, + reflect.String, + reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, + reflect.Float32, reflect.Float64: + return false + default: + return true + } +} + +// sanitizeParamValue returns the value of a setting as string prepared to be injected as environment variable +func sanitizeParamValue(v interface{}, secrets map[string]string) (string, error) { + t := reflect.TypeOf(v) + vv := reflect.ValueOf(v) + + switch t.Kind() { + case reflect.Bool: + return strconv.FormatBool(vv.Bool()), nil + + case reflect.String: + return vv.String(), nil + + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return fmt.Sprintf("%v", vv.Int()), nil + + case reflect.Float32, reflect.Float64: + return fmt.Sprintf("%v", vv.Float()), nil + + case reflect.Map: + switch v := v.(type) { + // gopkg.in/yaml.v3 only emits this map interface + case map[string]interface{}: + // check if it's a secret and return value if it's the case + value, isSecret, err := injectSecret(v, secrets) + if err != nil { + return "", err + } else if isSecret { + return value, nil + } + default: + return "", fmt.Errorf("could not handle: %#v", v) + } + + return handleComplex(vv.Interface(), secrets) + + case reflect.Slice, reflect.Array: + if vv.Len() == 0 { + return "", nil + } + + // if it's an interface unwrap and element check happen for each iteration later + if t.Elem().Kind() == reflect.Interface || + // else check directly if element is not complex + !isComplex(t.Elem().Kind()) { + containsComplex := false + in := make([]string, vv.Len()) + + for i := 0; i < vv.Len(); i++ { + v := vv.Index(i).Interface() + + // ensure each element is not complex + if isComplex(reflect.TypeOf(v).Kind()) { + containsComplex = true + break + } + + var err error + if in[i], err = sanitizeParamValue(v, secrets); err != nil { + return "", err + } + } + + if !containsComplex { + return strings.Join(in, ","), nil + } + } + } + + // handle all elements which are not primitives, string-maps containing secrets or arrays + return handleComplex(vv.Interface(), secrets) +} + +// handleComplex uses yaml2json to get json strings as values for environment variables +func handleComplex(v interface{}, secrets map[string]string) (string, error) { + v, err := injectSecretRecursive(v, secrets) + if err != nil { + return "", err + } + + out, err := yaml.Marshal(v) + if err != nil { + return "", err + } + out, err = yaml2json.Convert(out) + if err != nil { + return "", err + } + return string(out), nil +} + +// injectSecret probes if a map is a from_secret request. +// If it's a from_secret request it either returns the secret value or an error if the secret was not found +// else it just indicates to progress normally using the provided map as is +func injectSecret(v map[string]interface{}, secrets map[string]string) (string, bool, error) { + if secretNameI, ok := v["from_secret"]; ok { + if secretName, ok := secretNameI.(string); ok { + if secret, ok := secrets[strings.ToLower(secretName)]; ok { + return secret, true, nil + } + return "", false, fmt.Errorf("no secret found for %q", secretName) + } + return "", false, fmt.Errorf("from_secret has to be a string") + } + return "", false, nil +} + +// injectSecretRecursive iterates over all types and if they contain elements +// it iterates recursively over them too, using injectSecret internally +func injectSecretRecursive(v interface{}, secrets map[string]string) (interface{}, error) { + t := reflect.TypeOf(v) + + if !isComplex(t.Kind()) { + return v, nil + } + + switch t.Kind() { + case reflect.Map: + switch v := v.(type) { + // gopkg.in/yaml.v3 only emits this map interface + case map[string]interface{}: + // handle secrets + value, isSecret, err := injectSecret(v, secrets) + if err != nil { + return nil, err + } else if isSecret { + return value, nil + } + + for key, val := range v { + v[key], err = injectSecretRecursive(val, secrets) + if err != nil { + return nil, err + } + } + return v, nil + default: + return v, fmt.Errorf("could not handle: %#v", v) + } + + case reflect.Array, reflect.Slice: + vv := reflect.ValueOf(v) + vl := make([]interface{}, vv.Len()) + + for i := 0; i < vv.Len(); i++ { + v, err := injectSecretRecursive(vv.Index(i).Interface(), secrets) + if err != nil { + return nil, err + } + vl[i] = v + } + return vl, nil + + default: + return v, nil + } +} diff --git a/pipeline/frontend/yaml/compiler/params_test.go b/pipeline/frontend/yaml/compiler/settings/params_test.go similarity index 59% rename from pipeline/frontend/yaml/compiler/params_test.go rename to pipeline/frontend/yaml/compiler/settings/params_test.go index 82edbf583..22ab60aa5 100644 --- a/pipeline/frontend/yaml/compiler/params_test.go +++ b/pipeline/frontend/yaml/compiler/settings/params_test.go @@ -1,4 +1,18 @@ -package compiler +// Copyright 2022 Woodpecker Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package settings import ( "testing" @@ -15,7 +29,7 @@ func TestParamsToEnv(t *testing.T) { "float": 1.2, "bool": true, "slice": []int{1, 2, 3}, - "map": map[string]string{"hello": "world"}, + "map": map[string]interface{}{"hello": "world"}, "complex": []struct{ Name string }{{"Jack"}, {"Jill"}}, "complex2": struct{ Name string }{"Jack"}, "from.address": "noreply@example.com", @@ -39,14 +53,20 @@ func TestParamsToEnv(t *testing.T) { "PLUGIN_MY_SECRET": "FooBar", "PLUGIN_UPPERCASE_SECRET": "FooBar", } - secrets := map[string]Secret{ - "secret_token": {Name: "secret_token", Value: "FooBar", Match: nil}, + secrets := map[string]string{ + "secret_token": "FooBar", } got := map[string]string{} - assert.NoError(t, paramsToEnv(from, got, secrets)) + assert.NoError(t, ParamsToEnv(from, got, secrets)) assert.EqualValues(t, want, got, "Problem converting plugin parameters to environment variables") } +func TestSanitizeParamKey(t *testing.T) { + assert.EqualValues(t, "PLUGIN_DRY_RUN", sanitizeParamKey("dry-run")) + assert.EqualValues(t, "PLUGIN_DRY_RUN", sanitizeParamKey("dry_Run")) + assert.EqualValues(t, "PLUGIN_DRY_RUN", sanitizeParamKey("dry.run")) +} + func TestYAMLToParamsToEnv(t *testing.T) { fromYAML := []byte(`skip: ~ string: stringz @@ -56,6 +76,19 @@ bool: true slice: [1, 2, 3] my_secret: from_secret: secret_token +map: + key: "value" + entry2: + - "a" + - "b" + - 3 + secret: + from_secret: secret_token +list.map: + - registry: https://codeberg.org + username: "6543" + password: + from_secret: cb_password `) var from map[string]interface{} err := yaml.Unmarshal(fromYAML, &from) @@ -68,12 +101,15 @@ my_secret: "PLUGIN_BOOL": "true", "PLUGIN_SLICE": "1,2,3", "PLUGIN_MY_SECRET": "FooBar", + "PLUGIN_MAP": `{"entry2":["a","b",3],"key":"value","secret":"FooBar"}`, + "PLUGIN_LIST_MAP": `[{"password":"geheim","registry":"https://codeberg.org","username":"6543"}]`, } - secrets := map[string]Secret{ - "secret_token": {Name: "secret_token", Value: "FooBar", Match: nil}, + secrets := map[string]string{ + "secret_token": "FooBar", + "cb_password": "geheim", } got := map[string]string{} - assert.NoError(t, paramsToEnv(from, got, secrets)) + assert.NoError(t, ParamsToEnv(from, got, secrets)) assert.EqualValues(t, want, got, "Problem converting plugin parameters to environment variables") } @@ -84,10 +120,10 @@ func TestYAMLToParamsToEnvError(t *testing.T) { var from map[string]interface{} err := yaml.Unmarshal(fromYAML, &from) assert.NoError(t, err) - secrets := map[string]Secret{ - "secret_token": {Name: "secret_token", Value: "FooBar", Match: nil}, + secrets := map[string]string{ + "secret_token": "FooBar", } - assert.Error(t, paramsToEnv(from, make(map[string]string), secrets)) + assert.Error(t, ParamsToEnv(from, make(map[string]string), secrets)) } func stringsToInterface(val ...string) []interface{} {