diff --git a/Makefile b/Makefile index b8d916dc2..996f2ca5e 100644 --- a/Makefile +++ b/Makefile @@ -71,7 +71,7 @@ test-frontend: frontend-dependencies test-lib: $(DOCKER_RUN) go test -race -timeout 30s $(shell go list ./... | grep -v '/cmd\|/agent\|/cli\|/server') -test: test-agent test-server test-server-datastore test-cli test-frontend test-lib +test: test-agent test-server test-server-datastore test-cli test-lib test-frontend build-frontend: (cd web/; yarn install --frozen-lockfile; yarn build) diff --git a/pipeline/frontend/yaml/compiler/params.go b/pipeline/frontend/yaml/compiler/params.go index 4b170b7e8..d66c6a3a9 100644 --- a/pipeline/frontend/yaml/compiler/params.go +++ b/pipeline/frontend/yaml/compiler/params.go @@ -6,61 +6,94 @@ import ( "strconv" "strings" - json "github.com/woodpecker-ci/woodpecker/shared/yml" - "gopkg.in/yaml.v3" + + "github.com/woodpecker-ci/woodpecker/shared/yml" ) // paramsToEnv uses reflection to convert a map[string]interface to a list // of environment variables. -func paramsToEnv(from map[string]interface{}, to map[string]string) error { +func paramsToEnv(from map[string]interface{}, to map[string]string) (err error) { + if to == nil { + return fmt.Errorf("no map to write to") + } for k, v := range from { - if v == nil { + if v == nil || len(k) == 0 { continue } - - t := reflect.TypeOf(v) - vv := reflect.ValueOf(v) - - k = "PLUGIN_" + strings.ToUpper(k) - - switch t.Kind() { - case reflect.Bool: - to[k] = strconv.FormatBool(vv.Bool()) - - case reflect.String: - to[k] = vv.String() - - case reflect.Int, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Int8: - to[k] = fmt.Sprintf("%v", vv.Int()) - - case reflect.Float32, reflect.Float64: - to[k] = fmt.Sprintf("%v", vv.Float()) - - case reflect.Map: - yml, _ := yaml.Marshal(vv.Interface()) - out, _ := json.Yml2Json(yml) - to[k] = string(out) - - case reflect.Slice: - out, err := yaml.Marshal(vv.Interface()) - if err != nil { - return err - } - - var in []string - err = yaml.Unmarshal(out, &in) - if err == nil { - to[k] = strings.Join(in, ",") - } else { - out, err = json.Yml2Json(out) - if err != nil { - return err - } - to[k] = string(out) - } + to[sanitizeParamKey(k)], err = sanitizeParamValue(v) + if err != nil { + return err } } - return nil } + +func sanitizeParamKey(k string) string { + return "PLUGIN_" + + strings.ToUpper( + strings.ReplaceAll(k, ".", "_"), + ) +} + +func isComplex(t reflect.Kind) bool { + switch t { + case reflect.Bool, + reflect.String, + reflect.Int, reflect.Int16, reflect.Int32, reflect.Int64, + reflect.Float32, reflect.Float64: + return false + default: + return true + } +} + +func sanitizeParamValue(v interface{}) (string, error) { + t := reflect.TypeOf(v) + vv := reflect.ValueOf(v) + + switch t.Kind() { + case reflect.Bool: + return strconv.FormatBool(vv.Bool()), nil + + case reflect.String: + return vv.String(), nil + + case reflect.Int, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Int8: + return fmt.Sprintf("%v", vv.Int()), nil + + case reflect.Float32, reflect.Float64: + return fmt.Sprintf("%v", vv.Float()), nil + + case reflect.Map: + ymlOut, _ := yaml.Marshal(vv.Interface()) + out, _ := yml.ToJSON(ymlOut) + return string(out), nil + + case reflect.Slice, reflect.Array: + if !isComplex(t.Elem().Kind()) { + in := make([]string, vv.Len()) + for i := 0; i < vv.Len(); i++ { + var err error + if in[i], err = sanitizeParamValue(vv.Index(i).Interface()); err != nil { + return "", err + } + } + return strings.Join(in, ","), nil + } + + // it's complex use yml.ToJSON + fallthrough + + default: + out, err := yaml.Marshal(vv.Interface()) + if err != nil { + return "", err + } + out, err = yml.ToJSON(out) + if err != nil { + return "", err + } + return string(out), nil + } +} diff --git a/pipeline/frontend/yaml/compiler/params_test.go b/pipeline/frontend/yaml/compiler/params_test.go index 8700d7cc7..aeaf0b59a 100644 --- a/pipeline/frontend/yaml/compiler/params_test.go +++ b/pipeline/frontend/yaml/compiler/params_test.go @@ -8,23 +8,27 @@ import ( func TestParamsToEnv(t *testing.T) { from := map[string]interface{}{ - "skip": nil, - "string": "stringz", - "int": 1, - "float": 1.2, - "bool": true, - "map": map[string]string{"hello": "world"}, - "slice": []int{1, 2, 3}, - "complex": []struct{ Name string }{{"Jack"}, {"Jill"}}, + "skip": nil, + "string": "stringz", + "int": 1, + "float": 1.2, + "bool": true, + "slice": []int{1, 2, 3}, + "map": map[string]string{"hello": "world"}, + "complex": []struct{ Name string }{{"Jack"}, {"Jill"}}, + "complex2": struct{ Name string }{"Jack"}, + "from.address": "noreply@example.com", } want := map[string]string{ - "PLUGIN_STRING": "stringz", - "PLUGIN_INT": "1", - "PLUGIN_FLOAT": "1.2", - "PLUGIN_BOOL": "true", - "PLUGIN_MAP": `{"hello":"world"}`, - "PLUGIN_SLICE": "1,2,3", - "PLUGIN_COMPLEX": `[{"name":"Jack"},{"name":"Jill"}]`, + "PLUGIN_STRING": "stringz", + "PLUGIN_INT": "1", + "PLUGIN_FLOAT": "1.2", + "PLUGIN_BOOL": "true", + "PLUGIN_SLICE": "1,2,3", + "PLUGIN_MAP": `{"hello":"world"}`, + "PLUGIN_COMPLEX": `[{"name":"Jack"},{"name":"Jill"}]`, + "PLUGIN_COMPLEX2": `{"name":"Jack"}`, + "PLUGIN_FROM_ADDRESS": "noreply@example.com", } got := map[string]string{} assert.NoError(t, paramsToEnv(from, got)) diff --git a/shared/yml/yml.go b/shared/yml/yml.go index 7dc4d4125..90af53b1f 100644 --- a/shared/yml/yml.go +++ b/shared/yml/yml.go @@ -4,65 +4,72 @@ import ( "encoding/json" "fmt" "os" + "strconv" "gopkg.in/yaml.v3" ) -// Source: https://github.com/icza/dyno/blob/f1bafe5d99965c48cc9d5c7cf024eeb495facc1e/dyno.go#L563-L601 -// License: Apache 2.0 - Copyright 2017 Andras Belicza -// ConvertMapI2MapS walks the given dynamic object recursively, and -// converts maps with interface{} key type to maps with string key type. -// This function comes handy if you want to marshal a dynamic object into -// JSON where maps with interface{} key type are not allowed. -// -// Recursion is implemented into values of the following types: -// -map[interface{}]interface{} -// -map[string]interface{} -// -[]interface{} -// -// When converting map[interface{}]interface{} to map[string]interface{}, +// toJSON convert gopkg.in/yaml.v3 nodes to object that can be serialized as json // fmt.Sprint() with default formatting is used to convert the key to a string key. -func convertMapI2MapS(v interface{}) interface{} { - switch x := v.(type) { - case map[interface{}]interface{}: - m := map[string]interface{}{} - for k, v2 := range x { - switch k2 := k.(type) { - case string: // Fast check if it's already a string - m[k2] = convertMapI2MapS(v2) - default: - m[fmt.Sprint(k)] = convertMapI2MapS(v2) +func toJSON(node *yaml.Node) (interface{}, error) { + switch node.Kind { + case yaml.DocumentNode: + return toJSON(node.Content[0]) + + case yaml.SequenceNode: + val := make([]interface{}, len(node.Content)) + var err error + for i := range node.Content { + if val[i], err = toJSON(node.Content[i]); err != nil { + return nil, err } } - v = m + return val, nil - case []interface{}: - for i, v2 := range x { - x[i] = convertMapI2MapS(v2) + case yaml.MappingNode: + if (len(node.Content) % 2) != 0 { + return nil, fmt.Errorf("broken mapping node") } + val := make(map[string]interface{}, len(node.Content)%2) + for i := len(node.Content); i > 1; i = i - 2 { + k, err := toJSON(node.Content[i-2]) + if err != nil { + return nil, err + } + if val[fmt.Sprint(k)], err = toJSON(node.Content[i-1]); err != nil { + return nil, err + } + } + return val, nil - case map[string]interface{}: - for k, v2 := range x { - x[k] = convertMapI2MapS(v2) + case yaml.ScalarNode: + switch node.Tag { + case nullTag: + return nil, nil + case boolTag: + return strconv.ParseBool(node.Value) + case intTag: + return strconv.ParseInt(node.Value, 10, 64) + case floatTag: + return strconv.ParseFloat(node.Value, 64) } + return node.Value, nil } - return v + return nil, fmt.Errorf("do not support yaml node kind '%v'", node.Kind) } -func Yml2Json(data []byte) (j []byte, err error) { - m := make(map[interface{}]interface{}) - err = yaml.Unmarshal(data, &m) - if err != nil { +func ToJSON(data []byte) ([]byte, error) { + m := &yaml.Node{} + if err := yaml.Unmarshal(data, m); err != nil { return nil, err } - j, err = json.Marshal(convertMapI2MapS(m)) + d, err := toJSON(m) if err != nil { return nil, err } - - return j, nil + return json.Marshal(d) } func LoadYmlFileAsJSON(path string) (j []byte, err error) { @@ -71,10 +78,24 @@ func LoadYmlFileAsJSON(path string) (j []byte, err error) { return nil, err } - j, err = Yml2Json(data) + j, err = ToJSON(data) if err != nil { return nil, err } return j, nil } + +// Source: https://github.com/go-yaml/yaml/blob/3e3283e801afc229479d5fc68aa41df1137b8394/resolve.go#L70-L81 +const ( + nullTag = "!!null" + boolTag = "!!bool" + intTag = "!!int" + floatTag = "!!float" + // strTag = "!!str" // we dont have to parse it + // timestampTag = "!!timestamp" // TODO: do we have to parse this? + // seqTag = "!!seq" // TODO: do we have to parse this? + // mapTag = "!!map" // TODO: do we have to parse this? + // binaryTag = "!!binary" // TODO: do we have to parse this? + // mergeTag = "!!merge" // TODO: do we have to parse this? +) diff --git a/shared/yml/yml_test.go b/shared/yml/yml_test.go new file mode 100644 index 000000000..150b534da --- /dev/null +++ b/shared/yml/yml_test.go @@ -0,0 +1,45 @@ +package yml + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestToJSON(t *testing.T) { + tests := []struct { + yaml string + json string + }{{ + yaml: `- name: Jack +- name: Jill +`, + json: `[{"name":"Jack"},{"name":"Jill"}]`, + }, { + yaml: `name: Jack`, + json: `{"name":"Jack"}`, + }, { + yaml: `name: Jack +job: Butcher +`, + json: `{"job":"Butcher","name":"Jack"}`, + }, { + yaml: `- name: Jack + job: Butcher +- name: Jill + job: Cook + obj: + empty: false + data: | + some data 123 + with new line +`, + json: `[{"job":"Butcher","name":"Jack"},{"job":"Cook","name":"Jill","obj":{"data":"some data 123\nwith new line\n","empty":false}}]`, + }} + + for _, tc := range tests { + result, err := ToJSON([]byte(tc.yaml)) + assert.NoError(t, err) + assert.EqualValues(t, tc.json, string(result)) + } +}