Use external lib to convert yaml to json (#1028)

this move shared/yml/* into an independent lib
This commit is contained in:
6543 2022-07-17 17:23:31 +02:00 committed by GitHub
parent 86cbd63dc2
commit 31bad81979
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 20 additions and 163 deletions

7
go.mod
View file

@ -4,6 +4,7 @@ go 1.18
require (
code.gitea.io/sdk/gitea v0.15.1-0.20220501190934-319a978c6c71
codeberg.org/6543/go-yaml2json v0.1.0
github.com/bmatcuk/doublestar/v4 v4.0.2
github.com/docker/cli v20.10.14+incompatible
github.com/docker/distribution v2.8.1+incompatible
@ -29,7 +30,7 @@ require (
github.com/pkg/errors v0.9.1
github.com/prometheus/client_golang v1.12.1
github.com/rs/zerolog v1.26.1
github.com/stretchr/testify v1.7.1
github.com/stretchr/testify v1.8.0
github.com/tevino/abool v1.2.0
github.com/urfave/cli/v2 v2.5.1
github.com/xanzy/go-gitlab v0.64.0
@ -40,7 +41,7 @@ require (
golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f
google.golang.org/grpc v1.47.0
google.golang.org/protobuf v1.28.0
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
gopkg.in/yaml.v3 v3.0.1
xorm.io/builder v0.3.10
xorm.io/xorm v1.3.0
)
@ -93,7 +94,7 @@ require (
github.com/rogpeppe/go-internal v1.8.1 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/sirupsen/logrus v1.8.1 // indirect
github.com/stretchr/objx v0.3.0 // indirect
github.com/stretchr/objx v0.4.0 // indirect
github.com/syndtr/goleveldb v1.0.0 // indirect
github.com/ugorji/go/codec v1.2.7 // indirect
github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect

12
go.sum
View file

@ -32,6 +32,8 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX
cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
code.gitea.io/sdk/gitea v0.15.1-0.20220501190934-319a978c6c71 h1:+ZqwhfAftVOAd7AcLpfh4LBdTeJIyt60vGU39zhQPyA=
code.gitea.io/sdk/gitea v0.15.1-0.20220501190934-319a978c6c71/go.mod h1:MuMGvUxT8BmFHa0gHhHsrnz91QfmziXuFffm9AuhMCo=
codeberg.org/6543/go-yaml2json v0.1.0 h1:njuf3a8QgsmBXJFiH+7wNR01biBS4MU+XeWE7W3bnus=
codeberg.org/6543/go-yaml2json v0.1.0/go.mod h1:mz61q14LWF4ZABrgMEDMmk3t9dPi6zgR1uBh2VKV2RQ=
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
gitea.com/xorm/sqlfiddle v0.0.0-20180821085327-62ce714f951a h1:lSA0F4e9A2NcQSqGqTOXqu2aRi/XEQxDCBwM8yJtE6s=
gitea.com/xorm/sqlfiddle v0.0.0-20180821085327-62ce714f951a/go.mod h1:EXuID2Zs0pAQhH8yz+DNjUbjppKQzKFAn28TMYPB6IU=
@ -602,16 +604,17 @@ github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5J
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
github.com/stretchr/objx v0.3.0 h1:NGXK3lHquSN08v5vWalVI/L8XU9hdzE/G6xsrze47As=
github.com/stretchr/objx v0.3.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
github.com/stretchr/objx v0.4.0 h1:M2gUjqZET1qApGOWNSnZ49BAIMX4F/1plDv3+l31EJ4=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE=
github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
github.com/tevino/abool v1.2.0 h1:heAkClL8H6w+mK5md9dzsuohKeXHUpY7Vw0ZCKW+huA=
@ -1060,8 +1063,9 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=
gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0=
honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

View file

@ -6,9 +6,8 @@ import (
"strconv"
"strings"
"codeberg.org/6543/go-yaml2json"
"gopkg.in/yaml.v3"
"github.com/woodpecker-ci/woodpecker/shared/yml"
)
// paramsToEnv uses reflection to convert a map[string]interface to a list
@ -77,7 +76,7 @@ func sanitizeParamValue(v interface{}, secrets map[string]Secret) (string, error
}
}
ymlOut, _ := yaml.Marshal(vv.Interface())
out, _ := yml.ToJSON(ymlOut)
out, _ := yaml2json.Covnert(ymlOut)
return string(out), nil
case reflect.Slice, reflect.Array:
@ -103,7 +102,7 @@ func sanitizeParamValue(v interface{}, secrets map[string]Secret) (string, error
if err != nil {
return "", err
}
out, err = yml.ToJSON(out)
out, err = yaml2json.Covnert(out)
if err != nil {
return "", err
}

View file

@ -1,13 +1,13 @@
package schema
import (
"bytes"
_ "embed"
"fmt"
"io"
"codeberg.org/6543/go-yaml2json"
"github.com/xeipuuv/gojsonschema"
"github.com/woodpecker-ci/woodpecker/shared/yml"
)
//go:embed schema.json
@ -16,12 +16,13 @@ var schemaDefinition []byte
// Lint lints an io.Reader against the Woodpecker schema.json
func Lint(r io.Reader) ([]gojsonschema.ResultError, error) {
schemaLoader := gojsonschema.NewBytesLoader(schemaDefinition)
j, err := yml.LoadYmlReaderAsJSON(r)
buff := new(bytes.Buffer)
err := yaml2json.StreamConverter(r, buff)
if err != nil {
return nil, fmt.Errorf("Failed to load yml file %w", err)
}
documentLoader := gojsonschema.NewBytesLoader(j)
documentLoader := gojsonschema.NewBytesLoader(buff.Bytes())
result, err := gojsonschema.Validate(schemaLoader, documentLoader)
if err != nil {
return nil, fmt.Errorf("Validation failed %w", err)

View file

@ -1,103 +0,0 @@
package yml
import (
"encoding/json"
"fmt"
"io"
"strconv"
"gopkg.in/yaml.v3"
)
// toJSON convert gopkg.in/yaml.v3 nodes to object that can be serialized as json
// fmt.Sprint() with default formatting is used to convert the key to a string key.
func toJSON(node *yaml.Node) (interface{}, error) {
switch node.Kind {
case yaml.DocumentNode:
return toJSON(node.Content[0])
case yaml.SequenceNode:
val := make([]interface{}, len(node.Content))
var err error
for i := range node.Content {
if val[i], err = toJSON(node.Content[i]); err != nil {
return nil, err
}
}
return val, nil
case yaml.MappingNode:
if (len(node.Content) % 2) != 0 {
return nil, fmt.Errorf("broken mapping node")
}
val := make(map[string]interface{}, len(node.Content)%2)
for i := len(node.Content); i > 1; i = i - 2 {
k, err := toJSON(node.Content[i-2])
if err != nil {
return nil, err
}
if val[fmt.Sprint(k)], err = toJSON(node.Content[i-1]); err != nil {
return nil, err
}
}
return val, nil
case yaml.ScalarNode:
switch node.Tag {
case nullTag:
return nil, nil
case boolTag:
return strconv.ParseBool(node.Value)
case intTag:
return strconv.ParseInt(node.Value, 10, 64)
case floatTag:
return strconv.ParseFloat(node.Value, 64)
}
return node.Value, nil
}
return nil, fmt.Errorf("do not support yaml node kind '%v'", node.Kind)
}
// ToJSON converts YAML bytes to JSON
func ToJSON(data []byte) ([]byte, error) {
m := &yaml.Node{}
if err := yaml.Unmarshal(data, m); err != nil {
return nil, err
}
d, err := toJSON(m)
if err != nil {
return nil, err
}
return json.Marshal(d)
}
// LoadYmlReaderAsJSON reads from an io.Reader containing YAML and converts it to JSON
func LoadYmlReaderAsJSON(r io.Reader) (j []byte, err error) {
data, err := io.ReadAll(r)
if err != nil {
return nil, err
}
j, err = ToJSON(data)
if err != nil {
return nil, err
}
return j, nil
}
// Source: https://github.com/go-yaml/yaml/blob/3e3283e801afc229479d5fc68aa41df1137b8394/resolve.go#L70-L81
const (
nullTag = "!!null"
boolTag = "!!bool"
intTag = "!!int"
floatTag = "!!float"
// strTag = "!!str" // we dont have to parse it
// timestampTag = "!!timestamp" // TODO: do we have to parse this?
// seqTag = "!!seq" // TODO: do we have to parse this?
// mapTag = "!!map" // TODO: do we have to parse this?
// binaryTag = "!!binary" // TODO: do we have to parse this?
// mergeTag = "!!merge" // TODO: do we have to parse this?
)

View file

@ -1,45 +0,0 @@
package yml
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestToJSON(t *testing.T) {
tests := []struct {
yaml string
json string
}{{
yaml: `- name: Jack
- name: Jill
`,
json: `[{"name":"Jack"},{"name":"Jill"}]`,
}, {
yaml: `name: Jack`,
json: `{"name":"Jack"}`,
}, {
yaml: `name: Jack
job: Butcher
`,
json: `{"job":"Butcher","name":"Jack"}`,
}, {
yaml: `- name: Jack
job: Butcher
- name: Jill
job: Cook
obj:
empty: false
data: |
some data 123
with new line
`,
json: `[{"job":"Butcher","name":"Jack"},{"job":"Cook","name":"Jill","obj":{"data":"some data 123\nwith new line\n","empty":false}}]`,
}}
for _, tc := range tests {
result, err := ToJSON([]byte(tc.yaml))
assert.NoError(t, err)
assert.EqualValues(t, tc.json, string(result))
}
}