Use external lib to convert yaml to json (#1028)

this move shared/yml/* into an independent lib

go.mod

@@ -4,6 +4,7 @@ go 1.18
 
 require (
 	code.gitea.io/sdk/gitea v0.15.1-0.20220501190934-319a978c6c71
+	codeberg.org/6543/go-yaml2json v0.1.0
 	github.com/bmatcuk/doublestar/v4 v4.0.2
 	github.com/docker/cli v20.10.14+incompatible
 	github.com/docker/distribution v2.8.1+incompatible
@@ -29,7 +30,7 @@ require (
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.12.1
 	github.com/rs/zerolog v1.26.1
-	github.com/stretchr/testify v1.7.1
+	github.com/stretchr/testify v1.8.0
 	github.com/tevino/abool v1.2.0
 	github.com/urfave/cli/v2 v2.5.1
 	github.com/xanzy/go-gitlab v0.64.0
@@ -40,7 +41,7 @@ require (
 	golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f
 	google.golang.org/grpc v1.47.0
 	google.golang.org/protobuf v1.28.0
-	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
+	gopkg.in/yaml.v3 v3.0.1
 	xorm.io/builder v0.3.10
 	xorm.io/xorm v1.3.0
 )
@@ -93,7 +94,7 @@ require (
 	github.com/rogpeppe/go-internal v1.8.1 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/sirupsen/logrus v1.8.1 // indirect
-	github.com/stretchr/objx v0.3.0 // indirect
+	github.com/stretchr/objx v0.4.0 // indirect
 	github.com/syndtr/goleveldb v1.0.0 // indirect
 	github.com/ugorji/go/codec v1.2.7 // indirect
 	github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect

go.sum

@@ -32,6 +32,8 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX
 cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
 code.gitea.io/sdk/gitea v0.15.1-0.20220501190934-319a978c6c71 h1:+ZqwhfAftVOAd7AcLpfh4LBdTeJIyt60vGU39zhQPyA=
 code.gitea.io/sdk/gitea v0.15.1-0.20220501190934-319a978c6c71/go.mod h1:MuMGvUxT8BmFHa0gHhHsrnz91QfmziXuFffm9AuhMCo=
+codeberg.org/6543/go-yaml2json v0.1.0 h1:njuf3a8QgsmBXJFiH+7wNR01biBS4MU+XeWE7W3bnus=
+codeberg.org/6543/go-yaml2json v0.1.0/go.mod h1:mz61q14LWF4ZABrgMEDMmk3t9dPi6zgR1uBh2VKV2RQ=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 gitea.com/xorm/sqlfiddle v0.0.0-20180821085327-62ce714f951a h1:lSA0F4e9A2NcQSqGqTOXqu2aRi/XEQxDCBwM8yJtE6s=
 gitea.com/xorm/sqlfiddle v0.0.0-20180821085327-62ce714f951a/go.mod h1:EXuID2Zs0pAQhH8yz+DNjUbjppKQzKFAn28TMYPB6IU=
@@ -602,16 +604,17 @@ github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5J
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
-github.com/stretchr/objx v0.3.0 h1:NGXK3lHquSN08v5vWalVI/L8XU9hdzE/G6xsrze47As=
+github.com/stretchr/objx v0.4.0 h1:M2gUjqZET1qApGOWNSnZ49BAIMX4F/1plDv3+l31EJ4=
-github.com/stretchr/objx v0.3.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE=
 github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
 github.com/tevino/abool v1.2.0 h1:heAkClL8H6w+mK5md9dzsuohKeXHUpY7Vw0ZCKW+huA=
@@ -1060,8 +1063,9 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=
 gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0=
 honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

pipeline/frontend/yaml/compiler/params.go

@@ -6,9 +6,8 @@ import (
 	"strconv"
 	"strings"
 
+	"codeberg.org/6543/go-yaml2json"
 	"gopkg.in/yaml.v3"
-
-	"github.com/woodpecker-ci/woodpecker/shared/yml"
 )
 
 // paramsToEnv uses reflection to convert a map[string]interface to a list
@@ -77,7 +76,7 @@ func sanitizeParamValue(v interface{}, secrets map[string]Secret) (string, error
 			}
 		}
 		ymlOut, _ := yaml.Marshal(vv.Interface())
-		out, _ := yml.ToJSON(ymlOut)
+		out, _ := yaml2json.Covnert(ymlOut)
 		return string(out), nil
 
 	case reflect.Slice, reflect.Array:
@@ -103,7 +102,7 @@ func sanitizeParamValue(v interface{}, secrets map[string]Secret) (string, error
 		if err != nil {
 			return "", err
 		}
-		out, err = yml.ToJSON(out)
+		out, err = yaml2json.Covnert(out)
 		if err != nil {
 			return "", err
 		}

pipeline/schema/schema.go

@@ -1,13 +1,13 @@
 package schema
 
 import (
+	"bytes"
 	_ "embed"
 	"fmt"
 	"io"
 
+	"codeberg.org/6543/go-yaml2json"
 	"github.com/xeipuuv/gojsonschema"
-
-	"github.com/woodpecker-ci/woodpecker/shared/yml"
 )
 
 //go:embed schema.json
@@ -16,12 +16,13 @@ var schemaDefinition []byte
 // Lint lints an io.Reader against the Woodpecker schema.json
 func Lint(r io.Reader) ([]gojsonschema.ResultError, error) {
 	schemaLoader := gojsonschema.NewBytesLoader(schemaDefinition)
-	j, err := yml.LoadYmlReaderAsJSON(r)
+	buff := new(bytes.Buffer)
+	err := yaml2json.StreamConverter(r, buff)
 	if err != nil {
 		return nil, fmt.Errorf("Failed to load yml file %w", err)
 	}
 
-	documentLoader := gojsonschema.NewBytesLoader(j)
+	documentLoader := gojsonschema.NewBytesLoader(buff.Bytes())
 	result, err := gojsonschema.Validate(schemaLoader, documentLoader)
 	if err != nil {
 		return nil, fmt.Errorf("Validation failed %w", err)

shared/yml/yml.go

@@ -1,103 +0,0 @@
-package yml
-
-import (
-	"encoding/json"
-	"fmt"
-	"io"
-	"strconv"
-
-	"gopkg.in/yaml.v3"
-)
-
-// toJSON convert gopkg.in/yaml.v3 nodes to object that can be serialized as json
-// fmt.Sprint() with default formatting is used to convert the key to a string key.
-func toJSON(node *yaml.Node) (interface{}, error) {
-	switch node.Kind {
-	case yaml.DocumentNode:
-		return toJSON(node.Content[0])
-
-	case yaml.SequenceNode:
-		val := make([]interface{}, len(node.Content))
-		var err error
-		for i := range node.Content {
-			if val[i], err = toJSON(node.Content[i]); err != nil {
-				return nil, err
-			}
-		}
-		return val, nil
-
-	case yaml.MappingNode:
-		if (len(node.Content) % 2) != 0 {
-			return nil, fmt.Errorf("broken mapping node")
-		}
-		val := make(map[string]interface{}, len(node.Content)%2)
-		for i := len(node.Content); i > 1; i = i - 2 {
-			k, err := toJSON(node.Content[i-2])
-			if err != nil {
-				return nil, err
-			}
-			if val[fmt.Sprint(k)], err = toJSON(node.Content[i-1]); err != nil {
-				return nil, err
-			}
-		}
-		return val, nil
-
-	case yaml.ScalarNode:
-		switch node.Tag {
-		case nullTag:
-			return nil, nil
-		case boolTag:
-			return strconv.ParseBool(node.Value)
-		case intTag:
-			return strconv.ParseInt(node.Value, 10, 64)
-		case floatTag:
-			return strconv.ParseFloat(node.Value, 64)
-		}
-		return node.Value, nil
-	}
-
-	return nil, fmt.Errorf("do not support yaml node kind '%v'", node.Kind)
-}
-
-// ToJSON converts YAML bytes to JSON
-func ToJSON(data []byte) ([]byte, error) {
-	m := &yaml.Node{}
-	if err := yaml.Unmarshal(data, m); err != nil {
-		return nil, err
-	}
-
-	d, err := toJSON(m)
-	if err != nil {
-		return nil, err
-	}
-	return json.Marshal(d)
-}
-
-// LoadYmlReaderAsJSON reads from an io.Reader containing YAML and converts it to JSON
-func LoadYmlReaderAsJSON(r io.Reader) (j []byte, err error) {
-	data, err := io.ReadAll(r)
-	if err != nil {
-		return nil, err
-	}
-
-	j, err = ToJSON(data)
-	if err != nil {
-		return nil, err
-	}
-
-	return j, nil
-}
-
-// Source: https://github.com/go-yaml/yaml/blob/3e3283e801afc229479d5fc68aa41df1137b8394/resolve.go#L70-L81
-const (
-	nullTag  = "!!null"
-	boolTag  = "!!bool"
-	intTag   = "!!int"
-	floatTag = "!!float"
-	// strTag       = "!!str"       // we dont have to parse it
-	// timestampTag = "!!timestamp" // TODO: do we have to parse this?
-	// seqTag       = "!!seq"       // TODO: do we have to parse this?
-	// mapTag       = "!!map"       // TODO: do we have to parse this?
-	// binaryTag    = "!!binary"    // TODO: do we have to parse this?
-	// mergeTag     = "!!merge"     // TODO: do we have to parse this?
-)

shared/yml/yml_test.go

@@ -1,45 +0,0 @@
-package yml
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestToJSON(t *testing.T) {
-	tests := []struct {
-		yaml string
-		json string
-	}{{
-		yaml: `- name: Jack
-- name: Jill
-`,
-		json: `[{"name":"Jack"},{"name":"Jill"}]`,
-	}, {
-		yaml: `name: Jack`,
-		json: `{"name":"Jack"}`,
-	}, {
-		yaml: `name: Jack
-job: Butcher
-`,
-		json: `{"job":"Butcher","name":"Jack"}`,
-	}, {
-		yaml: `- name: Jack
-  job: Butcher
-- name: Jill
-  job: Cook
-  obj:
-    empty: false
-    data: |
-      some data 123
-      with new line
-`,
-		json: `[{"job":"Butcher","name":"Jack"},{"job":"Cook","name":"Jill","obj":{"data":"some data 123\nwith new line\n","empty":false}}]`,
-	}}
-
-	for _, tc := range tests {
-		result, err := ToJSON([]byte(tc.yaml))
-		assert.NoError(t, err)
-		assert.EqualValues(t, tc.json, string(result))
-	}
-}