mirror of
https://github.com/woodpecker-ci/woodpecker.git
synced 2024-11-27 12:21:03 +00:00
Merge pull request #1568 from bradrydzewski/master
backport branch, matrix parsing improvements
This commit is contained in:
commit
e584fb4201
13 changed files with 658 additions and 152 deletions
33
engine/expander/expand.go
Normal file
33
engine/expander/expand.go
Normal file
|
@ -0,0 +1,33 @@
|
|||
package expander
|
||||
|
||||
import "sort"
|
||||
|
||||
// Expand expands variables into the Yaml configuration using a
|
||||
// ${key} template parameter with limited support for bash string functions.
|
||||
func Expand(config []byte, envs map[string]string) []byte {
|
||||
return []byte(
|
||||
ExpandString(string(config), envs),
|
||||
)
|
||||
}
|
||||
|
||||
// ExpandString injects the variables into the Yaml configuration string using
|
||||
// a ${key} template parameter with limited support for bash string functions.
|
||||
func ExpandString(config string, envs map[string]string) string {
|
||||
if envs == nil || len(envs) == 0 {
|
||||
return config
|
||||
}
|
||||
keys := []string{}
|
||||
for k := range envs {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Sort(sort.Reverse(sort.StringSlice(keys)))
|
||||
expanded := config
|
||||
for _, k := range keys {
|
||||
v := envs[k]
|
||||
|
||||
for _, substitute := range substitutors {
|
||||
expanded = substitute(expanded, k, v)
|
||||
}
|
||||
}
|
||||
return expanded
|
||||
}
|
48
engine/expander/expand_test.go
Normal file
48
engine/expander/expand_test.go
Normal file
|
@ -0,0 +1,48 @@
|
|||
package expander
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/franela/goblin"
|
||||
)
|
||||
|
||||
func TestExpand(t *testing.T) {
|
||||
|
||||
g := goblin.Goblin(t)
|
||||
g.Describe("Expand params", func() {
|
||||
|
||||
g.It("Should replace vars with ${key}", func() {
|
||||
s := "echo ${FOO} $BAR"
|
||||
m := map[string]string{}
|
||||
m["FOO"] = "BAZ"
|
||||
g.Assert("echo BAZ $BAR").Equal(ExpandString(s, m))
|
||||
})
|
||||
|
||||
g.It("Should not replace vars in nil map", func() {
|
||||
s := "echo ${FOO} $BAR"
|
||||
g.Assert(s).Equal(ExpandString(s, nil))
|
||||
})
|
||||
|
||||
g.It("Should escape quoted variables", func() {
|
||||
s := `echo "${FOO}"`
|
||||
m := map[string]string{}
|
||||
m["FOO"] = "hello\nworld"
|
||||
g.Assert(`echo "hello\nworld"`).Equal(ExpandString(s, m))
|
||||
})
|
||||
|
||||
g.It("Should replace variable prefix", func() {
|
||||
s := `tag: ${TAG=${SHA:8}}`
|
||||
m := map[string]string{}
|
||||
m["TAG"] = ""
|
||||
m["SHA"] = "f36cbf54ee1a1eeab264c8e388f386218ab1701b"
|
||||
g.Assert("tag: f36cbf54").Equal(ExpandString(s, m))
|
||||
})
|
||||
|
||||
g.It("Should handle nested substitution operations", func() {
|
||||
s := `echo "${TAG##v}"`
|
||||
m := map[string]string{}
|
||||
m["TAG"] = "v1.0.0"
|
||||
g.Assert(`echo "1.0.0"`).Equal(ExpandString(s, m))
|
||||
})
|
||||
})
|
||||
}
|
172
engine/expander/func.go
Normal file
172
engine/expander/func.go
Normal file
|
@ -0,0 +1,172 @@
|
|||
package expander
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// these are helper functions that bring bash-substitution to the drone yaml file.
|
||||
// see http://tldp.org/LDP/abs/html/parameter-substitution.html
|
||||
|
||||
type substituteFunc func(str, key, val string) string
|
||||
|
||||
var substitutors = []substituteFunc{
|
||||
substituteQ,
|
||||
substitute,
|
||||
substitutePrefix,
|
||||
substituteSuffix,
|
||||
substituteDefault,
|
||||
substituteReplace,
|
||||
substituteLeft,
|
||||
substituteSubstr,
|
||||
}
|
||||
|
||||
// substitute is a helper function that substitutes a simple parameter using
|
||||
// ${parameter} notation.
|
||||
func substitute(str, key, val string) string {
|
||||
key = fmt.Sprintf("${%s}", key)
|
||||
return strings.Replace(str, key, val, -1)
|
||||
}
|
||||
|
||||
// substituteQ is a helper function that substitutes a simple parameter using
|
||||
// "${parameter}" notation with the escaped value, using %q.
|
||||
func substituteQ(str, key, val string) string {
|
||||
key = fmt.Sprintf(`"${%s}"`, key)
|
||||
val = fmt.Sprintf("%q", val)
|
||||
return strings.Replace(str, key, val, -1)
|
||||
}
|
||||
|
||||
// substitutePrefix is a helper function that substitutes paramters using
|
||||
// ${parameter##prefix} notation with the parameter value minus the trimmed prefix.
|
||||
func substitutePrefix(str, key, val string) string {
|
||||
key = fmt.Sprintf("\\${%s##(.+)}", key)
|
||||
reg, err := regexp.Compile(key)
|
||||
if err != nil {
|
||||
return str
|
||||
}
|
||||
for _, match := range reg.FindAllStringSubmatch(str, -1) {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
val_ := strings.TrimPrefix(val, match[1])
|
||||
str = strings.Replace(str, match[0], val_, -1)
|
||||
}
|
||||
return str
|
||||
}
|
||||
|
||||
// substituteSuffix is a helper function that substitutes paramters using
|
||||
// ${parameter%%suffix} notation with the parameter value minus the trimmed suffix.
|
||||
func substituteSuffix(str, key, val string) string {
|
||||
key = fmt.Sprintf("\\${%s%%%%(.+)}", key)
|
||||
reg, err := regexp.Compile(key)
|
||||
if err != nil {
|
||||
return str
|
||||
}
|
||||
for _, match := range reg.FindAllStringSubmatch(str, -1) {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
val_ := strings.TrimSuffix(val, match[1])
|
||||
str = strings.Replace(str, match[0], val_, -1)
|
||||
}
|
||||
return str
|
||||
}
|
||||
|
||||
// substituteDefault is a helper function that substitutes paramters using
|
||||
// ${parameter=default} notation with the parameter value. When empty the
|
||||
// default value is used.
|
||||
func substituteDefault(str, key, val string) string {
|
||||
key = fmt.Sprintf("\\${%s=(.+)}", key)
|
||||
reg, err := regexp.Compile(key)
|
||||
if err != nil {
|
||||
return str
|
||||
}
|
||||
for _, match := range reg.FindAllStringSubmatch(str, -1) {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
if len(val) == 0 {
|
||||
str = strings.Replace(str, match[0], match[1], -1)
|
||||
} else {
|
||||
str = strings.Replace(str, match[0], val, -1)
|
||||
}
|
||||
}
|
||||
return str
|
||||
}
|
||||
|
||||
// substituteReplace is a helper function that substitutes paramters using
|
||||
// ${parameter/old/new} notation with the parameter value. A find and replace
|
||||
// is performed before injecting the strings, replacing the old pattern with
|
||||
// the new value.
|
||||
func substituteReplace(str, key, val string) string {
|
||||
key = fmt.Sprintf("\\${%s/(.+)/(.+)}", key)
|
||||
reg, err := regexp.Compile(key)
|
||||
if err != nil {
|
||||
return str
|
||||
}
|
||||
for _, match := range reg.FindAllStringSubmatch(str, -1) {
|
||||
if len(match) != 3 {
|
||||
continue
|
||||
}
|
||||
with := strings.Replace(val, match[1], match[2], -1)
|
||||
str = strings.Replace(str, match[0], with, -1)
|
||||
}
|
||||
return str
|
||||
}
|
||||
|
||||
// substituteLeft is a helper function that substitutes paramters using
|
||||
// ${parameter:pos} notation with the parameter value, sliced up to the
|
||||
// specified position.
|
||||
func substituteLeft(str, key, val string) string {
|
||||
key = fmt.Sprintf("\\${%s:([0-9]*)}", key)
|
||||
reg, err := regexp.Compile(key)
|
||||
if err != nil {
|
||||
return str
|
||||
}
|
||||
for _, match := range reg.FindAllStringSubmatch(str, -1) {
|
||||
if len(match) != 2 {
|
||||
continue
|
||||
}
|
||||
index, err := strconv.Atoi(match[1])
|
||||
if err != nil {
|
||||
continue // skip
|
||||
}
|
||||
if index > len(val)-1 {
|
||||
continue // skip
|
||||
}
|
||||
|
||||
str = strings.Replace(str, match[0], val[:index], -1)
|
||||
}
|
||||
return str
|
||||
}
|
||||
|
||||
// substituteLeft is a helper function that substitutes paramters using
|
||||
// ${parameter:pos:len} notation with the parameter value as a substring,
|
||||
// starting at the specified position for the specified length.
|
||||
func substituteSubstr(str, key, val string) string {
|
||||
key = fmt.Sprintf("\\${%s:([0-9]*):([0-9]*)}", key)
|
||||
reg, err := regexp.Compile(key)
|
||||
if err != nil {
|
||||
return str
|
||||
}
|
||||
for _, match := range reg.FindAllStringSubmatch(str, -1) {
|
||||
if len(match) != 3 {
|
||||
continue
|
||||
}
|
||||
pos, err := strconv.Atoi(match[1])
|
||||
if err != nil {
|
||||
continue // skip
|
||||
}
|
||||
length, err := strconv.Atoi(match[2])
|
||||
if err != nil {
|
||||
continue // skip
|
||||
}
|
||||
if pos+length > len(val)-1 {
|
||||
continue // skip
|
||||
}
|
||||
str = strings.Replace(str, match[0], val[pos:pos+length], -1)
|
||||
}
|
||||
return str
|
||||
}
|
68
engine/expander/func_test.go
Normal file
68
engine/expander/func_test.go
Normal file
|
@ -0,0 +1,68 @@
|
|||
package expander
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/franela/goblin"
|
||||
)
|
||||
|
||||
func TestSubstitution(t *testing.T) {
|
||||
|
||||
g := goblin.Goblin(t)
|
||||
g.Describe("Parameter Substitution", func() {
|
||||
|
||||
g.It("Should substitute simple parameters", func() {
|
||||
before := "echo ${GREETING} WORLD"
|
||||
after := "echo HELLO WORLD"
|
||||
g.Assert(substitute(before, "GREETING", "HELLO")).Equal(after)
|
||||
})
|
||||
|
||||
g.It("Should substitute quoted parameters", func() {
|
||||
before := "echo \"${GREETING}\" WORLD"
|
||||
after := "echo \"HELLO\" WORLD"
|
||||
g.Assert(substituteQ(before, "GREETING", "HELLO")).Equal(after)
|
||||
})
|
||||
|
||||
g.It("Should substitute parameters and trim prefix", func() {
|
||||
before := "echo ${GREETING##asdf} WORLD"
|
||||
after := "echo HELLO WORLD"
|
||||
g.Assert(substitutePrefix(before, "GREETING", "asdfHELLO")).Equal(after)
|
||||
})
|
||||
|
||||
g.It("Should substitute parameters and trim suffix", func() {
|
||||
before := "echo ${GREETING%%asdf} WORLD"
|
||||
after := "echo HELLO WORLD"
|
||||
g.Assert(substituteSuffix(before, "GREETING", "HELLOasdf")).Equal(after)
|
||||
})
|
||||
|
||||
g.It("Should substitute parameters without using the default", func() {
|
||||
before := "echo ${GREETING=HOLA} WORLD"
|
||||
after := "echo HELLO WORLD"
|
||||
g.Assert(substituteDefault(before, "GREETING", "HELLO")).Equal(after)
|
||||
})
|
||||
|
||||
g.It("Should substitute parameters using the a default", func() {
|
||||
before := "echo ${GREETING=HOLA} WORLD"
|
||||
after := "echo HOLA WORLD"
|
||||
g.Assert(substituteDefault(before, "GREETING", "")).Equal(after)
|
||||
})
|
||||
|
||||
g.It("Should substitute parameters with replacement", func() {
|
||||
before := "echo ${GREETING/HE/A} MONDE"
|
||||
after := "echo ALLO MONDE"
|
||||
g.Assert(substituteReplace(before, "GREETING", "HELLO")).Equal(after)
|
||||
})
|
||||
|
||||
g.It("Should substitute parameters with left substr", func() {
|
||||
before := "echo ${FOO:4} IS COOL"
|
||||
after := "echo THIS IS COOL"
|
||||
g.Assert(substituteLeft(before, "FOO", "THIS IS A REALLY LONG STRING")).Equal(after)
|
||||
})
|
||||
|
||||
g.It("Should substitute parameters with substr", func() {
|
||||
before := "echo ${FOO:8:5} IS COOL"
|
||||
after := "echo DRONE IS COOL"
|
||||
g.Assert(substituteSubstr(before, "FOO", "THIS IS DRONE CI")).Equal(after)
|
||||
})
|
||||
})
|
||||
}
|
77
engine/parser/branch.go
Normal file
77
engine/parser/branch.go
Normal file
|
@ -0,0 +1,77 @@
|
|||
package parser
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
|
||||
"gopkg.in/yaml.v2"
|
||||
)
|
||||
|
||||
type Branch struct {
|
||||
Include []string `yaml:"include"`
|
||||
Exclude []string `yaml:"exclude"`
|
||||
}
|
||||
|
||||
// ParseBranch parses the branch section of the Yaml document.
|
||||
func ParseBranch(in []byte) *Branch {
|
||||
return parseBranch(in)
|
||||
}
|
||||
|
||||
// ParseBranchString parses the branch section of the Yaml document.
|
||||
func ParseBranchString(in string) *Branch {
|
||||
return ParseBranch([]byte(in))
|
||||
}
|
||||
|
||||
// Matches returns true if the branch matches the include patterns and
|
||||
// does not match any of the exclude patterns.
|
||||
func (b *Branch) Matches(branch string) bool {
|
||||
// when no includes or excludes automatically match
|
||||
if len(b.Include) == 0 && len(b.Exclude) == 0 {
|
||||
return true
|
||||
}
|
||||
|
||||
// exclusions are processed first. So we can include everything and
|
||||
// then selectively exclude certain sub-patterns.
|
||||
for _, pattern := range b.Exclude {
|
||||
if pattern == branch {
|
||||
return false
|
||||
}
|
||||
if ok, _ := filepath.Match(pattern, branch); ok {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
for _, pattern := range b.Include {
|
||||
if pattern == branch {
|
||||
return true
|
||||
}
|
||||
if ok, _ := filepath.Match(pattern, branch); ok {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
func parseBranch(in []byte) *Branch {
|
||||
out1 := struct {
|
||||
Branch struct {
|
||||
Include stringOrSlice `yaml:"include"`
|
||||
Exclude stringOrSlice `yaml:"exclude"`
|
||||
} `yaml:"branches"`
|
||||
}{}
|
||||
|
||||
out2 := struct {
|
||||
Include stringOrSlice `yaml:"branches"`
|
||||
}{}
|
||||
|
||||
yaml.Unmarshal(in, &out1)
|
||||
yaml.Unmarshal(in, &out2)
|
||||
|
||||
return &Branch{
|
||||
Exclude: out1.Branch.Exclude.Slice(),
|
||||
Include: append(
|
||||
out1.Branch.Include.Slice(),
|
||||
out2.Include.Slice()...,
|
||||
),
|
||||
}
|
||||
}
|
74
engine/parser/branch_test.go
Normal file
74
engine/parser/branch_test.go
Normal file
|
@ -0,0 +1,74 @@
|
|||
package parser
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/franela/goblin"
|
||||
)
|
||||
|
||||
func TestBranch(t *testing.T) {
|
||||
|
||||
g := goblin.Goblin(t)
|
||||
g.Describe("Branch filter", func() {
|
||||
|
||||
g.It("Should parse and match emtpy", func() {
|
||||
branch := ParseBranchString("")
|
||||
g.Assert(branch.Matches("master")).IsTrue()
|
||||
})
|
||||
|
||||
g.It("Should parse and match", func() {
|
||||
branch := ParseBranchString("branches: { include: [ master, develop ] }")
|
||||
g.Assert(branch.Matches("master")).IsTrue()
|
||||
})
|
||||
|
||||
g.It("Should parse and match shortand", func() {
|
||||
branch := ParseBranchString("branches: [ master, develop ]")
|
||||
g.Assert(branch.Matches("master")).IsTrue()
|
||||
})
|
||||
|
||||
g.It("Should parse and match shortand string", func() {
|
||||
branch := ParseBranchString("branches: master")
|
||||
g.Assert(branch.Matches("master")).IsTrue()
|
||||
})
|
||||
|
||||
g.It("Should parse and match exclude", func() {
|
||||
branch := ParseBranchString("branches: { exclude: [ master, develop ] }")
|
||||
g.Assert(branch.Matches("master")).IsFalse()
|
||||
})
|
||||
|
||||
g.It("Should parse and match exclude shorthand", func() {
|
||||
branch := ParseBranchString("branches: { exclude: master }")
|
||||
g.Assert(branch.Matches("master")).IsFalse()
|
||||
})
|
||||
|
||||
g.It("Should match include", func() {
|
||||
b := Branch{}
|
||||
b.Include = []string{"master"}
|
||||
g.Assert(b.Matches("master")).IsTrue()
|
||||
})
|
||||
|
||||
g.It("Should match include pattern", func() {
|
||||
b := Branch{}
|
||||
b.Include = []string{"feature/*"}
|
||||
g.Assert(b.Matches("feature/foo")).IsTrue()
|
||||
})
|
||||
|
||||
g.It("Should fail to match include pattern", func() {
|
||||
b := Branch{}
|
||||
b.Include = []string{"feature/*"}
|
||||
g.Assert(b.Matches("master")).IsFalse()
|
||||
})
|
||||
|
||||
g.It("Should match exclude", func() {
|
||||
b := Branch{}
|
||||
b.Exclude = []string{"master"}
|
||||
g.Assert(b.Matches("master")).IsFalse()
|
||||
})
|
||||
|
||||
g.It("Should match exclude pattern", func() {
|
||||
b := Branch{}
|
||||
b.Exclude = []string{"feature/*"}
|
||||
g.Assert(b.Matches("feature/foo")).IsFalse()
|
||||
})
|
||||
})
|
||||
}
|
100
engine/parser/matrix.go
Normal file
100
engine/parser/matrix.go
Normal file
|
@ -0,0 +1,100 @@
|
|||
package parser
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"gopkg.in/yaml.v2"
|
||||
)
|
||||
|
||||
const (
|
||||
limitTags = 10
|
||||
limitAxis = 25
|
||||
)
|
||||
|
||||
// Matrix represents the build matrix.
|
||||
type Matrix map[string][]string
|
||||
|
||||
// Axis represents a single permutation of entries from the build matrix.
|
||||
type Axis map[string]string
|
||||
|
||||
// String returns a string representation of an Axis as a comma-separated list
|
||||
// of environment variables.
|
||||
func (a Axis) String() string {
|
||||
var envs []string
|
||||
for k, v := range a {
|
||||
envs = append(envs, k+"="+v)
|
||||
}
|
||||
return strings.Join(envs, " ")
|
||||
}
|
||||
|
||||
// ParseMatrix parses the Yaml matrix definition.
|
||||
func ParseMatrix(data []byte) ([]Axis, error) {
|
||||
matrix, err := parseMatrix(data)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// if not a matrix build return an array with just the single axis.
|
||||
if len(matrix) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
return calcMatrix(matrix), nil
|
||||
}
|
||||
|
||||
// ParseMatrixString parses the Yaml string matrix definition.
|
||||
func ParseMatrixString(data string) ([]Axis, error) {
|
||||
return ParseMatrix([]byte(data))
|
||||
}
|
||||
|
||||
func calcMatrix(matrix Matrix) []Axis {
|
||||
// calculate number of permutations and extract the list of tags
|
||||
// (ie go_version, redis_version, etc)
|
||||
var perm int
|
||||
var tags []string
|
||||
for k, v := range matrix {
|
||||
perm *= len(v)
|
||||
if perm == 0 {
|
||||
perm = len(v)
|
||||
}
|
||||
tags = append(tags, k)
|
||||
}
|
||||
|
||||
// structure to hold the transformed result set
|
||||
axisList := []Axis{}
|
||||
|
||||
// for each axis calculate the uniqe set of values that should be used.
|
||||
for p := 0; p < perm; p++ {
|
||||
axis := map[string]string{}
|
||||
decr := perm
|
||||
for i, tag := range tags {
|
||||
elems := matrix[tag]
|
||||
decr = decr / len(elems)
|
||||
elem := p / decr % len(elems)
|
||||
axis[tag] = elems[elem]
|
||||
|
||||
// enforce a maximum number of tags in the build matrix.
|
||||
if i > limitTags {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
// append to the list of axis.
|
||||
axisList = append(axisList, axis)
|
||||
|
||||
// enforce a maximum number of axis that should be calculated.
|
||||
if p > limitAxis {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return axisList
|
||||
}
|
||||
|
||||
func parseMatrix(raw []byte) (Matrix, error) {
|
||||
data := struct {
|
||||
Matrix map[string][]string
|
||||
}{}
|
||||
err := yaml.Unmarshal(raw, &data)
|
||||
return data.Matrix, err
|
||||
}
|
|
@ -1,4 +1,4 @@
|
|||
package matrix
|
||||
package parser
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
@ -6,12 +6,12 @@ import (
|
|||
"github.com/franela/goblin"
|
||||
)
|
||||
|
||||
func Test_Matrix(t *testing.T) {
|
||||
func TestMatrix(t *testing.T) {
|
||||
|
||||
g := goblin.Goblin(t)
|
||||
g.Describe("Calculate matrix", func() {
|
||||
|
||||
axis, _ := Parse(fakeMatrix)
|
||||
axis, _ := ParseMatrixString(fakeMatrix)
|
||||
|
||||
g.It("Should calculate permutations", func() {
|
||||
g.Assert(len(axis)).Equal(24)
|
||||
|
@ -26,7 +26,7 @@ func Test_Matrix(t *testing.T) {
|
|||
})
|
||||
|
||||
g.It("Should return nil if no matrix", func() {
|
||||
axis, err := Parse("")
|
||||
axis, err := ParseMatrixString("")
|
||||
g.Assert(err == nil).IsTrue()
|
||||
g.Assert(axis == nil).IsTrue()
|
||||
})
|
28
engine/parser/types.go
Normal file
28
engine/parser/types.go
Normal file
|
@ -0,0 +1,28 @@
|
|||
package parser
|
||||
|
||||
// stringOrSlice represents a string or an array of strings.
|
||||
type stringOrSlice struct {
|
||||
parts []string
|
||||
}
|
||||
|
||||
func (s *stringOrSlice) UnmarshalYAML(unmarshal func(interface{}) error) error {
|
||||
var sliceType []string
|
||||
err := unmarshal(&sliceType)
|
||||
if err == nil {
|
||||
s.parts = sliceType
|
||||
return nil
|
||||
}
|
||||
|
||||
var stringType string
|
||||
err = unmarshal(&stringType)
|
||||
if err == nil {
|
||||
sliceType = make([]string, 0, 1)
|
||||
s.parts = append(sliceType, string(stringType))
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (s stringOrSlice) Slice() []string {
|
||||
return s.parts
|
||||
}
|
46
engine/parser/types_test.go
Normal file
46
engine/parser/types_test.go
Normal file
|
@ -0,0 +1,46 @@
|
|||
package parser
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/franela/goblin"
|
||||
"gopkg.in/yaml.v2"
|
||||
)
|
||||
|
||||
func TestTypes(t *testing.T) {
|
||||
g := goblin.Goblin(t)
|
||||
|
||||
g.Describe("Yaml types", func() {
|
||||
g.Describe("given a yaml file", func() {
|
||||
|
||||
g.It("should unmarshal a string", func() {
|
||||
in := []byte("foo")
|
||||
out := stringOrSlice{}
|
||||
err := yaml.Unmarshal(in, &out)
|
||||
if err != nil {
|
||||
g.Fail(err)
|
||||
}
|
||||
g.Assert(len(out.parts)).Equal(1)
|
||||
g.Assert(out.parts[0]).Equal("foo")
|
||||
})
|
||||
|
||||
g.It("should unmarshal a string slice", func() {
|
||||
in := []byte("[ foo ]")
|
||||
out := stringOrSlice{}
|
||||
err := yaml.Unmarshal(in, &out)
|
||||
if err != nil {
|
||||
g.Fail(err)
|
||||
}
|
||||
g.Assert(len(out.parts)).Equal(1)
|
||||
g.Assert(out.parts[0]).Equal("foo")
|
||||
})
|
||||
|
||||
g.It("should throw error when invalid string slice", func() {
|
||||
in := []byte("{ }") // string value should fail parse
|
||||
out := stringOrSlice{}
|
||||
err := yaml.Unmarshal(in, &out)
|
||||
g.Assert(err != nil).IsTrue("expects error")
|
||||
})
|
||||
})
|
||||
})
|
||||
}
|
31
web/hook.go
31
web/hook.go
|
@ -11,14 +11,13 @@ import (
|
|||
|
||||
log "github.com/Sirupsen/logrus"
|
||||
"github.com/drone/drone/engine"
|
||||
"github.com/drone/drone/engine/parser"
|
||||
"github.com/drone/drone/model"
|
||||
"github.com/drone/drone/remote"
|
||||
"github.com/drone/drone/router/middleware/context"
|
||||
"github.com/drone/drone/shared/httputil"
|
||||
"github.com/drone/drone/shared/token"
|
||||
"github.com/drone/drone/store"
|
||||
"github.com/drone/drone/yaml"
|
||||
"github.com/drone/drone/yaml/matrix"
|
||||
)
|
||||
|
||||
var (
|
||||
|
@ -149,41 +148,27 @@ func PostHook(c *gin.Context) {
|
|||
// NOTE we don't exit on failure. The sec file is optional
|
||||
}
|
||||
|
||||
axes, err := matrix.Parse(string(raw))
|
||||
axes, err := parser.ParseMatrix(raw)
|
||||
if err != nil {
|
||||
log.Errorf("failure to calculate matrix for %s. %s", repo.FullName, err)
|
||||
c.AbortWithError(400, err)
|
||||
c.String(500, "Failed to parse yaml file or calculate matrix. %s", err)
|
||||
return
|
||||
}
|
||||
if len(axes) == 0 {
|
||||
axes = append(axes, matrix.Axis{})
|
||||
axes = append(axes, parser.Axis{})
|
||||
}
|
||||
|
||||
netrc, err := remote_.Netrc(user, repo)
|
||||
if err != nil {
|
||||
log.Errorf("failure to generate netrc for %s. %s", repo.FullName, err)
|
||||
c.AbortWithError(500, err)
|
||||
c.String(500, "Failed to generate netrc file. %s", err)
|
||||
return
|
||||
}
|
||||
|
||||
key, _ := store.GetKey(c, repo)
|
||||
|
||||
// verify the branches can be built vs skipped
|
||||
yconfig, _ := yaml.Parse(string(raw))
|
||||
var match = false
|
||||
for _, branch := range yconfig.Branches {
|
||||
if branch == build.Branch {
|
||||
match = true
|
||||
break
|
||||
}
|
||||
match, _ = filepath.Match(branch, build.Branch)
|
||||
if match {
|
||||
break
|
||||
}
|
||||
}
|
||||
if !match && len(yconfig.Branches) != 0 {
|
||||
log.Infof("ignoring hook. yaml file excludes repo and branch %s %s", repo.FullName, build.Branch)
|
||||
c.AbortWithStatus(200)
|
||||
branches := parser.ParseBranch(raw)
|
||||
if !branches.Matches(build.Branch) {
|
||||
c.String(200, "Branch does not match restrictions defined in yaml")
|
||||
return
|
||||
}
|
||||
|
||||
|
|
|
@ -1,109 +0,0 @@
|
|||
package matrix
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"gopkg.in/yaml.v2"
|
||||
)
|
||||
|
||||
const (
|
||||
limitTags = 10
|
||||
limitAxis = 25
|
||||
)
|
||||
|
||||
// Matrix represents the build matrix.
|
||||
type Matrix map[string][]string
|
||||
|
||||
// Axis represents a single permutation of entries
|
||||
// from the build matrix.
|
||||
type Axis map[string]string
|
||||
|
||||
// String returns a string representation of an Axis as
|
||||
// a comma-separated list of environment variables.
|
||||
func (a Axis) String() string {
|
||||
var envs []string
|
||||
for k, v := range a {
|
||||
envs = append(envs, k+"="+v)
|
||||
}
|
||||
return strings.Join(envs, " ")
|
||||
}
|
||||
|
||||
// Parse parses the Matrix section of the yaml file and
|
||||
// returns a list of axis.
|
||||
func Parse(raw string) ([]Axis, error) {
|
||||
matrix, err := parseMatrix(raw)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// if not a matrix build return an array
|
||||
// with just the single axis.
|
||||
if len(matrix) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
return Calc(matrix), nil
|
||||
}
|
||||
|
||||
// Calc calculates the permutations for th build matrix.
|
||||
//
|
||||
// Note that this method will cap the number of permutations
|
||||
// to 25 to prevent an overly expensive calculation.
|
||||
func Calc(matrix Matrix) []Axis {
|
||||
// calculate number of permutations and
|
||||
// extract the list of tags
|
||||
// (ie go_version, redis_version, etc)
|
||||
var perm int
|
||||
var tags []string
|
||||
for k, v := range matrix {
|
||||
perm *= len(v)
|
||||
if perm == 0 {
|
||||
perm = len(v)
|
||||
}
|
||||
tags = append(tags, k)
|
||||
}
|
||||
|
||||
// structure to hold the transformed
|
||||
// result set
|
||||
axisList := []Axis{}
|
||||
|
||||
// for each axis calculate the uniqe
|
||||
// set of values that should be used.
|
||||
for p := 0; p < perm; p++ {
|
||||
axis := map[string]string{}
|
||||
decr := perm
|
||||
for i, tag := range tags {
|
||||
elems := matrix[tag]
|
||||
decr = decr / len(elems)
|
||||
elem := p / decr % len(elems)
|
||||
axis[tag] = elems[elem]
|
||||
|
||||
// enforce a maximum number of tags
|
||||
// in the build matrix.
|
||||
if i > limitTags {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
// append to the list of axis.
|
||||
axisList = append(axisList, axis)
|
||||
|
||||
// enforce a maximum number of axis
|
||||
// that should be calculated.
|
||||
if p > limitAxis {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return axisList
|
||||
}
|
||||
|
||||
// helper function to parse the Matrix data from
|
||||
// the raw yaml file.
|
||||
func parseMatrix(raw string) (Matrix, error) {
|
||||
data := struct {
|
||||
Matrix map[string][]string
|
||||
}{}
|
||||
err := yaml.Unmarshal([]byte(raw), &data)
|
||||
return data.Matrix, err
|
||||
}
|
16
yaml/yaml.go
16
yaml/yaml.go
|
@ -1,16 +0,0 @@
|
|||
package yaml
|
||||
|
||||
import (
|
||||
"gopkg.in/yaml.v2"
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
Debug bool `yaml:"debug"`
|
||||
Branches []string `yaml:"branches"`
|
||||
}
|
||||
|
||||
func Parse(raw string) (*Config, error) {
|
||||
c := &Config{}
|
||||
err := yaml.Unmarshal([]byte(raw), c)
|
||||
return c, err
|
||||
}
|
Loading…
Reference in a new issue