Merge pull request #1568 from bradrydzewski/master

backport branch, matrix parsing improvements
This commit is contained in:
Brad Rydzewski 2016-04-11 11:37:20 -07:00
commit e584fb4201
13 changed files with 658 additions and 152 deletions

33
engine/expander/expand.go Normal file
View file

@ -0,0 +1,33 @@
package expander
import "sort"
// Expand expands variables into the Yaml configuration using a
// ${key} template parameter with limited support for bash string functions.
func Expand(config []byte, envs map[string]string) []byte {
return []byte(
ExpandString(string(config), envs),
)
}
// ExpandString injects the variables into the Yaml configuration string using
// a ${key} template parameter with limited support for bash string functions.
func ExpandString(config string, envs map[string]string) string {
if envs == nil || len(envs) == 0 {
return config
}
keys := []string{}
for k := range envs {
keys = append(keys, k)
}
sort.Sort(sort.Reverse(sort.StringSlice(keys)))
expanded := config
for _, k := range keys {
v := envs[k]
for _, substitute := range substitutors {
expanded = substitute(expanded, k, v)
}
}
return expanded
}

View file

@ -0,0 +1,48 @@
package expander
import (
"testing"
"github.com/franela/goblin"
)
func TestExpand(t *testing.T) {
g := goblin.Goblin(t)
g.Describe("Expand params", func() {
g.It("Should replace vars with ${key}", func() {
s := "echo ${FOO} $BAR"
m := map[string]string{}
m["FOO"] = "BAZ"
g.Assert("echo BAZ $BAR").Equal(ExpandString(s, m))
})
g.It("Should not replace vars in nil map", func() {
s := "echo ${FOO} $BAR"
g.Assert(s).Equal(ExpandString(s, nil))
})
g.It("Should escape quoted variables", func() {
s := `echo "${FOO}"`
m := map[string]string{}
m["FOO"] = "hello\nworld"
g.Assert(`echo "hello\nworld"`).Equal(ExpandString(s, m))
})
g.It("Should replace variable prefix", func() {
s := `tag: ${TAG=${SHA:8}}`
m := map[string]string{}
m["TAG"] = ""
m["SHA"] = "f36cbf54ee1a1eeab264c8e388f386218ab1701b"
g.Assert("tag: f36cbf54").Equal(ExpandString(s, m))
})
g.It("Should handle nested substitution operations", func() {
s := `echo "${TAG##v}"`
m := map[string]string{}
m["TAG"] = "v1.0.0"
g.Assert(`echo "1.0.0"`).Equal(ExpandString(s, m))
})
})
}

172
engine/expander/func.go Normal file
View file

@ -0,0 +1,172 @@
package expander
import (
"fmt"
"regexp"
"strconv"
"strings"
)
// these are helper functions that bring bash-substitution to the drone yaml file.
// see http://tldp.org/LDP/abs/html/parameter-substitution.html
type substituteFunc func(str, key, val string) string
var substitutors = []substituteFunc{
substituteQ,
substitute,
substitutePrefix,
substituteSuffix,
substituteDefault,
substituteReplace,
substituteLeft,
substituteSubstr,
}
// substitute is a helper function that substitutes a simple parameter using
// ${parameter} notation.
func substitute(str, key, val string) string {
key = fmt.Sprintf("${%s}", key)
return strings.Replace(str, key, val, -1)
}
// substituteQ is a helper function that substitutes a simple parameter using
// "${parameter}" notation with the escaped value, using %q.
func substituteQ(str, key, val string) string {
key = fmt.Sprintf(`"${%s}"`, key)
val = fmt.Sprintf("%q", val)
return strings.Replace(str, key, val, -1)
}
// substitutePrefix is a helper function that substitutes paramters using
// ${parameter##prefix} notation with the parameter value minus the trimmed prefix.
func substitutePrefix(str, key, val string) string {
key = fmt.Sprintf("\\${%s##(.+)}", key)
reg, err := regexp.Compile(key)
if err != nil {
return str
}
for _, match := range reg.FindAllStringSubmatch(str, -1) {
if len(match) != 2 {
continue
}
val_ := strings.TrimPrefix(val, match[1])
str = strings.Replace(str, match[0], val_, -1)
}
return str
}
// substituteSuffix is a helper function that substitutes paramters using
// ${parameter%%suffix} notation with the parameter value minus the trimmed suffix.
func substituteSuffix(str, key, val string) string {
key = fmt.Sprintf("\\${%s%%%%(.+)}", key)
reg, err := regexp.Compile(key)
if err != nil {
return str
}
for _, match := range reg.FindAllStringSubmatch(str, -1) {
if len(match) != 2 {
continue
}
val_ := strings.TrimSuffix(val, match[1])
str = strings.Replace(str, match[0], val_, -1)
}
return str
}
// substituteDefault is a helper function that substitutes paramters using
// ${parameter=default} notation with the parameter value. When empty the
// default value is used.
func substituteDefault(str, key, val string) string {
key = fmt.Sprintf("\\${%s=(.+)}", key)
reg, err := regexp.Compile(key)
if err != nil {
return str
}
for _, match := range reg.FindAllStringSubmatch(str, -1) {
if len(match) != 2 {
continue
}
if len(val) == 0 {
str = strings.Replace(str, match[0], match[1], -1)
} else {
str = strings.Replace(str, match[0], val, -1)
}
}
return str
}
// substituteReplace is a helper function that substitutes paramters using
// ${parameter/old/new} notation with the parameter value. A find and replace
// is performed before injecting the strings, replacing the old pattern with
// the new value.
func substituteReplace(str, key, val string) string {
key = fmt.Sprintf("\\${%s/(.+)/(.+)}", key)
reg, err := regexp.Compile(key)
if err != nil {
return str
}
for _, match := range reg.FindAllStringSubmatch(str, -1) {
if len(match) != 3 {
continue
}
with := strings.Replace(val, match[1], match[2], -1)
str = strings.Replace(str, match[0], with, -1)
}
return str
}
// substituteLeft is a helper function that substitutes paramters using
// ${parameter:pos} notation with the parameter value, sliced up to the
// specified position.
func substituteLeft(str, key, val string) string {
key = fmt.Sprintf("\\${%s:([0-9]*)}", key)
reg, err := regexp.Compile(key)
if err != nil {
return str
}
for _, match := range reg.FindAllStringSubmatch(str, -1) {
if len(match) != 2 {
continue
}
index, err := strconv.Atoi(match[1])
if err != nil {
continue // skip
}
if index > len(val)-1 {
continue // skip
}
str = strings.Replace(str, match[0], val[:index], -1)
}
return str
}
// substituteLeft is a helper function that substitutes paramters using
// ${parameter:pos:len} notation with the parameter value as a substring,
// starting at the specified position for the specified length.
func substituteSubstr(str, key, val string) string {
key = fmt.Sprintf("\\${%s:([0-9]*):([0-9]*)}", key)
reg, err := regexp.Compile(key)
if err != nil {
return str
}
for _, match := range reg.FindAllStringSubmatch(str, -1) {
if len(match) != 3 {
continue
}
pos, err := strconv.Atoi(match[1])
if err != nil {
continue // skip
}
length, err := strconv.Atoi(match[2])
if err != nil {
continue // skip
}
if pos+length > len(val)-1 {
continue // skip
}
str = strings.Replace(str, match[0], val[pos:pos+length], -1)
}
return str
}

View file

@ -0,0 +1,68 @@
package expander
import (
"testing"
"github.com/franela/goblin"
)
func TestSubstitution(t *testing.T) {
g := goblin.Goblin(t)
g.Describe("Parameter Substitution", func() {
g.It("Should substitute simple parameters", func() {
before := "echo ${GREETING} WORLD"
after := "echo HELLO WORLD"
g.Assert(substitute(before, "GREETING", "HELLO")).Equal(after)
})
g.It("Should substitute quoted parameters", func() {
before := "echo \"${GREETING}\" WORLD"
after := "echo \"HELLO\" WORLD"
g.Assert(substituteQ(before, "GREETING", "HELLO")).Equal(after)
})
g.It("Should substitute parameters and trim prefix", func() {
before := "echo ${GREETING##asdf} WORLD"
after := "echo HELLO WORLD"
g.Assert(substitutePrefix(before, "GREETING", "asdfHELLO")).Equal(after)
})
g.It("Should substitute parameters and trim suffix", func() {
before := "echo ${GREETING%%asdf} WORLD"
after := "echo HELLO WORLD"
g.Assert(substituteSuffix(before, "GREETING", "HELLOasdf")).Equal(after)
})
g.It("Should substitute parameters without using the default", func() {
before := "echo ${GREETING=HOLA} WORLD"
after := "echo HELLO WORLD"
g.Assert(substituteDefault(before, "GREETING", "HELLO")).Equal(after)
})
g.It("Should substitute parameters using the a default", func() {
before := "echo ${GREETING=HOLA} WORLD"
after := "echo HOLA WORLD"
g.Assert(substituteDefault(before, "GREETING", "")).Equal(after)
})
g.It("Should substitute parameters with replacement", func() {
before := "echo ${GREETING/HE/A} MONDE"
after := "echo ALLO MONDE"
g.Assert(substituteReplace(before, "GREETING", "HELLO")).Equal(after)
})
g.It("Should substitute parameters with left substr", func() {
before := "echo ${FOO:4} IS COOL"
after := "echo THIS IS COOL"
g.Assert(substituteLeft(before, "FOO", "THIS IS A REALLY LONG STRING")).Equal(after)
})
g.It("Should substitute parameters with substr", func() {
before := "echo ${FOO:8:5} IS COOL"
after := "echo DRONE IS COOL"
g.Assert(substituteSubstr(before, "FOO", "THIS IS DRONE CI")).Equal(after)
})
})
}

77
engine/parser/branch.go Normal file
View file

@ -0,0 +1,77 @@
package parser
import (
"path/filepath"
"gopkg.in/yaml.v2"
)
type Branch struct {
Include []string `yaml:"include"`
Exclude []string `yaml:"exclude"`
}
// ParseBranch parses the branch section of the Yaml document.
func ParseBranch(in []byte) *Branch {
return parseBranch(in)
}
// ParseBranchString parses the branch section of the Yaml document.
func ParseBranchString(in string) *Branch {
return ParseBranch([]byte(in))
}
// Matches returns true if the branch matches the include patterns and
// does not match any of the exclude patterns.
func (b *Branch) Matches(branch string) bool {
// when no includes or excludes automatically match
if len(b.Include) == 0 && len(b.Exclude) == 0 {
return true
}
// exclusions are processed first. So we can include everything and
// then selectively exclude certain sub-patterns.
for _, pattern := range b.Exclude {
if pattern == branch {
return false
}
if ok, _ := filepath.Match(pattern, branch); ok {
return false
}
}
for _, pattern := range b.Include {
if pattern == branch {
return true
}
if ok, _ := filepath.Match(pattern, branch); ok {
return true
}
}
return false
}
func parseBranch(in []byte) *Branch {
out1 := struct {
Branch struct {
Include stringOrSlice `yaml:"include"`
Exclude stringOrSlice `yaml:"exclude"`
} `yaml:"branches"`
}{}
out2 := struct {
Include stringOrSlice `yaml:"branches"`
}{}
yaml.Unmarshal(in, &out1)
yaml.Unmarshal(in, &out2)
return &Branch{
Exclude: out1.Branch.Exclude.Slice(),
Include: append(
out1.Branch.Include.Slice(),
out2.Include.Slice()...,
),
}
}

View file

@ -0,0 +1,74 @@
package parser
import (
"testing"
"github.com/franela/goblin"
)
func TestBranch(t *testing.T) {
g := goblin.Goblin(t)
g.Describe("Branch filter", func() {
g.It("Should parse and match emtpy", func() {
branch := ParseBranchString("")
g.Assert(branch.Matches("master")).IsTrue()
})
g.It("Should parse and match", func() {
branch := ParseBranchString("branches: { include: [ master, develop ] }")
g.Assert(branch.Matches("master")).IsTrue()
})
g.It("Should parse and match shortand", func() {
branch := ParseBranchString("branches: [ master, develop ]")
g.Assert(branch.Matches("master")).IsTrue()
})
g.It("Should parse and match shortand string", func() {
branch := ParseBranchString("branches: master")
g.Assert(branch.Matches("master")).IsTrue()
})
g.It("Should parse and match exclude", func() {
branch := ParseBranchString("branches: { exclude: [ master, develop ] }")
g.Assert(branch.Matches("master")).IsFalse()
})
g.It("Should parse and match exclude shorthand", func() {
branch := ParseBranchString("branches: { exclude: master }")
g.Assert(branch.Matches("master")).IsFalse()
})
g.It("Should match include", func() {
b := Branch{}
b.Include = []string{"master"}
g.Assert(b.Matches("master")).IsTrue()
})
g.It("Should match include pattern", func() {
b := Branch{}
b.Include = []string{"feature/*"}
g.Assert(b.Matches("feature/foo")).IsTrue()
})
g.It("Should fail to match include pattern", func() {
b := Branch{}
b.Include = []string{"feature/*"}
g.Assert(b.Matches("master")).IsFalse()
})
g.It("Should match exclude", func() {
b := Branch{}
b.Exclude = []string{"master"}
g.Assert(b.Matches("master")).IsFalse()
})
g.It("Should match exclude pattern", func() {
b := Branch{}
b.Exclude = []string{"feature/*"}
g.Assert(b.Matches("feature/foo")).IsFalse()
})
})
}

100
engine/parser/matrix.go Normal file
View file

@ -0,0 +1,100 @@
package parser
import (
"strings"
"gopkg.in/yaml.v2"
)
const (
limitTags = 10
limitAxis = 25
)
// Matrix represents the build matrix.
type Matrix map[string][]string
// Axis represents a single permutation of entries from the build matrix.
type Axis map[string]string
// String returns a string representation of an Axis as a comma-separated list
// of environment variables.
func (a Axis) String() string {
var envs []string
for k, v := range a {
envs = append(envs, k+"="+v)
}
return strings.Join(envs, " ")
}
// ParseMatrix parses the Yaml matrix definition.
func ParseMatrix(data []byte) ([]Axis, error) {
matrix, err := parseMatrix(data)
if err != nil {
return nil, err
}
// if not a matrix build return an array with just the single axis.
if len(matrix) == 0 {
return nil, nil
}
return calcMatrix(matrix), nil
}
// ParseMatrixString parses the Yaml string matrix definition.
func ParseMatrixString(data string) ([]Axis, error) {
return ParseMatrix([]byte(data))
}
func calcMatrix(matrix Matrix) []Axis {
// calculate number of permutations and extract the list of tags
// (ie go_version, redis_version, etc)
var perm int
var tags []string
for k, v := range matrix {
perm *= len(v)
if perm == 0 {
perm = len(v)
}
tags = append(tags, k)
}
// structure to hold the transformed result set
axisList := []Axis{}
// for each axis calculate the uniqe set of values that should be used.
for p := 0; p < perm; p++ {
axis := map[string]string{}
decr := perm
for i, tag := range tags {
elems := matrix[tag]
decr = decr / len(elems)
elem := p / decr % len(elems)
axis[tag] = elems[elem]
// enforce a maximum number of tags in the build matrix.
if i > limitTags {
break
}
}
// append to the list of axis.
axisList = append(axisList, axis)
// enforce a maximum number of axis that should be calculated.
if p > limitAxis {
break
}
}
return axisList
}
func parseMatrix(raw []byte) (Matrix, error) {
data := struct {
Matrix map[string][]string
}{}
err := yaml.Unmarshal(raw, &data)
return data.Matrix, err
}

View file

@ -1,4 +1,4 @@
package matrix
package parser
import (
"testing"
@ -6,12 +6,12 @@ import (
"github.com/franela/goblin"
)
func Test_Matrix(t *testing.T) {
func TestMatrix(t *testing.T) {
g := goblin.Goblin(t)
g.Describe("Calculate matrix", func() {
axis, _ := Parse(fakeMatrix)
axis, _ := ParseMatrixString(fakeMatrix)
g.It("Should calculate permutations", func() {
g.Assert(len(axis)).Equal(24)
@ -26,7 +26,7 @@ func Test_Matrix(t *testing.T) {
})
g.It("Should return nil if no matrix", func() {
axis, err := Parse("")
axis, err := ParseMatrixString("")
g.Assert(err == nil).IsTrue()
g.Assert(axis == nil).IsTrue()
})

28
engine/parser/types.go Normal file
View file

@ -0,0 +1,28 @@
package parser
// stringOrSlice represents a string or an array of strings.
type stringOrSlice struct {
parts []string
}
func (s *stringOrSlice) UnmarshalYAML(unmarshal func(interface{}) error) error {
var sliceType []string
err := unmarshal(&sliceType)
if err == nil {
s.parts = sliceType
return nil
}
var stringType string
err = unmarshal(&stringType)
if err == nil {
sliceType = make([]string, 0, 1)
s.parts = append(sliceType, string(stringType))
return nil
}
return err
}
func (s stringOrSlice) Slice() []string {
return s.parts
}

View file

@ -0,0 +1,46 @@
package parser
import (
"testing"
"github.com/franela/goblin"
"gopkg.in/yaml.v2"
)
func TestTypes(t *testing.T) {
g := goblin.Goblin(t)
g.Describe("Yaml types", func() {
g.Describe("given a yaml file", func() {
g.It("should unmarshal a string", func() {
in := []byte("foo")
out := stringOrSlice{}
err := yaml.Unmarshal(in, &out)
if err != nil {
g.Fail(err)
}
g.Assert(len(out.parts)).Equal(1)
g.Assert(out.parts[0]).Equal("foo")
})
g.It("should unmarshal a string slice", func() {
in := []byte("[ foo ]")
out := stringOrSlice{}
err := yaml.Unmarshal(in, &out)
if err != nil {
g.Fail(err)
}
g.Assert(len(out.parts)).Equal(1)
g.Assert(out.parts[0]).Equal("foo")
})
g.It("should throw error when invalid string slice", func() {
in := []byte("{ }") // string value should fail parse
out := stringOrSlice{}
err := yaml.Unmarshal(in, &out)
g.Assert(err != nil).IsTrue("expects error")
})
})
})
}

View file

@ -11,14 +11,13 @@ import (
log "github.com/Sirupsen/logrus"
"github.com/drone/drone/engine"
"github.com/drone/drone/engine/parser"
"github.com/drone/drone/model"
"github.com/drone/drone/remote"
"github.com/drone/drone/router/middleware/context"
"github.com/drone/drone/shared/httputil"
"github.com/drone/drone/shared/token"
"github.com/drone/drone/store"
"github.com/drone/drone/yaml"
"github.com/drone/drone/yaml/matrix"
)
var (
@ -149,41 +148,27 @@ func PostHook(c *gin.Context) {
// NOTE we don't exit on failure. The sec file is optional
}
axes, err := matrix.Parse(string(raw))
axes, err := parser.ParseMatrix(raw)
if err != nil {
log.Errorf("failure to calculate matrix for %s. %s", repo.FullName, err)
c.AbortWithError(400, err)
c.String(500, "Failed to parse yaml file or calculate matrix. %s", err)
return
}
if len(axes) == 0 {
axes = append(axes, matrix.Axis{})
axes = append(axes, parser.Axis{})
}
netrc, err := remote_.Netrc(user, repo)
if err != nil {
log.Errorf("failure to generate netrc for %s. %s", repo.FullName, err)
c.AbortWithError(500, err)
c.String(500, "Failed to generate netrc file. %s", err)
return
}
key, _ := store.GetKey(c, repo)
// verify the branches can be built vs skipped
yconfig, _ := yaml.Parse(string(raw))
var match = false
for _, branch := range yconfig.Branches {
if branch == build.Branch {
match = true
break
}
match, _ = filepath.Match(branch, build.Branch)
if match {
break
}
}
if !match && len(yconfig.Branches) != 0 {
log.Infof("ignoring hook. yaml file excludes repo and branch %s %s", repo.FullName, build.Branch)
c.AbortWithStatus(200)
branches := parser.ParseBranch(raw)
if !branches.Matches(build.Branch) {
c.String(200, "Branch does not match restrictions defined in yaml")
return
}

View file

@ -1,109 +0,0 @@
package matrix
import (
"strings"
"gopkg.in/yaml.v2"
)
const (
limitTags = 10
limitAxis = 25
)
// Matrix represents the build matrix.
type Matrix map[string][]string
// Axis represents a single permutation of entries
// from the build matrix.
type Axis map[string]string
// String returns a string representation of an Axis as
// a comma-separated list of environment variables.
func (a Axis) String() string {
var envs []string
for k, v := range a {
envs = append(envs, k+"="+v)
}
return strings.Join(envs, " ")
}
// Parse parses the Matrix section of the yaml file and
// returns a list of axis.
func Parse(raw string) ([]Axis, error) {
matrix, err := parseMatrix(raw)
if err != nil {
return nil, err
}
// if not a matrix build return an array
// with just the single axis.
if len(matrix) == 0 {
return nil, nil
}
return Calc(matrix), nil
}
// Calc calculates the permutations for th build matrix.
//
// Note that this method will cap the number of permutations
// to 25 to prevent an overly expensive calculation.
func Calc(matrix Matrix) []Axis {
// calculate number of permutations and
// extract the list of tags
// (ie go_version, redis_version, etc)
var perm int
var tags []string
for k, v := range matrix {
perm *= len(v)
if perm == 0 {
perm = len(v)
}
tags = append(tags, k)
}
// structure to hold the transformed
// result set
axisList := []Axis{}
// for each axis calculate the uniqe
// set of values that should be used.
for p := 0; p < perm; p++ {
axis := map[string]string{}
decr := perm
for i, tag := range tags {
elems := matrix[tag]
decr = decr / len(elems)
elem := p / decr % len(elems)
axis[tag] = elems[elem]
// enforce a maximum number of tags
// in the build matrix.
if i > limitTags {
break
}
}
// append to the list of axis.
axisList = append(axisList, axis)
// enforce a maximum number of axis
// that should be calculated.
if p > limitAxis {
break
}
}
return axisList
}
// helper function to parse the Matrix data from
// the raw yaml file.
func parseMatrix(raw string) (Matrix, error) {
data := struct {
Matrix map[string][]string
}{}
err := yaml.Unmarshal([]byte(raw), &data)
return data.Matrix, err
}

View file

@ -1,16 +0,0 @@
package yaml
import (
"gopkg.in/yaml.v2"
)
type Config struct {
Debug bool `yaml:"debug"`
Branches []string `yaml:"branches"`
}
func Parse(raw string) (*Config, error) {
c := &Config{}
err := yaml.Unmarshal([]byte(raw), c)
return c, err
}