move to envsubst package

This commit is contained in:
Brad Rydzewski 2017-01-18 22:03:38 +07:00
parent 24ea9db99a
commit a03e962c2a
12 changed files with 442 additions and 340 deletions

View file

@ -13,8 +13,8 @@ import (
"github.com/drone/drone/model"
"github.com/drone/drone/version"
"github.com/drone/drone/yaml"
"github.com/drone/drone/yaml/expander"
"github.com/drone/drone/yaml/transform"
"github.com/drone/envsubst"
)
type Logger interface {
@ -93,7 +93,14 @@ func (a *Agent) Run(payload *model.Work, cancel <-chan bool) error {
func (a *Agent) prep(w *model.Work) (*yaml.Config, error) {
envs := toEnv(w)
w.Yaml = expander.ExpandString(w.Yaml, envs)
var err error
w.Yaml, err = envsubst.Eval(w.Yaml, func(s string) string {
return envs[s]
})
if err != nil {
return nil, err
}
// append secrets when verified or when a secret does not require
// verification

21
vendor/github.com/drone/envsubst/LICENSE generated vendored Normal file
View file

@ -0,0 +1,21 @@
MIT License
Copyright (c) 2017 drone.io
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

33
vendor/github.com/drone/envsubst/README generated vendored Normal file
View file

@ -0,0 +1,33 @@
Go package emulates bash environment variable substitution in a string using ${var} syntax. Includes support for bash string replacement functions.
Documentation:
http://godoc.org/github.com/drone/env
Supported Functions:
${var^}
${var^^}
${var,}
${var,,}
${var:position}
${var:position:length}
${var#substring}
${var##substring}
${var%substring}
${var%%substring}
${var/substring/replacement}
${var//substring/replacement}
${var/#substring/replacement}
${var/%substring/replacement}
${#var}
${var=default}
${var:=default}
${var:-default}
Unsupported Functions:
${var-default}
${var+default}
${var:?default}
${var:+default}

19
vendor/github.com/drone/envsubst/eval.go generated vendored Normal file
View file

@ -0,0 +1,19 @@
package envsubst
import "os"
// Eval replaces ${var} in the string based on the mapping function.
func Eval(s string, mapping func(string) string) (string, error) {
t, err := Parse(s)
if err != nil {
return s, err
}
return t.Execute(mapping)
}
// EvalEnv replaces ${var} in the string according to the values of the
// current environment variables. References to undefined variables are
// replaced by the empty string.
func EvalEnv(s string) (string, error) {
return Eval(s, os.Getenv)
}

177
vendor/github.com/drone/envsubst/funcs.go generated vendored Normal file
View file

@ -0,0 +1,177 @@
package envsubst
import (
"strconv"
"strings"
"unicode"
"unicode/utf8"
)
// defines a parameter substitution function.
type substituteFunc func(string, ...string) string
// toLen returns the length of string s.
func toLen(s string, args ...string) string {
return strconv.Itoa(len(s))
}
// toLower returns a copy of the string s with all characters
// mapped to their lower case.
func toLower(s string, args ...string) string {
return strings.ToLower(s)
}
// toUpper returns a copy of the string s with all characters
// mapped to their upper case.
func toUpper(s string, args ...string) string {
return strings.ToUpper(s)
}
// toLowerFirst returns a copy of the string s with the first
// character mapped to its lower case.
func toLowerFirst(s string, args ...string) string {
if s == "" {
return s
}
r, n := utf8.DecodeRuneInString(s)
return string(unicode.ToLower(r)) + s[n:]
}
// toUpperFirst returns a copy of the string s with the first
// character mapped to its upper case.
func toUpperFirst(s string, args ...string) string {
if s == "" {
return s
}
r, n := utf8.DecodeRuneInString(s)
return string(unicode.ToUpper(r)) + s[n:]
}
// toDefault returns a copy of the string s if not empty, else
// returns a copy of the first string arugment.
func toDefault(s string, args ...string) string {
if len(s) == 0 && len(args) == 1 {
s = args[0]
}
return s
}
// toSubstr returns a slice of the string s at the specified
// length and position.
func toSubstr(s string, args ...string) string {
if len(args) == 0 {
return s // should never happen
}
pos, err := strconv.Atoi(args[0])
if err != nil {
// bash returns the string if the position
// cannot be parsed.
return s
}
if len(args) == 1 {
if pos < len(s) {
return s[pos:]
}
// if the position exceeds the length of the
// string an empty string is returned
return ""
}
length, err := strconv.Atoi(args[1])
if err != nil {
// bash returns the string if the length
// cannot be parsed.
return s
}
if pos+length >= len(s) {
// if the position exceeds the length of the
// string an empty string is returned
return ""
}
return s[pos : pos+length]
}
// replaceAll returns a copy of the string s with all instances
// of the substring replaced with the replacement string.
func replaceAll(s string, args ...string) string {
switch len(args) {
case 0:
return s
case 1:
return strings.Replace(s, args[0], "", -1)
default:
return strings.Replace(s, args[0], args[1], -1)
}
}
// replaceFirst returns a copy of the string s with the first
// instance of the substring replaced with the replacement string.
func replaceFirst(s string, args ...string) string {
switch len(args) {
case 0:
return s
case 1:
return strings.Replace(s, args[0], "", 1)
default:
return strings.Replace(s, args[0], args[1], 1)
}
}
// replacePrefix returns a copy of the string s with the matching
// prefix replaced with the replacement string.
func replacePrefix(s string, args ...string) string {
if len(args) != 2 {
return s
}
if strings.HasPrefix(s, args[0]) {
return strings.Replace(s, args[0], args[1], 1)
}
return s
}
// replaceSuffix returns a copy of the string s with the matching
// suffix replaced with the replacement string.
func replaceSuffix(s string, args ...string) string {
if len(args) != 2 {
return s
}
if strings.HasSuffix(s, args[0]) {
s = strings.TrimSuffix(s, args[0])
s = s + args[1]
}
return s
}
// TODO
func trimShortestPrefix(s string, args ...string) string {
if len(args) != 0 {
s = strings.TrimPrefix(s, args[0])
}
return s
}
func trimShortestSuffix(s string, args ...string) string {
if len(args) != 0 {
s = strings.TrimSuffix(s, args[0])
}
return s
}
func trimLongestPrefix(s string, args ...string) string {
if len(args) != 0 {
s = strings.TrimPrefix(s, args[0])
}
return s
}
func trimLongestSuffix(s string, args ...string) string {
if len(args) != 0 {
s = strings.TrimSuffix(s, args[0])
}
return s
}

20
vendor/github.com/drone/envsubst/match.go generated vendored Normal file
View file

@ -0,0 +1,20 @@
package envsubst
func matches() {
}
// alnum
// alpha
// ascii
// blank
// cntrl
// digit
// graph
// lower
// print
// punct
// space
// upper
// word
// xdigit

157
vendor/github.com/drone/envsubst/template.go generated vendored Normal file
View file

@ -0,0 +1,157 @@
package envsubst
import (
"bytes"
"io"
"io/ioutil"
"github.com/drone/envsubst/parse"
)
// state represents the state of template execution. It is not part of the
// template so that multiple executions can run in parallel.
type state struct {
template *Template
writer io.Writer
node parse.Node // current node
// maps variable names to values
mapper func(string) string
}
// Template is the representation of a parsed shell format string.
type Template struct {
tree *parse.Tree
}
// Parse creates a new shell format template and parses the template
// definition from string s.
func Parse(s string) (t *Template, err error) {
t = new(Template)
t.tree, err = parse.Parse(s)
if err != nil {
return nil, err
}
return t, nil
}
// ParseFile creates a new shell format template and parses the template
// definition from the named file.
func ParseFile(path string) (*Template, error) {
b, err := ioutil.ReadFile(path)
if err != nil {
return nil, err
}
return Parse(string(b))
}
// Execute applies a parsed template to the specified data mapping.
func (t *Template) Execute(mapping func(string) string) (str string, err error) {
b := new(bytes.Buffer)
s := new(state)
s.node = t.tree.Root
s.mapper = mapping
s.writer = b
err = t.eval(s)
if err != nil {
return
}
return b.String(), nil
}
func (t *Template) eval(s *state) (err error) {
switch node := s.node.(type) {
case *parse.TextNode:
err = t.evalText(s, node)
case *parse.FuncNode:
err = t.evalFunc(s, node)
case *parse.ListNode:
err = t.evalList(s, node)
}
return err
}
func (t *Template) evalText(s *state, node *parse.TextNode) error {
_, err := io.WriteString(s.writer, node.Value)
return err
}
func (t *Template) evalList(s *state, node *parse.ListNode) (err error) {
for _, n := range node.Nodes {
s.node = n
err = t.eval(s)
if err != nil {
return err
}
}
return nil
}
func (t *Template) evalFunc(s *state, node *parse.FuncNode) error {
var w = s.writer
var buf bytes.Buffer
var args []string
for _, n := range node.Args {
buf.Reset()
s.writer = &buf
s.node = n
err := t.eval(s)
if err != nil {
return err
}
args = append(args, buf.String())
}
// restore the origin writer
s.writer = w
s.node = node
v := s.mapper(node.Param)
fn := lookupFunc(node.Name, len(args))
_, err := io.WriteString(s.writer, fn(v, args...))
return err
}
// lookupFunc returns the parameters substitution function by name. If the
// named function does not exists, a default function is returned.
func lookupFunc(name string, args int) substituteFunc {
switch name {
case ",":
return toLowerFirst
case ",,":
return toLower
case "^":
return toUpperFirst
case "^^":
return toUpper
case "#":
if args == 0 {
return toLen
}
return trimShortestPrefix
case "##":
return trimLongestPrefix
case "%":
return trimShortestSuffix
case "%%":
return trimLongestSuffix
case ":":
return toSubstr
case "/#":
return replacePrefix
case "/%":
return replaceSuffix
case "/":
return replaceFirst
case "//":
return replaceAll
case "=", ":=", ":-":
return toDefault
case ":?", ":+", "-", "+":
return toDefault
default:
return toDefault
}
}

7
vendor/vendor.json vendored
View file

@ -53,6 +53,12 @@
"revision": "5d2041e26a699eaca682e2ea41c8f891e1060444",
"revisionTime": "2016-01-25T09:48:45-08:00"
},
{
"checksumSHA1": "7tosn2Sxlubl+7ElXSZ6Mz8tAjY=",
"path": "github.com/drone/envsubst",
"revision": "3e65ae5fd2d944d56fdf52cb3f887247498d50e9",
"revisionTime": "2017-01-18T15:01:55Z"
},
{
"path": "github.com/eknkc/amber",
"revision": "144da19a9994994c069f0693294a66dd310e14a4",
@ -208,7 +214,6 @@
},
{
"checksumSHA1": "+HvW+k8YkDaPKwF0Lwcz+Tf2A+E=",
"origin": "github.com/drone/drone/vendor/github.com/samalba/dockerclient",
"path": "github.com/samalba/dockerclient",
"revision": "91d7393ff85980ba3a8966405871a3d446ca28f2",
"revisionTime": "2016-04-14T17:47:13Z"

View file

@ -1,33 +0,0 @@
package expander
import "sort"
// Expand expands variables into the Yaml configuration using a
// ${key} template parameter with limited support for bash string functions.
func Expand(config []byte, envs map[string]string) []byte {
return []byte(
ExpandString(string(config), envs),
)
}
// ExpandString injects the variables into the Yaml configuration string using
// a ${key} template parameter with limited support for bash string functions.
func ExpandString(config string, envs map[string]string) string {
if envs == nil || len(envs) == 0 {
return config
}
keys := []string{}
for k := range envs {
keys = append(keys, k)
}
sort.Sort(sort.Reverse(sort.StringSlice(keys)))
expanded := config
for _, k := range keys {
v := envs[k]
for _, substitute := range substitutors {
expanded = substitute(expanded, k, v)
}
}
return expanded
}

View file

@ -1,48 +0,0 @@
package expander
import (
"testing"
"github.com/franela/goblin"
)
func TestExpand(t *testing.T) {
g := goblin.Goblin(t)
g.Describe("Expand params", func() {
g.It("Should replace vars with ${key}", func() {
s := "echo ${FOO} $BAR"
m := map[string]string{}
m["FOO"] = "BAZ"
g.Assert("echo BAZ $BAR").Equal(ExpandString(s, m))
})
g.It("Should not replace vars in nil map", func() {
s := "echo ${FOO} $BAR"
g.Assert(s).Equal(ExpandString(s, nil))
})
g.It("Should escape quoted variables", func() {
s := `echo "${FOO}"`
m := map[string]string{}
m["FOO"] = "hello\nworld"
g.Assert(`echo "hello\nworld"`).Equal(ExpandString(s, m))
})
g.It("Should replace variable prefix", func() {
s := `tag: ${TAG=${SHA:8}}`
m := map[string]string{}
m["TAG"] = ""
m["SHA"] = "f36cbf54ee1a1eeab264c8e388f386218ab1701b"
g.Assert("tag: f36cbf54").Equal(ExpandString(s, m))
})
g.It("Should handle nested substitution operations", func() {
s := `echo "${TAG##v}"`
m := map[string]string{}
m["TAG"] = "v1.0.0"
g.Assert(`echo "1.0.0"`).Equal(ExpandString(s, m))
})
})
}

View file

@ -1,182 +0,0 @@
package expander
import (
"fmt"
"regexp"
"strconv"
"strings"
)
// these are helper functions that bring bash-substitution to the drone yaml file.
// see http://tldp.org/LDP/abs/html/parameter-substitution.html
type substituteFunc func(str, key, val string) string
var substitutors = []substituteFunc{
substituteQ,
substitute,
substitutePrefix,
substituteSuffix,
substituteDefault,
substituteReplace,
substituteLeft,
substituteSubstr,
}
// substitute is a helper function that substitutes a simple parameter using
// ${parameter} notation.
func substitute(str, key, val string) string {
key = fmt.Sprintf("${%s}", key)
return strings.Replace(str, key, val, -1)
}
// substituteQ is a helper function that substitutes a simple parameter using
// "${parameter}" notation with the escaped value, using %q.
func substituteQ(str, key, val string) string {
key = fmt.Sprintf(`"${%s}"`, key)
val = fmt.Sprintf("%q", val)
return strings.Replace(str, key, val, -1)
}
// substitutePrefix is a helper function that substitutes parameters using
// ${parameter##prefix} notation with the parameter value minus the trimmed prefix.
func substitutePrefix(str, key, val string) string {
key = fmt.Sprintf("\\${%s##(.+)}", key)
reg, err := regexp.Compile(key)
if err != nil {
return str
}
for _, match := range reg.FindAllStringSubmatch(str, -1) {
if len(match) != 2 {
continue
}
val_ := strings.TrimPrefix(val, match[1])
str = strings.Replace(str, match[0], val_, -1)
}
return str
}
// substituteSuffix is a helper function that substitutes parameters using
// ${parameter%%suffix} notation with the parameter value minus the trimmed suffix.
func substituteSuffix(str, key, val string) string {
key = fmt.Sprintf("\\${%s%%%%(.+)}", key)
reg, err := regexp.Compile(key)
if err != nil {
return str
}
for _, match := range reg.FindAllStringSubmatch(str, -1) {
if len(match) != 2 {
continue
}
val_ := strings.TrimSuffix(val, match[1])
str = strings.Replace(str, match[0], val_, -1)
}
return str
}
// substituteDefault is a helper function that substitutes parameters using
// ${parameter=default} notation with the parameter value. When empty the
// default value is used.
func substituteDefault(str, key, val string) string {
key = fmt.Sprintf("\\${%s=(.+)}", key)
reg, err := regexp.Compile(key)
if err != nil {
return str
}
for _, match := range reg.FindAllStringSubmatch(str, -1) {
if len(match) != 2 {
continue
}
if len(val) == 0 {
str = strings.Replace(str, match[0], match[1], -1)
} else {
str = strings.Replace(str, match[0], val, -1)
}
}
return str
}
// unescapeBackslash is a helper function to unescape any backslashes in str.
// Note that no actual literal conversions are done.
func unescapeBackslash(str string) string {
re := regexp.MustCompile(`\\(.)`)
return string(re.ReplaceAll([]byte(str), []byte("$1")))
}
// substituteReplace is a helper function that substitutes parameters using
// ${parameter/old/new} notation with the parameter value. A find and replace
// is performed before injecting the strings, replacing the old pattern with
// the new value.
func substituteReplace(str, key, val string) string {
key = fmt.Sprintf(`\${%s/((?:\\.|[^\\])+)/(.+)}`, key)
reg, err := regexp.Compile(key)
if err != nil {
return str
}
match := reg.FindStringSubmatch(str)
if match == nil {
return str
}
old := unescapeBackslash(match[1])
new := unescapeBackslash(match[2])
with := strings.Replace(val, old, new, -1)
return strings.Replace(str, match[0], with, -1)
}
// substituteLeft is a helper function that substitutes parameters using
// ${parameter:pos} notation with the parameter value, sliced up to the
// specified position.
func substituteLeft(str, key, val string) string {
key = fmt.Sprintf("\\${%s:([0-9]*)}", key)
reg, err := regexp.Compile(key)
if err != nil {
return str
}
for _, match := range reg.FindAllStringSubmatch(str, -1) {
if len(match) != 2 {
continue
}
index, err := strconv.Atoi(match[1])
if err != nil {
continue // skip
}
if index > len(val)-1 {
continue // skip
}
str = strings.Replace(str, match[0], val[:index], -1)
}
return str
}
// substituteLeft is a helper function that substitutes parameters using
// ${parameter:pos:len} notation with the parameter value as a substring,
// starting at the specified position for the specified length.
func substituteSubstr(str, key, val string) string {
key = fmt.Sprintf("\\${%s:([0-9]*):([0-9]*)}", key)
reg, err := regexp.Compile(key)
if err != nil {
return str
}
for _, match := range reg.FindAllStringSubmatch(str, -1) {
if len(match) != 3 {
continue
}
pos, err := strconv.Atoi(match[1])
if err != nil {
continue // skip
}
length, err := strconv.Atoi(match[2])
if err != nil {
continue // skip
}
if pos+length > len(val)-1 {
continue // skip
}
str = strings.Replace(str, match[0], val[pos:pos+length], -1)
}
return str
}

View file

@ -1,74 +0,0 @@
package expander
import (
"testing"
"github.com/franela/goblin"
)
func TestSubstitution(t *testing.T) {
g := goblin.Goblin(t)
g.Describe("Parameter Substitution", func() {
g.It("Should substitute simple parameters", func() {
before := "echo ${GREETING} WORLD"
after := "echo HELLO WORLD"
g.Assert(substitute(before, "GREETING", "HELLO")).Equal(after)
})
g.It("Should substitute quoted parameters", func() {
before := "echo \"${GREETING}\" WORLD"
after := "echo \"HELLO\" WORLD"
g.Assert(substituteQ(before, "GREETING", "HELLO")).Equal(after)
})
g.It("Should substitute parameters and trim prefix", func() {
before := "echo ${GREETING##asdf} WORLD"
after := "echo HELLO WORLD"
g.Assert(substitutePrefix(before, "GREETING", "asdfHELLO")).Equal(after)
})
g.It("Should substitute parameters and trim suffix", func() {
before := "echo ${GREETING%%asdf} WORLD"
after := "echo HELLO WORLD"
g.Assert(substituteSuffix(before, "GREETING", "HELLOasdf")).Equal(after)
})
g.It("Should substitute parameters without using the default", func() {
before := "echo ${GREETING=HOLA} WORLD"
after := "echo HELLO WORLD"
g.Assert(substituteDefault(before, "GREETING", "HELLO")).Equal(after)
})
g.It("Should substitute parameters using the a default", func() {
before := "echo ${GREETING=HOLA} WORLD"
after := "echo HOLA WORLD"
g.Assert(substituteDefault(before, "GREETING", "")).Equal(after)
})
g.It("Should substitute parameters with replacement", func() {
before := "echo ${GREETING/HE/A} MONDE"
after := "echo ALLO MONDE"
g.Assert(substituteReplace(before, "GREETING", "HELLO")).Equal(after)
})
g.It("Should substitute parameters with replacement, containing slashes", func() {
before := `echo ${GREETING/HE\//A} MONDE`
after := "echo ALLO MONDE"
g.Assert(substituteReplace(before, "GREETING", "HE/LLO")).Equal(after)
})
g.It("Should substitute parameters with left substr", func() {
before := "echo ${FOO:4} IS COOL"
after := "echo THIS IS COOL"
g.Assert(substituteLeft(before, "FOO", "THIS IS A REALLY LONG STRING")).Equal(after)
})
g.It("Should substitute parameters with substr", func() {
before := "echo ${FOO:8:5} IS COOL"
after := "echo DRONE IS COOL"
g.Assert(substituteSubstr(before, "FOO", "THIS IS DRONE CI")).Equal(after)
})
})
}