From d9b59dfa6fa969af358d1a0d6fdc077fdf20fedc Mon Sep 17 00:00:00 2001 From: Don Olmstead Date: Wed, 12 Aug 2015 18:21:51 -0700 Subject: [PATCH 1/2] Modifying transform behavior --- pkg/yaml/lint.go | 1 - pkg/yaml/parse.go | 15 +-- pkg/yaml/{trans.go => transform/transform.go} | 112 ++++++++++++++++-- .../transform_test.go} | 17 ++- 4 files changed, 125 insertions(+), 20 deletions(-) rename pkg/yaml/{trans.go => transform/transform.go} (63%) rename pkg/yaml/{trans_test.go => transform/transform_test.go} (94%) diff --git a/pkg/yaml/lint.go b/pkg/yaml/lint.go index 9141549af..27a07ed06 100644 --- a/pkg/yaml/lint.go +++ b/pkg/yaml/lint.go @@ -134,7 +134,6 @@ func LintPlugins(c *common.Config, opts *Opts) error { var images []string images = append(images, c.Setup.Image) images = append(images, c.Clone.Image) - c.Clone.Image = imageName(c.Clone.Image) for _, step := range c.Publish { images = append(images, step.Image) } diff --git a/pkg/yaml/parse.go b/pkg/yaml/parse.go index d68b672a5..2e9eb947b 100644 --- a/pkg/yaml/parse.go +++ b/pkg/yaml/parse.go @@ -4,6 +4,7 @@ import ( common "github.com/drone/drone/pkg/types" "github.com/drone/drone/pkg/yaml/inject" "github.com/drone/drone/pkg/yaml/matrix" + "github.com/drone/drone/pkg/yaml/transform" "github.com/drone/drone/Godeps/_workspace/src/gopkg.in/yaml.v2" ) @@ -77,20 +78,16 @@ func ParseSingle(raw string, opts *Opts) (*common.Config, error) { if err != nil { return nil, err } - // apply rules / transofms - transformSetup(conf) - transformClone(conf) - transformBuild(conf) - transformImages(conf) - transformDockerPlugin(conf) + // apply rules / transforms + transform.Transform(conf) if !opts.Network { - rmNetwork(conf) + transform.TransformRemoveNetwork(conf) } if !opts.Volumes { - rmVolumes(conf) + transform.TransformRemoveVolumes(conf) } if !opts.Privileged { - rmPrivileged(conf) + transform.TransformRemovePrivileged(conf) } err = LintPlugins(conf, opts) if err != nil { diff --git a/pkg/yaml/trans.go b/pkg/yaml/transform/transform.go similarity index 63% rename from pkg/yaml/trans.go rename to pkg/yaml/transform/transform.go index c38188f2c..88576f4f3 100644 --- a/pkg/yaml/trans.go +++ b/pkg/yaml/transform/transform.go @@ -1,6 +1,9 @@ -package parser +package transform import ( + "fmt" + "net/url" + "path/filepath" "strings" common "github.com/drone/drone/pkg/types" @@ -10,25 +13,60 @@ import ( // to the build configuration. type transformRule func(*common.Config) +var transformRules = [...]transformRule{ + transformSetup, + transformClone, + transformBuild, + transformImages, + transformDockerPlugin, +} + +var rmPrivilegedRules = [...]transformRule{ + rmPrivileged, + rmVolumes, + rmNetwork, +} + // Transform executes the default transformers that // ensure the minimal Yaml configuration is in place // and correctly configured. func Transform(c *common.Config) { - transformSetup(c) - transformClone(c) - transformBuild(c) - transformImages(c) - transformDockerPlugin(c) + for _, rule := range transformRules { + rule(c) + } } // TransformSafe executes all transformers that remove // privileged options from the Yaml. func TransformSafe(c *common.Config) { - rmPrivileged(c) - rmVolumes(c) + for _, rule := range rmPrivilegedRules { + rule(c) + } +} + +// TransformRemoveNetwork executes all transformers that +// remove network options from the Yaml. +func TransformRemoveNetwork(c *common.Config) { rmNetwork(c) } +// TransformRemoveVolumes executes all transformers that +// remove volume options from the Yaml. +func TransformRemoveVolumes(c *common.Config) { + rmVolumes(c) +} + +// TransformRemovePrivileged executes all transformers that +// remove privileged options from the Yaml. +func TransformRemovePrivileged(c *common.Config) { + rmPrivileged(c) +} + +func TransformRepo(c *common.Config, r *common.Repo) { + transformWorkspace(c, r) + transformCache(c, r) +} + // transformSetup is a transformer that adds a default // setup step if none exists. func transformSetup(c *common.Config) { @@ -82,7 +120,7 @@ func transformImages(c *common.Config) { } // transformDockerPlugin is a transformer that ensures the -// official Docker plugin can runs in privileged mode. It +// official Docker plugin can run in privileged mode. It // will disable volumes and network mode for added protection. func transformDockerPlugin(c *common.Config) { for _, step := range c.Publish { @@ -159,6 +197,49 @@ func rmNetwork(c *common.Config) { } } +// transformWorkspace is a transformer that adds the workspace +// directory to the configuration based on the repository +// information. +func transformWorkspace(c *common.Config, r *common.Repo) { + //c.Clone.Dir = workspaceRoot(r) +} + +// transformCache is a transformer that adds volumes +// to the configuration based on the cache. +func transformCache(c *common.Config, r *common.Repo) { + cacheCount := len(c.Build.Cache) + + if cacheCount != 0 { + volumes := make([]string, cacheCount) + + cache := cacheRoot(r) + workspace := workspaceRoot(r) + + for i, dir := range c.Build.Cache { + cacheDir := filepath.Join(cache, dir) + workspaceDir := filepath.Join(workspace, dir) + + volumes[i] = fmt.Sprintf("%s:%s", cacheDir, workspaceDir) + } + + c.Setup.Volumes = append(c.Setup.Volumes, volumes...) + c.Clone.Volumes = append(c.Clone.Volumes, volumes...) + c.Build.Volumes = append(c.Build.Volumes, volumes...) + for _, step := range c.Publish { + step.Volumes = append(step.Volumes, volumes...) + } + for _, step := range c.Deploy { + step.Volumes = append(step.Volumes, volumes...) + } + for _, step := range c.Notify { + step.Volumes = append(step.Volumes, volumes...) + } + for _, step := range c.Compose { + step.Volumes = append(step.Volumes, volumes...) + } + } +} + // imageName is a helper function that resolves the // image name. When using official drone plugins it // is possible to use an alias name. This converts to @@ -181,3 +262,16 @@ func imageNameDefault(name, defaultName string) string { } return imageName(name) } + +func workspaceRoot(r *common.Repo) string { + return filepath.Join("/drone/src", repoPath(r)) +} + +func cacheRoot(r *common.Repo) string { + return filepath.Join("/tmp/drone/cache", repoPath(r)) +} + +func repoPath(r *common.Repo) string { + parsed, _ := url.Parse(r.Link) + return filepath.Join(parsed.Host, r.FullName) +} diff --git a/pkg/yaml/trans_test.go b/pkg/yaml/transform/transform_test.go similarity index 94% rename from pkg/yaml/trans_test.go rename to pkg/yaml/transform/transform_test.go index 0e1431487..f438dcec1 100644 --- a/pkg/yaml/trans_test.go +++ b/pkg/yaml/transform/transform_test.go @@ -1,4 +1,4 @@ -package parser +package transform import ( "testing" @@ -142,5 +142,20 @@ func Test_Transform(t *testing.T) { g.Assert(imageName("azure")).Equal("plugins/drone-azure") g.Assert(imageName("azure_storage")).Equal("plugins/drone-azure-storage") }) + + g.It("Should have cached volumes", func() { + c := &common.Config{ + Build: &common.Step{ + Cache: []string{".git","foo","bar"}, + }, + } + r := &common.Repo{ + Link: "https://github.com/drone/drone", + FullName: "drone/drone", + } + transformCache(c, r) + + g.Assert(len(c.Build.Volumes)).Equal(3) + }) }) } From d27788e4b4e79f88b20d35a0ec34c164f7e699bf Mon Sep 17 00:00:00 2001 From: Don Olmstead Date: Mon, 17 Aug 2015 16:24:47 -0700 Subject: [PATCH 2/2] Adding caching into the build. Fixing style issues --- cmd/drone-build/run.go | 2 +- pkg/yaml/parse.go | 21 +++---- pkg/yaml/transform/transform.go | 90 ++++++++++++++++------------ pkg/yaml/transform/transform_test.go | 25 +++++++- 4 files changed, 87 insertions(+), 51 deletions(-) diff --git a/cmd/drone-build/run.go b/cmd/drone-build/run.go index ca67c4e73..6324c2b6c 100644 --- a/cmd/drone-build/run.go +++ b/cmd/drone-build/run.go @@ -45,7 +45,7 @@ func setup(c *Context) error { // inject the matrix parameters into the yaml injected := inject.Inject(string(c.Yaml), c.Job.Environment) - c.Conf, err = parser.ParseSingle(injected, &opts) + c.Conf, err = parser.ParseSingle(injected, &opts, c.Repo) if err != nil { return err } diff --git a/pkg/yaml/parse.go b/pkg/yaml/parse.go index 2e9eb947b..326c1acf7 100644 --- a/pkg/yaml/parse.go +++ b/pkg/yaml/parse.go @@ -28,14 +28,14 @@ var DefaultOpts = &Opts{ // Parse parses a build matrix and returns // a list of build configurations for each axis // using the default parsing options. -func Parse(raw string) ([]*common.Config, error) { - return ParseOpts(raw, DefaultOpts) +func Parse(raw string, r *common.Repo) ([]*common.Config, error) { + return ParseOpts(raw, DefaultOpts, r) } // ParseOpts parses a build matrix and returns // a list of build configurations for each axis // using the provided parsing options. -func ParseOpts(raw string, opts *Opts) ([]*common.Config, error) { +func ParseOpts(raw string, opts *Opts, r *common.Repo) ([]*common.Config, error) { axis, err := matrix.Parse(raw) if err != nil { return nil, err @@ -45,7 +45,7 @@ func ParseOpts(raw string, opts *Opts) ([]*common.Config, error) { // when no matrix values exist we should return // a single config value with an empty label. if len(axis) == 0 { - conf, err := ParseSingle(raw, opts) + conf, err := ParseSingle(raw, opts, r) if err != nil { return nil, err } @@ -55,7 +55,7 @@ func ParseOpts(raw string, opts *Opts) ([]*common.Config, error) { for _, ax := range axis { // inject the matrix values into the raw script injected := inject.Inject(raw, ax) - conf, err := ParseSingle(injected, opts) + conf, err := ParseSingle(injected, opts, r) if err != nil { return nil, err } @@ -67,7 +67,7 @@ func ParseOpts(raw string, opts *Opts) ([]*common.Config, error) { } // helper funtion to parse a yaml configuration file. -func ParseSingle(raw string, opts *Opts) (*common.Config, error) { +func ParseSingle(raw string, opts *Opts, r *common.Repo) (*common.Config, error) { conf := &common.Config{} err := yaml.Unmarshal([]byte(raw), conf) if err != nil { @@ -79,16 +79,17 @@ func ParseSingle(raw string, opts *Opts) (*common.Config, error) { return nil, err } // apply rules / transforms - transform.Transform(conf) + transform.Defaults(conf) if !opts.Network { - transform.TransformRemoveNetwork(conf) + transform.RemoveNetwork(conf) } if !opts.Volumes { - transform.TransformRemoveVolumes(conf) + transform.RemoveVolumes(conf) } if !opts.Privileged { - transform.TransformRemovePrivileged(conf) + transform.RemovePrivileged(conf) } + transform.Repo(conf, r) err = LintPlugins(conf, opts) if err != nil { return nil, err diff --git a/pkg/yaml/transform/transform.go b/pkg/yaml/transform/transform.go index 88576f4f3..a6a84953f 100644 --- a/pkg/yaml/transform/transform.go +++ b/pkg/yaml/transform/transform.go @@ -13,7 +13,7 @@ import ( // to the build configuration. type transformRule func(*common.Config) -var transformRules = [...]transformRule{ +var transformRules = []transformRule{ transformSetup, transformClone, transformBuild, @@ -21,48 +21,50 @@ var transformRules = [...]transformRule{ transformDockerPlugin, } -var rmPrivilegedRules = [...]transformRule{ +var rmPrivilegedRules = []transformRule{ rmPrivileged, rmVolumes, rmNetwork, } -// Transform executes the default transformers that +// Default executes the default transformers that // ensure the minimal Yaml configuration is in place // and correctly configured. -func Transform(c *common.Config) { +func Defaults(c *common.Config) { for _, rule := range transformRules { rule(c) } } -// TransformSafe executes all transformers that remove -// privileged options from the Yaml. -func TransformSafe(c *common.Config) { +// Safe executes all transformers that remove privileged +// options from the Yaml. +func Safe(c *common.Config) { for _, rule := range rmPrivilegedRules { rule(c) } } -// TransformRemoveNetwork executes all transformers that -// remove network options from the Yaml. -func TransformRemoveNetwork(c *common.Config) { +// RemoveNetwork executes all transformers that remove +// network options from the Yaml. +func RemoveNetwork(c *common.Config) { rmNetwork(c) } // TransformRemoveVolumes executes all transformers that // remove volume options from the Yaml. -func TransformRemoveVolumes(c *common.Config) { +func RemoveVolumes(c *common.Config) { rmVolumes(c) } -// TransformRemovePrivileged executes all transformers that -// remove privileged options from the Yaml. -func TransformRemovePrivileged(c *common.Config) { +// RemovePrivileged executes all transformers that remove +// privileged options from the Yaml. +func RemovePrivileged(c *common.Config) { rmPrivileged(c) } -func TransformRepo(c *common.Config, r *common.Repo) { +// Repo executes all transformers that rely on repository +// information. +func Repo(c *common.Config, r *common.Repo) { transformWorkspace(c, r) transformCache(c, r) } @@ -209,34 +211,38 @@ func transformWorkspace(c *common.Config, r *common.Repo) { func transformCache(c *common.Config, r *common.Repo) { cacheCount := len(c.Build.Cache) - if cacheCount != 0 { - volumes := make([]string, cacheCount) + if cacheCount == 0 { + return + } - cache := cacheRoot(r) - workspace := workspaceRoot(r) + volumes := make([]string, cacheCount) - for i, dir := range c.Build.Cache { - cacheDir := filepath.Join(cache, dir) - workspaceDir := filepath.Join(workspace, dir) + cache := cacheRoot(r) + workspace := workspaceRoot(r) - volumes[i] = fmt.Sprintf("%s:%s", cacheDir, workspaceDir) - } + for i, dir := range c.Build.Cache { + cacheDir := filepath.Join(cache, dir) + workspaceDir := filepath.Join(workspace, dir) - c.Setup.Volumes = append(c.Setup.Volumes, volumes...) - c.Clone.Volumes = append(c.Clone.Volumes, volumes...) - c.Build.Volumes = append(c.Build.Volumes, volumes...) - for _, step := range c.Publish { - step.Volumes = append(step.Volumes, volumes...) - } - for _, step := range c.Deploy { - step.Volumes = append(step.Volumes, volumes...) - } - for _, step := range c.Notify { - step.Volumes = append(step.Volumes, volumes...) - } - for _, step := range c.Compose { - step.Volumes = append(step.Volumes, volumes...) - } + volumes[i] = fmt.Sprintf("%s:%s", cacheDir, workspaceDir) + fmt.Printf("Volume %s", volumes[i]) + } + + c.Setup.Volumes = append(c.Setup.Volumes, volumes...) + c.Clone.Volumes = append(c.Clone.Volumes, volumes...) + c.Build.Volumes = append(c.Build.Volumes, volumes...) + + for _, step := range c.Publish { + step.Volumes = append(step.Volumes, volumes...) + } + for _, step := range c.Deploy { + step.Volumes = append(step.Volumes, volumes...) + } + for _, step := range c.Notify { + step.Volumes = append(step.Volumes, volumes...) + } + for _, step := range c.Compose { + step.Volumes = append(step.Volumes, volumes...) } } @@ -263,14 +269,20 @@ func imageNameDefault(name, defaultName string) string { return imageName(name) } +// workspaceRoot is a helper function that determines the +// default workspace the build runs in. func workspaceRoot(r *common.Repo) string { return filepath.Join("/drone/src", repoPath(r)) } +// cacheRoot is a helper function that deteremines the +// default caching root. func cacheRoot(r *common.Repo) string { return filepath.Join("/tmp/drone/cache", repoPath(r)) } +// repoPath is a helper function that creates a path based +// on the host and repository name. func repoPath(r *common.Repo) string { parsed, _ := url.Parse(r.Link) return filepath.Join(parsed.Host, r.FullName) diff --git a/pkg/yaml/transform/transform_test.go b/pkg/yaml/transform/transform_test.go index f438dcec1..2d29ba24c 100644 --- a/pkg/yaml/transform/transform_test.go +++ b/pkg/yaml/transform/transform_test.go @@ -145,9 +145,14 @@ func Test_Transform(t *testing.T) { g.It("Should have cached volumes", func() { c := &common.Config{ + Setup: &common.Step{}, + Clone: &common.Step{}, Build: &common.Step{ Cache: []string{".git","foo","bar"}, }, + Notify: map[string]*common.Step{}, + Deploy: map[string]*common.Step{}, + Publish: map[string]*common.Step{}, } r := &common.Repo{ Link: "https://github.com/drone/drone", @@ -155,7 +160,25 @@ func Test_Transform(t *testing.T) { } transformCache(c, r) - g.Assert(len(c.Build.Volumes)).Equal(3) + cacheCount := len(c.Build.Cache) + + test := func(s *common.Step) { + g.Assert(len(s.Volumes)).Equal(cacheCount) + } + + testRange := func(s map[string]*common.Step) { + for _, step := range s { + test(step) + } + } + + test(c.Setup) + test(c.Clone) + test(c.Build) + testRange(c.Publish) + testRange(c.Deploy) + testRange(c.Notify) + testRange(c.Compose) }) }) }