Merge pull request #1142 from donny-dont/features/add-caching

Adding caching

cmd/drone-build/run.go

@@ -45,7 +45,7 @@ func setup(c *Context) error {
 
 	// inject the matrix parameters into the yaml
 	injected := inject.Inject(string(c.Yaml), c.Job.Environment)
-	c.Conf, err = parser.ParseSingle(injected, &opts)
+	c.Conf, err = parser.ParseSingle(injected, &opts, c.Repo)
 	if err != nil {
 		return err
 	}

pkg/yaml/lint.go

@@ -134,7 +134,6 @@ func LintPlugins(c *common.Config, opts *Opts) error {
 	var images []string
 	images = append(images, c.Setup.Image)
 	images = append(images, c.Clone.Image)
-	c.Clone.Image = imageName(c.Clone.Image)
 	for _, step := range c.Publish {
 		images = append(images, step.Image)
 	}

pkg/yaml/parse.go

@@ -4,6 +4,7 @@ import (
 	common "github.com/drone/drone/pkg/types"
 	"github.com/drone/drone/pkg/yaml/inject"
 	"github.com/drone/drone/pkg/yaml/matrix"
+	"github.com/drone/drone/pkg/yaml/transform"
 
 	"github.com/drone/drone/Godeps/_workspace/src/gopkg.in/yaml.v2"
 )
@@ -27,14 +28,14 @@ var DefaultOpts = &Opts{
 // Parse parses a build matrix and returns
 // a list of build configurations for each axis
 // using the default parsing options.
-func Parse(raw string) ([]*common.Config, error) {
+func Parse(raw string, r *common.Repo) ([]*common.Config, error) {
-	return ParseOpts(raw, DefaultOpts)
+	return ParseOpts(raw, DefaultOpts, r)
 }
 
 // ParseOpts parses a build matrix and returns
 // a list of build configurations for each axis
 // using the provided parsing options.
-func ParseOpts(raw string, opts *Opts) ([]*common.Config, error) {
+func ParseOpts(raw string, opts *Opts, r *common.Repo) ([]*common.Config, error) {
 	axis, err := matrix.Parse(raw)
 	if err != nil {
 		return nil, err
@@ -44,7 +45,7 @@ func ParseOpts(raw string, opts *Opts) ([]*common.Config, error) {
 	// when no matrix values exist we should return
 	// a single config value with an empty label.
 	if len(axis) == 0 {
-		conf, err := ParseSingle(raw, opts)
+		conf, err := ParseSingle(raw, opts, r)
 		if err != nil {
 			return nil, err
 		}
@@ -54,7 +55,7 @@ func ParseOpts(raw string, opts *Opts) ([]*common.Config, error) {
 	for _, ax := range axis {
 		// inject the matrix values into the raw script
 		injected := inject.Inject(raw, ax)
-		conf, err := ParseSingle(injected, opts)
+		conf, err := ParseSingle(injected, opts, r)
 		if err != nil {
 			return nil, err
 		}
@@ -66,7 +67,7 @@ func ParseOpts(raw string, opts *Opts) ([]*common.Config, error) {
 }
 
 // helper funtion to parse a yaml configuration file.
-func ParseSingle(raw string, opts *Opts) (*common.Config, error) {
+func ParseSingle(raw string, opts *Opts, r *common.Repo) (*common.Config, error) {
 	conf := &common.Config{}
 	err := yaml.Unmarshal([]byte(raw), conf)
 	if err != nil {
@@ -77,21 +78,18 @@ func ParseSingle(raw string, opts *Opts) (*common.Config, error) {
 	if err != nil {
 		return nil, err
 	}
-	// apply rules / transofms
+	// apply rules / transforms
-	transformSetup(conf)
+	transform.Defaults(conf)
-	transformClone(conf)
-	transformBuild(conf)
-	transformImages(conf)
-	transformDockerPlugin(conf)
 	if !opts.Network {
-		rmNetwork(conf)
+		transform.RemoveNetwork(conf)
 	}
 	if !opts.Volumes {
-		rmVolumes(conf)
+		transform.RemoveVolumes(conf)
 	}
 	if !opts.Privileged {
-		rmPrivileged(conf)
+		transform.RemovePrivileged(conf)
 	}
+	transform.Repo(conf, r)
 	err = LintPlugins(conf, opts)
 	if err != nil {
 		return nil, err

pkg/yaml/transform/transform.go

@@ -1,6 +1,9 @@
-package parser
+package transform
 
 import (
+	"fmt"
+	"net/url"
+	"path/filepath"
 	"strings"
 
 	common "github.com/drone/drone/pkg/types"
@@ -10,25 +13,62 @@ import (
 // to the build configuration.
 type transformRule func(*common.Config)
 
-// Transform executes the default transformers that
+var transformRules = []transformRule{
-// ensure the minimal Yaml configuration is in place
+	transformSetup,
-// and correctly configured.
+	transformClone,
-func Transform(c *common.Config) {
+	transformBuild,
-	transformSetup(c)
+	transformImages,
-	transformClone(c)
+	transformDockerPlugin,
-	transformBuild(c)
-	transformImages(c)
-	transformDockerPlugin(c)
 }
 
-// TransformSafe executes all transformers that remove
+var rmPrivilegedRules = []transformRule{
-// privileged options from the Yaml.
+	rmPrivileged,
-func TransformSafe(c *common.Config) {
+	rmVolumes,
-	rmPrivileged(c)
+	rmNetwork,
-	rmVolumes(c)
+}
+
+// Default executes the default transformers that
+// ensure the minimal Yaml configuration is in place
+// and correctly configured.
+func Defaults(c *common.Config) {
+	for _, rule := range transformRules {
+		rule(c)
+	}
+}
+
+// Safe executes all transformers that remove privileged
+// options from the Yaml.
+func Safe(c *common.Config) {
+	for _, rule := range rmPrivilegedRules {
+		rule(c)
+	}
+}
+
+// RemoveNetwork executes all transformers that remove
+// network options from the Yaml.
+func RemoveNetwork(c *common.Config) {
 	rmNetwork(c)
 }
 
+// TransformRemoveVolumes executes all transformers that
+// remove volume options from the Yaml.
+func RemoveVolumes(c *common.Config) {
+	rmVolumes(c)
+}
+
+// RemovePrivileged executes all transformers that remove
+// privileged options from the Yaml.
+func RemovePrivileged(c *common.Config) {
+	rmPrivileged(c)
+}
+
+// Repo executes all transformers that rely on repository
+// information.
+func Repo(c *common.Config, r *common.Repo) {
+  transformWorkspace(c, r)
+	transformCache(c, r)
+}
+
 // transformSetup is a transformer that adds a default
 // setup step if none exists.
 func transformSetup(c *common.Config) {
@@ -82,7 +122,7 @@ func transformImages(c *common.Config) {
 }
 
 // transformDockerPlugin is a transformer that ensures the
-// official Docker plugin can runs in privileged mode. It
+// official Docker plugin can run in privileged mode. It
 // will disable volumes and network mode for added protection.
 func transformDockerPlugin(c *common.Config) {
 	for _, step := range c.Publish {
@@ -159,6 +199,53 @@ func rmNetwork(c *common.Config) {
 	}
 }
 
+// transformWorkspace is a transformer that adds the workspace
+// directory to the configuration based on the repository
+// information.
+func transformWorkspace(c *common.Config, r *common.Repo) {
+	//c.Clone.Dir = workspaceRoot(r)
+}
+
+// transformCache is a transformer that adds volumes
+// to the configuration based on the cache.
+func transformCache(c *common.Config, r *common.Repo) {
+	cacheCount := len(c.Build.Cache)
+
+  if cacheCount == 0 {
+		return
+	}
+
+	volumes := make([]string, cacheCount)
+
+	cache := cacheRoot(r)
+	workspace := workspaceRoot(r)
+
+	for i, dir := range c.Build.Cache {
+		cacheDir := filepath.Join(cache, dir)
+		workspaceDir := filepath.Join(workspace, dir)
+
+		volumes[i] = fmt.Sprintf("%s:%s", cacheDir, workspaceDir)
+		fmt.Printf("Volume %s", volumes[i])
+	}
+
+	c.Setup.Volumes = append(c.Setup.Volumes, volumes...)
+	c.Clone.Volumes = append(c.Clone.Volumes, volumes...)
+	c.Build.Volumes = append(c.Build.Volumes, volumes...)
+
+	for _, step := range c.Publish {
+		step.Volumes = append(step.Volumes, volumes...)
+	}
+	for _, step := range c.Deploy {
+		step.Volumes = append(step.Volumes, volumes...)
+	}
+	for _, step := range c.Notify {
+		step.Volumes = append(step.Volumes, volumes...)
+	}
+	for _, step := range c.Compose {
+		step.Volumes = append(step.Volumes, volumes...)
+	}
+}
+
 // imageName is a helper function that resolves the
 // image name. When using official drone plugins it
 // is possible to use an alias name. This converts to
@@ -181,3 +268,22 @@ func imageNameDefault(name, defaultName string) string {
 	}
 	return imageName(name)
 }
+
+// workspaceRoot is a helper function that determines the
+// default workspace the build runs in.
+func workspaceRoot(r *common.Repo) string {
+  return filepath.Join("/drone/src", repoPath(r))
+}
+
+// cacheRoot is a helper function that deteremines the
+// default caching root.
+func cacheRoot(r *common.Repo) string {
+	return filepath.Join("/tmp/drone/cache", repoPath(r))
+}
+
+// repoPath is a helper function that creates a path based
+// on the host and repository name.
+func repoPath(r *common.Repo) string {
+	parsed, _ := url.Parse(r.Link)
+	return filepath.Join(parsed.Host, r.FullName)
+}

pkg/yaml/transform/transform_test.go

@@ -1,4 +1,4 @@
-package parser
+package transform
 
 import (
 	"testing"
@@ -142,5 +142,43 @@ func Test_Transform(t *testing.T) {
 			g.Assert(imageName("azure")).Equal("plugins/drone-azure")
 			g.Assert(imageName("azure_storage")).Equal("plugins/drone-azure-storage")
 		})
+
+		g.It("Should have cached volumes", func() {
+			c := &common.Config{
+				Setup: &common.Step{},
+				Clone: &common.Step{},
+				Build: &common.Step{
+					Cache: []string{".git","foo","bar"},
+				},
+				Notify:  map[string]*common.Step{},
+				Deploy:  map[string]*common.Step{},
+				Publish: map[string]*common.Step{},
+			}
+			r := &common.Repo{
+				Link: "https://github.com/drone/drone",
+				FullName: "drone/drone",
+			}
+			transformCache(c, r)
+
+      cacheCount := len(c.Build.Cache)
+
+			test := func(s *common.Step) {
+				g.Assert(len(s.Volumes)).Equal(cacheCount)
+			}
+
+			testRange := func(s map[string]*common.Step) {
+				for _, step := range s {
+					test(step)
+				}
+			}
+
+      test(c.Setup)
+			test(c.Clone)
+			test(c.Build)
+			testRange(c.Publish)
+			testRange(c.Deploy)
+			testRange(c.Notify)
+			testRange(c.Compose)
+		})
 	})
 }