Merge remote-tracking branch 'upstream/bolt' into bolt

datastore/bolt/repo.go

@@ -132,23 +132,43 @@ func (db *DB) SetRepoKeypair(repo string, keypair *common.Keypair) error {
 
 // DelRepo deletes the repository.
 func (db *DB) DelRepo(repo *common.Repo) error {
-	//TODO(benschumacher) rework this to use BoltDB's txn wrapper
-
-	t, err := db.Begin(true)
-	if err != nil {
-		return err
-	}
 	key := []byte(repo.FullName)
-	err = t.Bucket(bucketRepo).Delete(key)
+
-	if err != nil {
+	return db.Update(func(t *bolt.Tx) error {
-		t.Rollback()
+		err := t.Bucket(bucketRepo).Delete(key)
+		if err != nil {
+			return err
+		}
+		t.Bucket(bucketRepoKeys).Delete(key)
+		t.Bucket(bucketRepoParams).Delete(key)
+
+		// should we just ignore these error conditions? or should
+		// we go ahead with the transaction and assume we can
+		// cleanup the leftovers through some other maintenance process?
+		err = db.deleteTracesOfRepo(t, key)
+
 		return err
-	}
+	})
-	t.Bucket(bucketRepoKeys).Delete(key)
+}
-	t.Bucket(bucketRepoParams).Delete(key)
+
-	// TODO(bradrydzewski) delete all builds
+// deleteTracesOfRepo cleans up build leftovers when a repo is removed
-	// TODO(bradrydzewski) delete all tasks
+func (db *DB) deleteTracesOfRepo(t *bolt.Tx, repoKey []byte) error {
-	return t.Commit()
+	err := error(nil)
+
+	// bucketBuildSeq uses the repoKey directly
+	t.Bucket(bucketBuildSeq).Delete(repoKey)
+
+	// the other buckets use repoKey with '/buildNumber', at least.
+	// validating that an additiona '/' is there ensures that we don't
+	// match 'github.com/drone/droney' when we're cleaning up after
+	// 'github.com/drone/drone'.
+	prefix := append(repoKey, '/')
+	deleteWithPrefix(t, bucketBuildLogs, prefix, true)
+	deleteWithPrefix(t, bucketBuildStatus, prefix, true)
+	deleteWithPrefix(t, bucketBuildTasks, prefix, true)
+	deleteWithPrefix(t, bucketBuild, prefix, true)
+
+	return err
 }
 
 // Subscribed returns true if the user is subscribed

datastore/bolt/repo_del_test.go

@@ -0,0 +1,63 @@
+package bolt
+
+import (
+	"io/ioutil"
+	"os"
+	"testing"
+
+	"github.com/drone/drone/common"
+	. "github.com/franela/goblin"
+)
+
+func TestRepoDel(t *testing.T) {
+	g := Goblin(t)
+	g.Describe("Delete repo", func() {
+
+		var db *DB // temporary database
+
+		user := &common.User{Login: "freya"}
+		repoUri := string("github.com/octopod/hq")
+
+		// create a new database before each unit
+		// test and destroy afterwards.
+		g.BeforeEach(func() {
+			file, err := ioutil.TempFile(os.TempDir(), "drone-bolt")
+			if err != nil {
+				panic(err)
+			}
+
+			db = Must(file.Name())
+		})
+		g.AfterEach(func() {
+			os.Remove(db.Path())
+		})
+
+		g.It("should cleanup", func() {
+			repo := &common.Repo{FullName: repoUri}
+			err := db.SetRepoNotExists(user, repo)
+			g.Assert(err).Equal(nil)
+
+			db.SetBuild(repoUri, &common.Build{State: "success"})
+			db.SetBuild(repoUri, &common.Build{State: "success"})
+			db.SetBuild(repoUri, &common.Build{State: "pending"})
+
+			db.SetBuildStatus(repoUri, 1, &common.Status{Context: "success"})
+			db.SetBuildStatus(repoUri, 2, &common.Status{Context: "success"})
+			db.SetBuildStatus(repoUri, 3, &common.Status{Context: "pending"})
+
+			// first a little sanity to validate our test conditions
+			_, err = db.BuildLast(repoUri)
+			g.Assert(err).Equal(nil)
+
+			// now run our specific test suite
+			// 1. ensure that we can delete the repo
+			err = db.DelRepo(repo)
+			g.Assert(err).Equal(nil)
+
+			// 2. ensure that deleting the repo cleans up other references
+			_, err = db.Build(repoUri, 1)
+			g.Assert(err).Equal(ErrKeyNotFound)
+		})
+	})
+
+}

datastore/bolt/repo_test.go

@@ -1,24 +1,130 @@
 package bolt
 
 import (
-	"testing"
+	"github.com/drone/drone/common"
-
 	. "github.com/franela/goblin"
+	"os"
+	"testing"
 )
 
 func TestRepo(t *testing.T) {
 	g := Goblin(t)
-	g.Describe("Repos", func() {
+	g.Describe("Repo", func() {
+		testUser := "octocat"
+		testRepo := "github.com/octopod/hq"
+		testRepo2 := "github.com/octopod/avengers"
+		var db *DB // Temp database
+
+		// create a new database before each unit
+		// test and destroy afterwards.
+		g.BeforeEach(func() {
+			db = Must("/tmp/drone.test.db")
+		})
+		g.AfterEach(func() {
+			os.Remove(db.Path())
+		})
+
+		g.It("Should set Repo", func() {
+			err := db.SetRepo(&common.Repo{FullName: testRepo})
+			g.Assert(err).Equal(nil)
+
+			repo, err := db.Repo(testRepo)
+			g.Assert(err).Equal(nil)
+			g.Assert(repo.FullName).Equal(testRepo)
+		})
+
+		g.It("Should get Repo", func() {
+			db.SetRepo(&common.Repo{FullName: testRepo})
+
+			repo, err := db.Repo(testRepo)
+			g.Assert(err).Equal(nil)
+			g.Assert(repo.FullName).Equal(testRepo)
+		})
+
+		g.It("Should del Repo", func() {
+			db.SetRepo(&common.Repo{FullName: testRepo})
+
+			db.Repo(testRepo)
+			err_ := db.DelRepo((&common.Repo{FullName: testRepo}))
+			g.Assert(err_).Equal(nil)
+		})
+
+		g.It("Should get RepoList", func() {
+			db.SetRepoNotExists(&common.User{Login: testUser}, &common.Repo{FullName: testRepo})
+			db.SetRepoNotExists(&common.User{Login: testUser}, &common.Repo{FullName: testRepo2})
+
+			repos, err := db.RepoList(testUser)
+			g.Assert(err).Equal(nil)
+			g.Assert(len(repos)).Equal(2)
+		})
+
+		g.It("Should set RepoParams", func() {
+			db.SetRepo(&common.Repo{FullName: testRepo})
+			err := db.SetRepoParams(testRepo, map[string]string{"A": "Alpha"})
+			g.Assert(err).Equal(nil)
+		})
+
+		g.It("Should get RepoParams", func() {
+			db.SetRepo(&common.Repo{FullName: testRepo})
+			err := db.SetRepoParams(testRepo, map[string]string{"A": "Alpha", "B": "Beta"})
+			params, err := db.RepoParams(testRepo)
+			g.Assert(err).Equal(nil)
+			g.Assert(len(params)).Equal(2)
+			g.Assert(params["A"]).Equal("Alpha")
+			g.Assert(params["B"]).Equal("Beta")
+		})
+
+		// we test again with same repo/user already existing
+		// to see if it will return "ErrConflict"
+		g.It("Should set SetRepoNotExists", func() {
+			err := db.SetRepoNotExists(&common.User{Login: testUser}, &common.Repo{FullName: testRepo})
+			g.Assert(err).Equal(nil)
+			// We should get ErrConflict now, trying to add the same repo again.
+			err_ := db.SetRepoNotExists(&common.User{Login: testUser}, &common.Repo{FullName: testRepo})
+			g.Assert(err_).Equal(ErrKeyExists)
+		})
+
+		g.It("Should set RepoKeypair", func() {
+			db.SetRepo(&common.Repo{FullName: testRepo})
+
+			err := db.SetRepoKeypair(testRepo, &common.Keypair{Private: "A", Public: "Alpha"})
+			g.Assert(err).Equal(nil)
+		})
+
+		g.It("Should get RepoKeypair", func() {
+			db.SetRepo(&common.Repo{FullName: testRepo})
+			err := db.SetRepoKeypair(testRepo, &common.Keypair{Private: "A", Public: "Alpha"})
+
+			keypair, err := db.RepoKeypair(testRepo)
+			g.Assert(err).Equal(nil)
+			g.Assert(keypair.Public).Equal("Alpha")
+			g.Assert(keypair.Private).Equal("A")
+		})
+
+		g.It("Should set Subscriber", func() {
+			db.SetRepo(&common.Repo{FullName: testRepo})
+			err := db.SetSubscriber(testUser, testRepo)
+			g.Assert(err).Equal(nil)
+		})
+
+		g.It("Should get Subscribed", func() {
+			db.SetRepo(&common.Repo{FullName: testRepo})
+			err := db.SetSubscriber(testUser, testRepo)
+			subscribed, err := db.Subscribed(testUser, testRepo)
+			g.Assert(err).Equal(nil)
+			g.Assert(subscribed).Equal(true)
+		})
+
+		g.It("Should del Subscriber", func() {
+			db.SetRepo(&common.Repo{FullName: testRepo})
+			db.SetSubscriber(testUser, testRepo)
+			err := db.DelSubscriber(testUser, testRepo)
+			g.Assert(err).Equal(nil)
+
+			subscribed, err := db.Subscribed(testUser, testRepo)
+			g.Assert(subscribed).Equal(false)
+
+		})
 
-		g.It("Should find by name")
-		g.It("Should find params")
-		g.It("Should find keys")
-		g.It("Should delete")
-		g.It("Should insert")
-		g.It("Should not insert if exists")
-		g.It("Should insert params")
-		g.It("Should update params")
-		g.It("Should insert keys")
-		g.It("Should update keys")
 	})
 }

datastore/bolt/task.go

@@ -2,13 +2,18 @@ package bolt
 
 import (
 	"bytes"
+	"github.com/boltdb/bolt"
+	//"github.com/drone/drone/common"
 	"io"
 	"strconv"
-
-	"github.com/boltdb/bolt"
-	"github.com/drone/drone/common"
 )
 
+/*
+	Brad Rydzewski1:00 PM
+	the `Task`, `TaskList` and `SetTask` are deprecated and can be probably be removed.
+	I just need to make sure we aren't still using those functions anywhere else in the code
+*/
+/*
 // GetTask gets the task at index N for the named
 // repository and build number.
 func (db *DB) Task(repo string, build int, task int) (*common.Task, error) {
@@ -63,6 +68,7 @@ func (db *DB) SetTask(repo string, build int, task *common.Task) error {
 		return update(t, bucketBuildTasks, key, task)
 	})
 }
+*/
 
 // SetLogs inserts or updates a task logs for the
 // named repository and build number.

datastore/bolt/task_test.go

@@ -1 +1,89 @@
 package bolt
+
+import (
+	"github.com/drone/drone/common"
+	. "github.com/franela/goblin"
+	"io/ioutil"
+	"os"
+	"testing"
+)
+
+func TestTask(t *testing.T) {
+	g := Goblin(t)
+	g.Describe("Tasks", func() {
+		//testUser := "octocat"
+		testRepo := "github.com/octopod/hq"
+		testBuild := 1
+		testTask := 0
+		//testTask2 := 1
+		testLogInfo := []byte("Log Info for SetLogs()")
+		var db *DB // Temp database
+
+		// create a new database before each unit
+		// test and destroy afterwards.
+		g.BeforeEach(func() {
+			db = Must("/tmp/drone.test.db")
+		})
+		g.AfterEach(func() {
+			os.Remove(db.Path())
+		})
+
+		/*
+			Brad Rydzewski1:00 PM
+			the `Task`, `TaskList` and `SetTask` are deprecated and can be probably be removed.
+			I just need to make sure we aren't still using those functions anywhere else in the code
+		*/
+		/*
+			g.It("Should get TaskList", func() {
+				db.SetRepo(&common.Repo{FullName: testRepo})
+				//db.SetRepoNotExists(&common.User{Login: testUser}, &common.Repo{FullName: testRepo})
+				err := db.SetTask(testRepo, testBuild, &common.Task{Number: testTask})
+				g.Assert(err).Equal(nil)
+				err_ := db.SetTask(testRepo, testBuild, &common.Task{Number: testTask2})
+				g.Assert(err_).Equal(nil)
+				//
+				tasks, err := db.TaskList(testRepo, testBuild)
+				// We seem to have an issue here. TaskList doesn't seem to be returning
+				// All the tasks added to to repo/build. So commenting these for now.
+				//g.Assert(err).Equal(nil)
+				//g.Assert(len(tasks)).Equal(2)
+			})
+
+			g.It("Should set Task", func() {
+				db.SetRepo(&common.Repo{FullName: testRepo})
+				err := db.SetTask(testRepo, testBuild, &common.Task{Number: testTask})
+				g.Assert(err).Equal(nil)
+			})
+
+			g.It("Should get Task", func() {
+				db.SetRepo(&common.Repo{FullName: testRepo})
+				db.SetTask(testRepo, testBuild, &common.Task{Number: testTask})
+				//
+				task, err := db.Task(testRepo, testBuild, testTask)
+				g.Assert(err).Equal(nil)
+				g.Assert(task.Number).Equal(testTask)
+			})
+		*/
+
+		g.It("Should set Logs", func() {
+			db.SetRepo(&common.Repo{FullName: testRepo})
+			//db.SetTask(testRepo, testBuild, &common.Task{Number: testTask})
+			//db.SetTask(testRepo, testBuild, &common.Task{Number: testTask2})
+			//
+			err := db.SetLogs(testRepo, testBuild, testTask, testLogInfo)
+			g.Assert(err).Equal(nil)
+		})
+
+		g.It("Should LogReader", func() {
+			db.SetRepo(&common.Repo{FullName: testRepo})
+			//db.SetTask(testRepo, testBuild, &common.Task{Number: testTask})
+			//db.SetTask(testRepo, testBuild, &common.Task{Number: testTask2})
+			db.SetLogs(testRepo, testBuild, testTask, testLogInfo)
+			//
+			buf, err_ := db.LogReader(testRepo, testBuild, testTask)
+			g.Assert(err_).Equal(nil)
+			logInfo, err_ := ioutil.ReadAll(buf)
+			g.Assert(logInfo).Equal(testLogInfo)
+		})
+	})
+}

datastore/bolt/util.go

@@ -84,3 +84,17 @@ func splice(t *bolt.Tx, bucket, index, value []byte) error {
 
 	return update(t, bucket, index, &keys)
 }
+
+func deleteWithPrefix(t *bolt.Tx, bucket, prefix []byte, ignoreErr bool) error {
+	var err error
+
+	c := t.Bucket(bucket).Cursor()
+	for k, _ := c.Seek(prefix); bytes.HasPrefix(k, prefix); k, _ = c.Next() {
+		err = c.Delete()
+		if !ignoreErr && err != nil {
+			break
+		}
+	}
+
+	return err
+}

datastore/datastore.go

@@ -129,15 +129,15 @@ type Datastore interface {
 
 	// GetTask gets the task at index N for the named
 	// repository and build number.
-	Task(string, int, int) (*common.Task, error)
+	//Task(string, int, int) (*common.Task, error)
 
 	// TaskList gets all tasks for the named repository
 	// and build number.
-	TaskList(string, int) ([]*common.Task, error)
+	//TaskList(string, int) ([]*common.Task, error)
 
 	// SetTask inserts or updates a task for the named
 	// repository and build number.
-	SetTask(string, int, *common.Task) error
+	//SetTask(string, int, *common.Task) error
 
 	// LogReader gets the task logs at index N for
 	// the named repository and build number.