Merge pull request #83 from fudanchii/migration

Migration system for sqlite
This commit is contained in:
Brad Rydzewski 2014-02-17 20:46:13 -08:00
commit 7ce1e55ac3
12 changed files with 597 additions and 16 deletions

View file

@ -44,6 +44,7 @@ test:
go test -v github.com/drone/drone/pkg/channel
go test -v github.com/drone/drone/pkg/database
go test -v github.com/drone/drone/pkg/database/encrypt
go test -v github.com/drone/drone/pkg/database/migrate
go test -v github.com/drone/drone/pkg/database/testing
go test -v github.com/drone/drone/pkg/mail
go test -v github.com/drone/drone/pkg/model

View file

@ -15,6 +15,7 @@ import (
"github.com/drone/drone/pkg/channel"
"github.com/drone/drone/pkg/database"
"github.com/drone/drone/pkg/database/migrate"
"github.com/drone/drone/pkg/handler"
)
@ -55,8 +56,9 @@ func main() {
// setup the database connection and register with the
// global database package.
func setupDatabase() {
// inform meddler we're using sqlite
// inform meddler and migration we're using sqlite
meddler.Default = meddler.SQLite
migrate.Driver = migrate.SQLite
// connect to the SQLite database
db, err := sql.Open(driver, datasource)
@ -65,6 +67,9 @@ func setupDatabase() {
}
database.Set(db)
migration := migrate.New(db)
migration.All().Migrate()
}
// setup routes for static assets. These assets may

View file

@ -0,0 +1,23 @@
package migrate
type Rev1 struct{}
var RenamePrivelegedToPrivileged = &Rev1{}
func (r *Rev1) Revision() int64 {
return 201402200603
}
func (r *Rev1) Up(op Operation) error {
_, err := op.RenameColumns("repos", map[string]string{
"priveleged": "privileged",
})
return err
}
func (r *Rev1) Down(op Operation) error {
_, err := op.RenameColumns("repos", map[string]string{
"privileged": "priveleged",
})
return err
}

View file

@ -0,0 +1,11 @@
package migrate
func (m *Migration) All() *Migration {
// List all migrations here
m.Add(RenamePrivelegedToPrivileged)
// m.Add(...)
// ...
return m
}

View file

@ -49,17 +49,40 @@ const deleteRevisionStmt = `
DELETE FROM migration where revision = ?
`
// Operation interface covers basic migration operations.
// Implementation details is specific for each database,
// see migrate/sqlite.go for implementation reference.
type Operation interface {
CreateTable(tableName string, args []string) (sql.Result, error)
RenameTable(tableName, newName string) (sql.Result, error)
DropTable(tableName string) (sql.Result, error)
AddColumn(tableName, columnSpec string) (sql.Result, error)
DropColumns(tableName string, columnsToDrop []string) (sql.Result, error)
RenameColumns(tableName string, columnChanges map[string]string) (sql.Result, error)
}
type Revision interface {
Up(tx *sql.Tx) error
Down(tx *sql.Tx) error
Up(op Operation) error
Down(op Operation) error
Revision() int64
}
type MigrationDriver struct {
Tx *sql.Tx
}
type Migration struct {
db *sql.DB
revs []Revision
}
var Driver func(tx *sql.Tx) Operation
func New(db *sql.DB) *Migration {
return &Migration{db: db}
}
@ -119,12 +142,14 @@ func (m *Migration) up(target, current int64) error {
return err
}
op := Driver(tx)
// loop through and execute revisions
for _, rev := range m.revs {
if rev.Revision() >= target {
if rev.Revision() > current && rev.Revision() <= target {
current = rev.Revision()
// execute the revision Upgrade.
if err := rev.Up(tx); err != nil {
if err := rev.Up(op); err != nil {
log.Printf("Failed to upgrade to Revision Number %v\n", current)
log.Println(err)
return tx.Rollback()
@ -150,6 +175,8 @@ func (m *Migration) down(target, current int64) error {
return err
}
op := Driver(tx)
// reverse the list of revisions
revs := []Revision{}
for _, rev := range m.revs {
@ -162,8 +189,8 @@ func (m *Migration) down(target, current int64) error {
if rev.Revision() > target {
current = rev.Revision()
// execute the revision Upgrade.
if err := rev.Down(tx); err != nil {
log.Printf("Failed to downgrade to Revision Number %v\n", current)
if err := rev.Down(op); err != nil {
log.Printf("Failed to downgrade from Revision Number %v\n", current)
log.Println(err)
return tx.Rollback()
}
@ -174,7 +201,7 @@ func (m *Migration) down(target, current int64) error {
return tx.Rollback()
}
log.Printf("Successfully downgraded to Revision %v\n", current)
log.Printf("Successfully downgraded from Revision %v\n", current)
}
}

View file

@ -0,0 +1,152 @@
package migrate
import (
"database/sql"
"fmt"
"strings"
"github.com/dchest/uniuri"
_ "github.com/mattn/go-sqlite3"
)
type SQLiteDriver MigrationDriver
func SQLite(tx *sql.Tx) Operation {
return &SQLiteDriver{Tx: tx}
}
func (s *SQLiteDriver) CreateTable(tableName string, args []string) (sql.Result, error) {
return s.Tx.Exec(fmt.Sprintf("CREATE TABLE %s (%s);", tableName, strings.Join(args, ", ")))
}
func (s *SQLiteDriver) RenameTable(tableName, newName string) (sql.Result, error) {
return s.Tx.Exec(fmt.Sprintf("ALTER TABLE %s RENAME TO %s;", tableName, newName))
}
func (s *SQLiteDriver) DropTable(tableName string) (sql.Result, error) {
return s.Tx.Exec(fmt.Sprintf("DROP TABLE IF EXISTS %s;", tableName))
}
func (s *SQLiteDriver) AddColumn(tableName, columnSpec string) (sql.Result, error) {
return s.Tx.Exec(fmt.Sprintf("ALTER TABLE %s ADD COLUMN %s;", tableName, columnSpec))
}
func (s *SQLiteDriver) DropColumns(tableName string, columnsToDrop []string) (sql.Result, error) {
if len(columnsToDrop) == 0 {
return nil, fmt.Errorf("No columns to drop.")
}
sql, err := s.getDDLFromTable(tableName)
if err != nil {
return nil, err
}
columns, err := fetchColumns(sql)
if err != nil {
return nil, err
}
columnNames := selectName(columns)
var preparedColumns []string
for k, column := range columnNames {
listed := false
for _, dropped := range columnsToDrop {
if column == dropped {
listed = true
break
}
}
if !listed {
preparedColumns = append(preparedColumns, columns[k])
}
}
if len(preparedColumns) == 0 {
return nil, fmt.Errorf("No columns match, drops nothing.")
}
// Rename old table, here's our proxy
proxyName := fmt.Sprintf("%s_%s", tableName, uniuri.NewLen(16))
if result, err := s.RenameTable(tableName, proxyName); err != nil {
return result, err
}
// Recreate table with dropped columns omitted
if result, err := s.CreateTable(tableName, preparedColumns); err != nil {
return result, err
}
// Move data from old table
if result, err := s.Tx.Exec(fmt.Sprintf("INSERT INTO %s SELECT %s FROM %s;", tableName,
strings.Join(selectName(preparedColumns), ", "), proxyName)); err != nil {
return result, err
}
// Clean up proxy table
return s.DropTable(proxyName)
}
func (s *SQLiteDriver) RenameColumns(tableName string, columnChanges map[string]string) (sql.Result, error) {
sql, err := s.getDDLFromTable(tableName)
if err != nil {
return nil, err
}
columns, err := fetchColumns(sql)
if err != nil {
return nil, err
}
// We need a list of columns name to migrate data to the new table
var oldColumnsName = selectName(columns)
// newColumns will be used to create the new table
var newColumns []string
for k, column := range oldColumnsName {
added := false
for Old, New := range columnChanges {
if column == Old {
columnToAdd := strings.Replace(columns[k], Old, New, 1)
newColumns = append(newColumns, columnToAdd)
added = true
break
}
}
if !added {
newColumns = append(newColumns, columns[k])
}
}
// Rename current table
proxyName := fmt.Sprintf("%s_%s", tableName, uniuri.NewLen(16))
if result, err := s.RenameTable(tableName, proxyName); err != nil {
return result, err
}
// Create new table with the new columns
if result, err := s.CreateTable(tableName, newColumns); err != nil {
return result, err
}
// Migrate data
if result, err := s.Tx.Exec(fmt.Sprintf("INSERT INTO %s SELECT %s FROM %s", tableName,
strings.Join(oldColumnsName, ", "), proxyName)); err != nil {
return result, err
}
// Clean up proxy table
return s.DropTable(proxyName)
}
func (s *SQLiteDriver) getDDLFromTable(tableName string) (string, error) {
var sql string
query := `SELECT sql FROM sqlite_master WHERE type='table' and name=?;`
err := s.Tx.QueryRow(query, tableName).Scan(&sql)
if err != nil {
return "", err
}
return sql, nil
}

View file

@ -0,0 +1,325 @@
package migrate
import (
"database/sql"
"os"
"testing"
"github.com/russross/meddler"
)
type Sample struct {
ID int64 `meddler:"id,pk"`
Imel string `meddler:"imel"`
Name string `meddler:"name"`
}
type RenameSample struct {
ID int64 `meddler:"id,pk"`
Email string `meddler:"email"`
Name string `meddler:"name"`
}
type AddColumnSample struct {
ID int64 `meddler:"id,pk"`
Imel string `meddler:"imel"`
Name string `meddler:"name"`
Url string `meddler:"url"`
Num int64 `meddler:"num"`
}
// ---------- revision 1
type revision1 struct{}
func (r *revision1) Up(op Operation) error {
_, err := op.CreateTable("samples", []string{
"id INTEGER PRIMARY KEY AUTOINCREMENT",
"imel VARCHAR(255) UNIQUE",
"name VARCHAR(255)",
})
return err
}
func (r *revision1) Down(op Operation) error {
_, err := op.DropTable("samples")
return err
}
func (r *revision1) Revision() int64 {
return 1
}
// ---------- end of revision 1
// ---------- revision 2
type revision2 struct{}
func (r *revision2) Up(op Operation) error {
_, err := op.RenameTable("samples", "examples")
return err
}
func (r *revision2) Down(op Operation) error {
_, err := op.RenameTable("examples", "samples")
return err
}
func (r *revision2) Revision() int64 {
return 2
}
// ---------- end of revision 2
// ---------- revision 3
type revision3 struct{}
func (r *revision3) Up(op Operation) error {
if _, err := op.AddColumn("samples", "url VARCHAR(255)"); err != nil {
return err
}
_, err := op.AddColumn("samples", "num INTEGER")
return err
}
func (r *revision3) Down(op Operation) error {
_, err := op.DropColumns("samples", []string{"num", "url"})
return err
}
func (r *revision3) Revision() int64 {
return 3
}
// ---------- end of revision 3
// ---------- revision 4
type revision4 struct{}
func (r *revision4) Up(op Operation) error {
_, err := op.RenameColumns("samples", map[string]string{
"imel": "email",
})
return err
}
func (r *revision4) Down(op Operation) error {
_, err := op.RenameColumns("samples", map[string]string{
"email": "imel",
})
return err
}
func (r *revision4) Revision() int64 {
return 4
}
// ----------
var db *sql.DB
var testSchema = `
CREATE TABLE samples (
id INTEGER PRIMARY KEY AUTOINCREMENT,
imel VARCHAR(255) UNIQUE,
name VARCHAR(255)
);
`
var dataDump = []string{
`INSERT INTO samples (imel, name) VALUES ('test@example.com', 'Test Tester');`,
`INSERT INTO samples (imel, name) VALUES ('foo@bar.com', 'Foo Bar');`,
`INSERT INTO samples (imel, name) VALUES ('crash@bandicoot.io', 'Crash Bandicoot');`,
}
func TestMigrateCreateTable(t *testing.T) {
defer tearDown()
if err := setUp(); err != nil {
t.Fatalf("Error preparing database: %q", err)
}
Driver = SQLite
mgr := New(db)
if err := mgr.Add(&revision1{}).Migrate(); err != nil {
t.Errorf("Can not migrate: %q", err)
}
sample := Sample{
ID: 1,
Imel: "test@example.com",
Name: "Test Tester",
}
if err := meddler.Save(db, "samples", &sample); err != nil {
t.Errorf("Can not save data: %q", err)
}
}
func TestMigrateRenameTable(t *testing.T) {
defer tearDown()
if err := setUp(); err != nil {
t.Fatalf("Error preparing database: %q", err)
}
Driver = SQLite
mgr := New(db)
if err := mgr.Add(&revision1{}).Migrate(); err != nil {
t.Errorf("Can not migrate: %q", err)
}
loadFixture(t)
if err := mgr.Add(&revision2{}).Migrate(); err != nil {
t.Errorf("Can not migrate: %q", err)
}
sample := Sample{}
if err := meddler.QueryRow(db, &sample, `SELECT * FROM examples WHERE id = ?`, 2); err != nil {
t.Errorf("Can not fetch data: %q", err)
}
if sample.Imel != "foo@bar.com" {
t.Errorf("Column doesn't match. Expect: %s, got: %s", "foo@bar.com", sample.Imel)
}
}
type TableInfo struct {
CID int64 `meddler:"cid,pk"`
Name string `meddler:"name"`
Type string `meddler:"type"`
Notnull bool `meddler:"notnull"`
DfltValue interface{} `meddler:"dflt_value"`
PK bool `meddler:"pk"`
}
func TestMigrateAddRemoveColumns(t *testing.T) {
defer tearDown()
if err := setUp(); err != nil {
t.Fatalf("Error preparing database: %q", err)
}
Driver = SQLite
mgr := New(db)
if err := mgr.Add(&revision1{}, &revision3{}).Migrate(); err != nil {
t.Errorf("Can not migrate: %q", err)
}
var columns []*TableInfo
if err := meddler.QueryAll(db, &columns, `PRAGMA table_info(samples);`); err != nil {
t.Errorf("Can not access table info: %q", err)
}
if len(columns) < 5 {
t.Errorf("Expect length columns: %d\nGot: %d", 5, len(columns))
}
var row = AddColumnSample{
ID: 33,
Name: "Foo",
Imel: "foo@bar.com",
Url: "http://example.com",
Num: 42,
}
if err := meddler.Save(db, "samples", &row); err != nil {
t.Errorf("Can not save into database: %q", err)
}
if err := mgr.MigrateTo(1); err != nil {
t.Errorf("Can not migrate: %q", err)
}
var another_columns []*TableInfo
if err := meddler.QueryAll(db, &another_columns, `PRAGMA table_info(samples);`); err != nil {
t.Errorf("Can not access table info: %q", err)
}
if len(another_columns) != 3 {
t.Errorf("Expect length columns = %d, got: %d", 3, len(columns))
}
}
func TestRenameColumn(t *testing.T) {
defer tearDown()
if err := setUp(); err != nil {
t.Fatalf("Error preparing database: %q", err)
}
Driver = SQLite
mgr := New(db)
if err := mgr.Add(&revision1{}, &revision4{}).MigrateTo(1); err != nil {
t.Errorf("Can not migrate: %q", err)
}
loadFixture(t)
if err := mgr.MigrateTo(4); err != nil {
t.Errorf("Can not migrate: %q", err)
}
row := RenameSample{}
if err := meddler.QueryRow(db, &row, `SELECT * FROM samples WHERE id = 3;`); err != nil {
t.Errorf("Can not query database: %q", err)
}
if row.Email != "crash@bandicoot.io" {
t.Errorf("Expect %s, got %s", "crash@bandicoot.io", row.Email)
}
}
func TestMigrateExistingTable(t *testing.T) {
defer tearDown()
if err := setUp(); err != nil {
t.Fatalf("Error preparing database: %q", err)
}
Driver = SQLite
if _, err := db.Exec(testSchema); err != nil {
t.Errorf("Can not create database: %q", err)
}
loadFixture(t)
mgr := New(db)
if err := mgr.Add(&revision4{}).Migrate(); err != nil {
t.Errorf("Can not migrate: %q", err)
}
var rows []*RenameSample
if err := meddler.QueryAll(db, &rows, `SELECT * from samples;`); err != nil {
t.Errorf("Can not query database: %q", err)
}
if len(rows) != 3 {
t.Errorf("Expect rows length = %d, got %d", 3, len(rows))
}
if rows[1].Email != "foo@bar.com" {
t.Errorf("Expect email = %s, got %s", "foo@bar.com", rows[1].Email)
}
}
func setUp() error {
var err error
db, err = sql.Open("sqlite3", "migration_tests.sqlite")
return err
}
func tearDown() {
db.Close()
os.Remove("migration_tests.sqlite")
}
func loadFixture(t *testing.T) {
for _, sql := range dataDump {
if _, err := db.Exec(sql); err != nil {
t.Errorf("Can not insert into database: %q", err)
}
}
}

View file

@ -0,0 +1,32 @@
package migrate
import (
"fmt"
"strings"
)
func fetchColumns(sql string) ([]string, error) {
if !strings.HasPrefix(sql, "CREATE TABLE ") {
return []string{}, fmt.Errorf("Sql input is not a DDL statement.")
}
parenIdx := strings.Index(sql, "(")
return strings.Split(sql[parenIdx+1:len(sql)-1], ","), nil
}
func selectName(columns []string) []string {
var results []string
for _, column := range columns {
col := strings.SplitN(strings.Trim(column, " \n\t"), " ", 2)
results = append(results, col[0])
}
return results
}
func setForUpdate(left []string, right []string) string {
var results []string
for k, str := range left {
results = append(results, fmt.Sprintf("%s = %s", str, right[k]))
}
return strings.Join(results, ", ")
}

View file

@ -13,7 +13,7 @@ const repoTable = "repos"
// SQL Queries to retrieve a list of all repos belonging to a User.
const repoStmt = `
SELECT id, slug, host, owner, name, private, disabled, disabled_pr, scm, url, username, password,
public_key, private_key, params, timeout, priveleged, created, updated, user_id, team_id
public_key, private_key, params, timeout, privileged, created, updated, user_id, team_id
FROM repos
WHERE user_id = ? AND team_id = 0
ORDER BY slug ASC
@ -22,7 +22,7 @@ ORDER BY slug ASC
// SQL Queries to retrieve a list of all repos belonging to a Team.
const repoTeamStmt = `
SELECT id, slug, host, owner, name, private, disabled, disabled_pr, scm, url, username, password,
public_key, private_key, params, timeout, priveleged, created, updated, user_id, team_id
public_key, private_key, params, timeout, privileged, created, updated, user_id, team_id
FROM repos
WHERE team_id = ?
ORDER BY slug ASC
@ -31,7 +31,7 @@ ORDER BY slug ASC
// SQL Queries to retrieve a repo by id.
const repoFindStmt = `
SELECT id, slug, host, owner, name, private, disabled, disabled_pr, scm, url, username, password,
public_key, private_key, params, timeout, priveleged, created, updated, user_id, team_id
public_key, private_key, params, timeout, privileged, created, updated, user_id, team_id
FROM repos
WHERE id = ?
`
@ -39,7 +39,7 @@ WHERE id = ?
// SQL Queries to retrieve a repo by name.
const repoFindSlugStmt = `
SELECT id, slug, host, owner, name, private, disabled, disabled_pr, scm, url, username, password,
public_key, private_key, params, timeout, priveleged, created, updated, user_id, team_id
public_key, private_key, params, timeout, privileged, created, updated, user_id, team_id
FROM repos
WHERE slug = ?
`

View file

@ -7,6 +7,7 @@ import (
"github.com/drone/drone/pkg/database"
"github.com/drone/drone/pkg/database/encrypt"
"github.com/drone/drone/pkg/database/migrate"
. "github.com/drone/drone/pkg/model"
_ "github.com/mattn/go-sqlite3"
@ -31,6 +32,7 @@ func init() {
// notify meddler that we are working with sqlite
meddler.Default = meddler.SQLite
migrate.Driver = migrate.SQLite
}
func Setup() {
@ -40,6 +42,9 @@ func Setup() {
// make sure all the tables and indexes are created
database.Set(db)
migration := migrate.New(db)
migration.All().Migrate()
// create dummy user data
user1 := User{
Password: "$2a$10$b8d63QsTL38vx7lj0HEHfOdbu1PCAg6Gfca74UavkXooIBx9YxopS",

View file

@ -41,7 +41,7 @@ func (h UserHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
// AdminHandler wraps the default http.HandlerFunc to include
// the currently authenticated User in the method signature,
// in addition to handling an error as the return value. It also
// verifies the user has Administrative priveleges.
// verifies the user has Administrative privileges.
type AdminHandler func(w http.ResponseWriter, r *http.Request, user *User) error
func (h AdminHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
@ -51,7 +51,7 @@ func (h AdminHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
return
}
// User MUST have administrative priveleges in order
// User MUST have administrative privileges in order
// to execute the handler.
if user.Admin == false {
RenderNotFound(w)

View file

@ -88,9 +88,9 @@ type Repo struct {
// before exceeding its timelimit and being killed.
Timeout int64 `meddler:"timeout" json:"timeout"`
// Indicates the build should be executed in priveleged
// Indicates the build should be executed in privileged
// mode. This could, for example, be used to run Docker in Docker.
Priveleged bool `meddler:"priveleged" json:"priveleged"`
Privileged bool `meddler:"privileged" json:"privileged"`
// Foreign keys signify the User that created
// the repository and team account linked to