migration improvements

store/datastore/ddl/migrate.go

@@ -2,6 +2,7 @@ package ddl
 
 import (
 	"database/sql"
+	"errors"
 
 	"github.com/drone/drone/store/datastore/ddl/mysql"
 	"github.com/drone/drone/store/datastore/ddl/postgres"
@@ -20,6 +21,9 @@ const (
 func Migrate(driver string, db *sql.DB) error {
 	switch driver {
 	case DriverMysql:
+		if err := checkPriorMigration(db); err != nil {
+			return err
+		}
 		return mysql.Migrate(db)
 	case DriverPostgres:
 		return postgres.Migrate(db)
@@ -27,3 +31,75 @@ func Migrate(driver string, db *sql.DB) error {
 		return sqlite.Migrate(db)
 	}
 }
+
+// we need to check and see if there was a previous migration
+// for drone 0.6 or prior and migrate to the new migration
+// system. Attempting to migrate from 0.5 or below to 0.7 or
+// above will result in an error.
+//
+// this can be removed once we get to 1.0 with the reasonable
+// expectation that people are no longer using 0.5.
+func checkPriorMigration(db *sql.DB) error {
+	var none int
+	if err := db.QueryRow(legacyMigrationsExist).Scan(&none); err != nil {
+		// if no legacy migrations exist, this is a fresh install
+		// and we can proceed as normal.
+		return nil
+	}
+	if err := db.QueryRow(legacyMigrationsCurrent).Scan(&none); err != nil {
+		// this indicates an attempted upgrade from 0.5 or lower to
+		// version 0.7 or higher and will fail.
+		return errors.New("Please upgrade to 0.6 before upgrading to 0.7+")
+	}
+	if _, err := db.Exec(createMigrationsTable); err != nil {
+		return err
+	}
+	if _, err := db.Exec(legacyMigrationsImport); err != nil {
+		return err
+	}
+	return nil
+}
+
+var legacyMigrationsExist = `
+SELECT 1
+FROM gorp_migrations
+LIMIT 1
+`
+
+var legacyMigrationsCurrent = `
+SELECT 1
+FROM gorp_migrations
+WHERE id = '16.sql'
+LIMIT 1
+`
+
+var legacyMigrationsImport = `
+INSERT OR IGNORE INTO migrations (name) VALUES
+ ('create-table-users')
+,('create-table-repos')
+,('create-table-builds')
+,('create-index-builds-repo')
+,('create-index-builds-author')
+,('create-table-procs')
+,('create-index-procs-build')
+,('create-table-logs')
+,('create-table-files')
+,('create-index-files-builds')
+,('create-index-files-procs')
+,('create-table-secrets')
+,('create-index-secrets-repo')
+,('create-table-registry')
+,('create-index-registry-repo')
+,('create-table-config')
+,('create-table-tasks')
+,('create-table-agents')
+,('create-table-senders')
+,('create-index-sender-repos')
+`
+
+var createMigrationsTable = `
+CREATE TABLE IF NOT EXISTS migrations (
+ name VARCHAR(512)
+,UNIQUE(name)
+)
+`

store/datastore/ddl/mysql/ddl_gen.go

@@ -6,87 +6,87 @@ import (
 
 var migrations = []struct {
 	name string
-	stmt []string
+	stmt string
 }{
 	{
-		name: "001_create_table_users.sql",
+		name: "create-table-users",
-		stmt: []string{
+		stmt: createTableUsers,
-			createTableUsers,
-		},
 	},
 	{
-		name: "002_create_table_repos.sql",
+		name: "create-table-repos",
-		stmt: []string{
+		stmt: createTableRepos,
-			createTableRepos,
-		},
 	},
 	{
-		name: "003_create_table_builds.sql",
+		name: "create-table-builds",
-		stmt: []string{
+		stmt: createTableBuilds,
-			createTableBuilds,
-			createIndexBuildsRepo,
-			createIndexBuildsAuthor,
-		},
 	},
 	{
-		name: "004_create_table_procs.sql",
+		name: "create-index-builds-repo",
-		stmt: []string{
+		stmt: createIndexBuildsRepo,
-			createTableProcs,
-			createIndexProcsBuild,
-		},
 	},
 	{
-		name: "005_create_table_logs.sql",
+		name: "create-index-builds-author",
-		stmt: []string{
+		stmt: createIndexBuildsAuthor,
-			createTableLogs,
-		},
 	},
 	{
-		name: "006_create_table_files.sql",
+		name: "create-table-procs",
-		stmt: []string{
+		stmt: createTableProcs,
-			createTableFiles,
-			createIndexFilesBuilds,
-			createIndexFilesProcs,
-		},
 	},
 	{
-		name: "007_create_table_secets.sql",
+		name: "create-index-procs-build",
-		stmt: []string{
+		stmt: createIndexProcsBuild,
-			createTableSecrets,
-			createIndexSecretsRepo,
-		},
 	},
 	{
-		name: "008_create_table_registry.sql",
+		name: "create-table-logs",
-		stmt: []string{
+		stmt: createTableLogs,
-			createTableRegistry,
-			createIndexRegistryRepo,
-		},
 	},
 	{
-		name: "009_create_table_config.sql",
+		name: "create-table-files",
-		stmt: []string{
+		stmt: createTableFiles,
-			createTableConfig,
-		},
 	},
 	{
-		name: "010_create_table_tasks.sql",
+		name: "create-index-files-builds",
-		stmt: []string{
+		stmt: createIndexFilesBuilds,
-			createTableTasks,
-		},
 	},
 	{
-		name: "011_create_table_agents.sql",
+		name: "create-index-files-procs",
-		stmt: []string{
+		stmt: createIndexFilesProcs,
-			createTableAgents,
-		},
 	},
 	{
-		name: "012_create_table_senders.sql",
+		name: "create-table-secrets",
-		stmt: []string{
+		stmt: createTableSecrets,
-			createTableSenders,
-			createIndexSenderRepos,
 	},
+	{
+		name: "create-index-secrets-repo",
+		stmt: createIndexSecretsRepo,
+	},
+	{
+		name: "create-table-registry",
+		stmt: createTableRegistry,
+	},
+	{
+		name: "create-index-registry-repo",
+		stmt: createIndexRegistryRepo,
+	},
+	{
+		name: "create-table-config",
+		stmt: createTableConfig,
+	},
+	{
+		name: "create-table-tasks",
+		stmt: createTableTasks,
+	},
+	{
+		name: "create-table-agents",
+		stmt: createTableAgents,
+	},
+	{
+		name: "create-table-senders",
+		stmt: createTableSenders,
+	},
+	{
+		name: "create-index-sender-repos",
+		stmt: createIndexSenderRepos,
 	},
 }
 
@@ -101,18 +101,18 @@ func Migrate(db *sql.DB) error {
 		return err
 	}
 	for _, migration := range migrations {
-		_, ok := completed[migration.name]
+		if _, ok := completed[migration.name]; ok {
-		if ok {
+
 			continue
 		}
-		for _, stmt := range migration.stmt {
+
-			if _, err := db.Exec(stmt); err != nil {
+		if _, err := db.Exec(migration.stmt); err != nil {
 			return err
 		}
-		}
 		if err := insertMigration(db, migration.name); err != nil {
 			return err
 		}
+
 	}
 	return nil
 }

store/datastore/ddl/postgres/ddl_gen.go

@@ -6,87 +6,87 @@ import (
 
 var migrations = []struct {
 	name string
-	stmt []string
+	stmt string
 }{
 	{
-		name: "001_create_table_users.sql",
+		name: "create-table-users",
-		stmt: []string{
+		stmt: createTableUsers,
-			createTableUsers,
-		},
 	},
 	{
-		name: "002_create_table_repos.sql",
+		name: "create-table-repos",
-		stmt: []string{
+		stmt: createTableRepos,
-			createTableRepos,
-		},
 	},
 	{
-		name: "003_create_table_builds.sql",
+		name: "create-table-builds",
-		stmt: []string{
+		stmt: createTableBuilds,
-			createTableBuilds,
-			createIndexBuildsRepo,
-			createIndexBuildsAuthor,
-		},
 	},
 	{
-		name: "004_create_table_procs.sql",
+		name: "create-index-builds-repo",
-		stmt: []string{
+		stmt: createIndexBuildsRepo,
-			createTableProcs,
-			createIndexProcsBuild,
-		},
 	},
 	{
-		name: "005_create_table_logs.sql",
+		name: "create-index-builds-author",
-		stmt: []string{
+		stmt: createIndexBuildsAuthor,
-			createTableLogs,
-		},
 	},
 	{
-		name: "006_create_table_files.sql",
+		name: "create-table-procs",
-		stmt: []string{
+		stmt: createTableProcs,
-			createTableFiles,
-			createIndexFilesBuilds,
-			createIndexFilesProcs,
-		},
 	},
 	{
-		name: "007_create_table_secets.sql",
+		name: "create-index-procs-build",
-		stmt: []string{
+		stmt: createIndexProcsBuild,
-			createTableSecrets,
-			createIndexSecretsRepo,
-		},
 	},
 	{
-		name: "008_create_table_registry.sql",
+		name: "create-table-logs",
-		stmt: []string{
+		stmt: createTableLogs,
-			createTableRegistry,
-			createIndexRegistryRepo,
-		},
 	},
 	{
-		name: "009_create_table_config.sql",
+		name: "create-table-files",
-		stmt: []string{
+		stmt: createTableFiles,
-			createTableConfig,
-		},
 	},
 	{
-		name: "010_create_table_tasks.sql",
+		name: "create-index-files-builds",
-		stmt: []string{
+		stmt: createIndexFilesBuilds,
-			createTableTasks,
-		},
 	},
 	{
-		name: "011_create_table_agents.sql",
+		name: "create-index-files-procs",
-		stmt: []string{
+		stmt: createIndexFilesProcs,
-			createTableAgents,
-		},
 	},
 	{
-		name: "012_create_table_senders.sql",
+		name: "create-table-secrets",
-		stmt: []string{
+		stmt: createTableSecrets,
-			createTableSenders,
-			createIndexSenderRepos,
 	},
+	{
+		name: "create-index-secrets-repo",
+		stmt: createIndexSecretsRepo,
+	},
+	{
+		name: "create-table-registry",
+		stmt: createTableRegistry,
+	},
+	{
+		name: "create-index-registry-repo",
+		stmt: createIndexRegistryRepo,
+	},
+	{
+		name: "create-table-config",
+		stmt: createTableConfig,
+	},
+	{
+		name: "create-table-tasks",
+		stmt: createTableTasks,
+	},
+	{
+		name: "create-table-agents",
+		stmt: createTableAgents,
+	},
+	{
+		name: "create-table-senders",
+		stmt: createTableSenders,
+	},
+	{
+		name: "create-index-sender-repos",
+		stmt: createIndexSenderRepos,
 	},
 }
 
@@ -101,18 +101,18 @@ func Migrate(db *sql.DB) error {
 		return err
 	}
 	for _, migration := range migrations {
-		_, ok := completed[migration.name]
+		if _, ok := completed[migration.name]; ok {
-		if ok {
+
 			continue
 		}
-		for _, stmt := range migration.stmt {
+
-			if _, err := db.Exec(stmt); err != nil {
+		if _, err := db.Exec(migration.stmt); err != nil {
 			return err
 		}
-		}
 		if err := insertMigration(db, migration.name); err != nil {
 			return err
 		}
+
 	}
 	return nil
 }

store/datastore/ddl/sqlite/ddl_gen.go

@@ -6,88 +6,91 @@ import (
 
 var migrations = []struct {
 	name string
-	stmt []string
+	stmt string
 }{
 	{
-		name: "001_create_table_users.sql",
+		name: "create-table-users",
-		stmt: []string{
+		stmt: createTableUsers,
-			createTableUsers,
-		},
 	},
 	{
-		name: "002_create_table_repos.sql",
+		name: "create-table-repos",
-		stmt: []string{
+		stmt: createTableRepos,
-			createTableRepos,
-		},
 	},
 	{
-		name: "003_create_table_builds.sql",
+		name: "create-table-builds",
-		stmt: []string{
+		stmt: createTableBuilds,
-			createTableBuilds,
-			createIndexBuildsRepo,
-			createIndexBuildsAuthor,
-			createIndexBuildsStatus,
-		},
 	},
 	{
-		name: "004_create_table_procs.sql",
+		name: "create-index-builds-repo",
-		stmt: []string{
+		stmt: createIndexBuildsRepo,
-			createTableProcs,
-			createIndexProcsBuild,
-		},
 	},
 	{
-		name: "005_create_table_logs.sql",
+		name: "create-index-builds-author",
-		stmt: []string{
+		stmt: createIndexBuildsAuthor,
-			createTableLogs,
-		},
 	},
 	{
-		name: "006_create_table_files.sql",
+		name: "create-index-builds-status",
-		stmt: []string{
+		stmt: createIndexBuildsStatus,
-			createTableFiles,
-			createIndexFilesBuilds,
-			createIndexFilesProcs,
-		},
 	},
 	{
-		name: "007_create_table_secets.sql",
+		name: "create-table-procs",
-		stmt: []string{
+		stmt: createTableProcs,
-			createTableSecrets,
-			createIndexSecretsRepo,
-		},
 	},
 	{
-		name: "008_create_table_registry.sql",
+		name: "create-index-procs-build",
-		stmt: []string{
+		stmt: createIndexProcsBuild,
-			createTableRegistry,
-			createIndexRegistryRepo,
-		},
 	},
 	{
-		name: "009_create_table_config.sql",
+		name: "create-table-logs",
-		stmt: []string{
+		stmt: createTableLogs,
-			createTableConfig,
-		},
 	},
 	{
-		name: "010_create_table_tasks.sql",
+		name: "create-table-files",
-		stmt: []string{
+		stmt: createTableFiles,
-			createTableTasks,
-		},
 	},
 	{
-		name: "011_create_table_agents.sql",
+		name: "create-index-files-builds",
-		stmt: []string{
+		stmt: createIndexFilesBuilds,
-			createTableAgents,
-		},
 	},
 	{
-		name: "012_create_table_senders.sql",
+		name: "create-index-files-procs",
-		stmt: []string{
+		stmt: createIndexFilesProcs,
-			createTableSenders,
-			createIndexSenderRepos,
 	},
+	{
+		name: "create-table-secrets",
+		stmt: createTableSecrets,
+	},
+	{
+		name: "create-index-secrets-repo",
+		stmt: createIndexSecretsRepo,
+	},
+	{
+		name: "create-table-registry",
+		stmt: createTableRegistry,
+	},
+	{
+		name: "create-index-registry-repo",
+		stmt: createIndexRegistryRepo,
+	},
+	{
+		name: "create-table-config",
+		stmt: createTableConfig,
+	},
+	{
+		name: "create-table-tasks",
+		stmt: createTableTasks,
+	},
+	{
+		name: "create-table-agents",
+		stmt: createTableAgents,
+	},
+	{
+		name: "create-table-senders",
+		stmt: createTableSenders,
+	},
+	{
+		name: "create-index-sender-repos",
+		stmt: createIndexSenderRepos,
 	},
 }
 
@@ -102,18 +105,18 @@ func Migrate(db *sql.DB) error {
 		return err
 	}
 	for _, migration := range migrations {
-		_, ok := completed[migration.name]
+		if _, ok := completed[migration.name]; ok {
-		if ok {
+
 			continue
 		}
-		for _, stmt := range migration.stmt {
+
-			if _, err := db.Exec(stmt); err != nil {
+		if _, err := db.Exec(migration.stmt); err != nil {
 			return err
 		}
-		}
 		if err := insertMigration(db, migration.name); err != nil {
 			return err
 		}
+
 	}
 	return nil
 }