From 7bacbd56995c775243e420dcf25d256893cee23d Mon Sep 17 00:00:00 2001 From: qwerty287 <80460567+qwerty287@users.noreply.github.com> Date: Tue, 28 Nov 2023 10:31:54 +0100 Subject: [PATCH] Migrate to Xormigrate (#2711) Co-authored-by: Anbraten --- cmd/server/setup.go | 4 +- go.mod | 1 + go.sum | 7 + server/config.go | 5 +- server/store/datastore/engine.go | 5 +- .../migration/000_legacy_to_xormigrate.go | 33 ++++ ...egacy_to_xorm.go => 001_legacy_to_xorm.go} | 10 +- ...fallback.go => 002_repos_drop_fallback.go} | 7 +- ...03_repos_drop_allow_deploys_allow_tags.go} | 7 +- ...ame.go => 004_fix_pr_secret_event_name.go} | 7 +- ...nter.go => 005_repos_drop_repo_counter.go} | 7 +- ...05_drop_senders.go => 006_drop_senders.go} | 7 +- ..._log_data_type.go => 007_log_data_type.go} | 7 +- ...ts_add_user.go => 008_secrets_add_user.go} | 13 +- ...names.go => 009_lowercase_secret_names.go} | 7 +- ..._table.go => 009_recreate_agents_table.go} | 7 +- ...ne.go => 010_rename_builds_to_pipeline.go} | 8 +- ... 011_columns_rename_builds_to_pipeline.go} | 8 +- ...o => 012_columns_rename_procs_to_steps.go} | 8 +- ...forge.go => 013_rename_remote_to_forge.go} | 14 +- ...014_rename_forge_id_to_forge_remote_id.go} | 8 +- ...ers.go => 015_remove_active_from_users.go} | 8 +- ..._repos.go => 016_remove_inactive_repos.go} | 8 +- ...les_table.go => 017_remove_files_table.go} | 7 +- ...chine_col.go => 018_remove_machine_col.go} | 13 +- ..._drop_old_cols.go => 019_drop_old_cols.go} | 13 +- ..._logs_table.go => 020_alter_logs_table.go} | 60 +++---- ...ws.go => 021_parent_steps_to_workflows.go} | 18 +- .../{021_add_orgs.go => 022_add_orgs.go} | 28 +-- .../{022_add_org_id.go => 023_add_org_id.go} | 8 +- ...ask_data_type.go => 024_task_data_type.go} | 7 +- ...g_data_type.go => 025_config_data_type.go} | 7 +- ... => 026_remove_secrets_plugin_only_col.go} | 13 +- ..._convert_to_new_pipeline_errors_format.go} | 29 +-- ...{027_link_to_url.go => 028_link_to_url.go} | 8 +- server/store/datastore/migration/logger.go | 55 ++++++ server/store/datastore/migration/migration.go | 169 ++---------------- .../datastore/migration/migration_test.go | 9 +- server/store/mocks/store.go | 10 +- server/store/store.go | 2 +- 40 files changed, 305 insertions(+), 347 deletions(-) create mode 100644 server/store/datastore/migration/000_legacy_to_xormigrate.go rename server/store/datastore/migration/{000_legacy_to_xorm.go => 001_legacy_to_xorm.go} (96%) rename server/store/datastore/migration/{001_repos_drop_fallback.go => 002_repos_drop_fallback.go} (79%) rename server/store/datastore/migration/{002_repos_drop_allow_deploys_allow_tags.go => 003_repos_drop_allow_deploys_allow_tags.go} (78%) rename server/store/datastore/migration/{003_fix_pr_secret_event_name.go => 004_fix_pr_secret_event_name.go} (88%) rename server/store/datastore/migration/{004_repos_drop_repo_counter.go => 005_repos_drop_repo_counter.go} (80%) rename server/store/datastore/migration/{005_drop_senders.go => 006_drop_senders.go} (82%) rename server/store/datastore/migration/{006_log_data_type.go => 007_log_data_type.go} (83%) rename server/store/datastore/migration/{007_secrets_add_user.go => 008_secrets_add_user.go} (80%) rename server/store/datastore/migration/{008_lowercase_secret_names.go => 009_lowercase_secret_names.go} (81%) rename server/store/datastore/migration/{008_recreate_agents_table.go => 009_recreate_agents_table.go} (82%) rename server/store/datastore/migration/{009_rename_builds_to_pipeline.go => 010_rename_builds_to_pipeline.go} (83%) rename server/store/datastore/migration/{010_columns_rename_builds_to_pipeline.go => 011_columns_rename_builds_to_pipeline.go} (91%) rename server/store/datastore/migration/{011_columns_rename_procs_to_steps.go => 012_columns_rename_procs_to_steps.go} (91%) rename server/store/datastore/migration/{012_rename_remote_to_forge.go => 013_rename_remote_to_forge.go} (78%) rename server/store/datastore/migration/{013_rename_forge_id_to_forge_remote_id.go => 014_rename_forge_id_to_forge_remote_id.go} (79%) rename server/store/datastore/migration/{014_remove_active_from_users.go => 015_remove_active_from_users.go} (80%) rename server/store/datastore/migration/{015_remove_inactive_repos.go => 016_remove_inactive_repos.go} (84%) rename server/store/datastore/migration/{016_remove_files_table.go => 017_remove_files_table.go} (82%) rename server/store/datastore/migration/{017_remove_machine_col.go => 018_remove_machine_col.go} (76%) rename server/store/datastore/migration/{018_drop_old_cols.go => 019_drop_old_cols.go} (79%) rename server/store/datastore/migration/{019_alter_logs_table.go => 020_alter_logs_table.go} (67%) rename server/store/datastore/migration/{020_parent_steps_to_workflows.go => 021_parent_steps_to_workflows.go} (85%) rename server/store/datastore/migration/{021_add_orgs.go => 022_add_orgs.go} (86%) rename server/store/datastore/migration/{022_add_org_id.go => 023_add_org_id.go} (91%) rename server/store/datastore/migration/{023_task_data_type.go => 024_task_data_type.go} (81%) rename server/store/datastore/migration/{024_config_data_type.go => 025_config_data_type.go} (81%) rename server/store/datastore/migration/{025_remove_secrets_plugin_only_col.go => 026_remove_secrets_plugin_only_col.go} (78%) rename server/store/datastore/migration/{026_convert_to_new_pipeline_errors_format.go => 027_convert_to_new_pipeline_errors_format.go} (74%) rename server/store/datastore/migration/{027_link_to_url.go => 028_link_to_url.go} (83%) create mode 100644 server/store/datastore/migration/logger.go diff --git a/cmd/server/setup.go b/cmd/server/setup.go index 570def50e..8caa0f995 100644 --- a/cmd/server/setup.go +++ b/cmd/server/setup.go @@ -52,8 +52,6 @@ import ( ) func setupStore(c *cli.Context) (store.Store, error) { - // TODO: find a better way than global var to pass down to allow long migrations - server.Config.Server.Migrations.AllowLong = c.Bool("migrations-allow-long") datasource := c.String("datasource") driver := c.String("driver") xorm := store.XORM{ @@ -90,7 +88,7 @@ func setupStore(c *cli.Context) (store.Store, error) { log.Fatal().Err(err).Msg("could not open datastore") } - if err := store.Migrate(); err != nil { + if err := store.Migrate(c.Bool("migrations-allow-long")); err != nil { log.Fatal().Err(err).Msg("could not migrate datastore") } diff --git a/go.mod b/go.mod index fc2c9f07f..f38f3539a 100644 --- a/go.mod +++ b/go.mod @@ -61,6 +61,7 @@ require ( k8s.io/api v0.28.4 k8s.io/apimachinery v0.28.4 k8s.io/client-go v0.28.4 + src.techknowlogick.com/xormigrate v1.7.1 xorm.io/builder v0.3.13 xorm.io/xorm v1.3.4 ) diff --git a/go.sum b/go.sum index 6edd89e75..c56b01939 100644 --- a/go.sum +++ b/go.sum @@ -62,6 +62,7 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davidmz/go-pageant v1.0.2 h1:bPblRCh5jGU+Uptpz6LgMZGD5hJoOt7otgT454WvHn0= github.com/davidmz/go-pageant v1.0.2/go.mod h1:P2EDDnMqIwG5Rrp05dTRITj9z2zpGcD9efWSkTNKLIE= +github.com/denisenkom/go-mssqldb v0.12.3 h1:pBSGx9Tq67pBOTLmxNuirNTeB8Vjmf886Kx+8Y+8shw= github.com/denisenkom/go-mssqldb v0.12.3/go.mod h1:k0mtMFOnU+AihqFxPMiF05rtiDrorD1Vrm1KEz5hxDo= github.com/distribution/reference v0.5.0 h1:/FUIFXtfc/x2gpa5/VGfiGLuOIdYa1t65IKK2OFGvA0= github.com/distribution/reference v0.5.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= @@ -145,7 +146,9 @@ github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v5 v5.1.0 h1:UGKbA/IPjtS6zLcdB7i5TyACMgSbOTiR8qzXgw8HWQU= github.com/golang-jwt/jwt/v5 v5.1.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= +github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe h1:lXe2qZdvpiX5WZkZR4hgp4KJVfY3nMkvmwbVkpv1rVY= github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= +github.com/golang-sql/sqlexp v0.1.0 h1:ZCD6MBpcuOVfGVqsEmY5/4FtYiKz6tSyUv9LPEDei6A= github.com/golang-sql/sqlexp v0.1.0/go.mod h1:J4ad9Vo8ZCWQ2GMrC4UCQy1JpCbwU9m3EOqtpKwwwHI= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= @@ -236,6 +239,7 @@ github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dv github.com/jackc/puddle v1.3.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jellydator/ttlcache/v3 v3.1.0 h1:0gPFG0IHHP6xyUyXq+JaD8fwkDCqgqwohXNJBcYE71g= github.com/jellydator/ttlcache/v3 v3.1.0/go.mod h1:hi7MGFdMAwZna5n2tuvh63DvFLzVKySzCVW6+0gA2n4= +github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= @@ -671,8 +675,11 @@ sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kF sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= +src.techknowlogick.com/xormigrate v1.7.1 h1:RKGLLUAqJ+zO8iZ7eOc7oLH7f0cs2gfXSZSvBRBHnlY= +src.techknowlogick.com/xormigrate v1.7.1/go.mod h1:YGNBdj8prENlySwIKmfoEXp7ILGjAltyKFXD0qLgD7U= xorm.io/builder v0.3.11-0.20220531020008-1bd24a7dc978/go.mod h1:aUW0S9eb9VCaPohFCH3j7czOx1PMW3i1HrSzbLYGBSE= xorm.io/builder v0.3.13 h1:a3jmiVVL19psGeXx8GIurTp7p0IIgqeDmwhcR6BAOAo= xorm.io/builder v0.3.13/go.mod h1:aUW0S9eb9VCaPohFCH3j7czOx1PMW3i1HrSzbLYGBSE= +xorm.io/xorm v1.3.3/go.mod h1:qFJGFoVYbbIdnz2vaL5OxSQ2raleMpyRRalnq3n9OJo= xorm.io/xorm v1.3.4 h1:vWFKzR3DhGUDl5b4srhUjhDwjxkZAc4C7BFszpu0swI= xorm.io/xorm v1.3.4/go.mod h1:qFJGFoVYbbIdnz2vaL5OxSQ2raleMpyRRalnq3n9OJo= diff --git a/server/config.go b/server/config.go index af381b7a9..0c450375f 100644 --- a/server/config.go +++ b/server/config.go @@ -70,10 +70,7 @@ var Config = struct { RootPath string CustomCSSFile string CustomJsFile string - Migrations struct { - AllowLong bool - } - EnableSwagger bool + EnableSwagger bool // Open bool // Orgs map[string]struct{} // Admins map[string]struct{} diff --git a/server/store/datastore/engine.go b/server/store/datastore/engine.go index aba8856ff..e996471a3 100644 --- a/server/store/datastore/engine.go +++ b/server/store/datastore/engine.go @@ -16,6 +16,7 @@ package datastore import ( "github.com/rs/zerolog" + "go.woodpecker-ci.org/woodpecker/server/store" "go.woodpecker-ci.org/woodpecker/server/store/datastore/migration" @@ -54,8 +55,8 @@ func (s storage) Ping() error { } // Migrate old storage or init new one -func (s storage) Migrate() error { - return migration.Migrate(s.engine) +func (s storage) Migrate(allowLong bool) error { + return migration.Migrate(s.engine, allowLong) } func (s storage) Close() error { diff --git a/server/store/datastore/migration/000_legacy_to_xormigrate.go b/server/store/datastore/migration/000_legacy_to_xormigrate.go new file mode 100644 index 000000000..260e06321 --- /dev/null +++ b/server/store/datastore/migration/000_legacy_to_xormigrate.go @@ -0,0 +1,33 @@ +package migration + +import ( + "src.techknowlogick.com/xormigrate" + "xorm.io/xorm" +) + +type v000Migrations struct { + Name string `xorm:"UNIQUE"` +} + +func (m *v000Migrations) TableName() string { + return "migrations" +} + +var legacyToXormigrate = xormigrate.Migration{ + ID: "legacy-to-xormigrate", + MigrateSession: func(sess *xorm.Session) error { + var mig []*v000Migrations + if err := sess.Find(&mig); err != nil { + return err + } + for _, m := range mig { + if _, err := sess.Insert(&xormigrate.Migration{ + ID: m.Name, + }); err != nil { + return err + } + } + + return sess.DropTable("migrations") + }, +} diff --git a/server/store/datastore/migration/000_legacy_to_xorm.go b/server/store/datastore/migration/001_legacy_to_xorm.go similarity index 96% rename from server/store/datastore/migration/000_legacy_to_xorm.go rename to server/store/datastore/migration/001_legacy_to_xorm.go index a56bde1cd..47a526147 100644 --- a/server/store/datastore/migration/000_legacy_to_xorm.go +++ b/server/store/datastore/migration/001_legacy_to_xorm.go @@ -18,14 +18,14 @@ import ( "fmt" "github.com/rs/zerolog/log" + "src.techknowlogick.com/xormigrate" "xorm.io/xorm" "xorm.io/xorm/schemas" ) -var legacy2Xorm = task{ - name: "xorm", - required: true, - fn: func(sess *xorm.Session) error { +var legacy2Xorm = xormigrate.Migration{ + ID: "xorm", + MigrateSession: func(sess *xorm.Session) error { // make sure we have required migrations - else fail and point to last major version for _, mig := range []string{ // users @@ -74,7 +74,7 @@ var legacy2Xorm = task{ "create-table-build-config", "populate-build-config", } { - exist, err := sess.Exist(&migrations{mig}) + exist, err := sess.Exist(&xormigrate.Migration{ID: mig}) if err != nil { return fmt.Errorf("test migration existence: %w", err) } diff --git a/server/store/datastore/migration/001_repos_drop_fallback.go b/server/store/datastore/migration/002_repos_drop_fallback.go similarity index 79% rename from server/store/datastore/migration/001_repos_drop_fallback.go rename to server/store/datastore/migration/002_repos_drop_fallback.go index 4ad5d56b6..111bd3eb0 100644 --- a/server/store/datastore/migration/001_repos_drop_fallback.go +++ b/server/store/datastore/migration/002_repos_drop_fallback.go @@ -15,12 +15,13 @@ package migration import ( + "src.techknowlogick.com/xormigrate" "xorm.io/xorm" ) -var alterTableReposDropFallback = task{ - name: "alter-table-drop-repo-fallback", - fn: func(sess *xorm.Session) error { +var alterTableReposDropFallback = xormigrate.Migration{ + ID: "alter-table-drop-repo-fallback", + MigrateSession: func(sess *xorm.Session) error { return dropTableColumns(sess, "repos", "repo_fallback") }, } diff --git a/server/store/datastore/migration/002_repos_drop_allow_deploys_allow_tags.go b/server/store/datastore/migration/003_repos_drop_allow_deploys_allow_tags.go similarity index 78% rename from server/store/datastore/migration/002_repos_drop_allow_deploys_allow_tags.go rename to server/store/datastore/migration/003_repos_drop_allow_deploys_allow_tags.go index f13be0948..d99610f56 100644 --- a/server/store/datastore/migration/002_repos_drop_allow_deploys_allow_tags.go +++ b/server/store/datastore/migration/003_repos_drop_allow_deploys_allow_tags.go @@ -15,12 +15,13 @@ package migration import ( + "src.techknowlogick.com/xormigrate" "xorm.io/xorm" ) -var alterTableReposDropAllowDeploysAllowTags = task{ - name: "drop-allow-push-tags-deploys-columns", - fn: func(sess *xorm.Session) error { +var alterTableReposDropAllowDeploysAllowTags = xormigrate.Migration{ + ID: "drop-allow-push-tags-deploys-columns", + MigrateSession: func(sess *xorm.Session) error { return dropTableColumns(sess, "repos", "repo_allow_deploys", "repo_allow_tags", diff --git a/server/store/datastore/migration/003_fix_pr_secret_event_name.go b/server/store/datastore/migration/004_fix_pr_secret_event_name.go similarity index 88% rename from server/store/datastore/migration/003_fix_pr_secret_event_name.go rename to server/store/datastore/migration/004_fix_pr_secret_event_name.go index 7f84df3e4..3ec8a464d 100644 --- a/server/store/datastore/migration/003_fix_pr_secret_event_name.go +++ b/server/store/datastore/migration/004_fix_pr_secret_event_name.go @@ -15,14 +15,15 @@ package migration import ( + "src.techknowlogick.com/xormigrate" "xorm.io/xorm" "go.woodpecker-ci.org/woodpecker/server/model" ) -var fixPRSecretEventName = task{ - name: "fix-pr-secret-event-name", - fn: func(sess *xorm.Session) error { +var fixPRSecretEventName = xormigrate.Migration{ + ID: "fix-pr-secret-event-name", + MigrateSession: func(sess *xorm.Session) error { const batchSize = 100 for start := 0; ; start += batchSize { secrets := make([]*model.Secret, 0, batchSize) diff --git a/server/store/datastore/migration/004_repos_drop_repo_counter.go b/server/store/datastore/migration/005_repos_drop_repo_counter.go similarity index 80% rename from server/store/datastore/migration/004_repos_drop_repo_counter.go rename to server/store/datastore/migration/005_repos_drop_repo_counter.go index b2f82d5b6..dc1785e0c 100644 --- a/server/store/datastore/migration/004_repos_drop_repo_counter.go +++ b/server/store/datastore/migration/005_repos_drop_repo_counter.go @@ -15,12 +15,13 @@ package migration import ( + "src.techknowlogick.com/xormigrate" "xorm.io/xorm" ) -var alterTableReposDropCounter = task{ - name: "alter-table-drop-counter", - fn: func(sess *xorm.Session) error { +var alterTableReposDropCounter = xormigrate.Migration{ + ID: "alter-table-drop-counter", + MigrateSession: func(sess *xorm.Session) error { return dropTableColumns(sess, "repos", "repo_counter") }, } diff --git a/server/store/datastore/migration/005_drop_senders.go b/server/store/datastore/migration/006_drop_senders.go similarity index 82% rename from server/store/datastore/migration/005_drop_senders.go rename to server/store/datastore/migration/006_drop_senders.go index 736b18f01..f547bda09 100644 --- a/server/store/datastore/migration/005_drop_senders.go +++ b/server/store/datastore/migration/006_drop_senders.go @@ -15,12 +15,13 @@ package migration import ( + "src.techknowlogick.com/xormigrate" "xorm.io/xorm" ) -var dropSenders = task{ - name: "drop-senders", - fn: func(sess *xorm.Session) error { +var dropSenders = xormigrate.Migration{ + ID: "drop-senders", + MigrateSession: func(sess *xorm.Session) error { return sess.DropTable("senders") }, } diff --git a/server/store/datastore/migration/006_log_data_type.go b/server/store/datastore/migration/007_log_data_type.go similarity index 83% rename from server/store/datastore/migration/006_log_data_type.go rename to server/store/datastore/migration/007_log_data_type.go index 72efef147..cb70c5d2c 100644 --- a/server/store/datastore/migration/006_log_data_type.go +++ b/server/store/datastore/migration/007_log_data_type.go @@ -15,13 +15,14 @@ package migration import ( + "src.techknowlogick.com/xormigrate" "xorm.io/xorm" "xorm.io/xorm/schemas" ) -var alterTableLogUpdateColumnLogDataType = task{ - name: "alter-table-logs-update-type-of-data", - fn: func(sess *xorm.Session) (err error) { +var alterTableLogUpdateColumnLogDataType = xormigrate.Migration{ + ID: "alter-table-logs-update-type-of-data", + MigrateSession: func(sess *xorm.Session) (err error) { dialect := sess.Engine().Dialect().URI().DBType switch dialect { diff --git a/server/store/datastore/migration/007_secrets_add_user.go b/server/store/datastore/migration/008_secrets_add_user.go similarity index 80% rename from server/store/datastore/migration/007_secrets_add_user.go rename to server/store/datastore/migration/008_secrets_add_user.go index 4c1f91ee2..b77208f75 100644 --- a/server/store/datastore/migration/007_secrets_add_user.go +++ b/server/store/datastore/migration/008_secrets_add_user.go @@ -15,24 +15,25 @@ package migration import ( + "src.techknowlogick.com/xormigrate" "xorm.io/xorm" ) -type SecretV007 struct { +type SecretV008 struct { Owner string `json:"-" xorm:"NOT NULL DEFAULT '' UNIQUE(s) INDEX 'secret_owner'"` RepoID int64 `json:"-" xorm:"NOT NULL DEFAULT 0 UNIQUE(s) INDEX 'secret_repo_id'"` Name string `json:"name" xorm:"NOT NULL UNIQUE(s) INDEX 'secret_name'"` } // TableName return database table name for xorm -func (SecretV007) TableName() string { +func (SecretV008) TableName() string { return "secrets" } -var alterTableSecretsAddUserCol = task{ - name: "alter-table-add-secrets-user-id", - fn: func(sess *xorm.Session) error { - if err := sess.Sync(new(SecretV007)); err != nil { +var alterTableSecretsAddUserCol = xormigrate.Migration{ + ID: "alter-table-add-secrets-user-id", + MigrateSession: func(sess *xorm.Session) error { + if err := sess.Sync(new(SecretV008)); err != nil { return err } if err := alterColumnDefault(sess, "secrets", "secret_repo_id", "0"); err != nil { diff --git a/server/store/datastore/migration/008_lowercase_secret_names.go b/server/store/datastore/migration/009_lowercase_secret_names.go similarity index 81% rename from server/store/datastore/migration/008_lowercase_secret_names.go rename to server/store/datastore/migration/009_lowercase_secret_names.go index 24dfabcd2..5402dc36c 100644 --- a/server/store/datastore/migration/008_lowercase_secret_names.go +++ b/server/store/datastore/migration/009_lowercase_secret_names.go @@ -15,12 +15,13 @@ package migration import ( + "src.techknowlogick.com/xormigrate" "xorm.io/xorm" ) -var lowercaseSecretNames = task{ - name: "lowercase-secret-names", - fn: func(sess *xorm.Session) (err error) { +var lowercaseSecretNames = xormigrate.Migration{ + ID: "lowercase-secret-names", + MigrateSession: func(sess *xorm.Session) (err error) { _, err = sess.Exec("UPDATE secrets SET secret_name = LOWER(secret_name);") return err }, diff --git a/server/store/datastore/migration/008_recreate_agents_table.go b/server/store/datastore/migration/009_recreate_agents_table.go similarity index 82% rename from server/store/datastore/migration/008_recreate_agents_table.go rename to server/store/datastore/migration/009_recreate_agents_table.go index cdd7fe6e3..32d5b4d31 100644 --- a/server/store/datastore/migration/008_recreate_agents_table.go +++ b/server/store/datastore/migration/009_recreate_agents_table.go @@ -15,14 +15,15 @@ package migration import ( + "src.techknowlogick.com/xormigrate" "xorm.io/xorm" "go.woodpecker-ci.org/woodpecker/server/model" ) -var recreateAgentsTable = task{ - name: "recreate-agents-table", - fn: func(sess *xorm.Session) error { +var recreateAgentsTable = xormigrate.Migration{ + ID: "recreate-agents-table", + MigrateSession: func(sess *xorm.Session) error { if err := sess.DropTable("agents"); err != nil { return err } diff --git a/server/store/datastore/migration/009_rename_builds_to_pipeline.go b/server/store/datastore/migration/010_rename_builds_to_pipeline.go similarity index 83% rename from server/store/datastore/migration/009_rename_builds_to_pipeline.go rename to server/store/datastore/migration/010_rename_builds_to_pipeline.go index 9f263c6ed..9fd042425 100644 --- a/server/store/datastore/migration/009_rename_builds_to_pipeline.go +++ b/server/store/datastore/migration/010_rename_builds_to_pipeline.go @@ -15,13 +15,13 @@ package migration import ( + "src.techknowlogick.com/xormigrate" "xorm.io/xorm" ) -var renameBuildsToPipeline = task{ - name: "rename-builds-to-pipeline", - required: true, - fn: func(sess *xorm.Session) error { +var renameBuildsToPipeline = xormigrate.Migration{ + ID: "rename-builds-to-pipeline", + MigrateSession: func(sess *xorm.Session) error { err := renameTable(sess, "builds", "pipelines") if err != nil { return err diff --git a/server/store/datastore/migration/010_columns_rename_builds_to_pipeline.go b/server/store/datastore/migration/011_columns_rename_builds_to_pipeline.go similarity index 91% rename from server/store/datastore/migration/010_columns_rename_builds_to_pipeline.go rename to server/store/datastore/migration/011_columns_rename_builds_to_pipeline.go index dd4ca2e68..b80a465e5 100644 --- a/server/store/datastore/migration/010_columns_rename_builds_to_pipeline.go +++ b/server/store/datastore/migration/011_columns_rename_builds_to_pipeline.go @@ -17,6 +17,7 @@ package migration import ( "strings" + "src.techknowlogick.com/xormigrate" "xorm.io/xorm" ) @@ -25,10 +26,9 @@ type oldTable struct { columns []string } -var renameColumnsBuildsToPipeline = task{ - name: "rename-columns-builds-to-pipeline", - required: true, - fn: func(sess *xorm.Session) error { +var renameColumnsBuildsToPipeline = xormigrate.Migration{ + ID: "rename-columns-builds-to-pipeline", + MigrateSession: func(sess *xorm.Session) error { var oldColumns []*oldTable oldColumns = append(oldColumns, &oldTable{ diff --git a/server/store/datastore/migration/011_columns_rename_procs_to_steps.go b/server/store/datastore/migration/012_columns_rename_procs_to_steps.go similarity index 91% rename from server/store/datastore/migration/011_columns_rename_procs_to_steps.go rename to server/store/datastore/migration/012_columns_rename_procs_to_steps.go index 8af344e64..015bdac4f 100644 --- a/server/store/datastore/migration/011_columns_rename_procs_to_steps.go +++ b/server/store/datastore/migration/012_columns_rename_procs_to_steps.go @@ -17,13 +17,13 @@ package migration import ( "strings" + "src.techknowlogick.com/xormigrate" "xorm.io/xorm" ) -var renameTableProcsToSteps = task{ - name: "rename-procs-to-steps", - required: true, - fn: func(sess *xorm.Session) error { +var renameTableProcsToSteps = xormigrate.Migration{ + ID: "rename-procs-to-steps", + MigrateSession: func(sess *xorm.Session) error { err := renameTable(sess, "procs", "steps") if err != nil { return err diff --git a/server/store/datastore/migration/012_rename_remote_to_forge.go b/server/store/datastore/migration/013_rename_remote_to_forge.go similarity index 78% rename from server/store/datastore/migration/012_rename_remote_to_forge.go rename to server/store/datastore/migration/013_rename_remote_to_forge.go index ddcc43ca0..78eff328b 100644 --- a/server/store/datastore/migration/012_rename_remote_to_forge.go +++ b/server/store/datastore/migration/013_rename_remote_to_forge.go @@ -15,28 +15,28 @@ package migration import ( + "src.techknowlogick.com/xormigrate" "xorm.io/xorm" ) -type oldRepo012 struct { +type oldRepo013 struct { ID int64 `xorm:"pk autoincr 'repo_id'"` RemoteID string `xorm:"remote_id"` } -func (oldRepo012) TableName() string { +func (oldRepo013) TableName() string { return "repos" } -var renameRemoteToForge = task{ - name: "rename-remote-to-forge", - required: true, - fn: func(sess *xorm.Session) error { +var renameRemoteToForge = xormigrate.Migration{ + ID: "rename-remote-to-forge", + MigrateSession: func(sess *xorm.Session) error { if err := renameColumn(sess, "pipelines", "pipeline_remote", "pipeline_clone_url"); err != nil { return err } // make sure the column exist before rename it - if err := sess.Sync(new(oldRepo012)); err != nil { + if err := sess.Sync(new(oldRepo013)); err != nil { return err } diff --git a/server/store/datastore/migration/013_rename_forge_id_to_forge_remote_id.go b/server/store/datastore/migration/014_rename_forge_id_to_forge_remote_id.go similarity index 79% rename from server/store/datastore/migration/013_rename_forge_id_to_forge_remote_id.go rename to server/store/datastore/migration/014_rename_forge_id_to_forge_remote_id.go index 4065b3003..60a58660d 100644 --- a/server/store/datastore/migration/013_rename_forge_id_to_forge_remote_id.go +++ b/server/store/datastore/migration/014_rename_forge_id_to_forge_remote_id.go @@ -15,13 +15,13 @@ package migration import ( + "src.techknowlogick.com/xormigrate" "xorm.io/xorm" ) -var renameForgeIDToForgeRemoteID = task{ - name: "rename-forge-id-to-forge-remote-id", - required: true, - fn: func(sess *xorm.Session) error { +var renameForgeIDToForgeRemoteID = xormigrate.Migration{ + ID: "rename-forge-id-to-forge-remote-id", + MigrateSession: func(sess *xorm.Session) error { return renameColumn(sess, "repos", "forge_id", "forge_remote_id") }, } diff --git a/server/store/datastore/migration/014_remove_active_from_users.go b/server/store/datastore/migration/015_remove_active_from_users.go similarity index 80% rename from server/store/datastore/migration/014_remove_active_from_users.go rename to server/store/datastore/migration/015_remove_active_from_users.go index 41bb21af0..d0eb785d1 100644 --- a/server/store/datastore/migration/014_remove_active_from_users.go +++ b/server/store/datastore/migration/015_remove_active_from_users.go @@ -15,13 +15,13 @@ package migration import ( + "src.techknowlogick.com/xormigrate" "xorm.io/xorm" ) -var removeActiveFromUsers = task{ - name: "remove-active-from-users", - required: true, - fn: func(sess *xorm.Session) error { +var removeActiveFromUsers = xormigrate.Migration{ + ID: "remove-active-from-users", + MigrateSession: func(sess *xorm.Session) error { return dropTableColumns(sess, "users", "user_active") }, } diff --git a/server/store/datastore/migration/015_remove_inactive_repos.go b/server/store/datastore/migration/016_remove_inactive_repos.go similarity index 84% rename from server/store/datastore/migration/015_remove_inactive_repos.go rename to server/store/datastore/migration/016_remove_inactive_repos.go index 1c6de6b29..48742f89b 100644 --- a/server/store/datastore/migration/015_remove_inactive_repos.go +++ b/server/store/datastore/migration/016_remove_inactive_repos.go @@ -15,13 +15,13 @@ package migration import ( + "src.techknowlogick.com/xormigrate" "xorm.io/xorm" ) -var removeInactiveRepos = task{ - name: "remove-inactive-repos", - required: true, - fn: func(sess *xorm.Session) error { +var removeInactiveRepos = xormigrate.Migration{ + ID: "remove-inactive-repos", + MigrateSession: func(sess *xorm.Session) error { // If the timeout is 0, the repo was never activated, so we remove it. _, err := sess.Table("repos").Where("repo_active = ?", false).And("repo_timeout = ?", 0).Delete() if err != nil { diff --git a/server/store/datastore/migration/016_remove_files_table.go b/server/store/datastore/migration/017_remove_files_table.go similarity index 82% rename from server/store/datastore/migration/016_remove_files_table.go rename to server/store/datastore/migration/017_remove_files_table.go index 9396ba030..8cc99b806 100644 --- a/server/store/datastore/migration/016_remove_files_table.go +++ b/server/store/datastore/migration/017_remove_files_table.go @@ -15,12 +15,13 @@ package migration import ( + "src.techknowlogick.com/xormigrate" "xorm.io/xorm" ) -var dropFiles = task{ - name: "drop-files", - fn: func(sess *xorm.Session) error { +var dropFiles = xormigrate.Migration{ + ID: "drop-files", + MigrateSession: func(sess *xorm.Session) error { return sess.DropTable("files") }, } diff --git a/server/store/datastore/migration/017_remove_machine_col.go b/server/store/datastore/migration/018_remove_machine_col.go similarity index 76% rename from server/store/datastore/migration/017_remove_machine_col.go rename to server/store/datastore/migration/018_remove_machine_col.go index 9c02121ce..791104479 100644 --- a/server/store/datastore/migration/017_remove_machine_col.go +++ b/server/store/datastore/migration/018_remove_machine_col.go @@ -15,23 +15,24 @@ package migration import ( + "src.techknowlogick.com/xormigrate" "xorm.io/xorm" ) -type oldStep017 struct { +type oldStep018 struct { ID int64 `xorm:"pk autoincr 'step_id'"` Machine string `xorm:"step_machine"` } -func (oldStep017) TableName() string { +func (oldStep018) TableName() string { return "steps" } -var removeMachineCol = task{ - name: "remove-machine-col", - fn: func(sess *xorm.Session) error { +var removeMachineCol = xormigrate.Migration{ + ID: "remove-machine-col", + MigrateSession: func(sess *xorm.Session) error { // make sure step_machine column exists - if err := sess.Sync(new(oldStep017)); err != nil { + if err := sess.Sync(new(oldStep018)); err != nil { return err } return dropTableColumns(sess, "steps", "step_machine") diff --git a/server/store/datastore/migration/018_drop_old_cols.go b/server/store/datastore/migration/019_drop_old_cols.go similarity index 79% rename from server/store/datastore/migration/018_drop_old_cols.go rename to server/store/datastore/migration/019_drop_old_cols.go index c13a4091d..f380c924e 100644 --- a/server/store/datastore/migration/018_drop_old_cols.go +++ b/server/store/datastore/migration/019_drop_old_cols.go @@ -15,24 +15,25 @@ package migration import ( + "src.techknowlogick.com/xormigrate" "xorm.io/xorm" ) -type oldPipeline018 struct { +type oldPipeline019 struct { ID int64 `xorm:"pk autoincr 'pipeline_id'"` Signed bool `xorm:"pipeline_signed"` Verified bool `xorm:"pipeline_verified"` } -func (oldPipeline018) TableName() string { +func (oldPipeline019) TableName() string { return "pipelines" } -var dropOldCols = task{ - name: "drop-old-col", - fn: func(sess *xorm.Session) error { +var dropOldCols = xormigrate.Migration{ + ID: "drop-old-col", + MigrateSession: func(sess *xorm.Session) error { // make sure columns on pipelines exist - if err := sess.Sync(new(oldPipeline018)); err != nil { + if err := sess.Sync(new(oldPipeline019)); err != nil { return err } if err := dropTableColumns(sess, "steps", "step_pgid"); err != nil { diff --git a/server/store/datastore/migration/019_alter_logs_table.go b/server/store/datastore/migration/020_alter_logs_table.go similarity index 67% rename from server/store/datastore/migration/019_alter_logs_table.go rename to server/store/datastore/migration/020_alter_logs_table.go index b9c6fd97d..431e5a18c 100644 --- a/server/store/datastore/migration/019_alter_logs_table.go +++ b/server/store/datastore/migration/020_alter_logs_table.go @@ -22,29 +22,26 @@ import ( "github.com/rs/zerolog/log" "github.com/tevino/abool/v2" + "src.techknowlogick.com/xormigrate" "xorm.io/xorm" - "go.woodpecker-ci.org/woodpecker/server" "go.woodpecker-ci.org/woodpecker/shared/utils" ) -// maxDefaultSqliteItems set the threshold at witch point the migration will fail by default -var maxDefaultSqliteItems019 = 5000 +// perPage020 sets the size of the slice to read per page +var perPage020 = 100 -// perPage019 set the size of the slice to read per page -var perPage019 = 100 - -type oldLogs019 struct { +type oldLogs020 struct { ID int64 `xorm:"pk autoincr 'log_id'"` StepID int64 `xorm:"UNIQUE 'log_step_id'"` Data []byte `xorm:"LONGBLOB 'log_data'"` } -func (oldLogs019) TableName() string { +func (oldLogs020) TableName() string { return "logs" } -type oldLogEntry019 struct { +type oldLogEntry020 struct { Step string `json:"step,omitempty"` Time int64 `json:"time,omitempty"` Type int `json:"type,omitempty"` @@ -52,7 +49,7 @@ type oldLogEntry019 struct { Out string `json:"out,omitempty"` } -type newLogEntry019 struct { +type newLogEntry020 struct { ID int64 `xorm:"pk autoincr 'id'"` StepID int64 `xorm:"'step_id'"` Time int64 @@ -62,38 +59,27 @@ type newLogEntry019 struct { Type int } -func (newLogEntry019) TableName() string { +func (newLogEntry020) TableName() string { return "log_entries" } -var initLogsEntriesTable = task{ - name: "init-log_entries", - required: true, - fn: func(sess *xorm.Session) error { - return sess.Sync(new(newLogEntry019)) +var initLogsEntriesTable = xormigrate.Migration{ + ID: "init-log_entries", + MigrateSession: func(sess *xorm.Session) error { + return sess.Sync(new(newLogEntry020)) }, } -var migrateLogs2LogEntries = task{ - name: "migrate-logs-to-log_entries", - required: false, - engineFn: func(e *xorm.Engine) error { +var migrateLogs2LogEntries = xormigrate.Migration{ + ID: "migrate-logs-to-log_entries", + Long: true, + Migrate: func(e *xorm.Engine) error { // make sure old logs table exists - if exist, err := e.IsTableExist(new(oldLogs019)); !exist || err != nil { + if exist, err := e.IsTableExist(new(oldLogs020)); !exist || err != nil { return err } - // first we check if we have just 1000 entries to migrate - toMigrate, err := e.Count(new(oldLogs019)) - if err != nil { - return err - } - - if toMigrate > int64(maxDefaultSqliteItems019) && !server.Config.Server.Migrations.AllowLong { - return fmt.Errorf("Migrating logs to log_entries is skipped, as we have %d entries to convert. Set 'WOODPECKER_MIGRATIONS_ALLOW_LONG' to 'true' to migrate anyway", toMigrate) - } - - if err := e.Sync(new(oldLogs019)); err != nil { + if err := e.Sync(new(oldLogs020)); err != nil { return err } @@ -101,8 +87,8 @@ var migrateLogs2LogEntries = task{ page := 0 offset := 0 - logs := make([]*oldLogs019, 0, perPage019) - logEntries := make([]*oldLogEntry019, 0, 50) + logs := make([]*oldLogs020, 0, perPage020) + logEntries := make([]*oldLogEntry020, 0, 50) sigterm := abool.New() ctx, cancelCtx := context.WithCancelCause(context.Background()) @@ -124,7 +110,7 @@ var migrateLogs2LogEntries = task{ } logs = logs[:0] - err := sess.Limit(perPage019, offset).Find(&logs) + err := sess.Limit(perPage020, offset).Find(&logs) if err != nil { return err } @@ -146,7 +132,7 @@ var migrateLogs2LogEntries = task{ time = logEntry.Time } - log := &newLogEntry019{ + log := &newLogEntry020{ StepID: l.StepID, Data: []byte(logEntry.Out), Line: logEntry.Pos, @@ -168,7 +154,7 @@ var migrateLogs2LogEntries = task{ return err } - if len(logs) < perPage019 { + if len(logs) < perPage020 { break } diff --git a/server/store/datastore/migration/020_parent_steps_to_workflows.go b/server/store/datastore/migration/021_parent_steps_to_workflows.go similarity index 85% rename from server/store/datastore/migration/020_parent_steps_to_workflows.go rename to server/store/datastore/migration/021_parent_steps_to_workflows.go index e763d9338..ac0325e8b 100644 --- a/server/store/datastore/migration/020_parent_steps_to_workflows.go +++ b/server/store/datastore/migration/021_parent_steps_to_workflows.go @@ -15,12 +15,13 @@ package migration import ( + "src.techknowlogick.com/xormigrate" "xorm.io/xorm" "go.woodpecker-ci.org/woodpecker/server/model" ) -type oldStep020 struct { +type oldStep021 struct { ID int64 `xorm:"pk autoincr 'step_id'"` PipelineID int64 `xorm:"UNIQUE(s) INDEX 'step_pipeline_id'"` PID int `xorm:"UNIQUE(s) 'step_pid'"` @@ -35,23 +36,22 @@ type oldStep020 struct { Environ map[string]string `xorm:"json 'step_environ'"` } -func (oldStep020) TableName() string { +func (oldStep021) TableName() string { return "steps" } -var parentStepsToWorkflows = task{ - name: "parent-steps-to-workflows", - required: true, - fn: func(sess *xorm.Session) error { +var parentStepsToWorkflows = xormigrate.Migration{ + ID: "parent-steps-to-workflows", + MigrateSession: func(sess *xorm.Session) error { if err := sess.Sync(new(model.Workflow)); err != nil { return err } // make sure the columns exist before removing them - if err := sess.Sync(new(oldStep020)); err != nil { + if err := sess.Sync(new(oldStep021)); err != nil { return err } - var parentSteps []*oldStep020 + var parentSteps []*oldStep021 err := sess.Where("step_ppid = ?", 0).Find(&parentSteps) if err != nil { return err @@ -76,7 +76,7 @@ var parentStepsToWorkflows = task{ return err } - _, err = sess.Delete(&oldStep020{ID: p.ID}) + _, err = sess.Delete(&oldStep021{ID: p.ID}) if err != nil { return err } diff --git a/server/store/datastore/migration/021_add_orgs.go b/server/store/datastore/migration/022_add_orgs.go similarity index 86% rename from server/store/datastore/migration/021_add_orgs.go rename to server/store/datastore/migration/022_add_orgs.go index 6cea0f88d..8f56eb93f 100644 --- a/server/store/datastore/migration/021_add_orgs.go +++ b/server/store/datastore/migration/022_add_orgs.go @@ -18,13 +18,14 @@ import ( "fmt" "strings" + "src.techknowlogick.com/xormigrate" "xorm.io/builder" "xorm.io/xorm" "go.woodpecker-ci.org/woodpecker/server/model" ) -type oldSecret021 struct { +type oldSecret022 struct { ID int64 `xorm:"pk autoincr 'secret_id'"` Owner string `xorm:"'secret_owner'"` OrgID int64 `xorm:"NOT NULL DEFAULT 0 'secret_org_id'"` @@ -32,51 +33,50 @@ type oldSecret021 struct { Name string `xorm:"NOT NULL INDEX 'secret_name'"` } -func (oldSecret021) TableName() string { +func (oldSecret022) TableName() string { return "secrets" } -type syncRepo021 struct { +type syncRepo022 struct { OrgID int64 `json:"org_id" xorm:"repo_org_id"` } // TableName return database table name for xorm -func (syncRepo021) TableName() string { +func (syncRepo022) TableName() string { return "repos" } -type repo021 struct { +type repo022 struct { ID int64 `json:"id,omitempty" xorm:"pk autoincr 'repo_id'"` OrgID int64 `json:"org_id" xorm:"repo_org_id"` Owner string `json:"owner" xorm:"UNIQUE(name) 'repo_owner'"` } // TableName return database table name for xorm -func (repo021) TableName() string { +func (repo022) TableName() string { return "repos" } -var addOrgs = task{ - name: "add-orgs", - required: true, - fn: func(sess *xorm.Session) error { +var addOrgs = xormigrate.Migration{ + ID: "add-orgs", + MigrateSession: func(sess *xorm.Session) error { if exist, err := sess.IsTableExist("orgs"); exist && err == nil { if err := sess.DropTable("orgs"); err != nil { return fmt.Errorf("drop old orgs table failed: %w", err) } } - if err := sess.Sync(new(model.Org), new(syncRepo021), new(model.User)); err != nil { + if err := sess.Sync(new(model.Org), new(syncRepo022), new(model.User)); err != nil { return fmt.Errorf("sync new models failed: %w", err) } // make sure the columns exist before removing them - if _, err := sess.SyncWithOptions(xorm.SyncOptions{IgnoreConstrains: true, IgnoreIndices: true}, new(oldSecret021)); err != nil { + if _, err := sess.SyncWithOptions(xorm.SyncOptions{IgnoreConstrains: true, IgnoreIndices: true}, new(oldSecret022)); err != nil { return fmt.Errorf("sync old secrets models failed: %w", err) } // get all org names from repos - var repos []*repo021 + var repos []*repo022 if err := sess.Find(&repos); err != nil { return fmt.Errorf("find all repos failed: %w", err) } @@ -107,7 +107,7 @@ var addOrgs = task{ orgs[orgName] = org // update org secrets - var secrets []*oldSecret021 + var secrets []*oldSecret022 if err := sess.Where(builder.Eq{"secret_owner": orgName, "secret_repo_id": 0}).Find(&secrets); err != nil { return fmt.Errorf("get org secrets failed: %w", err) } diff --git a/server/store/datastore/migration/022_add_org_id.go b/server/store/datastore/migration/023_add_org_id.go similarity index 91% rename from server/store/datastore/migration/022_add_org_id.go rename to server/store/datastore/migration/023_add_org_id.go index 4f130610b..c9e49f4b9 100644 --- a/server/store/datastore/migration/022_add_org_id.go +++ b/server/store/datastore/migration/023_add_org_id.go @@ -17,15 +17,15 @@ package migration import ( "fmt" + "src.techknowlogick.com/xormigrate" "xorm.io/xorm" "go.woodpecker-ci.org/woodpecker/server/model" ) -var addOrgID = task{ - name: "add-org-id", - required: true, - fn: func(sess *xorm.Session) error { +var addOrgID = xormigrate.Migration{ + ID: "add-org-id", + MigrateSession: func(sess *xorm.Session) error { if err := sess.Sync(new(model.User)); err != nil { return fmt.Errorf("sync new models failed: %w", err) } diff --git a/server/store/datastore/migration/023_task_data_type.go b/server/store/datastore/migration/024_task_data_type.go similarity index 81% rename from server/store/datastore/migration/023_task_data_type.go rename to server/store/datastore/migration/024_task_data_type.go index 9b788c7e4..e2e1a50a2 100644 --- a/server/store/datastore/migration/023_task_data_type.go +++ b/server/store/datastore/migration/024_task_data_type.go @@ -15,13 +15,14 @@ package migration import ( + "src.techknowlogick.com/xormigrate" "xorm.io/xorm" "xorm.io/xorm/schemas" ) -var alterTableTasksUpdateColumnTaskDataType = task{ - name: "alter-table-tasks-update-type-of-task-data", - fn: func(sess *xorm.Session) (err error) { +var alterTableTasksUpdateColumnTaskDataType = xormigrate.Migration{ + ID: "alter-table-tasks-update-type-of-task-data", + MigrateSession: func(sess *xorm.Session) (err error) { dialect := sess.Engine().Dialect().URI().DBType switch dialect { diff --git a/server/store/datastore/migration/024_config_data_type.go b/server/store/datastore/migration/025_config_data_type.go similarity index 81% rename from server/store/datastore/migration/024_config_data_type.go rename to server/store/datastore/migration/025_config_data_type.go index a0fcfa91d..296d20d0f 100644 --- a/server/store/datastore/migration/024_config_data_type.go +++ b/server/store/datastore/migration/025_config_data_type.go @@ -15,13 +15,14 @@ package migration import ( + "src.techknowlogick.com/xormigrate" "xorm.io/xorm" "xorm.io/xorm/schemas" ) -var alterTableConfigUpdateColumnConfigDataType = task{ - name: "alter-table-config-update-type-of-config-data", - fn: func(sess *xorm.Session) (err error) { +var alterTableConfigUpdateColumnConfigDataType = xormigrate.Migration{ + ID: "alter-table-config-update-type-of-config-data", + MigrateSession: func(sess *xorm.Session) (err error) { dialect := sess.Engine().Dialect().URI().DBType switch dialect { diff --git a/server/store/datastore/migration/025_remove_secrets_plugin_only_col.go b/server/store/datastore/migration/026_remove_secrets_plugin_only_col.go similarity index 78% rename from server/store/datastore/migration/025_remove_secrets_plugin_only_col.go rename to server/store/datastore/migration/026_remove_secrets_plugin_only_col.go index 5978de16d..082744cca 100644 --- a/server/store/datastore/migration/025_remove_secrets_plugin_only_col.go +++ b/server/store/datastore/migration/026_remove_secrets_plugin_only_col.go @@ -15,10 +15,11 @@ package migration import ( + "src.techknowlogick.com/xormigrate" "xorm.io/xorm" ) -type oldSecret025 struct { +type oldSecret026 struct { ID int64 `json:"id" xorm:"pk autoincr 'secret_id'"` PluginsOnly bool `json:"plugins_only" xorm:"secret_plugins_only"` SkipVerify bool `json:"-" xorm:"secret_skip_verify"` @@ -26,15 +27,15 @@ type oldSecret025 struct { Images []string `json:"images" xorm:"json 'secret_images'"` } -func (oldSecret025) TableName() string { +func (oldSecret026) TableName() string { return "secrets" } -var removePluginOnlyOptionFromSecretsTable = task{ - name: "remove-plugin-only-option-from-secrets-table", - fn: func(sess *xorm.Session) (err error) { +var removePluginOnlyOptionFromSecretsTable = xormigrate.Migration{ + ID: "remove-plugin-only-option-from-secrets-table", + MigrateSession: func(sess *xorm.Session) (err error) { // make sure plugin_only column exists - if err := sess.Sync(new(oldSecret025)); err != nil { + if err := sess.Sync(new(oldSecret026)); err != nil { return err } diff --git a/server/store/datastore/migration/026_convert_to_new_pipeline_errors_format.go b/server/store/datastore/migration/027_convert_to_new_pipeline_errors_format.go similarity index 74% rename from server/store/datastore/migration/026_convert_to_new_pipeline_errors_format.go rename to server/store/datastore/migration/027_convert_to_new_pipeline_errors_format.go index c52000101..8d3cc05d9 100644 --- a/server/store/datastore/migration/026_convert_to_new_pipeline_errors_format.go +++ b/server/store/datastore/migration/027_convert_to_new_pipeline_errors_format.go @@ -15,53 +15,54 @@ package migration import ( + "src.techknowlogick.com/xormigrate" "xorm.io/xorm" "go.woodpecker-ci.org/woodpecker/pipeline/errors" ) -// perPage026 set the size of the slice to read per page -var perPage026 = 100 +// perPage027 set the size of the slice to read per page +var perPage027 = 100 -type pipeline026 struct { +type pipeline027 struct { ID int64 `json:"id" xorm:"pk autoincr 'pipeline_id'"` Error string `json:"error" xorm:"LONGTEXT 'pipeline_error'"` // old error format Errors []*errors.PipelineError `json:"errors" xorm:"json 'pipeline_errors'"` // new error format } -func (pipeline026) TableName() string { +func (pipeline027) TableName() string { return "pipelines" } -type PipelineError026 struct { +type PipelineError027 struct { Type string `json:"type"` Message string `json:"message"` IsWarning bool `json:"is_warning"` Data any `json:"data"` } -var convertToNewPipelineErrorFormat = task{ - name: "convert-to-new-pipeline-error-format", - required: true, - fn: func(sess *xorm.Session) (err error) { +var convertToNewPipelineErrorFormat = xormigrate.Migration{ + ID: "convert-to-new-pipeline-error-format", + Long: true, + MigrateSession: func(sess *xorm.Session) (err error) { // make sure pipeline_error column exists - if err := sess.Sync(new(pipeline026)); err != nil { + if err := sess.Sync(new(pipeline027)); err != nil { return err } page := 0 - oldPipelines := make([]*pipeline026, 0, perPage026) + oldPipelines := make([]*pipeline027, 0, perPage027) for { oldPipelines = oldPipelines[:0] - err := sess.Limit(perPage026, page*perPage026).Cols("pipeline_id", "pipeline_error").Where("pipeline_error != ''").Find(&oldPipelines) + err := sess.Limit(perPage027, page*perPage027).Cols("pipeline_id", "pipeline_error").Where("pipeline_error != ''").Find(&oldPipelines) if err != nil { return err } for _, oldPipeline := range oldPipelines { - var newPipeline pipeline026 + var newPipeline pipeline027 newPipeline.ID = oldPipeline.ID newPipeline.Errors = []*errors.PipelineError{{ Type: "generic", @@ -73,7 +74,7 @@ var convertToNewPipelineErrorFormat = task{ } } - if len(oldPipelines) < perPage026 { + if len(oldPipelines) < perPage027 { break } diff --git a/server/store/datastore/migration/027_link_to_url.go b/server/store/datastore/migration/028_link_to_url.go similarity index 83% rename from server/store/datastore/migration/027_link_to_url.go rename to server/store/datastore/migration/028_link_to_url.go index 51b3deb05..aaf5e65ce 100644 --- a/server/store/datastore/migration/027_link_to_url.go +++ b/server/store/datastore/migration/028_link_to_url.go @@ -15,13 +15,13 @@ package migration import ( + "src.techknowlogick.com/xormigrate" "xorm.io/xorm" ) -var renameLinkToURL = task{ - name: "rename-link-to-url", - required: true, - fn: func(sess *xorm.Session) (err error) { +var renameLinkToURL = xormigrate.Migration{ + ID: "rename-link-to-url", + MigrateSession: func(sess *xorm.Session) (err error) { if err := renameColumn(sess, "pipelines", "pipeline_link", "pipeline_forge_url"); err != nil { return err } diff --git a/server/store/datastore/migration/logger.go b/server/store/datastore/migration/logger.go new file mode 100644 index 000000000..a9493307d --- /dev/null +++ b/server/store/datastore/migration/logger.go @@ -0,0 +1,55 @@ +// Copyright 2023 Woodpecker Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package migration + +import ( + "fmt" + + "github.com/rs/zerolog/log" +) + +type xormigrateLogger struct{} + +func (l *xormigrateLogger) Debug(v ...interface{}) { + log.Debug().Msg(fmt.Sprint(v...)) +} + +func (l *xormigrateLogger) Debugf(format string, v ...interface{}) { + log.Debug().Msgf(format, v...) +} + +func (l *xormigrateLogger) Info(v ...interface{}) { + log.Info().Msg(fmt.Sprint(v...)) +} + +func (l *xormigrateLogger) Infof(format string, v ...interface{}) { + log.Info().Msgf(format, v...) +} + +func (l *xormigrateLogger) Warn(v ...interface{}) { + log.Warn().Msg(fmt.Sprint(v...)) +} + +func (l *xormigrateLogger) Warnf(format string, v ...interface{}) { + log.Warn().Msgf(format, v...) +} + +func (l *xormigrateLogger) Error(v ...interface{}) { + log.Error().Msg(fmt.Sprint(v...)) +} + +func (l *xormigrateLogger) Errorf(format string, v ...interface{}) { + log.Error().Msgf(format, v...) +} diff --git a/server/store/datastore/migration/migration.go b/server/store/datastore/migration/migration.go index 0346e7c27..a56168623 100644 --- a/server/store/datastore/migration/migration.go +++ b/server/store/datastore/migration/migration.go @@ -15,21 +15,19 @@ package migration import ( - "context" - "errors" "fmt" "reflect" - "time" - "github.com/rs/zerolog/log" + "src.techknowlogick.com/xormigrate" "xorm.io/xorm" "go.woodpecker-ci.org/woodpecker/server/model" ) // APPEND NEW MIGRATIONS -// they are executed in order and if one fails woodpecker will try to rollback that specific one and quits -var migrationTasks = []*task{ +// they are executed in order and if one fails Xormigrate will try to rollback that specific one and quits +var migrationTasks = []*xormigrate.Migration{ + &legacyToXormigrate, &legacy2Xorm, &alterTableReposDropFallback, &alterTableReposDropAllowDeploysAllowTags, @@ -82,70 +80,25 @@ var allBeans = []any{ new(model.Org), } -type migrations struct { - Name string `xorm:"UNIQUE"` -} - -type task struct { - name string - required bool - fn func(sess *xorm.Session) error - // engineFn does manage session on it's own. only use it if you really need to - engineFn func(e *xorm.Engine) error -} - -// initNew create tables for new instance -func initNew(sess *xorm.Session) error { - if err := syncAll(sess); err != nil { - return err - } - - // dummy run migrations - for _, task := range migrationTasks { - if _, err := sess.Insert(&migrations{task.name}); err != nil { - return err - } - } - - return nil -} - -func Migrate(e *xorm.Engine) error { +func Migrate(e *xorm.Engine, allowLong bool) error { e.SetDisableGlobalCache(true) - if err := e.Sync(new(migrations)); err != nil { - return fmt.Errorf("error to create migrations table: %w", err) + m := xormigrate.New(e, migrationTasks) + m.AllowLong(allowLong) + oldCount, err := e.Table("migrations").Count() + if oldCount < 1 || err != nil { + // allow new schema initialization if old migrations table is empty or it does not exist (err != nil) + // schema initialization will always run if we call `InitSchema` + m.InitSchema(func(engine *xorm.Engine) error { + // do nothing on schema init, models are synced in any case below + return nil + }) } - sess := e.NewSession() - defer sess.Close() - if err := sess.Begin(); err != nil { - return fmt.Errorf("could not create initial migration session: %w", err) - } + m.SetLogger(&xormigrateLogger{}) - // check if we have a fresh installation or need to check for migrations - c, err := sess.Count(new(migrations)) - if err != nil { - return fmt.Errorf("could not count migrations: %w", err) - } - - if c == 0 { - if err := initNew(sess); err != nil { - return fmt.Errorf("could not init a new database: %w", err) - } - if err := sess.Commit(); err != nil { - return fmt.Errorf("could not commit initial migration session: %w", err) - } - - return nil - } - - if err := sess.Commit(); err != nil { - return fmt.Errorf("could not commit initial migration session: %w", err) - } - - if err := runTasks(e, migrationTasks); err != nil { - return fmt.Errorf("run tasks failed: %w", err) + if err := m.Migrate(); err != nil { + return err } e.SetDisableGlobalCache(false) @@ -157,74 +110,7 @@ func Migrate(e *xorm.Engine) error { return nil } -func runTasks(e *xorm.Engine, tasks []*task) error { - // cache migrations in db - migCache := make(map[string]bool) - var migList []*migrations - if err := e.Find(&migList); err != nil { - return err - } - for i := range migList { - migCache[migList[i].Name] = true - } - - for _, task := range tasks { - if migCache[task.name] { - log.Trace().Msgf("migration task '%s' already applied", task.name) - continue - } - - log.Trace().Msgf("start migration task '%s'", task.name) - aliveMsgCancel := showBeAliveSign(task.name) - defer aliveMsgCancel(nil) - var taskErr error - if task.fn != nil { - sess := e.NewSession().NoCache() - defer sess.Close() - if err := sess.Begin(); err != nil { - return fmt.Errorf("could not begin session for '%s': %w", task.name, err) - } - - if taskErr = task.fn(sess); taskErr != nil { - aliveMsgCancel(nil) - if err := sess.Rollback(); err != nil { - taskErr = errors.Join(taskErr, err) - } - } else if err := sess.Commit(); err != nil { - return fmt.Errorf("could not commit session for '%s': %w", task.name, err) - } - } else if task.engineFn != nil { - taskErr = task.engineFn(e) - } else { - log.Trace().Msgf("skip migration task '%s'", task.name) - aliveMsgCancel(nil) - continue - } - - aliveMsgCancel(nil) - if taskErr != nil { - if task.required { - return fmt.Errorf("migration task '%s' failed: %w", task.name, taskErr) - } - log.Error().Err(taskErr).Msgf("migration task '%s' failed but is not required", task.name) - continue - } - log.Debug().Msgf("migration task '%s' done", task.name) - - if _, err := e.Insert(&migrations{task.name}); err != nil { - return fmt.Errorf("migration task '%s' could not be marked as finished: %w", task.name, err) - } - - migCache[task.name] = true - } - return nil -} - -type syncEngine interface { - Sync(beans ...any) error -} - -func syncAll(sess syncEngine) error { +func syncAll(sess *xorm.Engine) error { for _, bean := range allBeans { if err := sess.Sync(bean); err != nil { return fmt.Errorf("Sync error '%s': %w", reflect.TypeOf(bean), err) @@ -232,20 +118,3 @@ func syncAll(sess syncEngine) error { } return nil } - -var showBeAliveSignDelay = time.Second * 20 - -func showBeAliveSign(taskName string) context.CancelCauseFunc { - ctx, cancel := context.WithCancelCause(context.Background()) - go func() { - for { - select { - case <-ctx.Done(): - return - case <-time.After(showBeAliveSignDelay): - log.Info().Msgf("Migration '%s' is still running, please be patient", taskName) - } - } - }() - return cancel -} diff --git a/server/store/datastore/migration/migration_test.go b/server/store/datastore/migration/migration_test.go index 32e14a5b5..d8985d590 100644 --- a/server/store/datastore/migration/migration_test.go +++ b/server/store/datastore/migration/migration_test.go @@ -94,14 +94,9 @@ func testDB(t *testing.T, new bool) (engine *xorm.Engine, closeDB func()) { } func TestMigrate(t *testing.T) { - // make all tasks required for tests - for _, task := range migrationTasks { - task.required = true - } - // init new db engine, closeDB := testDB(t, true) - assert.NoError(t, Migrate(engine)) + assert.NoError(t, Migrate(engine, true)) closeDB() dbType := engine.Dialect().URI().DBType @@ -112,6 +107,6 @@ func TestMigrate(t *testing.T) { // migrate old db engine, closeDB = testDB(t, false) - assert.NoError(t, Migrate(engine)) + assert.NoError(t, Migrate(engine, true)) closeDB() } diff --git a/server/store/mocks/store.go b/server/store/mocks/store.go index 66c63405a..d40b3947c 100644 --- a/server/store/mocks/store.go +++ b/server/store/mocks/store.go @@ -1155,13 +1155,13 @@ func (_m *Store) LogSave(_a0 *model.Step, _a1 []*model.LogEntry) error { return r0 } -// Migrate provides a mock function with given fields: -func (_m *Store) Migrate() error { - ret := _m.Called() +// Migrate provides a mock function with given fields: _a0 +func (_m *Store) Migrate(_a0 bool) error { + ret := _m.Called(_a0) var r0 error - if rf, ok := ret.Get(0).(func() error); ok { - r0 = rf() + if rf, ok := ret.Get(0).(func(bool) error); ok { + r0 = rf(_a0) } else { r0 = ret.Error(0) } diff --git a/server/store/store.go b/server/store/store.go index 39ace982e..9e8b87fa5 100644 --- a/server/store/store.go +++ b/server/store/store.go @@ -197,5 +197,5 @@ type Store interface { // Store operations Ping() error Close() error - Migrate() error + Migrate(bool) error }