From b2ce1f5da5ec0c5b43b012089b39dee7adfed747 Mon Sep 17 00:00:00 2001
From: 6543 <6543@obermui.de>
Date: Mon, 20 Dec 2021 09:59:27 +0100
Subject: [PATCH] fix sqlite migration on column drop of uncommon schemas
 (#629)

* fix sqlite schema normalising

* sqlite robuster alg to extract columns form schema
---
 server/store/datastore/migration/common.go    | 45 ++++++++++++++--
 .../store/datastore/migration/common_test.go  | 51 +++++++++++++++++++
 2 files changed, 91 insertions(+), 5 deletions(-)
 create mode 100644 server/store/datastore/migration/common_test.go

diff --git a/server/store/datastore/migration/common.go b/server/store/datastore/migration/common.go
index 0c15e9d42..6afa6a04e 100644
--- a/server/store/datastore/migration/common.go
+++ b/server/store/datastore/migration/common.go
@@ -82,15 +82,13 @@ func dropTableColumns(sess *xorm.Session, tableName string, columnNames ...strin
 		if err != nil {
 			return err
 		}
-		tableSQL := string(res[0]["sql"])
+		tableSQL := normalizeSQLiteTableSchema(string(res[0]["sql"]))
 
 		// Separate out the column definitions
 		tableSQL = tableSQL[strings.Index(tableSQL, "("):]
 
 		// Remove the required columnNames
-		for _, name := range columnNames {
-			tableSQL = regexp.MustCompile(regexp.QuoteMeta("`"+name+"`")+"[^`,)]*?[,)]").ReplaceAllString(tableSQL, "")
-		}
+		tableSQL = removeColumnFromSQLITETableSchema(tableSQL, columnNames...)
 
 		// Ensure the query is ended properly
 		tableSQL = strings.TrimSpace(tableSQL)
@@ -102,7 +100,16 @@ func dropTableColumns(sess *xorm.Session, tableName string, columnNames ...strin
 		}
 
 		// Find all the columns in the table
-		columns := regexp.MustCompile("`([^`]*)`").FindAllString(tableSQL, -1)
+		var columns []string
+		for _, rawColumn := range strings.Split(strings.ReplaceAll(tableSQL[1:len(tableSQL)-1], ", ", ",\n"), "\n") {
+			if strings.ContainsAny(rawColumn, "()") {
+				continue
+			}
+			rawColumn = strings.TrimSpace(rawColumn)
+			columns = append(columns,
+				strings.ReplaceAll(rawColumn[0:strings.Index(rawColumn, " ")], "`", ""),
+			)
+		}
 
 		tableSQL = fmt.Sprintf("CREATE TABLE `new_%s_new` ", tableName) + tableSQL
 		if _, err := sess.Exec(tableSQL); err != nil {
@@ -204,3 +211,31 @@ func dropTableColumns(sess *xorm.Session, tableName string, columnNames ...strin
 
 	return nil
 }
+
+var whitespaces = regexp.MustCompile(`\s+`)
+var columnSeparator = regexp.MustCompile(`\s?,\s?`)
+
+func removeColumnFromSQLITETableSchema(schema string, names ...string) string {
+	if len(names) == 0 {
+		return schema
+	}
+	for i := range names {
+		if len(names[i]) == 0 {
+			continue
+		}
+		schema = regexp.MustCompile(`\s(`+
+			regexp.QuoteMeta("`"+names[i]+"`")+
+			"|"+
+			regexp.QuoteMeta(names[i])+
+			")[^`,)]*?[,)]").ReplaceAllString(schema, "")
+	}
+	return schema
+}
+
+func normalizeSQLiteTableSchema(schema string) string {
+	return columnSeparator.ReplaceAllString(
+		whitespaces.ReplaceAllString(
+			strings.ReplaceAll(schema, "\n", " "),
+			" "),
+		", ")
+}
diff --git a/server/store/datastore/migration/common_test.go b/server/store/datastore/migration/common_test.go
new file mode 100644
index 000000000..76e46e2e3
--- /dev/null
+++ b/server/store/datastore/migration/common_test.go
@@ -0,0 +1,51 @@
+package migration
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestRemoveColumnFromSQLITETableSchema(t *testing.T) {
+	schema := "CREATE TABLE repos ( repo_id INTEGER PRIMARY KEY AUTOINCREMENT, repo_user_id INTEGER, repo_owner TEXT, " +
+		"repo_name TEXT, repo_full_name TEXT, `repo_avatar` TEXT, repo_branch TEXT, repo_timeout INTEGER, " +
+		"repo_allow_pr BOOLEAN, repo_config_path TEXT, repo_visibility TEXT, repo_counter INTEGER, repo_active BOOLEAN, " +
+		"repo_fallback BOOLEAN, UNIQUE(repo_full_name) )"
+
+	assert.EqualValues(t, schema, removeColumnFromSQLITETableSchema(schema, ""))
+
+	assert.EqualValues(t, "CREATE TABLE repos ( repo_id INTEGER PRIMARY KEY AUTOINCREMENT, repo_user_id INTEGER, repo_owner TEXT, "+
+		"repo_name TEXT, repo_full_name TEXT, repo_branch TEXT, repo_timeout INTEGER, "+
+		"repo_allow_pr BOOLEAN, repo_config_path TEXT, repo_visibility TEXT, repo_counter INTEGER, repo_active BOOLEAN, "+
+		"repo_fallback BOOLEAN, UNIQUE(repo_full_name) )", removeColumnFromSQLITETableSchema(schema, "repo_avatar"))
+
+	assert.EqualValues(t, "CREATE TABLE repos ( repo_user_id INTEGER, repo_owner TEXT, "+
+		"repo_name TEXT, repo_full_name TEXT, `repo_avatar` TEXT, repo_timeout INTEGER, "+
+		"repo_allow_pr BOOLEAN, repo_config_path TEXT, repo_visibility TEXT, repo_counter INTEGER, repo_active BOOLEAN, "+
+		"repo_fallback BOOLEAN, UNIQUE(repo_full_name) )", removeColumnFromSQLITETableSchema(schema, "repo_id", "repo_branch", "invalid", ""))
+}
+
+func TestNormalizeSQLiteTableSchema(t *testing.T) {
+	assert.EqualValues(t, "", normalizeSQLiteTableSchema(``))
+	assert.EqualValues(t,
+		"CREATE TABLE repos ( repo_id INTEGER PRIMARY KEY AUTOINCREMENT, "+
+			"repo_user_id INTEGER, repo_owner TEXT, repo_name TEXT, repo_full_name TEXT, "+
+			"`repo_avatar` TEXT, repo_link TEXT, repo_clone TEXT, repo_branch TEXT, "+
+			"repo_timeout INTEGER, repo_allow_pr BOOLEAN, repo_config_path TEXT, "+
+			"repo_visibility TEXT, repo_counter INTEGER, repo_active BOOLEAN, "+
+			"repo_fallback BOOLEAN, UNIQUE(repo_full_name) )",
+		normalizeSQLiteTableSchema(`CREATE TABLE repos (
+ repo_id            INTEGER PRIMARY KEY AUTOINCREMENT
+,repo_user_id       INTEGER
+,repo_owner         TEXT,
+  repo_name         TEXT
+,repo_full_name     TEXT
+,`+"`"+`repo_avatar`+"`"+`        TEXT
+,repo_link          TEXT
+,repo_clone         TEXT
+,repo_branch        TEXT ,repo_timeout			INTEGER
+,repo_allow_pr      BOOLEAN
+,repo_config_path   TEXT
+, repo_visibility TEXT, repo_counter INTEGER, repo_active BOOLEAN, repo_fallback BOOLEAN,UNIQUE(repo_full_name)
+)`))
+}