Move database settings from models to setting (#7806)

* move database settings from models to setting

* update docs

* fix checkout pr

* fix tests

* fix lint

* remove unsupported tidb options

* correct wrong variable name

* remove tidb totally
This commit is contained in:
Lunny Xiao 2019-08-24 17:24:45 +08:00 committed by Antoine GIRARD
parent 26af3401c3
commit f83db078f0
35 changed files with 423 additions and 376 deletions

View file

@ -38,7 +38,7 @@ func initDB() error {
func initDBDisableConsole(disableConsole bool) error { func initDBDisableConsole(disableConsole bool) error {
setting.NewContext() setting.NewContext()
models.LoadConfigs() setting.InitDBConfig()
setting.NewXORMLogService(disableConsole) setting.NewXORMLogService(disableConsole)
if err := models.SetEngine(); err != nil { if err := models.SetEngine(); err != nil {

View file

@ -31,9 +31,9 @@ func runConvert(ctx *cli.Context) error {
log.Trace("AppWorkPath: %s", setting.AppWorkPath) log.Trace("AppWorkPath: %s", setting.AppWorkPath)
log.Trace("Custom path: %s", setting.CustomPath) log.Trace("Custom path: %s", setting.CustomPath)
log.Trace("Log path: %s", setting.LogRootPath) log.Trace("Log path: %s", setting.LogRootPath)
models.LoadConfigs() setting.InitDBConfig()
if models.DbCfg.Type != "mysql" { if !setting.Database.UseMySQL {
fmt.Println("This command can only be used with a MySQL database") fmt.Println("This command can only be used with a MySQL database")
return nil return nil
} }

View file

@ -58,7 +58,6 @@ It can be used for backup and capture Gitea server image to send to maintainer`,
func runDump(ctx *cli.Context) error { func runDump(ctx *cli.Context) error {
setting.NewContext() setting.NewContext()
setting.NewServices() // cannot access session settings otherwise setting.NewServices() // cannot access session settings otherwise
models.LoadConfigs()
err := models.SetEngine() err := models.SetEngine()
if err != nil { if err != nil {
@ -104,8 +103,8 @@ func runDump(ctx *cli.Context) error {
} }
targetDBType := ctx.String("database") targetDBType := ctx.String("database")
if len(targetDBType) > 0 && targetDBType != models.DbCfg.Type { if len(targetDBType) > 0 && targetDBType != setting.Database.Type {
log.Printf("Dumping database %s => %s...", models.DbCfg.Type, targetDBType) log.Printf("Dumping database %s => %s...", setting.Database.Type, targetDBType)
} else { } else {
log.Printf("Dumping database...") log.Printf("Dumping database...")
} }

View file

@ -30,7 +30,7 @@ func runMigrate(ctx *cli.Context) error {
log.Trace("AppWorkPath: %s", setting.AppWorkPath) log.Trace("AppWorkPath: %s", setting.AppWorkPath)
log.Trace("Custom path: %s", setting.CustomPath) log.Trace("Custom path: %s", setting.CustomPath)
log.Trace("Log path: %s", setting.LogRootPath) log.Trace("Log path: %s", setting.LogRootPath)
models.LoadConfigs() setting.InitDBConfig()
if err := models.NewEngine(migrations.Migrate); err != nil { if err := models.NewEngine(migrations.Migrate); err != nil {
log.Fatal("Failed to initialize ORM engine: %v", err) log.Fatal("Failed to initialize ORM engine: %v", err)

View file

@ -79,16 +79,16 @@ func runPR() {
setting.CheckLFSVersion() setting.CheckLFSVersion()
//models.LoadConfigs() //models.LoadConfigs()
/* /*
models.DbCfg.Type = "sqlite3" setting.Database.Type = "sqlite3"
models.DbCfg.Path = ":memory:" setting.Database.Path = ":memory:"
models.DbCfg.Timeout = 500 setting.Database.Timeout = 500
*/ */
db := setting.Cfg.Section("database") db := setting.Cfg.Section("database")
db.NewKey("DB_TYPE", "sqlite3") db.NewKey("DB_TYPE", "sqlite3")
db.NewKey("PATH", ":memory:") db.NewKey("PATH", ":memory:")
setting.LogSQL = true
models.LoadConfigs()
routers.NewServices() routers.NewServices()
setting.Database.LogSQL = true
//x, err = xorm.NewEngine("sqlite3", "file::memory:?cache=shared") //x, err = xorm.NewEngine("sqlite3", "file::memory:?cache=shared")
var helper testfixtures.Helper = &testfixtures.SQLite{} var helper testfixtures.Helper = &testfixtures.SQLite{}

View file

@ -275,6 +275,10 @@ LOG_SQL = true
DB_RETRIES = 10 DB_RETRIES = 10
; Backoff time per DB retry (time.Duration) ; Backoff time per DB retry (time.Duration)
DB_RETRY_BACKOFF = 3s DB_RETRY_BACKOFF = 3s
; Max idle database connections on connnection pool, default is 0
MAX_IDLE_CONNS = 0
; Database connection max life time, default is 3s
CONN_MAX_LIFETIME = 3s
[indexer] [indexer]
; Issue indexer type, currently support: bleve or db, default is bleve ; Issue indexer type, currently support: bleve or db, default is bleve

View file

@ -166,6 +166,8 @@ Values containing `#` or `;` must be quoted using `` ` `` or `"""`.
- `LOG_SQL`: **true**: Log the executed SQL. - `LOG_SQL`: **true**: Log the executed SQL.
- `DB_RETRIES`: **10**: How many ORM init / DB connect attempts allowed. - `DB_RETRIES`: **10**: How many ORM init / DB connect attempts allowed.
- `DB_RETRY_BACKOFF`: **3s**: time.Duration to wait before trying another ORM init / DB connect attempt, if failure occured. - `DB_RETRY_BACKOFF`: **3s**: time.Duration to wait before trying another ORM init / DB connect attempt, if failure occured.
- `MAX_IDLE_CONNS` **0**: Max idle database connections on connnection pool, default is 0
- `CONN_MAX_LIFETIME` **3s**: Database connection max lifetime
## Indexer (`indexer`) ## Indexer (`indexer`)

View file

@ -82,6 +82,8 @@ menu:
- `CHARSET`: **utf8**: 仅当数据库为 MySQL 时有效, 可以为 "utf8" 或 "utf8mb4"。注意:如果使用 "utf8mb4",你的 MySQL InnoDB 版本必须在 5.6 以上。 - `CHARSET`: **utf8**: 仅当数据库为 MySQL 时有效, 可以为 "utf8" 或 "utf8mb4"。注意:如果使用 "utf8mb4",你的 MySQL InnoDB 版本必须在 5.6 以上。
- `PATH`: Tidb 或者 SQLite3 数据文件存放路径。 - `PATH`: Tidb 或者 SQLite3 数据文件存放路径。
- `LOG_SQL`: **true**: 显示生成的SQL默认为真。 - `LOG_SQL`: **true**: 显示生成的SQL默认为真。
- `MAX_IDLE_CONNS` **0**: 最大空闲数据库连接
- `CONN_MAX_LIFETIME` **3s**: 数据库连接最大存活时间
## Indexer (`indexer`) ## Indexer (`indexer`)

View file

@ -59,13 +59,13 @@ func TestMain(m *testing.M) {
routes.RegisterRoutes(mac) routes.RegisterRoutes(mac)
var helper testfixtures.Helper var helper testfixtures.Helper
if setting.UseMySQL { if setting.Database.UseMySQL {
helper = &testfixtures.MySQL{} helper = &testfixtures.MySQL{}
} else if setting.UsePostgreSQL { } else if setting.Database.UsePostgreSQL {
helper = &testfixtures.PostgreSQL{} helper = &testfixtures.PostgreSQL{}
} else if setting.UseSQLite3 { } else if setting.Database.UseSQLite3 {
helper = &testfixtures.SQLite{} helper = &testfixtures.SQLite{}
} else if setting.UseMSSQL { } else if setting.Database.UseMSSQL {
helper = &testfixtures.SQLServer{} helper = &testfixtures.SQLServer{}
} else { } else {
fmt.Println("Unsupported RDBMS for integration tests") fmt.Println("Unsupported RDBMS for integration tests")
@ -121,12 +121,12 @@ func initIntegrationTest() {
setting.SetCustomPathAndConf("", "", "") setting.SetCustomPathAndConf("", "", "")
setting.NewContext() setting.NewContext()
setting.CheckLFSVersion() setting.CheckLFSVersion()
models.LoadConfigs() setting.InitDBConfig()
switch { switch {
case setting.UseMySQL: case setting.Database.UseMySQL:
db, err := sql.Open("mysql", fmt.Sprintf("%s:%s@tcp(%s)/", db, err := sql.Open("mysql", fmt.Sprintf("%s:%s@tcp(%s)/",
models.DbCfg.User, models.DbCfg.Passwd, models.DbCfg.Host)) setting.Database.User, setting.Database.Passwd, setting.Database.Host))
defer db.Close() defer db.Close()
if err != nil { if err != nil {
log.Fatalf("sql.Open: %v", err) log.Fatalf("sql.Open: %v", err)
@ -134,14 +134,14 @@ func initIntegrationTest() {
if _, err = db.Exec("CREATE DATABASE IF NOT EXISTS testgitea"); err != nil { if _, err = db.Exec("CREATE DATABASE IF NOT EXISTS testgitea"); err != nil {
log.Fatalf("db.Exec: %v", err) log.Fatalf("db.Exec: %v", err)
} }
case setting.UsePostgreSQL: case setting.Database.UsePostgreSQL:
db, err := sql.Open("postgres", fmt.Sprintf("postgres://%s:%s@%s/?sslmode=%s", db, err := sql.Open("postgres", fmt.Sprintf("postgres://%s:%s@%s/?sslmode=%s",
models.DbCfg.User, models.DbCfg.Passwd, models.DbCfg.Host, models.DbCfg.SSLMode)) setting.Database.User, setting.Database.Passwd, setting.Database.Host, setting.Database.SSLMode))
defer db.Close() defer db.Close()
if err != nil { if err != nil {
log.Fatalf("sql.Open: %v", err) log.Fatalf("sql.Open: %v", err)
} }
rows, err := db.Query(fmt.Sprintf("SELECT 1 FROM pg_database WHERE datname = '%s'", models.DbCfg.Name)) rows, err := db.Query(fmt.Sprintf("SELECT 1 FROM pg_database WHERE datname = '%s'", setting.Database.Name))
if err != nil { if err != nil {
log.Fatalf("db.Query: %v", err) log.Fatalf("db.Query: %v", err)
} }
@ -153,10 +153,10 @@ func initIntegrationTest() {
if _, err = db.Exec("CREATE DATABASE testgitea"); err != nil { if _, err = db.Exec("CREATE DATABASE testgitea"); err != nil {
log.Fatalf("db.Exec: %v", err) log.Fatalf("db.Exec: %v", err)
} }
case setting.UseMSSQL: case setting.Database.UseMSSQL:
host, port := models.ParseMSSQLHostPort(models.DbCfg.Host) host, port := setting.ParseMSSQLHostPort(setting.Database.Host)
db, err := sql.Open("mssql", fmt.Sprintf("server=%s; port=%s; database=%s; user id=%s; password=%s;", db, err := sql.Open("mssql", fmt.Sprintf("server=%s; port=%s; database=%s; user id=%s; password=%s;",
host, port, "master", models.DbCfg.User, models.DbCfg.Passwd)) host, port, "master", setting.Database.User, setting.Database.Passwd))
if err != nil { if err != nil {
log.Fatalf("sql.Open: %v", err) log.Fatalf("sql.Open: %v", err)
} }

View file

@ -39,7 +39,7 @@ func storeObjectInRepo(t *testing.T, repositoryID int64, content *[]byte) string
assert.NoError(t, err) assert.NoError(t, err)
var lfsMetaObject *models.LFSMetaObject var lfsMetaObject *models.LFSMetaObject
if setting.UsePostgreSQL { if setting.Database.UsePostgreSQL {
lfsMetaObject = &models.LFSMetaObject{ID: lfsID, Oid: oid, Size: int64(len(*content)), RepositoryID: repositoryID} lfsMetaObject = &models.LFSMetaObject{ID: lfsID, Oid: oid, Size: int64(len(*content)), RepositoryID: repositoryID}
} else { } else {
lfsMetaObject = &models.LFSMetaObject{Oid: oid, Size: int64(len(*content)), RepositoryID: repositoryID} lfsMetaObject = &models.LFSMetaObject{Oid: oid, Size: int64(len(*content)), RepositoryID: repositoryID}

View file

@ -54,7 +54,7 @@ func initMigrationTest(t *testing.T) {
setting.NewContext() setting.NewContext()
setting.CheckLFSVersion() setting.CheckLFSVersion()
models.LoadConfigs() setting.InitDBConfig()
setting.NewLogServices(true) setting.NewLogServices(true)
} }
@ -64,7 +64,7 @@ func availableVersions() ([]string, error) {
return nil, err return nil, err
} }
defer migrationsDir.Close() defer migrationsDir.Close()
versionRE, err := regexp.Compile("gitea-v(?P<version>.+)\\." + regexp.QuoteMeta(models.DbCfg.Type) + "\\.sql.gz") versionRE, err := regexp.Compile("gitea-v(?P<version>.+)\\." + regexp.QuoteMeta(setting.Database.Type) + "\\.sql.gz")
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -85,7 +85,7 @@ func availableVersions() ([]string, error) {
} }
func readSQLFromFile(version string) (string, error) { func readSQLFromFile(version string) (string, error) {
filename := fmt.Sprintf("integrations/migration-test/gitea-v%s.%s.sql.gz", version, models.DbCfg.Type) filename := fmt.Sprintf("integrations/migration-test/gitea-v%s.%s.sql.gz", version, setting.Database.Type)
if _, err := os.Stat(filename); os.IsNotExist(err) { if _, err := os.Stat(filename); os.IsNotExist(err) {
return "", nil return "", nil
@ -114,17 +114,17 @@ func restoreOldDB(t *testing.T, version string) bool {
data, err := readSQLFromFile(version) data, err := readSQLFromFile(version)
assert.NoError(t, err) assert.NoError(t, err)
if len(data) == 0 { if len(data) == 0 {
integrations.Printf("No db found to restore for %s version: %s\n", models.DbCfg.Type, version) integrations.Printf("No db found to restore for %s version: %s\n", setting.Database.Type, version)
return false return false
} }
switch { switch {
case setting.UseSQLite3: case setting.Database.UseSQLite3:
os.Remove(models.DbCfg.Path) os.Remove(setting.Database.Path)
err := os.MkdirAll(path.Dir(models.DbCfg.Path), os.ModePerm) err := os.MkdirAll(path.Dir(setting.Database.Path), os.ModePerm)
assert.NoError(t, err) assert.NoError(t, err)
db, err := sql.Open("sqlite3", fmt.Sprintf("file:%s?cache=shared&mode=rwc&_busy_timeout=%d", models.DbCfg.Path, models.DbCfg.Timeout)) db, err := sql.Open("sqlite3", fmt.Sprintf("file:%s?cache=shared&mode=rwc&_busy_timeout=%d", setting.Database.Path, setting.Database.Timeout))
assert.NoError(t, err) assert.NoError(t, err)
defer db.Close() defer db.Close()
@ -132,20 +132,20 @@ func restoreOldDB(t *testing.T, version string) bool {
assert.NoError(t, err) assert.NoError(t, err)
db.Close() db.Close()
case setting.UseMySQL: case setting.Database.UseMySQL:
db, err := sql.Open("mysql", fmt.Sprintf("%s:%s@tcp(%s)/", db, err := sql.Open("mysql", fmt.Sprintf("%s:%s@tcp(%s)/",
models.DbCfg.User, models.DbCfg.Passwd, models.DbCfg.Host)) setting.Database.User, setting.Database.Passwd, setting.Database.Host))
assert.NoError(t, err) assert.NoError(t, err)
defer db.Close() defer db.Close()
_, err = db.Exec(fmt.Sprintf("DROP DATABASE IF EXISTS %s", models.DbCfg.Name)) _, err = db.Exec(fmt.Sprintf("DROP DATABASE IF EXISTS %s", setting.Database.Name))
assert.NoError(t, err) assert.NoError(t, err)
_, err = db.Exec(fmt.Sprintf("CREATE DATABASE IF NOT EXISTS %s", models.DbCfg.Name)) _, err = db.Exec(fmt.Sprintf("CREATE DATABASE IF NOT EXISTS %s", setting.Database.Name))
assert.NoError(t, err) assert.NoError(t, err)
db, err = sql.Open("mysql", fmt.Sprintf("%s:%s@tcp(%s)/%s?multiStatements=true", db, err = sql.Open("mysql", fmt.Sprintf("%s:%s@tcp(%s)/%s?multiStatements=true",
models.DbCfg.User, models.DbCfg.Passwd, models.DbCfg.Host, models.DbCfg.Name)) setting.Database.User, setting.Database.Passwd, setting.Database.Host, setting.Database.Name))
assert.NoError(t, err) assert.NoError(t, err)
defer db.Close() defer db.Close()
@ -153,21 +153,21 @@ func restoreOldDB(t *testing.T, version string) bool {
assert.NoError(t, err) assert.NoError(t, err)
db.Close() db.Close()
case setting.UsePostgreSQL: case setting.Database.UsePostgreSQL:
db, err := sql.Open("postgres", fmt.Sprintf("postgres://%s:%s@%s/?sslmode=%s", db, err := sql.Open("postgres", fmt.Sprintf("postgres://%s:%s@%s/?sslmode=%s",
models.DbCfg.User, models.DbCfg.Passwd, models.DbCfg.Host, models.DbCfg.SSLMode)) setting.Database.User, setting.Database.Passwd, setting.Database.Host, setting.Database.SSLMode))
assert.NoError(t, err) assert.NoError(t, err)
defer db.Close() defer db.Close()
_, err = db.Exec(fmt.Sprintf("DROP DATABASE IF EXISTS %s", models.DbCfg.Name)) _, err = db.Exec(fmt.Sprintf("DROP DATABASE IF EXISTS %s", setting.Database.Name))
assert.NoError(t, err) assert.NoError(t, err)
_, err = db.Exec(fmt.Sprintf("CREATE DATABASE %s", models.DbCfg.Name)) _, err = db.Exec(fmt.Sprintf("CREATE DATABASE %s", setting.Database.Name))
assert.NoError(t, err) assert.NoError(t, err)
db.Close() db.Close()
db, err = sql.Open("postgres", fmt.Sprintf("postgres://%s:%s@%s/%s?sslmode=%s", db, err = sql.Open("postgres", fmt.Sprintf("postgres://%s:%s@%s/%s?sslmode=%s",
models.DbCfg.User, models.DbCfg.Passwd, models.DbCfg.Host, models.DbCfg.Name, models.DbCfg.SSLMode)) setting.Database.User, setting.Database.Passwd, setting.Database.Host, setting.Database.Name, setting.Database.SSLMode))
assert.NoError(t, err) assert.NoError(t, err)
defer db.Close() defer db.Close()
@ -175,10 +175,10 @@ func restoreOldDB(t *testing.T, version string) bool {
assert.NoError(t, err) assert.NoError(t, err)
db.Close() db.Close()
case setting.UseMSSQL: case setting.Database.UseMSSQL:
host, port := models.ParseMSSQLHostPort(models.DbCfg.Host) host, port := setting.ParseMSSQLHostPort(setting.Database.Host)
db, err := sql.Open("mssql", fmt.Sprintf("server=%s; port=%s; database=%s; user id=%s; password=%s;", db, err := sql.Open("mssql", fmt.Sprintf("server=%s; port=%s; database=%s; user id=%s; password=%s;",
host, port, "master", models.DbCfg.User, models.DbCfg.Passwd)) host, port, "master", setting.Database.User, setting.Database.Passwd))
assert.NoError(t, err) assert.NoError(t, err)
defer db.Close() defer db.Close()
@ -191,7 +191,7 @@ func restoreOldDB(t *testing.T, version string) bool {
dbname := statement[5 : len(statement)-1] dbname := statement[5 : len(statement)-1]
db.Close() db.Close()
db, err = sql.Open("mssql", fmt.Sprintf("server=%s; port=%s; database=%s; user id=%s; password=%s;", db, err = sql.Open("mssql", fmt.Sprintf("server=%s; port=%s; database=%s; user id=%s; password=%s;",
host, port, dbname, models.DbCfg.User, models.DbCfg.Passwd)) host, port, dbname, setting.Database.User, setting.Database.Passwd))
assert.NoError(t, err) assert.NoError(t, err)
defer db.Close() defer db.Close()
} }
@ -210,7 +210,7 @@ func wrappedMigrate(x *xorm.Engine) error {
func doMigrationTest(t *testing.T, version string) { func doMigrationTest(t *testing.T, version string) {
integrations.PrintCurrentTest(t) integrations.PrintCurrentTest(t)
integrations.Printf("Performing migration test for %s version: %s\n", models.DbCfg.Type, version) integrations.Printf("Performing migration test for %s version: %s\n", setting.Database.Type, version)
if !restoreOldDB(t, version) { if !restoreOldDB(t, version) {
return return
} }
@ -227,7 +227,7 @@ func doMigrationTest(t *testing.T, version string) {
func TestMigrations(t *testing.T) { func TestMigrations(t *testing.T) {
initMigrationTest(t) initMigrationTest(t)
dialect := models.DbCfg.Type dialect := setting.Database.Type
versions, err := availableVersions() versions, err := availableVersions()
assert.NoError(t, err) assert.NoError(t, err)

View file

@ -4,11 +4,15 @@
package models package models
import "fmt" import (
"fmt"
"code.gitea.io/gitea/modules/setting"
)
// ConvertUtf8ToUtf8mb4 converts database and tables from utf8 to utf8mb4 if it's mysql // ConvertUtf8ToUtf8mb4 converts database and tables from utf8 to utf8mb4 if it's mysql
func ConvertUtf8ToUtf8mb4() error { func ConvertUtf8ToUtf8mb4() error {
_, err := x.Exec(fmt.Sprintf("ALTER DATABASE `%s` CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci", DbCfg.Name)) _, err := x.Exec(fmt.Sprintf("ALTER DATABASE `%s` CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci", setting.Database.Name))
if err != nil { if err != nil {
return err return err
} }

View file

@ -296,7 +296,7 @@ func dropTableColumns(sess *xorm.Session, tableName string, columnNames ...strin
// TODO: This will not work if there are foreign keys // TODO: This will not work if there are foreign keys
switch { switch {
case setting.UseSQLite3: case setting.Database.UseSQLite3:
// First drop the indexes on the columns // First drop the indexes on the columns
res, errIndex := sess.Query(fmt.Sprintf("PRAGMA index_list(`%s`)", tableName)) res, errIndex := sess.Query(fmt.Sprintf("PRAGMA index_list(`%s`)", tableName))
if errIndex != nil { if errIndex != nil {
@ -372,7 +372,7 @@ func dropTableColumns(sess *xorm.Session, tableName string, columnNames ...strin
return err return err
} }
case setting.UsePostgreSQL: case setting.Database.UsePostgreSQL:
cols := "" cols := ""
for _, col := range columnNames { for _, col := range columnNames {
if cols != "" { if cols != "" {
@ -383,7 +383,7 @@ func dropTableColumns(sess *xorm.Session, tableName string, columnNames ...strin
if _, err := sess.Exec(fmt.Sprintf("ALTER TABLE `%s` %s", tableName, cols)); err != nil { if _, err := sess.Exec(fmt.Sprintf("ALTER TABLE `%s` %s", tableName, cols)); err != nil {
return fmt.Errorf("Drop table `%s` columns %v: %v", tableName, columnNames, err) return fmt.Errorf("Drop table `%s` columns %v: %v", tableName, columnNames, err)
} }
case setting.UseMySQL, setting.UseTiDB: case setting.Database.UseMySQL:
// Drop indexes on columns first // Drop indexes on columns first
sql := fmt.Sprintf("SHOW INDEX FROM %s WHERE column_name IN ('%s')", tableName, strings.Join(columnNames, "','")) sql := fmt.Sprintf("SHOW INDEX FROM %s WHERE column_name IN ('%s')", tableName, strings.Join(columnNames, "','"))
res, err := sess.Query(sql) res, err := sess.Query(sql)
@ -409,7 +409,7 @@ func dropTableColumns(sess *xorm.Session, tableName string, columnNames ...strin
if _, err := sess.Exec(fmt.Sprintf("ALTER TABLE `%s` %s", tableName, cols)); err != nil { if _, err := sess.Exec(fmt.Sprintf("ALTER TABLE `%s` %s", tableName, cols)); err != nil {
return fmt.Errorf("Drop table `%s` columns %v: %v", tableName, columnNames, err) return fmt.Errorf("Drop table `%s` columns %v: %v", tableName, columnNames, err)
} }
case setting.UseMSSQL: case setting.Database.UseMSSQL:
cols := "" cols := ""
for _, col := range columnNames { for _, col := range columnNames {
if cols != "" { if cols != "" {

View file

@ -42,7 +42,7 @@ func generateAndMigrateGitHooks(x *xorm.Engine) (err error) {
} }
) )
return x.Where("id > 0").BufferSize(setting.IterateBufferSize).Iterate(new(Repository), return x.Where("id > 0").BufferSize(setting.Database.IterateBufferSize).Iterate(new(Repository),
func(idx int, bean interface{}) error { func(idx int, bean interface{}) error {
repo := bean.(*Repository) repo := bean.(*Repository)
user := new(User) user := new(User)

View file

@ -42,7 +42,7 @@ func generateAndMigrateWikiGitHooks(x *xorm.Engine) (err error) {
} }
) )
return x.Where("id > 0").BufferSize(setting.IterateBufferSize).Iterate(new(Repository), return x.Where("id > 0").BufferSize(setting.Database.IterateBufferSize).Iterate(new(Repository),
func(idx int, bean interface{}) error { func(idx int, bean interface{}) error {
repo := bean.(*Repository) repo := bean.(*Repository)
user := new(User) user := new(User)

View file

@ -36,7 +36,7 @@ func generateAndMigrateGitHookChains(x *xorm.Engine) (err error) {
hookTpl = fmt.Sprintf("#!/usr/bin/env %s\ndata=$(cat)\nexitcodes=\"\"\nhookname=$(basename $0)\nGIT_DIR=${GIT_DIR:-$(dirname $0)}\n\nfor hook in ${GIT_DIR}/hooks/${hookname}.d/*; do\ntest -x \"${hook}\" || continue\necho \"${data}\" | \"${hook}\"\nexitcodes=\"${exitcodes} $?\"\ndone\n\nfor i in ${exitcodes}; do\n[ ${i} -eq 0 ] || exit ${i}\ndone\n", setting.ScriptType) hookTpl = fmt.Sprintf("#!/usr/bin/env %s\ndata=$(cat)\nexitcodes=\"\"\nhookname=$(basename $0)\nGIT_DIR=${GIT_DIR:-$(dirname $0)}\n\nfor hook in ${GIT_DIR}/hooks/${hookname}.d/*; do\ntest -x \"${hook}\" || continue\necho \"${data}\" | \"${hook}\"\nexitcodes=\"${exitcodes} $?\"\ndone\n\nfor i in ${exitcodes}; do\n[ ${i} -eq 0 ] || exit ${i}\ndone\n", setting.ScriptType)
) )
return x.Where("id > 0").BufferSize(setting.IterateBufferSize).Iterate(new(Repository), return x.Where("id > 0").BufferSize(setting.Database.IterateBufferSize).Iterate(new(Repository),
func(idx int, bean interface{}) error { func(idx int, bean interface{}) error {
repo := bean.(*Repository) repo := bean.(*Repository)
user := new(User) user := new(User)

View file

@ -41,8 +41,6 @@ func convertIntervalToDuration(x *xorm.Engine) (err error) {
_, err = sess.Exec("ALTER TABLE mirror MODIFY `interval` BIGINT") _, err = sess.Exec("ALTER TABLE mirror MODIFY `interval` BIGINT")
case "postgres": case "postgres":
_, err = sess.Exec("ALTER TABLE mirror ALTER COLUMN \"interval\" SET DATA TYPE bigint") _, err = sess.Exec("ALTER TABLE mirror ALTER COLUMN \"interval\" SET DATA TYPE bigint")
case "tidb":
_, err = sess.Exec("ALTER TABLE mirror MODIFY `interval` BIGINT")
case "mssql": case "mssql":
_, err = sess.Exec("ALTER TABLE mirror ALTER COLUMN \"interval\" BIGINT") _, err = sess.Exec("ALTER TABLE mirror ALTER COLUMN \"interval\" BIGINT")
case "sqlite3": case "sqlite3":

View file

@ -15,9 +15,9 @@ import (
func removeActionColumns(x *xorm.Engine) error { func removeActionColumns(x *xorm.Engine) error {
switch { switch {
case setting.UseSQLite3: case setting.Database.UseSQLite3:
log.Warn("Unable to drop columns in SQLite") log.Warn("Unable to drop columns in SQLite")
case setting.UseMySQL, setting.UsePostgreSQL, setting.UseMSSQL, setting.UseTiDB: case setting.Database.UseMySQL, setting.Database.UsePostgreSQL, setting.Database.UseMSSQL:
if _, err := x.Exec("ALTER TABLE action DROP COLUMN act_user_name"); err != nil { if _, err := x.Exec("ALTER TABLE action DROP COLUMN act_user_name"); err != nil {
return fmt.Errorf("DROP COLUMN act_user_name: %v", err) return fmt.Errorf("DROP COLUMN act_user_name: %v", err)
} else if _, err = x.Exec("ALTER TABLE action DROP COLUMN repo_user_name"); err != nil { } else if _, err = x.Exec("ALTER TABLE action DROP COLUMN repo_user_name"); err != nil {

View file

@ -13,9 +13,9 @@ import (
func removeIndexColumnFromRepoUnitTable(x *xorm.Engine) (err error) { func removeIndexColumnFromRepoUnitTable(x *xorm.Engine) (err error) {
switch { switch {
case setting.UseSQLite3: case setting.Database.UseSQLite3:
log.Warn("Unable to drop columns in SQLite") log.Warn("Unable to drop columns in SQLite")
case setting.UseMySQL, setting.UsePostgreSQL, setting.UseMSSQL, setting.UseTiDB: case setting.Database.UseMySQL, setting.Database.UsePostgreSQL, setting.Database.UseMSSQL:
if _, err := x.Exec("ALTER TABLE repo_unit DROP COLUMN `index`"); err != nil { if _, err := x.Exec("ALTER TABLE repo_unit DROP COLUMN `index`"); err != nil {
// Ignoring this error in case we run this migration second time (after migration reordering) // Ignoring this error in case we run this migration second time (after migration reordering)
log.Warn("DROP COLUMN index: %v", err) log.Warn("DROP COLUMN index: %v", err)

View file

@ -40,9 +40,9 @@ func migrateProtectedBranchStruct(x *xorm.Engine) error {
} }
switch { switch {
case setting.UseSQLite3: case setting.Database.UseSQLite3:
log.Warn("Unable to drop columns in SQLite") log.Warn("Unable to drop columns in SQLite")
case setting.UseMySQL, setting.UsePostgreSQL, setting.UseMSSQL, setting.UseTiDB: case setting.Database.UseMySQL, setting.Database.UsePostgreSQL, setting.Database.UseMSSQL:
if _, err := x.Exec("ALTER TABLE protected_branch DROP COLUMN can_push"); err != nil { if _, err := x.Exec("ALTER TABLE protected_branch DROP COLUMN can_push"); err != nil {
// Ignoring this error in case we run this migration second time (after migration reordering) // Ignoring this error in case we run this migration second time (after migration reordering)
log.Warn("DROP COLUMN can_push (skipping): %v", err) log.Warn("DROP COLUMN can_push (skipping): %v", err)

View file

@ -80,7 +80,7 @@ func removeStaleWatches(x *xorm.Engine) error {
} }
repoCache := make(map[int64]*Repository) repoCache := make(map[int64]*Repository)
err := sess.BufferSize(setting.IterateBufferSize).Iterate(new(Watch), err := sess.BufferSize(setting.Database.IterateBufferSize).Iterate(new(Watch),
func(idx int, bean interface{}) error { func(idx int, bean interface{}) error {
watch := bean.(*Watch) watch := bean.(*Watch)
@ -117,7 +117,7 @@ func removeStaleWatches(x *xorm.Engine) error {
} }
repoCache = make(map[int64]*Repository) repoCache = make(map[int64]*Repository)
err = sess.BufferSize(setting.IterateBufferSize). err = sess.BufferSize(setting.Database.IterateBufferSize).
Distinct("issue_watch.user_id", "issue.repo_id"). Distinct("issue_watch.user_id", "issue.repo_id").
Join("INNER", "issue", "issue_watch.issue_id = issue.id"). Join("INNER", "issue", "issue_watch.issue_id = issue.id").
Where("issue_watch.is_watching = ?", true). Where("issue_watch.is_watching = ?", true).

View file

@ -14,8 +14,6 @@ func changeU2FCounterType(x *xorm.Engine) error {
var err error var err error
switch x.Dialect().DriverName() { switch x.Dialect().DriverName() {
case "tidb":
fallthrough
case "mysql": case "mysql":
_, err = x.Exec("ALTER TABLE `u2f_registration` MODIFY `counter` BIGINT") _, err = x.Exec("ALTER TABLE `u2f_registration` MODIFY `counter` BIGINT")
case "postgres": case "postgres":

View file

@ -9,12 +9,6 @@ import (
"database/sql" "database/sql"
"errors" "errors"
"fmt" "fmt"
"net/url"
"os"
"path"
"path/filepath"
"strings"
"time"
"code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/setting"
@ -53,23 +47,10 @@ type Engine interface {
var ( var (
x *xorm.Engine x *xorm.Engine
supportedDatabases = []string{"mysql", "postgres", "mssql"}
tables []interface{} tables []interface{}
// HasEngine specifies if we have a xorm.Engine // HasEngine specifies if we have a xorm.Engine
HasEngine bool HasEngine bool
// DbCfg holds the database settings
DbCfg struct {
Type, Host, Name, User, Passwd, Path, SSLMode, Charset string
Timeout int
}
// EnableSQLite3 use SQLite3
EnableSQLite3 bool
// EnableTiDB enable TiDB
EnableTiDB bool
) )
func init() { func init() {
@ -139,120 +120,13 @@ func init() {
} }
} }
// LoadConfigs loads the database settings
func LoadConfigs() {
sec := setting.Cfg.Section("database")
DbCfg.Type = sec.Key("DB_TYPE").String()
switch DbCfg.Type {
case "sqlite3":
setting.UseSQLite3 = true
case "mysql":
setting.UseMySQL = true
case "postgres":
setting.UsePostgreSQL = true
case "tidb":
setting.UseTiDB = true
case "mssql":
setting.UseMSSQL = true
}
DbCfg.Host = sec.Key("HOST").String()
DbCfg.Name = sec.Key("NAME").String()
DbCfg.User = sec.Key("USER").String()
if len(DbCfg.Passwd) == 0 {
DbCfg.Passwd = sec.Key("PASSWD").String()
}
DbCfg.SSLMode = sec.Key("SSL_MODE").MustString("disable")
DbCfg.Charset = sec.Key("CHARSET").In("utf8", []string{"utf8", "utf8mb4"})
DbCfg.Path = sec.Key("PATH").MustString(filepath.Join(setting.AppDataPath, "gitea.db"))
DbCfg.Timeout = sec.Key("SQLITE_TIMEOUT").MustInt(500)
}
// parsePostgreSQLHostPort parses given input in various forms defined in
// https://www.postgresql.org/docs/current/static/libpq-connect.html#LIBPQ-CONNSTRING
// and returns proper host and port number.
func parsePostgreSQLHostPort(info string) (string, string) {
host, port := "127.0.0.1", "5432"
if strings.Contains(info, ":") && !strings.HasSuffix(info, "]") {
idx := strings.LastIndex(info, ":")
host = info[:idx]
port = info[idx+1:]
} else if len(info) > 0 {
host = info
}
return host, port
}
func getPostgreSQLConnectionString(dbHost, dbUser, dbPasswd, dbName, dbParam, dbsslMode string) (connStr string) {
host, port := parsePostgreSQLHostPort(dbHost)
if host[0] == '/' { // looks like a unix socket
connStr = fmt.Sprintf("postgres://%s:%s@:%s/%s%ssslmode=%s&host=%s",
url.PathEscape(dbUser), url.PathEscape(dbPasswd), port, dbName, dbParam, dbsslMode, host)
} else {
connStr = fmt.Sprintf("postgres://%s:%s@%s:%s/%s%ssslmode=%s",
url.PathEscape(dbUser), url.PathEscape(dbPasswd), host, port, dbName, dbParam, dbsslMode)
}
return
}
// ParseMSSQLHostPort splits the host into host and port
func ParseMSSQLHostPort(info string) (string, string) {
host, port := "127.0.0.1", "1433"
if strings.Contains(info, ":") {
host = strings.Split(info, ":")[0]
port = strings.Split(info, ":")[1]
} else if strings.Contains(info, ",") {
host = strings.Split(info, ",")[0]
port = strings.TrimSpace(strings.Split(info, ",")[1])
} else if len(info) > 0 {
host = info
}
return host, port
}
func getEngine() (*xorm.Engine, error) { func getEngine() (*xorm.Engine, error) {
connStr := "" connStr, err := setting.DBConnStr()
var Param = "?" if err != nil {
if strings.Contains(DbCfg.Name, Param) { return nil, err
Param = "&"
}
switch DbCfg.Type {
case "mysql":
connType := "tcp"
if DbCfg.Host[0] == '/' { // looks like a unix socket
connType = "unix"
}
tls := DbCfg.SSLMode
if tls == "disable" { // allow (Postgres-inspired) default value to work in MySQL
tls = "false"
}
connStr = fmt.Sprintf("%s:%s@%s(%s)/%s%scharset=%s&parseTime=true&tls=%s",
DbCfg.User, DbCfg.Passwd, connType, DbCfg.Host, DbCfg.Name, Param, DbCfg.Charset, tls)
case "postgres":
connStr = getPostgreSQLConnectionString(DbCfg.Host, DbCfg.User, DbCfg.Passwd, DbCfg.Name, Param, DbCfg.SSLMode)
case "mssql":
host, port := ParseMSSQLHostPort(DbCfg.Host)
connStr = fmt.Sprintf("server=%s; port=%s; database=%s; user id=%s; password=%s;", host, port, DbCfg.Name, DbCfg.User, DbCfg.Passwd)
case "sqlite3":
if !EnableSQLite3 {
return nil, errors.New("this binary version does not build support for SQLite3")
}
if err := os.MkdirAll(path.Dir(DbCfg.Path), os.ModePerm); err != nil {
return nil, fmt.Errorf("Failed to create directories: %v", err)
}
connStr = fmt.Sprintf("file:%s?cache=shared&mode=rwc&_busy_timeout=%d", DbCfg.Path, DbCfg.Timeout)
case "tidb":
if !EnableTiDB {
return nil, errors.New("this binary version does not build support for TiDB")
}
if err := os.MkdirAll(path.Dir(DbCfg.Path), os.ModePerm); err != nil {
return nil, fmt.Errorf("Failed to create directories: %v", err)
}
connStr = "goleveldb://" + DbCfg.Path
default:
return nil, fmt.Errorf("Unknown database type: %s", DbCfg.Type)
} }
return xorm.NewEngine(DbCfg.Type, connStr) return xorm.NewEngine(setting.Database.Type, connStr)
} }
// NewTestEngine sets a new test xorm.Engine // NewTestEngine sets a new test xorm.Engine
@ -280,11 +154,11 @@ func SetEngine() (err error) {
x.SetMapper(core.GonicMapper{}) x.SetMapper(core.GonicMapper{})
// WARNING: for serv command, MUST remove the output to os.stdout, // WARNING: for serv command, MUST remove the output to os.stdout,
// so use log file to instead print to stdout. // so use log file to instead print to stdout.
x.SetLogger(NewXORMLogger(setting.LogSQL)) x.SetLogger(NewXORMLogger(setting.Database.LogSQL))
x.ShowSQL(setting.LogSQL) x.ShowSQL(setting.Database.LogSQL)
if DbCfg.Type == "mysql" { if setting.Database.UseMySQL {
x.SetMaxIdleConns(0) x.SetMaxIdleConns(setting.Database.MaxIdleConns)
x.SetConnMaxLifetime(3 * time.Second) x.SetConnMaxLifetime(setting.Database.ConnMaxLifetime)
} }
return nil return nil

View file

@ -1,5 +1,4 @@
// Copyright 2016 The Gogs Authors. All rights reserved. // Copyright 2019 The Gitea Authors. All rights reserved.
// Copyright 2018 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style // Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file. // license that can be found in the LICENSE file.
@ -11,99 +10,19 @@ import (
"path/filepath" "path/filepath"
"testing" "testing"
"code.gitea.io/gitea/modules/setting"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
) )
func Test_parsePostgreSQLHostPort(t *testing.T) {
tests := []struct {
HostPort string
Host string
Port string
}{
{
HostPort: "127.0.0.1:1234",
Host: "127.0.0.1",
Port: "1234",
},
{
HostPort: "127.0.0.1",
Host: "127.0.0.1",
Port: "5432",
},
{
HostPort: "[::1]:1234",
Host: "[::1]",
Port: "1234",
},
{
HostPort: "[::1]",
Host: "[::1]",
Port: "5432",
},
{
HostPort: "/tmp/pg.sock:1234",
Host: "/tmp/pg.sock",
Port: "1234",
},
{
HostPort: "/tmp/pg.sock",
Host: "/tmp/pg.sock",
Port: "5432",
},
}
for _, test := range tests {
host, port := parsePostgreSQLHostPort(test.HostPort)
assert.Equal(t, test.Host, host)
assert.Equal(t, test.Port, port)
}
}
func Test_getPostgreSQLConnectionString(t *testing.T) {
tests := []struct {
Host string
Port string
User string
Passwd string
Name string
Param string
SSLMode string
Output string
}{
{
Host: "/tmp/pg.sock",
Port: "4321",
User: "testuser",
Passwd: "space space !#$%^^%^```-=?=",
Name: "gitea",
Param: "",
SSLMode: "false",
Output: "postgres://testuser:space%20space%20%21%23$%25%5E%5E%25%5E%60%60%60-=%3F=@:5432/giteasslmode=false&host=/tmp/pg.sock",
},
{
Host: "localhost",
Port: "1234",
User: "pgsqlusername",
Passwd: "I love Gitea!",
Name: "gitea",
Param: "",
SSLMode: "true",
Output: "postgres://pgsqlusername:I%20love%20Gitea%21@localhost:5432/giteasslmode=true",
},
}
for _, test := range tests {
connStr := getPostgreSQLConnectionString(test.Host, test.User, test.Passwd, test.Name, test.Param, test.SSLMode)
assert.Equal(t, test.Output, connStr)
}
}
func TestDumpDatabase(t *testing.T) { func TestDumpDatabase(t *testing.T) {
assert.NoError(t, PrepareTestDatabase()) assert.NoError(t, PrepareTestDatabase())
dir, err := ioutil.TempDir(os.TempDir(), "dump") dir, err := ioutil.TempDir(os.TempDir(), "dump")
assert.NoError(t, err) assert.NoError(t, err)
for _, dbType := range supportedDatabases { for _, dbName := range setting.SupportedDatabases {
dbType := setting.GetDBTypeByName(dbName)
assert.NoError(t, DumpDatabase(filepath.Join(dir, dbType+".sql"), dbType)) assert.NoError(t, DumpDatabase(filepath.Join(dir, dbType+".sql"), dbType))
} }
} }

View file

@ -2201,7 +2201,7 @@ func GitFsck() {
log.Trace("Doing: GitFsck") log.Trace("Doing: GitFsck")
if err := x. if err := x.
Where("id>0 AND is_fsck_enabled=?", true).BufferSize(setting.IterateBufferSize). Where("id>0 AND is_fsck_enabled=?", true).BufferSize(setting.Database.IterateBufferSize).
Iterate(new(Repository), Iterate(new(Repository),
func(idx int, bean interface{}) error { func(idx int, bean interface{}) error {
repo := bean.(*Repository) repo := bean.(*Repository)
@ -2225,7 +2225,7 @@ func GitFsck() {
func GitGcRepos() error { func GitGcRepos() error {
args := append([]string{"gc"}, setting.Git.GCArgs...) args := append([]string{"gc"}, setting.Git.GCArgs...)
return x. return x.
Where("id > 0").BufferSize(setting.IterateBufferSize). Where("id > 0").BufferSize(setting.Database.IterateBufferSize).
Iterate(new(Repository), Iterate(new(Repository),
func(idx int, bean interface{}) error { func(idx int, bean interface{}) error {
repo := bean.(*Repository) repo := bean.(*Repository)
@ -2568,7 +2568,7 @@ func (repo *Repository) generateRandomAvatar(e Engine) error {
// RemoveRandomAvatars removes the randomly generated avatars that were created for repositories // RemoveRandomAvatars removes the randomly generated avatars that were created for repositories
func RemoveRandomAvatars() error { func RemoveRandomAvatars() error {
return x. return x.
Where("id > 0").BufferSize(setting.IterateBufferSize). Where("id > 0").BufferSize(setting.Database.IterateBufferSize).
Iterate(new(Repository), Iterate(new(Repository),
func(idx int, bean interface{}) error { func(idx int, bean interface{}) error {
repository := bean.(*Repository) repository := bean.(*Repository)

View file

@ -49,7 +49,7 @@ func MainTest(m *testing.M, pathToGiteaRoot string) {
setting.RunUser = "runuser" setting.RunUser = "runuser"
setting.SSH.Port = 3000 setting.SSH.Port = 3000
setting.SSH.Domain = "try.gitea.io" setting.SSH.Domain = "try.gitea.io"
setting.UseSQLite3 = true setting.Database.UseSQLite3 = true
setting.RepoRootPath, err = ioutil.TempDir(os.TempDir(), "repos") setting.RepoRootPath, err = ioutil.TempDir(os.TempDir(), "repos")
if err != nil { if err != nil {
fatalTestError("TempDir: %v\n", err) fatalTestError("TempDir: %v\n", err)

View file

@ -40,7 +40,6 @@ import (
"golang.org/x/crypto/scrypt" "golang.org/x/crypto/scrypt"
"golang.org/x/crypto/ssh" "golang.org/x/crypto/ssh"
"xorm.io/builder" "xorm.io/builder"
"xorm.io/core"
) )
// UserType defines the user type // UserType defines the user type
@ -1432,9 +1431,9 @@ func (opts *SearchUserOptions) toConds() builder.Cond {
if opts.OwnerID > 0 { if opts.OwnerID > 0 {
var exprCond builder.Cond var exprCond builder.Cond
if DbCfg.Type == core.MYSQL { if setting.Database.UseMySQL {
exprCond = builder.Expr("org_user.org_id = user.id") exprCond = builder.Expr("org_user.org_id = user.id")
} else if DbCfg.Type == core.MSSQL { } else if setting.Database.UseMSSQL {
exprCond = builder.Expr("org_user.org_id = [user].id") exprCond = builder.Expr("org_user.org_id = [user].id")
} else { } else {
exprCond = builder.Expr("org_user.org_id = \"user\".id") exprCond = builder.Expr("org_user.org_id = \"user\".id")

View file

@ -21,13 +21,13 @@ func GetUserHeatmapDataByUser(user *User) ([]*UserHeatmapData, error) {
var groupBy string var groupBy string
var groupByName = "timestamp" // We need this extra case because mssql doesn't allow grouping by alias var groupByName = "timestamp" // We need this extra case because mssql doesn't allow grouping by alias
switch { switch {
case setting.UseSQLite3: case setting.Database.UseSQLite3:
groupBy = "strftime('%s', strftime('%Y-%m-%d', created_unix, 'unixepoch'))" groupBy = "strftime('%s', strftime('%Y-%m-%d', created_unix, 'unixepoch'))"
case setting.UseMySQL: case setting.Database.UseMySQL:
groupBy = "UNIX_TIMESTAMP(DATE(FROM_UNIXTIME(created_unix)))" groupBy = "UNIX_TIMESTAMP(DATE(FROM_UNIXTIME(created_unix)))"
case setting.UsePostgreSQL: case setting.Database.UsePostgreSQL:
groupBy = "extract(epoch from date_trunc('day', to_timestamp(created_unix)))" groupBy = "extract(epoch from date_trunc('day', to_timestamp(created_unix)))"
case setting.UseMSSQL: case setting.Database.UseMSSQL:
groupBy = "datediff(SECOND, '19700101', dateadd(DAY, 0, datediff(day, 0, dateadd(s, created_unix, '19700101'))))" groupBy = "datediff(SECOND, '19700101', dateadd(DAY, 0, datediff(day, 0, dateadd(s, created_unix, '19700101'))))"
groupByName = groupBy groupByName = groupBy
} }

171
modules/setting/database.go Normal file
View file

@ -0,0 +1,171 @@
// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package setting
import (
"errors"
"fmt"
"net/url"
"os"
"path"
"path/filepath"
"strings"
"time"
)
var (
// SupportedDatabases includes all supported databases type
SupportedDatabases = []string{"MySQL", "PostgreSQL", "MSSQL"}
dbTypes = map[string]string{"MySQL": "mysql", "PostgreSQL": "postgres", "MSSQL": "mssql", "SQLite3": "sqlite3"}
// EnableSQLite3 use SQLite3, set by build flag
EnableSQLite3 bool
// Database holds the database settings
Database = struct {
Type string
Host string
Name string
User string
Passwd string
SSLMode string
Path string
LogSQL bool
Charset string
Timeout int // seconds
UseSQLite3 bool
UseMySQL bool
UseMSSQL bool
UsePostgreSQL bool
DBConnectRetries int
DBConnectBackoff time.Duration
MaxIdleConns int
ConnMaxLifetime time.Duration
IterateBufferSize int
}{
Timeout: 500,
MaxIdleConns: 0,
ConnMaxLifetime: 3 * time.Second,
}
)
// GetDBTypeByName returns the dataase type as it defined on XORM according the given name
func GetDBTypeByName(name string) string {
return dbTypes[name]
}
// InitDBConfig loads the database settings
func InitDBConfig() {
sec := Cfg.Section("database")
Database.Type = sec.Key("DB_TYPE").String()
switch Database.Type {
case "sqlite3":
Database.UseSQLite3 = true
case "mysql":
Database.UseMySQL = true
case "postgres":
Database.UsePostgreSQL = true
case "mssql":
Database.UseMSSQL = true
}
Database.Host = sec.Key("HOST").String()
Database.Name = sec.Key("NAME").String()
Database.User = sec.Key("USER").String()
if len(Database.Passwd) == 0 {
Database.Passwd = sec.Key("PASSWD").String()
}
Database.SSLMode = sec.Key("SSL_MODE").MustString("disable")
Database.Charset = sec.Key("CHARSET").In("utf8", []string{"utf8", "utf8mb4"})
Database.Path = sec.Key("PATH").MustString(filepath.Join(AppDataPath, "gitea.db"))
Database.Timeout = sec.Key("SQLITE_TIMEOUT").MustInt(500)
Database.MaxIdleConns = sec.Key("MAX_IDLE_CONNS").MustInt(0)
Database.ConnMaxLifetime = sec.Key("CONN_MAX_LIFE_TIME").MustDuration(3 * time.Second)
Database.IterateBufferSize = sec.Key("ITERATE_BUFFER_SIZE").MustInt(50)
Database.LogSQL = sec.Key("LOG_SQL").MustBool(true)
Database.DBConnectRetries = sec.Key("DB_RETRIES").MustInt(10)
Database.DBConnectBackoff = sec.Key("DB_RETRY_BACKOFF").MustDuration(3 * time.Second)
}
// DBConnStr returns database connection string
func DBConnStr() (string, error) {
connStr := ""
var Param = "?"
if strings.Contains(Database.Name, Param) {
Param = "&"
}
switch Database.Type {
case "mysql":
connType := "tcp"
if Database.Host[0] == '/' { // looks like a unix socket
connType = "unix"
}
tls := Database.SSLMode
if tls == "disable" { // allow (Postgres-inspired) default value to work in MySQL
tls = "false"
}
connStr = fmt.Sprintf("%s:%s@%s(%s)/%s%scharset=%s&parseTime=true&tls=%s",
Database.User, Database.Passwd, connType, Database.Host, Database.Name, Param, Database.Charset, tls)
case "postgres":
connStr = getPostgreSQLConnectionString(Database.Host, Database.User, Database.Passwd, Database.Name, Param, Database.SSLMode)
case "mssql":
host, port := ParseMSSQLHostPort(Database.Host)
connStr = fmt.Sprintf("server=%s; port=%s; database=%s; user id=%s; password=%s;", host, port, Database.Name, Database.User, Database.Passwd)
case "sqlite3":
if !EnableSQLite3 {
return "", errors.New("this binary version does not build support for SQLite3")
}
if err := os.MkdirAll(path.Dir(Database.Path), os.ModePerm); err != nil {
return "", fmt.Errorf("Failed to create directories: %v", err)
}
connStr = fmt.Sprintf("file:%s?cache=shared&mode=rwc&_busy_timeout=%d", Database.Path, Database.Timeout)
default:
return "", fmt.Errorf("Unknown database type: %s", Database.Type)
}
return connStr, nil
}
// parsePostgreSQLHostPort parses given input in various forms defined in
// https://www.postgresql.org/docs/current/static/libpq-connect.html#LIBPQ-CONNSTRING
// and returns proper host and port number.
func parsePostgreSQLHostPort(info string) (string, string) {
host, port := "127.0.0.1", "5432"
if strings.Contains(info, ":") && !strings.HasSuffix(info, "]") {
idx := strings.LastIndex(info, ":")
host = info[:idx]
port = info[idx+1:]
} else if len(info) > 0 {
host = info
}
return host, port
}
func getPostgreSQLConnectionString(dbHost, dbUser, dbPasswd, dbName, dbParam, dbsslMode string) (connStr string) {
host, port := parsePostgreSQLHostPort(dbHost)
if host[0] == '/' { // looks like a unix socket
connStr = fmt.Sprintf("postgres://%s:%s@:%s/%s%ssslmode=%s&host=%s",
url.PathEscape(dbUser), url.PathEscape(dbPasswd), port, dbName, dbParam, dbsslMode, host)
} else {
connStr = fmt.Sprintf("postgres://%s:%s@%s:%s/%s%ssslmode=%s",
url.PathEscape(dbUser), url.PathEscape(dbPasswd), host, port, dbName, dbParam, dbsslMode)
}
return
}
// ParseMSSQLHostPort splits the host into host and port
func ParseMSSQLHostPort(info string) (string, string) {
host, port := "127.0.0.1", "1433"
if strings.Contains(info, ":") {
host = strings.Split(info, ":")[0]
port = strings.Split(info, ":")[1]
} else if strings.Contains(info, ",") {
host = strings.Split(info, ",")[0]
port = strings.TrimSpace(strings.Split(info, ",")[1])
} else if len(info) > 0 {
host = info
}
return host, port
}

View file

@ -4,7 +4,7 @@
// Use of this source code is governed by a MIT-style // Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file. // license that can be found in the LICENSE file.
package models package setting
import ( import (
_ "github.com/mattn/go-sqlite3" _ "github.com/mattn/go-sqlite3"
@ -12,5 +12,5 @@ import (
func init() { func init() {
EnableSQLite3 = true EnableSQLite3 = true
supportedDatabases = append(supportedDatabases, "sqlite3") SupportedDatabases = append(SupportedDatabases, "SQLite3")
} }

View file

@ -0,0 +1,94 @@
// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package setting
import (
"testing"
"github.com/stretchr/testify/assert"
)
func Test_parsePostgreSQLHostPort(t *testing.T) {
tests := []struct {
HostPort string
Host string
Port string
}{
{
HostPort: "127.0.0.1:1234",
Host: "127.0.0.1",
Port: "1234",
},
{
HostPort: "127.0.0.1",
Host: "127.0.0.1",
Port: "5432",
},
{
HostPort: "[::1]:1234",
Host: "[::1]",
Port: "1234",
},
{
HostPort: "[::1]",
Host: "[::1]",
Port: "5432",
},
{
HostPort: "/tmp/pg.sock:1234",
Host: "/tmp/pg.sock",
Port: "1234",
},
{
HostPort: "/tmp/pg.sock",
Host: "/tmp/pg.sock",
Port: "5432",
},
}
for _, test := range tests {
host, port := parsePostgreSQLHostPort(test.HostPort)
assert.Equal(t, test.Host, host)
assert.Equal(t, test.Port, port)
}
}
func Test_getPostgreSQLConnectionString(t *testing.T) {
tests := []struct {
Host string
Port string
User string
Passwd string
Name string
Param string
SSLMode string
Output string
}{
{
Host: "/tmp/pg.sock",
Port: "4321",
User: "testuser",
Passwd: "space space !#$%^^%^```-=?=",
Name: "gitea",
Param: "",
SSLMode: "false",
Output: "postgres://testuser:space%20space%20%21%23$%25%5E%5E%25%5E%60%60%60-=%3F=@:5432/giteasslmode=false&host=/tmp/pg.sock",
},
{
Host: "localhost",
Port: "1234",
User: "pgsqlusername",
Passwd: "I love Gitea!",
Name: "gitea",
Param: "",
SSLMode: "true",
Output: "postgres://pgsqlusername:I%20love%20Gitea%21@localhost:5432/giteasslmode=true",
},
}
for _, test := range tests {
connStr := getPostgreSQLConnectionString(test.Host, test.User, test.Passwd, test.Name, test.Param, test.SSLMode)
assert.Equal(t, test.Output, connStr)
}
}

View file

@ -148,16 +148,6 @@ var (
DisableGitHooks bool DisableGitHooks bool
PasswordHashAlgo string PasswordHashAlgo string
// Database settings
UseSQLite3 bool
UseMySQL bool
UseMSSQL bool
UsePostgreSQL bool
UseTiDB bool
LogSQL bool
DBConnectRetries int
DBConnectBackoff time.Duration
// UI settings // UI settings
UI = struct { UI = struct {
ExplorePagingNum int ExplorePagingNum int
@ -357,7 +347,10 @@ var (
IsWindows bool IsWindows bool
HasRobotsTxt bool HasRobotsTxt bool
InternalToken string // internal access token InternalToken string // internal access token
IterateBufferSize int
// UILocation is the location on the UI, so that we can display the time on UI.
// Currently only show the default time.Local, it could be added to app.ini after UI is ready
UILocation = time.Local
) )
// DateLang transforms standard language locale name to corresponding value in datetime plugin. // DateLang transforms standard language locale name to corresponding value in datetime plugin.
@ -775,10 +768,6 @@ func NewContext() {
CSRFCookieHTTPOnly = sec.Key("CSRF_COOKIE_HTTP_ONLY").MustBool(true) CSRFCookieHTTPOnly = sec.Key("CSRF_COOKIE_HTTP_ONLY").MustBool(true)
InternalToken = loadInternalToken(sec) InternalToken = loadInternalToken(sec)
IterateBufferSize = Cfg.Section("database").Key("ITERATE_BUFFER_SIZE").MustInt(50)
LogSQL = Cfg.Section("database").Key("LOG_SQL").MustBool(true)
DBConnectRetries = Cfg.Section("database").Key("DB_RETRIES").MustInt(10)
DBConnectBackoff = Cfg.Section("database").Key("DB_RETRY_BACKOFF").MustDuration(3 * time.Second)
sec = Cfg.Section("attachment") sec = Cfg.Section("attachment")
AttachmentPath = sec.Key("PATH").MustString(path.Join(AppDataPath, "attachments")) AttachmentPath = sec.Key("PATH").MustString(path.Join(AppDataPath, "attachments"))
@ -1037,6 +1026,7 @@ func loadOrGenerateInternalToken(sec *ini.Section) string {
// NewServices initializes the services // NewServices initializes the services
func NewServices() { func NewServices() {
InitDBConfig()
newService() newService()
NewLogServices(false) NewLogServices(false)
newCacheService() newCacheService()

View file

@ -289,7 +289,7 @@ func Config(ctx *context.Context) {
ctx.Data["LFS"] = setting.LFS ctx.Data["LFS"] = setting.LFS
ctx.Data["Service"] = setting.Service ctx.Data["Service"] = setting.Service
ctx.Data["DbCfg"] = models.DbCfg ctx.Data["DbCfg"] = setting.Database
ctx.Data["Webhook"] = setting.Webhook ctx.Data["Webhook"] = setting.Webhook
ctx.Data["MailerEnabled"] = false ctx.Data["MailerEnabled"] = false
@ -333,7 +333,7 @@ func Config(ctx *context.Context) {
ctx.Data["AccessLogTemplate"] = setting.AccessLogTemplate ctx.Data["AccessLogTemplate"] = setting.AccessLogTemplate
ctx.Data["DisableRouterLog"] = setting.DisableRouterLog ctx.Data["DisableRouterLog"] = setting.DisableRouterLog
ctx.Data["EnableXORMLog"] = setting.EnableXORMLog ctx.Data["EnableXORMLog"] = setting.EnableXORMLog
ctx.Data["LogSQL"] = setting.LogSQL ctx.Data["LogSQL"] = setting.Database.LogSQL
ctx.HTML(200, tplConfig) ctx.HTML(200, tplConfig)
} }

View file

@ -47,16 +47,16 @@ func NewServices() {
// In case of problems connecting to DB, retry connection. Eg, PGSQL in Docker Container on Synology // In case of problems connecting to DB, retry connection. Eg, PGSQL in Docker Container on Synology
func initDBEngine() (err error) { func initDBEngine() (err error) {
log.Info("Beginning ORM engine initialization.") log.Info("Beginning ORM engine initialization.")
for i := 0; i < setting.DBConnectRetries; i++ { for i := 0; i < setting.Database.DBConnectRetries; i++ {
log.Info("ORM engine initialization attempt #%d/%d...", i+1, setting.DBConnectRetries) log.Info("ORM engine initialization attempt #%d/%d...", i+1, setting.Database.DBConnectRetries)
if err = models.NewEngine(migrations.Migrate); err == nil { if err = models.NewEngine(migrations.Migrate); err == nil {
break break
} else if i == setting.DBConnectRetries-1 { } else if i == setting.Database.DBConnectRetries-1 {
return err return err
} }
log.Error("ORM engine initialization attempt #%d/%d failed. Error: %v", i+1, setting.DBConnectRetries, err) log.Error("ORM engine initialization attempt #%d/%d failed. Error: %v", i+1, setting.Database.DBConnectRetries, err)
log.Info("Backing off for %d seconds", int64(setting.DBConnectBackoff/time.Second)) log.Info("Backing off for %d seconds", int64(setting.Database.DBConnectBackoff/time.Second))
time.Sleep(setting.DBConnectBackoff) time.Sleep(setting.Database.DBConnectBackoff)
} }
models.HasEngine = true models.HasEngine = true
return nil return nil
@ -73,7 +73,7 @@ func GlobalInit() {
log.Trace("AppWorkPath: %s", setting.AppWorkPath) log.Trace("AppWorkPath: %s", setting.AppWorkPath)
log.Trace("Custom path: %s", setting.CustomPath) log.Trace("Custom path: %s", setting.CustomPath)
log.Trace("Log path: %s", setting.LogRootPath) log.Trace("Log path: %s", setting.LogRootPath)
models.LoadConfigs()
NewServices() NewServices()
if setting.InstallLock { if setting.InstallLock {
@ -102,12 +102,9 @@ func GlobalInit() {
models.InitDeliverHooks() models.InitDeliverHooks()
models.InitTestPullRequests() models.InitTestPullRequests()
} }
if models.EnableSQLite3 { if setting.EnableSQLite3 {
log.Info("SQLite3 Supported") log.Info("SQLite3 Supported")
} }
if models.EnableTiDB {
log.Info("TiDB Supported")
}
checkRunMode() checkRunMode()
if setting.InstallLock && setting.SSH.StartBuiltinServer { if setting.InstallLock && setting.SSH.StartBuiltinServer {

View file

@ -40,11 +40,7 @@ func InstallInit(ctx *context.Context) {
ctx.Data["Title"] = ctx.Tr("install.install") ctx.Data["Title"] = ctx.Tr("install.install")
ctx.Data["PageIsInstall"] = true ctx.Data["PageIsInstall"] = true
dbOpts := []string{"MySQL", "PostgreSQL", "MSSQL"} ctx.Data["DbOptions"] = setting.SupportedDatabases
if models.EnableSQLite3 {
dbOpts = append(dbOpts, "SQLite3")
}
ctx.Data["DbOptions"] = dbOpts
} }
// Install render installation page // Install render installation page
@ -52,21 +48,21 @@ func Install(ctx *context.Context) {
form := auth.InstallForm{} form := auth.InstallForm{}
// Database settings // Database settings
form.DbHost = models.DbCfg.Host form.DbHost = setting.Database.Host
form.DbUser = models.DbCfg.User form.DbUser = setting.Database.User
form.DbPasswd = models.DbCfg.Passwd form.DbPasswd = setting.Database.Passwd
form.DbName = models.DbCfg.Name form.DbName = setting.Database.Name
form.DbPath = models.DbCfg.Path form.DbPath = setting.Database.Path
form.Charset = models.DbCfg.Charset form.Charset = setting.Database.Charset
ctx.Data["CurDbOption"] = "MySQL" ctx.Data["CurDbOption"] = "MySQL"
switch models.DbCfg.Type { switch setting.Database.Type {
case "postgres": case "postgres":
ctx.Data["CurDbOption"] = "PostgreSQL" ctx.Data["CurDbOption"] = "PostgreSQL"
case "mssql": case "mssql":
ctx.Data["CurDbOption"] = "MSSQL" ctx.Data["CurDbOption"] = "MSSQL"
case "sqlite3": case "sqlite3":
if models.EnableSQLite3 { if setting.EnableSQLite3 {
ctx.Data["CurDbOption"] = "SQLite3" ctx.Data["CurDbOption"] = "SQLite3"
} }
} }
@ -144,18 +140,18 @@ func InstallPost(ctx *context.Context, form auth.InstallForm) {
// Pass basic check, now test configuration. // Pass basic check, now test configuration.
// Test database setting. // Test database setting.
dbTypes := map[string]string{"MySQL": "mysql", "PostgreSQL": "postgres", "MSSQL": "mssql", "SQLite3": "sqlite3"}
models.DbCfg.Type = dbTypes[form.DbType]
models.DbCfg.Host = form.DbHost
models.DbCfg.User = form.DbUser
models.DbCfg.Passwd = form.DbPasswd
models.DbCfg.Name = form.DbName
models.DbCfg.SSLMode = form.SSLMode
models.DbCfg.Charset = form.Charset
models.DbCfg.Path = form.DbPath
if (models.DbCfg.Type == "sqlite3") && setting.Database.Type = setting.GetDBTypeByName(form.DbType)
len(models.DbCfg.Path) == 0 { setting.Database.Host = form.DbHost
setting.Database.User = form.DbUser
setting.Database.Passwd = form.DbPasswd
setting.Database.Name = form.DbName
setting.Database.SSLMode = form.SSLMode
setting.Database.Charset = form.Charset
setting.Database.Path = form.DbPath
if (setting.Database.Type == "sqlite3") &&
len(setting.Database.Path) == 0 {
ctx.Data["Err_DbPath"] = true ctx.Data["Err_DbPath"] = true
ctx.RenderWithErr(ctx.Tr("install.err_empty_db_path"), tplInstall, &form) ctx.RenderWithErr(ctx.Tr("install.err_empty_db_path"), tplInstall, &form)
return return
@ -265,14 +261,14 @@ func InstallPost(ctx *context.Context, form auth.InstallForm) {
log.Error("Failed to load custom conf '%s': %v", setting.CustomConf, err) log.Error("Failed to load custom conf '%s': %v", setting.CustomConf, err)
} }
} }
cfg.Section("database").Key("DB_TYPE").SetValue(models.DbCfg.Type) cfg.Section("database").Key("DB_TYPE").SetValue(setting.Database.Type)
cfg.Section("database").Key("HOST").SetValue(models.DbCfg.Host) cfg.Section("database").Key("HOST").SetValue(setting.Database.Host)
cfg.Section("database").Key("NAME").SetValue(models.DbCfg.Name) cfg.Section("database").Key("NAME").SetValue(setting.Database.Name)
cfg.Section("database").Key("USER").SetValue(models.DbCfg.User) cfg.Section("database").Key("USER").SetValue(setting.Database.User)
cfg.Section("database").Key("PASSWD").SetValue(models.DbCfg.Passwd) cfg.Section("database").Key("PASSWD").SetValue(setting.Database.Passwd)
cfg.Section("database").Key("SSL_MODE").SetValue(models.DbCfg.SSLMode) cfg.Section("database").Key("SSL_MODE").SetValue(setting.Database.SSLMode)
cfg.Section("database").Key("CHARSET").SetValue(models.DbCfg.Charset) cfg.Section("database").Key("CHARSET").SetValue(setting.Database.Charset)
cfg.Section("database").Key("PATH").SetValue(models.DbCfg.Path) cfg.Section("database").Key("PATH").SetValue(setting.Database.Path)
cfg.Section("").Key("APP_NAME").SetValue(form.AppName) cfg.Section("").Key("APP_NAME").SetValue(form.AppName)
cfg.Section("repository").Key("ROOT").SetValue(form.RepoRootPath) cfg.Section("repository").Key("ROOT").SetValue(form.RepoRootPath)