forgejo/modules/setting/database.go

234 lines
7.2 KiB
Go
Raw Normal View History

// Copyright 2019 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package setting
import (
"errors"
"fmt"
"net"
"net/url"
"os"
"path/filepath"
"strings"
"time"
)
var (
// SupportedDatabaseTypes includes all XORM supported databases type, sqlite3 maybe added by `database_sqlite3.go`
SupportedDatabaseTypes = []string{"mysql", "postgres", "mssql"}
// DatabaseTypeNames contains the friendly names for all database types
DatabaseTypeNames = map[string]string{"mysql": "MySQL", "postgres": "PostgreSQL", "mssql": "MSSQL", "sqlite3": "SQLite3"}
// EnableSQLite3 use SQLite3, set by build flag
EnableSQLite3 bool
// Database holds the database settings
Database = struct {
Type DatabaseType
Host string
Name string
User string
Passwd string
Schema string
SSLMode string
Path string
LogSQL bool
MysqlCharset string
CharsetCollation string
Timeout int // seconds
SQLiteJournalMode string
DBConnectRetries int
DBConnectBackoff time.Duration
MaxIdleConns int
MaxOpenConns int
ConnMaxIdleTime time.Duration
ConnMaxLifetime time.Duration
IterateBufferSize int
AutoMigration bool
SlowQueryThreshold time.Duration
}{
Timeout: 500,
IterateBufferSize: 50,
}
)
// LoadDBSetting loads the database settings
func LoadDBSetting() {
loadDBSetting(CfgProvider)
}
func loadDBSetting(rootCfg ConfigProvider) {
sec := rootCfg.Section("database")
Database.Type = DatabaseType(sec.Key("DB_TYPE").String())
Database.Host = sec.Key("HOST").String()
Database.Name = sec.Key("NAME").String()
Database.User = sec.Key("USER").String()
if len(Database.Passwd) == 0 {
Database.Passwd = sec.Key("PASSWD").String()
}
Database.Schema = sec.Key("SCHEMA").String()
Database.SSLMode = sec.Key("SSL_MODE").MustString("disable")
Database.CharsetCollation = sec.Key("CHARSET_COLLATION").String()
[BRANDING] Rebrand default config settings for new installs (#140) Replaces `Gitea` with `Forgejo` in the default config settings for new installs. This will not affect existing installs. Co-authored-by: Caesar Schinas <caesar@caesarschinas.com> Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/140 Co-authored-by: Caesar Schinas <caesar@noreply.codeberg.org> Co-committed-by: Caesar Schinas <caesar@noreply.codeberg.org> (cherry picked from commit ca1319aa16128516e50dabdc8e8cadc23eb71d2f) (cherry picked from commit 52a4d238a0b5bbea28b28e55e35f28c51ecbb2de) (cherry picked from commit f63536538cfe4b010ebb5a6323f4b5e5b6ec7232) Conflicts: web_src/js/features/install.js (cherry picked from commit 861cc434e129f3fbd932ee36067c560e754dab9a) (cherry picked from commit 0e6ea60c802d6cfd95dea4aad5df96bb6e4bc7a5) (cherry picked from commit 0cbc0ec15d9d952b0ecfb68a36bf58cbae0c43dd) (cherry picked from commit 3cc19b0ae214ae163f23efa52ab0aa53eb7c935b) (cherry picked from commit 50fcb885fe2f87a27e6ff778cdd0c7cd97bbe9e9) (cherry picked from commit f6039d4df481fc58b8db3e863158665d24cac847) (cherry picked from commit 5ae5c6ba2deefe829f768900f5e1bbcbe9389e15) (cherry picked from commit f0b565e0ed7fe52f0264e92c49736b487b9bff8a) (cherry picked from commit adbd4d2015e8e2c4789849c84cfa4032acd40b7f) (cherry picked from commit d26c540ffdbfb7ec83412635576ab39653d1b531) (cherry picked from commit 6df6781b42ea126d029ba9e85485dbcb9bf6601d) (cherry picked from commit b6fb56e1c407195bedfe09f91ecb6537024d5189) (cherry picked from commit bb4f98a0ca7515aa6c44529df0573195f779a643) (cherry picked from commit 6779229f2732f6791aba3bd1ba51a338a88ca1f3) (cherry picked from commit c216c85aee552aa15e9d6316002ee9e170de64d7) (cherry picked from commit dff780bced1dc78ddd7fa4952766969ee528c538) (cherry picked from commit 4e036aa3b6d1bc5f2041c3b30ec289cb082fe824) (cherry picked from commit 8b3bc3e8a64edaba64cb759ea31248eea7307937) (cherry picked from commit 1e4d8523321efaf6c5febdd77ba5150effe5c1ac) (cherry picked from commit 07a15d18447bb03bb04001f1f65305670d1467ef) (cherry picked from commit fb44b3e10d685dd180f37678a3e2a64ee641f2b2) (cherry picked from commit b212d833190ce59230b3fc288c1aac5106cf33a0) (cherry picked from commit 5754971be5c37e97f3165878af3760117e40af01) (cherry picked from commit 0c43b4e82cab028fae1b709c549251d63a3ffc04) Conflicts: routers/install/install.go https://codeberg.org/forgejo/forgejo/pulls/1351 (cherry picked from commit 2e22a7208a0f149afc3794425c7b5b2b0181a939) (cherry picked from commit 676b0a8a48f99d751b677b9eca5b57238b5003a5) (cherry picked from commit bc4a8bf9bcb75e92b7802c477b1bb14a047344a6) (cherry picked from commit 5e09a4e174cc6d62b08fa718d0309275effadbd8) (cherry picked from commit 712c52a32a3b6ce9aafaefaf9e63729522d1e0c5) (cherry picked from commit ba3d93cc4a5dca5d0daef647fab557e0136925e2) (cherry picked from commit a5a0396abc512834ca7b0b247277e4d753a87659) (cherry picked from commit 09b205f30b17b8f22688f39b5f98d06b44745a4b) (cherry picked from commit 52b7729e86aade4faa9577abb53401a4040131f0) (cherry picked from commit bfe78735e84312bccb8adb402041830da93ba646) (cherry picked from commit 915c4f7a7b1e20ba4299a8631c873c7aed288992) (cherry picked from commit 3b1af856bbc604511bdd42eda65dbf0808a12abc) (cherry picked from commit ffe2c0a3535fa510ba416d12324d00406678075e) (cherry picked from commit fd5cc72128272572a84b011f67a05999e60e931a) (cherry picked from commit 95d3555e47cdca47e603692e669ebbdc024dc6d3) (cherry picked from commit b9b853f4b0a9a0587a1b2fce7c63bb315aba0af2) (cherry picked from commit 967453eb6f5776d65883797dd9ad24b57eecbbd3) (cherry picked from commit 3298f502dda8570884494e557df5800ba0f2150f) (cherry picked from commit 3481d1f816fe87db000cde4ef1e6d952c520efcf) (cherry picked from commit 7508c24a0c6d44f98ff1ccb96285309e640b16bb) (cherry picked from commit 54e97f7c35bbb1800defdfdf99934f22505c0c4d) (cherry picked from commit 492876992f59c597882c69fef6880d7981e33046)
2022-12-19 20:01:46 +00:00
Database.Path = sec.Key("PATH").MustString(filepath.Join(AppDataPath, "forgejo.db"))
Database.Timeout = sec.Key("SQLITE_TIMEOUT").MustInt(500)
Database.SQLiteJournalMode = sec.Key("SQLITE_JOURNAL_MODE").MustString("")
Database.MaxIdleConns = sec.Key("MAX_IDLE_CONNS").MustInt(2)
if Database.Type.IsMySQL() {
Database.ConnMaxLifetime = sec.Key("CONN_MAX_LIFETIME").MustDuration(3 * time.Second)
} else {
Database.ConnMaxLifetime = sec.Key("CONN_MAX_LIFETIME").MustDuration(0)
}
Database.ConnMaxIdleTime = sec.Key("CONN_MAX_IDLETIME").MustDuration(0)
Database.MaxOpenConns = sec.Key("MAX_OPEN_CONNS").MustInt(0)
Database.IterateBufferSize = sec.Key("ITERATE_BUFFER_SIZE").MustInt(50)
Rewrite logger system (#24726) ## ⚠️ Breaking The `log.<mode>.<logger>` style config has been dropped. If you used it, please check the new config manual & app.example.ini to make your instance output logs as expected. Although many legacy options still work, it's encouraged to upgrade to the new options. The SMTP logger is deleted because SMTP is not suitable to collect logs. If you have manually configured Gitea log options, please confirm the logger system works as expected after upgrading. ## Description Close #12082 and maybe more log-related issues, resolve some related FIXMEs in old code (which seems unfixable before) Just like rewriting queue #24505 : make code maintainable, clear legacy bugs, and add the ability to support more writers (eg: JSON, structured log) There is a new document (with examples): `logging-config.en-us.md` This PR is safer than the queue rewriting, because it's just for logging, it won't break other logic. ## The old problems The logging system is quite old and difficult to maintain: * Unclear concepts: Logger, NamedLogger, MultiChannelledLogger, SubLogger, EventLogger, WriterLogger etc * Some code is diffuclt to konw whether it is right: `log.DelNamedLogger("console")` vs `log.DelNamedLogger(log.DEFAULT)` vs `log.DelLogger("console")` * The old system heavily depends on ini config system, it's difficult to create new logger for different purpose, and it's very fragile. * The "color" trick is difficult to use and read, many colors are unnecessary, and in the future structured log could help * It's difficult to add other log formats, eg: JSON format * The log outputer doesn't have full control of its goroutine, it's difficult to make outputer have advanced behaviors * The logs could be lost in some cases: eg: no Fatal error when using CLI. * Config options are passed by JSON, which is quite fragile. * INI package makes the KEY in `[log]` section visible in `[log.sub1]` and `[log.sub1.subA]`, this behavior is quite fragile and would cause more unclear problems, and there is no strong requirement to support `log.<mode>.<logger>` syntax. ## The new design See `logger.go` for documents. ## Screenshot <details> ![image](https://github.com/go-gitea/gitea/assets/2114189/4462d713-ba39-41f5-bb08-de912e67e1ff) ![image](https://github.com/go-gitea/gitea/assets/2114189/b188035e-f691-428b-8b2d-ff7b2199b2f9) ![image](https://github.com/go-gitea/gitea/assets/2114189/132e9745-1c3b-4e00-9e0d-15eaea495dee) </details> ## TODO * [x] add some new tests * [x] fix some tests * [x] test some sub-commands (manually ....) --------- Co-authored-by: Jason Song <i@wolfogre.com> Co-authored-by: delvh <dev.lh@web.de> Co-authored-by: Giteabot <teabot@gitea.io>
2023-05-21 22:35:11 +00:00
Database.LogSQL = sec.Key("LOG_SQL").MustBool(false)
Database.DBConnectRetries = sec.Key("DB_RETRIES").MustInt(10)
Database.DBConnectBackoff = sec.Key("DB_RETRY_BACKOFF").MustDuration(3 * time.Second)
Database.AutoMigration = sec.Key("AUTO_MIGRATION").MustBool(true)
deprecatedSetting(rootCfg, "database", "SLOW_QUERY_TRESHOLD", "database", "SLOW_QUERY_THRESHOLD", "1.23")
if sec.HasKey("SLOW_QUERY_TRESHOLD") && !sec.HasKey("SLOW_QUERY_THRESHOLD") {
Database.SlowQueryThreshold = sec.Key("SLOW_QUERY_TRESHOLD").MustDuration(5 * time.Second)
} else {
Database.SlowQueryThreshold = sec.Key("SLOW_QUERY_THRESHOLD").MustDuration(5 * time.Second)
}
}
// DBConnStr returns database connection string
func DBConnStr() (string, error) {
var connStr string
paramSep := "?"
if strings.Contains(Database.Name, paramSep) {
paramSep = "&"
}
switch Database.Type {
case "mysql":
connType := "tcp"
if len(Database.Host) > 0 && Database.Host[0] == '/' { // looks like a unix socket
connType = "unix"
}
tls := Database.SSLMode
if tls == "disable" { // allow (Postgres-inspired) default value to work in MySQL
tls = "false"
}
connStr = fmt.Sprintf("%s:%s@%s(%s)/%s%sparseTime=true&tls=%s",
Database.User, Database.Passwd, connType, Database.Host, Database.Name, paramSep, tls)
case "postgres":
connStr = getPostgreSQLConnectionString(Database.Host, Database.User, Database.Passwd, Database.Name, Database.SSLMode)
case "mssql":
host, port := ParseMSSQLHostPort(Database.Host)
connStr = fmt.Sprintf("server=%s; port=%s; database=%s; user id=%s; password=%s;", host, port, Database.Name, Database.User, Database.Passwd)
case "sqlite3":
if !EnableSQLite3 {
return "", errors.New("this Gitea binary was not built with SQLite3 support")
}
if err := os.MkdirAll(filepath.Dir(Database.Path), os.ModePerm); err != nil {
return "", fmt.Errorf("Failed to create directories: %w", err)
}
journalMode := ""
if Database.SQLiteJournalMode != "" {
journalMode = "&_journal_mode=" + Database.SQLiteJournalMode
}
connStr = fmt.Sprintf("file:%s?cache=shared&mode=rwc&_busy_timeout=%d&_txlock=immediate%s",
Database.Path, Database.Timeout, journalMode)
default:
return "", fmt.Errorf("unknown database type: %s", Database.Type)
}
return connStr, nil
}
// parsePostgreSQLHostPort parses given input in various forms defined in
// https://www.postgresql.org/docs/current/static/libpq-connect.html#LIBPQ-CONNSTRING
// and returns proper host and port number.
func parsePostgreSQLHostPort(info string) (host, port string) {
if h, p, err := net.SplitHostPort(info); err == nil {
host, port = h, p
} else {
// treat the "info" as "host", if it's an IPv6 address, remove the wrapper
host = info
if strings.HasPrefix(host, "[") && strings.HasSuffix(host, "]") {
host = host[1 : len(host)-1]
}
}
// set fallback values
if host == "" {
host = "127.0.0.1"
}
if port == "" {
port = "5432"
}
return host, port
}
func getPostgreSQLConnectionString(dbHost, dbUser, dbPasswd, dbName, dbsslMode string) (connStr string) {
dbName, dbParam, _ := strings.Cut(dbName, "?")
host, port := parsePostgreSQLHostPort(dbHost)
connURL := url.URL{
Scheme: "postgres",
User: url.UserPassword(dbUser, dbPasswd),
Host: net.JoinHostPort(host, port),
Path: dbName,
OmitHost: false,
RawQuery: dbParam,
}
query := connURL.Query()
if strings.HasPrefix(host, "/") { // looks like a unix socket
query.Add("host", host)
connURL.Host = ":" + port
}
query.Set("sslmode", dbsslMode)
connURL.RawQuery = query.Encode()
return connURL.String()
}
// ParseMSSQLHostPort splits the host into host and port
func ParseMSSQLHostPort(info string) (string, string) {
// the default port "0" might be related to MSSQL's dynamic port, maybe it should be double-confirmed in the future
host, port := "127.0.0.1", "0"
if strings.Contains(info, ":") {
host = strings.Split(info, ":")[0]
port = strings.Split(info, ":")[1]
} else if strings.Contains(info, ",") {
host = strings.Split(info, ",")[0]
port = strings.TrimSpace(strings.Split(info, ",")[1])
} else if len(info) > 0 {
host = info
}
if host == "" {
host = "127.0.0.1"
}
if port == "" {
port = "0"
}
return host, port
}
type DatabaseType string
func (t DatabaseType) String() string {
return string(t)
}
func (t DatabaseType) IsSQLite3() bool {
return t == "sqlite3"
}
func (t DatabaseType) IsMySQL() bool {
return t == "mysql"
}
func (t DatabaseType) IsMSSQL() bool {
return t == "mssql"
}
func (t DatabaseType) IsPostgreSQL() bool {
return t == "postgres"
}