forgejo/modules/setting/database.go

225 lines
6.8 KiB
Go
Raw Normal View History

// Copyright 2019 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package setting
import (
"errors"
"fmt"
"net"
"net/url"
"os"
"path/filepath"
"strings"
"time"
)
var (
// SupportedDatabaseTypes includes all XORM supported databases type, sqlite3 maybe added by `database_sqlite3.go`
SupportedDatabaseTypes = []string{"mysql", "postgres", "mssql"}
// DatabaseTypeNames contains the friendly names for all database types
DatabaseTypeNames = map[string]string{"mysql": "MySQL", "postgres": "PostgreSQL", "mssql": "MSSQL", "sqlite3": "SQLite3"}
// EnableSQLite3 use SQLite3, set by build flag
EnableSQLite3 bool
// Database holds the database settings
Database = struct {
Type DatabaseType
Host string
Name string
User string
Passwd string
Schema string
SSLMode string
Path string
LogSQL bool
MysqlCharset string
Timeout int // seconds
SQLiteJournalMode string
DBConnectRetries int
DBConnectBackoff time.Duration
MaxIdleConns int
MaxOpenConns int
ConnMaxLifetime time.Duration
IterateBufferSize int
AutoMigration bool
[GITEA] Add slow SQL query warning - Databases are one of the most important parts of Forgejo, every interaction with Forgejo uses the database in one way or another. Therefore, it is important to maintain the database and recognize when Forgejo is not doing well with the database. Forgejo already has the option to log *every* SQL query along with its execution time, but monitoring becomes impractical for larger instances and takes up unnecessary storage in the logs. - Add a QoL enhancement that allows instance administrators to specify a threshold value beyond which query execution time is logged as a warning in the xorm logger. The default value is a conservative five seconds to avoid this becoming a source of spam in the logs. - The use case for this patch is that with an instance the size of Codeberg, monitoring SQL logs is not very fruitful and most of them are uninteresting. Recently, in the context of persistent deadlock issues (https://codeberg.org/forgejo/forgejo/issues/220), I have noticed that certain queries hold locks on tables like comment and issue for several seconds. This patch helps to identify which queries these are and when they happen. - Added unit test. (cherry picked from commit 24bbe7886fb4cb9a38c8dab8c44f4c9cbfa25481) (cherry picked from commit 6e29145b3c1455498531593d38e6a914941a12cb) (cherry picked from commit 63731e30712872bd2395eb3cf36d9996e5793645) (cherry picked from commit 3ce1a097369c132654de70df707b867e47bd1c40) (cherry picked from commit a64426907de788cc0937a7a2b16af4d2f26f7fe6) (cherry picked from commit 4b1921569156445c58d9889602733da5934c7b95) (cherry picked from commit e6356744359fa947c049827d60c2ea0e277e03dc) (cherry picked from commit 9cf501f1af4cd870221cef6af489618785b71186) (cherry picked from commit 0d6b934eba1c0e9b27b364791113aae816b6b366) (cherry picked from commit 4b6c2738795002887844a106f2fed2ef1673eed1) (cherry picked from commit 89b1315338b0c7a726a36a84e9844013a13560b8) (cherry picked from commit edd8e66ce991c395bb0af7720631c3cd26caaa51) [GITEA] Add slow SQL query warning (squash) document the setting (cherry picked from commit ce38599c5141c7fc6bc054819f5ff1c1b45bda1f) (cherry picked from commit 794aa67c68c8e24ac7301eb7ef767c6e2499a78d) (cherry picked from commit a4c2c6b004c21488e90f637ca7920f49108ed75d) (cherry picked from commit 97912752bc802db79bb26a6591aec885aea30ee4) (cherry picked from commit 00b5327c9750215a290238516e7b6fb1e6601e14) (cherry picked from commit 1069c860e78c11225b4d74ff3044df7786562821) (cherry picked from commit 84241f42c83852918b57c8bd25364697037fe42f) (cherry picked from commit e4bda0e8457d00c01b83f153ed5a4a8ea4cf85c8) (cherry picked from commit 7357fb91bff87045b133c3a7ac9fc70eea781bc4) (cherry picked from commit a8dd7f6da278ae112200b5efa5bf27e3961f5996)
2023-08-18 02:39:23 +00:00
SlowQueryTreshold time.Duration
}{
Timeout: 500,
IterateBufferSize: 50,
}
)
// LoadDBSetting loads the database settings
func LoadDBSetting() {
loadDBSetting(CfgProvider)
}
func loadDBSetting(rootCfg ConfigProvider) {
sec := rootCfg.Section("database")
Database.Type = DatabaseType(sec.Key("DB_TYPE").String())
Database.Host = sec.Key("HOST").String()
Database.Name = sec.Key("NAME").String()
Database.User = sec.Key("USER").String()
if len(Database.Passwd) == 0 {
Database.Passwd = sec.Key("PASSWD").String()
}
Database.Schema = sec.Key("SCHEMA").String()
Database.SSLMode = sec.Key("SSL_MODE").MustString("disable")
Database.MysqlCharset = sec.Key("MYSQL_CHARSET").MustString("utf8mb4") // do not document it, end users won't need it.
Database.Path = sec.Key("PATH").MustString(filepath.Join(AppDataPath, "gitea.db"))
Database.Timeout = sec.Key("SQLITE_TIMEOUT").MustInt(500)
Database.SQLiteJournalMode = sec.Key("SQLITE_JOURNAL_MODE").MustString("")
Database.MaxIdleConns = sec.Key("MAX_IDLE_CONNS").MustInt(2)
if Database.Type.IsMySQL() {
Database.ConnMaxLifetime = sec.Key("CONN_MAX_LIFETIME").MustDuration(3 * time.Second)
} else {
Database.ConnMaxLifetime = sec.Key("CONN_MAX_LIFETIME").MustDuration(0)
}
Database.MaxOpenConns = sec.Key("MAX_OPEN_CONNS").MustInt(0)
Database.IterateBufferSize = sec.Key("ITERATE_BUFFER_SIZE").MustInt(50)
Rewrite logger system (#24726) ## ⚠️ Breaking The `log.<mode>.<logger>` style config has been dropped. If you used it, please check the new config manual & app.example.ini to make your instance output logs as expected. Although many legacy options still work, it's encouraged to upgrade to the new options. The SMTP logger is deleted because SMTP is not suitable to collect logs. If you have manually configured Gitea log options, please confirm the logger system works as expected after upgrading. ## Description Close #12082 and maybe more log-related issues, resolve some related FIXMEs in old code (which seems unfixable before) Just like rewriting queue #24505 : make code maintainable, clear legacy bugs, and add the ability to support more writers (eg: JSON, structured log) There is a new document (with examples): `logging-config.en-us.md` This PR is safer than the queue rewriting, because it's just for logging, it won't break other logic. ## The old problems The logging system is quite old and difficult to maintain: * Unclear concepts: Logger, NamedLogger, MultiChannelledLogger, SubLogger, EventLogger, WriterLogger etc * Some code is diffuclt to konw whether it is right: `log.DelNamedLogger("console")` vs `log.DelNamedLogger(log.DEFAULT)` vs `log.DelLogger("console")` * The old system heavily depends on ini config system, it's difficult to create new logger for different purpose, and it's very fragile. * The "color" trick is difficult to use and read, many colors are unnecessary, and in the future structured log could help * It's difficult to add other log formats, eg: JSON format * The log outputer doesn't have full control of its goroutine, it's difficult to make outputer have advanced behaviors * The logs could be lost in some cases: eg: no Fatal error when using CLI. * Config options are passed by JSON, which is quite fragile. * INI package makes the KEY in `[log]` section visible in `[log.sub1]` and `[log.sub1.subA]`, this behavior is quite fragile and would cause more unclear problems, and there is no strong requirement to support `log.<mode>.<logger>` syntax. ## The new design See `logger.go` for documents. ## Screenshot <details> ![image](https://github.com/go-gitea/gitea/assets/2114189/4462d713-ba39-41f5-bb08-de912e67e1ff) ![image](https://github.com/go-gitea/gitea/assets/2114189/b188035e-f691-428b-8b2d-ff7b2199b2f9) ![image](https://github.com/go-gitea/gitea/assets/2114189/132e9745-1c3b-4e00-9e0d-15eaea495dee) </details> ## TODO * [x] add some new tests * [x] fix some tests * [x] test some sub-commands (manually ....) --------- Co-authored-by: Jason Song <i@wolfogre.com> Co-authored-by: delvh <dev.lh@web.de> Co-authored-by: Giteabot <teabot@gitea.io>
2023-05-21 22:35:11 +00:00
Database.LogSQL = sec.Key("LOG_SQL").MustBool(false)
Database.DBConnectRetries = sec.Key("DB_RETRIES").MustInt(10)
Database.DBConnectBackoff = sec.Key("DB_RETRY_BACKOFF").MustDuration(3 * time.Second)
Database.AutoMigration = sec.Key("AUTO_MIGRATION").MustBool(true)
[GITEA] Add slow SQL query warning - Databases are one of the most important parts of Forgejo, every interaction with Forgejo uses the database in one way or another. Therefore, it is important to maintain the database and recognize when Forgejo is not doing well with the database. Forgejo already has the option to log *every* SQL query along with its execution time, but monitoring becomes impractical for larger instances and takes up unnecessary storage in the logs. - Add a QoL enhancement that allows instance administrators to specify a threshold value beyond which query execution time is logged as a warning in the xorm logger. The default value is a conservative five seconds to avoid this becoming a source of spam in the logs. - The use case for this patch is that with an instance the size of Codeberg, monitoring SQL logs is not very fruitful and most of them are uninteresting. Recently, in the context of persistent deadlock issues (https://codeberg.org/forgejo/forgejo/issues/220), I have noticed that certain queries hold locks on tables like comment and issue for several seconds. This patch helps to identify which queries these are and when they happen. - Added unit test. (cherry picked from commit 24bbe7886fb4cb9a38c8dab8c44f4c9cbfa25481) (cherry picked from commit 6e29145b3c1455498531593d38e6a914941a12cb) (cherry picked from commit 63731e30712872bd2395eb3cf36d9996e5793645) (cherry picked from commit 3ce1a097369c132654de70df707b867e47bd1c40) (cherry picked from commit a64426907de788cc0937a7a2b16af4d2f26f7fe6) (cherry picked from commit 4b1921569156445c58d9889602733da5934c7b95) (cherry picked from commit e6356744359fa947c049827d60c2ea0e277e03dc) (cherry picked from commit 9cf501f1af4cd870221cef6af489618785b71186) (cherry picked from commit 0d6b934eba1c0e9b27b364791113aae816b6b366) (cherry picked from commit 4b6c2738795002887844a106f2fed2ef1673eed1) (cherry picked from commit 89b1315338b0c7a726a36a84e9844013a13560b8) (cherry picked from commit edd8e66ce991c395bb0af7720631c3cd26caaa51) [GITEA] Add slow SQL query warning (squash) document the setting (cherry picked from commit ce38599c5141c7fc6bc054819f5ff1c1b45bda1f) (cherry picked from commit 794aa67c68c8e24ac7301eb7ef767c6e2499a78d) (cherry picked from commit a4c2c6b004c21488e90f637ca7920f49108ed75d) (cherry picked from commit 97912752bc802db79bb26a6591aec885aea30ee4) (cherry picked from commit 00b5327c9750215a290238516e7b6fb1e6601e14) (cherry picked from commit 1069c860e78c11225b4d74ff3044df7786562821) (cherry picked from commit 84241f42c83852918b57c8bd25364697037fe42f) (cherry picked from commit e4bda0e8457d00c01b83f153ed5a4a8ea4cf85c8) (cherry picked from commit 7357fb91bff87045b133c3a7ac9fc70eea781bc4) (cherry picked from commit a8dd7f6da278ae112200b5efa5bf27e3961f5996)
2023-08-18 02:39:23 +00:00
Database.SlowQueryTreshold = sec.Key("SLOW_QUERY_TRESHOLD").MustDuration(5 * time.Second)
}
// DBConnStr returns database connection string
func DBConnStr() (string, error) {
var connStr string
paramSep := "?"
if strings.Contains(Database.Name, paramSep) {
paramSep = "&"
}
switch Database.Type {
case "mysql":
connType := "tcp"
if len(Database.Host) > 0 && Database.Host[0] == '/' { // looks like a unix socket
connType = "unix"
}
tls := Database.SSLMode
if tls == "disable" { // allow (Postgres-inspired) default value to work in MySQL
tls = "false"
}
connStr = fmt.Sprintf("%s:%s@%s(%s)/%s%scharset=%s&parseTime=true&tls=%s",
Database.User, Database.Passwd, connType, Database.Host, Database.Name, paramSep, Database.MysqlCharset, tls)
case "postgres":
connStr = getPostgreSQLConnectionString(Database.Host, Database.User, Database.Passwd, Database.Name, Database.SSLMode)
case "mssql":
host, port := ParseMSSQLHostPort(Database.Host)
connStr = fmt.Sprintf("server=%s; port=%s; database=%s; user id=%s; password=%s;", host, port, Database.Name, Database.User, Database.Passwd)
case "sqlite3":
if !EnableSQLite3 {
return "", errors.New("this Gitea binary was not built with SQLite3 support")
}
if err := os.MkdirAll(filepath.Dir(Database.Path), os.ModePerm); err != nil {
return "", fmt.Errorf("Failed to create directories: %w", err)
}
journalMode := ""
if Database.SQLiteJournalMode != "" {
journalMode = "&_journal_mode=" + Database.SQLiteJournalMode
}
connStr = fmt.Sprintf("file:%s?cache=shared&mode=rwc&_busy_timeout=%d&_txlock=immediate%s",
Database.Path, Database.Timeout, journalMode)
default:
return "", fmt.Errorf("unknown database type: %s", Database.Type)
}
return connStr, nil
}
// parsePostgreSQLHostPort parses given input in various forms defined in
// https://www.postgresql.org/docs/current/static/libpq-connect.html#LIBPQ-CONNSTRING
// and returns proper host and port number.
func parsePostgreSQLHostPort(info string) (host, port string) {
if h, p, err := net.SplitHostPort(info); err == nil {
host, port = h, p
} else {
// treat the "info" as "host", if it's an IPv6 address, remove the wrapper
host = info
if strings.HasPrefix(host, "[") && strings.HasSuffix(host, "]") {
host = host[1 : len(host)-1]
}
}
// set fallback values
if host == "" {
host = "127.0.0.1"
}
if port == "" {
port = "5432"
}
return host, port
}
func getPostgreSQLConnectionString(dbHost, dbUser, dbPasswd, dbName, dbsslMode string) (connStr string) {
dbName, dbParam, _ := strings.Cut(dbName, "?")
host, port := parsePostgreSQLHostPort(dbHost)
connURL := url.URL{
Scheme: "postgres",
User: url.UserPassword(dbUser, dbPasswd),
Host: net.JoinHostPort(host, port),
Path: dbName,
OmitHost: false,
RawQuery: dbParam,
}
query := connURL.Query()
if dbHost[0] == '/' { // looks like a unix socket
query.Add("host", dbHost)
connURL.Host = ":" + port
}
query.Set("sslmode", dbsslMode)
connURL.RawQuery = query.Encode()
return connURL.String()
}
// ParseMSSQLHostPort splits the host into host and port
func ParseMSSQLHostPort(info string) (string, string) {
// the default port "0" might be related to MSSQL's dynamic port, maybe it should be double-confirmed in the future
host, port := "127.0.0.1", "0"
if strings.Contains(info, ":") {
host = strings.Split(info, ":")[0]
port = strings.Split(info, ":")[1]
} else if strings.Contains(info, ",") {
host = strings.Split(info, ",")[0]
port = strings.TrimSpace(strings.Split(info, ",")[1])
} else if len(info) > 0 {
host = info
}
if host == "" {
host = "127.0.0.1"
}
if port == "" {
port = "0"
}
return host, port
}
type DatabaseType string
func (t DatabaseType) String() string {
return string(t)
}
func (t DatabaseType) IsSQLite3() bool {
return t == "sqlite3"
}
func (t DatabaseType) IsMySQL() bool {
return t == "mysql"
}
func (t DatabaseType) IsMSSQL() bool {
return t == "mssql"
}
func (t DatabaseType) IsPostgreSQL() bool {
return t == "postgres"
}