forgejo/tests/integration/migration-test/migration_test.go

324 lines
9.8 KiB
Go
Raw Normal View History

// Copyright 2019 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package migrations
import (
"compress/gzip"
"context"
"database/sql"
"fmt"
"io"
"os"
"path"
"path/filepath"
"regexp"
"sort"
2024-07-15 15:32:36 +00:00
"strings"
"testing"
"code.gitea.io/gitea/models/db"
"code.gitea.io/gitea/models/migrations"
migrate_base "code.gitea.io/gitea/models/migrations/base"
"code.gitea.io/gitea/models/unittest"
"code.gitea.io/gitea/modules/base"
"code.gitea.io/gitea/modules/charset"
"code.gitea.io/gitea/modules/git"
Rewrite logger system (#24726) ## ⚠️ Breaking The `log.<mode>.<logger>` style config has been dropped. If you used it, please check the new config manual & app.example.ini to make your instance output logs as expected. Although many legacy options still work, it's encouraged to upgrade to the new options. The SMTP logger is deleted because SMTP is not suitable to collect logs. If you have manually configured Gitea log options, please confirm the logger system works as expected after upgrading. ## Description Close #12082 and maybe more log-related issues, resolve some related FIXMEs in old code (which seems unfixable before) Just like rewriting queue #24505 : make code maintainable, clear legacy bugs, and add the ability to support more writers (eg: JSON, structured log) There is a new document (with examples): `logging-config.en-us.md` This PR is safer than the queue rewriting, because it's just for logging, it won't break other logic. ## The old problems The logging system is quite old and difficult to maintain: * Unclear concepts: Logger, NamedLogger, MultiChannelledLogger, SubLogger, EventLogger, WriterLogger etc * Some code is diffuclt to konw whether it is right: `log.DelNamedLogger("console")` vs `log.DelNamedLogger(log.DEFAULT)` vs `log.DelLogger("console")` * The old system heavily depends on ini config system, it's difficult to create new logger for different purpose, and it's very fragile. * The "color" trick is difficult to use and read, many colors are unnecessary, and in the future structured log could help * It's difficult to add other log formats, eg: JSON format * The log outputer doesn't have full control of its goroutine, it's difficult to make outputer have advanced behaviors * The logs could be lost in some cases: eg: no Fatal error when using CLI. * Config options are passed by JSON, which is quite fragile. * INI package makes the KEY in `[log]` section visible in `[log.sub1]` and `[log.sub1.subA]`, this behavior is quite fragile and would cause more unclear problems, and there is no strong requirement to support `log.<mode>.<logger>` syntax. ## The new design See `logger.go` for documents. ## Screenshot <details> ![image](https://github.com/go-gitea/gitea/assets/2114189/4462d713-ba39-41f5-bb08-de912e67e1ff) ![image](https://github.com/go-gitea/gitea/assets/2114189/b188035e-f691-428b-8b2d-ff7b2199b2f9) ![image](https://github.com/go-gitea/gitea/assets/2114189/132e9745-1c3b-4e00-9e0d-15eaea495dee) </details> ## TODO * [x] add some new tests * [x] fix some tests * [x] test some sub-commands (manually ....) --------- Co-authored-by: Jason Song <i@wolfogre.com> Co-authored-by: delvh <dev.lh@web.de> Co-authored-by: Giteabot <teabot@gitea.io>
2023-05-21 22:35:11 +00:00
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
Rewrite logger system (#24726) ## ⚠️ Breaking The `log.<mode>.<logger>` style config has been dropped. If you used it, please check the new config manual & app.example.ini to make your instance output logs as expected. Although many legacy options still work, it's encouraged to upgrade to the new options. The SMTP logger is deleted because SMTP is not suitable to collect logs. If you have manually configured Gitea log options, please confirm the logger system works as expected after upgrading. ## Description Close #12082 and maybe more log-related issues, resolve some related FIXMEs in old code (which seems unfixable before) Just like rewriting queue #24505 : make code maintainable, clear legacy bugs, and add the ability to support more writers (eg: JSON, structured log) There is a new document (with examples): `logging-config.en-us.md` This PR is safer than the queue rewriting, because it's just for logging, it won't break other logic. ## The old problems The logging system is quite old and difficult to maintain: * Unclear concepts: Logger, NamedLogger, MultiChannelledLogger, SubLogger, EventLogger, WriterLogger etc * Some code is diffuclt to konw whether it is right: `log.DelNamedLogger("console")` vs `log.DelNamedLogger(log.DEFAULT)` vs `log.DelLogger("console")` * The old system heavily depends on ini config system, it's difficult to create new logger for different purpose, and it's very fragile. * The "color" trick is difficult to use and read, many colors are unnecessary, and in the future structured log could help * It's difficult to add other log formats, eg: JSON format * The log outputer doesn't have full control of its goroutine, it's difficult to make outputer have advanced behaviors * The logs could be lost in some cases: eg: no Fatal error when using CLI. * Config options are passed by JSON, which is quite fragile. * INI package makes the KEY in `[log]` section visible in `[log.sub1]` and `[log.sub1.subA]`, this behavior is quite fragile and would cause more unclear problems, and there is no strong requirement to support `log.<mode>.<logger>` syntax. ## The new design See `logger.go` for documents. ## Screenshot <details> ![image](https://github.com/go-gitea/gitea/assets/2114189/4462d713-ba39-41f5-bb08-de912e67e1ff) ![image](https://github.com/go-gitea/gitea/assets/2114189/b188035e-f691-428b-8b2d-ff7b2199b2f9) ![image](https://github.com/go-gitea/gitea/assets/2114189/132e9745-1c3b-4e00-9e0d-15eaea495dee) </details> ## TODO * [x] add some new tests * [x] fix some tests * [x] test some sub-commands (manually ....) --------- Co-authored-by: Jason Song <i@wolfogre.com> Co-authored-by: delvh <dev.lh@web.de> Co-authored-by: Giteabot <teabot@gitea.io>
2023-05-21 22:35:11 +00:00
"code.gitea.io/gitea/modules/testlogger"
"code.gitea.io/gitea/modules/util"
"code.gitea.io/gitea/tests"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
2019-10-17 09:26:49 +00:00
"xorm.io/xorm"
)
var currentEngine *xorm.Engine
func initMigrationTest(t *testing.T) func() {
Rewrite logger system (#24726) ## ⚠️ Breaking The `log.<mode>.<logger>` style config has been dropped. If you used it, please check the new config manual & app.example.ini to make your instance output logs as expected. Although many legacy options still work, it's encouraged to upgrade to the new options. The SMTP logger is deleted because SMTP is not suitable to collect logs. If you have manually configured Gitea log options, please confirm the logger system works as expected after upgrading. ## Description Close #12082 and maybe more log-related issues, resolve some related FIXMEs in old code (which seems unfixable before) Just like rewriting queue #24505 : make code maintainable, clear legacy bugs, and add the ability to support more writers (eg: JSON, structured log) There is a new document (with examples): `logging-config.en-us.md` This PR is safer than the queue rewriting, because it's just for logging, it won't break other logic. ## The old problems The logging system is quite old and difficult to maintain: * Unclear concepts: Logger, NamedLogger, MultiChannelledLogger, SubLogger, EventLogger, WriterLogger etc * Some code is diffuclt to konw whether it is right: `log.DelNamedLogger("console")` vs `log.DelNamedLogger(log.DEFAULT)` vs `log.DelLogger("console")` * The old system heavily depends on ini config system, it's difficult to create new logger for different purpose, and it's very fragile. * The "color" trick is difficult to use and read, many colors are unnecessary, and in the future structured log could help * It's difficult to add other log formats, eg: JSON format * The log outputer doesn't have full control of its goroutine, it's difficult to make outputer have advanced behaviors * The logs could be lost in some cases: eg: no Fatal error when using CLI. * Config options are passed by JSON, which is quite fragile. * INI package makes the KEY in `[log]` section visible in `[log.sub1]` and `[log.sub1.subA]`, this behavior is quite fragile and would cause more unclear problems, and there is no strong requirement to support `log.<mode>.<logger>` syntax. ## The new design See `logger.go` for documents. ## Screenshot <details> ![image](https://github.com/go-gitea/gitea/assets/2114189/4462d713-ba39-41f5-bb08-de912e67e1ff) ![image](https://github.com/go-gitea/gitea/assets/2114189/b188035e-f691-428b-8b2d-ff7b2199b2f9) ![image](https://github.com/go-gitea/gitea/assets/2114189/132e9745-1c3b-4e00-9e0d-15eaea495dee) </details> ## TODO * [x] add some new tests * [x] fix some tests * [x] test some sub-commands (manually ....) --------- Co-authored-by: Jason Song <i@wolfogre.com> Co-authored-by: delvh <dev.lh@web.de> Co-authored-by: Giteabot <teabot@gitea.io>
2023-05-21 22:35:11 +00:00
log.RegisterEventWriter("test", testlogger.NewTestLoggerWriter)
deferFn := tests.PrintCurrentTest(t, 2)
giteaRoot := base.SetupGiteaRoot()
if giteaRoot == "" {
tests.Printf("Environment variable $GITEA_ROOT not set\n")
os.Exit(1)
}
setting.AppPath = path.Join(giteaRoot, "gitea")
if _, err := os.Stat(setting.AppPath); err != nil {
tests.Printf("Could not find gitea binary at %s\n", setting.AppPath)
os.Exit(1)
}
giteaConf := os.Getenv("GITEA_CONF")
if giteaConf == "" {
tests.Printf("Environment variable $GITEA_CONF not set\n")
os.Exit(1)
} else if !path.IsAbs(giteaConf) {
setting.CustomConf = path.Join(giteaRoot, giteaConf)
} else {
setting.CustomConf = giteaConf
}
unittest.InitSettings()
assert.NotEmpty(t, setting.RepoRootPath)
require.NoError(t, util.RemoveAll(setting.RepoRootPath))
require.NoError(t, unittest.CopyDir(path.Join(filepath.Dir(setting.AppPath), "tests/gitea-repositories-meta"), setting.RepoRootPath))
ownerDirs, err := os.ReadDir(setting.RepoRootPath)
if err != nil {
require.NoError(t, err, "unable to read the new repo root: %v\n", err)
}
for _, ownerDir := range ownerDirs {
if !ownerDir.Type().IsDir() {
continue
}
repoDirs, err := os.ReadDir(filepath.Join(setting.RepoRootPath, ownerDir.Name()))
if err != nil {
require.NoError(t, err, "unable to read the new repo root: %v\n", err)
}
for _, repoDir := range repoDirs {
_ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "objects", "pack"), 0o755)
_ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "objects", "info"), 0o755)
_ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "refs", "heads"), 0o755)
_ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "refs", "tag"), 0o755)
}
}
require.NoError(t, git.InitFull(context.Background()))
setting.LoadDBSetting()
Rewrite logger system (#24726) ## ⚠️ Breaking The `log.<mode>.<logger>` style config has been dropped. If you used it, please check the new config manual & app.example.ini to make your instance output logs as expected. Although many legacy options still work, it's encouraged to upgrade to the new options. The SMTP logger is deleted because SMTP is not suitable to collect logs. If you have manually configured Gitea log options, please confirm the logger system works as expected after upgrading. ## Description Close #12082 and maybe more log-related issues, resolve some related FIXMEs in old code (which seems unfixable before) Just like rewriting queue #24505 : make code maintainable, clear legacy bugs, and add the ability to support more writers (eg: JSON, structured log) There is a new document (with examples): `logging-config.en-us.md` This PR is safer than the queue rewriting, because it's just for logging, it won't break other logic. ## The old problems The logging system is quite old and difficult to maintain: * Unclear concepts: Logger, NamedLogger, MultiChannelledLogger, SubLogger, EventLogger, WriterLogger etc * Some code is diffuclt to konw whether it is right: `log.DelNamedLogger("console")` vs `log.DelNamedLogger(log.DEFAULT)` vs `log.DelLogger("console")` * The old system heavily depends on ini config system, it's difficult to create new logger for different purpose, and it's very fragile. * The "color" trick is difficult to use and read, many colors are unnecessary, and in the future structured log could help * It's difficult to add other log formats, eg: JSON format * The log outputer doesn't have full control of its goroutine, it's difficult to make outputer have advanced behaviors * The logs could be lost in some cases: eg: no Fatal error when using CLI. * Config options are passed by JSON, which is quite fragile. * INI package makes the KEY in `[log]` section visible in `[log.sub1]` and `[log.sub1.subA]`, this behavior is quite fragile and would cause more unclear problems, and there is no strong requirement to support `log.<mode>.<logger>` syntax. ## The new design See `logger.go` for documents. ## Screenshot <details> ![image](https://github.com/go-gitea/gitea/assets/2114189/4462d713-ba39-41f5-bb08-de912e67e1ff) ![image](https://github.com/go-gitea/gitea/assets/2114189/b188035e-f691-428b-8b2d-ff7b2199b2f9) ![image](https://github.com/go-gitea/gitea/assets/2114189/132e9745-1c3b-4e00-9e0d-15eaea495dee) </details> ## TODO * [x] add some new tests * [x] fix some tests * [x] test some sub-commands (manually ....) --------- Co-authored-by: Jason Song <i@wolfogre.com> Co-authored-by: delvh <dev.lh@web.de> Co-authored-by: Giteabot <teabot@gitea.io>
2023-05-21 22:35:11 +00:00
setting.InitLoggersForTest()
return deferFn
}
func availableVersions() ([]string, error) {
migrationsDir, err := os.Open("tests/integration/migration-test")
if err != nil {
return nil, err
}
defer migrationsDir.Close()
[DB] run all Forgejo migrations in integration tests The tests at tests/integration/migration-test/migration_test.go will not run any Forgejo migration when using the gitea-*.sql.gz files because they do not contain a ForgejoVersion row which is interpreted as a new Forgejo installation for which there is no need for migration. Create a situation by which the ForgejoVersion table exists and has a version of 0 in tests/integration/migration-test/forgejo-v1.19.0.*.sql.gz thus ensuring all Forgejo migrations are run. The forgejo*.sql.gz files do not have any Gitea related records, which will be interpreted by the Gitea migrations as a new installation that does not need any migration. As a consequence the migration tests run when using forgejo-v1.19.0.*.sql.gz are exclusively about Forgejo migrations. (cherry picked from commit ec8003859c920ac05a071ad9b1d9d8af5a694ac0) (cherry picked from commit 697570ff390656930d41bcdc4194aacc5c7a54f4) (cherry picked from commit f041aec17265ca34943f961d9640e60c96c0f24d) (cherry picked from commit 60463e3bab9dfa6aae7f379fc77309ab52b1d250) (cherry picked from commit b2fc2a7c1338ad2a9bd0a6887443e604dccda53f) (cherry picked from commit fb2759b6afc213d5e2db8f6e523391927c611b23) (cherry picked from commit 37cfc3b2275afd523f53e149781d69a2914fd83b) (cherry picked from commit 832607500a576a40ee1e35897e58ef319dba9788) (cherry picked from commit 143d4007b1b60ad1880b64ce1d4998e77854a05a) (cherry picked from commit a17e803fbfb7908729c2d92fd58360806cce6fbe) (cherry picked from commit 72ffd49bc3c218acf8c87f7464f2c028860adba8) (cherry picked from commit 9b92a5fd725b34d7ed18b5f1ebc78e1a45de07eb) (cherry picked from commit 0a334d0a9b02617360aaa76d55dbb0f6fd5547f4) (cherry picked from commit 3add683c9411018f735c0a19827354870310b9c8) (cherry picked from commit 8ed3f3f86b0ddab8fc6adc0557b3f5d5e3b8ed4a) (cherry picked from commit 97dc65852b215d85c8608033bd4180d91aef1a54) (cherry picked from commit e0503cbfead8e5f7f48d0eb0f32bd6881437bb9b) (cherry picked from commit f25e6a196b32dcb500fa7df65f21124e5e6a10c4) (cherry picked from commit cca19c7a4e35982c5fec4de4fcb53690f22d1212) (cherry picked from commit 2c89d7f18c9493e720e0d06c4b2cd072445eff47) (cherry picked from commit 4eb8695c40e5d1c64113ce19bb3deae1eafe64c9) (cherry picked from commit e4c42a68ffccb83fcd7379baa402daae15d6f57b) (cherry picked from commit f76d5e0a820fbce8c7be968d8318a5a543e43e85) (cherry picked from commit c6e5a2962cae3eee0b39469bcf39396ee6efddcd) (cherry picked from commit f9927a2253263110b60c4a49771bf72f660b3ad5)
2023-08-09 11:00:36 +00:00
versionRE, err := regexp.Compile(".*-v(?P<version>.+)\\." + regexp.QuoteMeta(setting.Database.Type.String()) + "\\.sql.gz")
if err != nil {
return nil, err
}
filenames, err := migrationsDir.Readdirnames(-1)
if err != nil {
return nil, err
}
versions := []string{}
for _, filename := range filenames {
if versionRE.MatchString(filename) {
substrings := versionRE.FindStringSubmatch(filename)
versions = append(versions, substrings[1])
}
}
sort.Strings(versions)
return versions, nil
}
func readSQLFromFile(version string) (string, error) {
filename := fmt.Sprintf("tests/integration/migration-test/gitea-v%s.%s.sql.gz", version, setting.Database.Type)
if _, err := os.Stat(filename); os.IsNotExist(err) {
[DB] run all Forgejo migrations in integration tests The tests at tests/integration/migration-test/migration_test.go will not run any Forgejo migration when using the gitea-*.sql.gz files because they do not contain a ForgejoVersion row which is interpreted as a new Forgejo installation for which there is no need for migration. Create a situation by which the ForgejoVersion table exists and has a version of 0 in tests/integration/migration-test/forgejo-v1.19.0.*.sql.gz thus ensuring all Forgejo migrations are run. The forgejo*.sql.gz files do not have any Gitea related records, which will be interpreted by the Gitea migrations as a new installation that does not need any migration. As a consequence the migration tests run when using forgejo-v1.19.0.*.sql.gz are exclusively about Forgejo migrations. (cherry picked from commit ec8003859c920ac05a071ad9b1d9d8af5a694ac0) (cherry picked from commit 697570ff390656930d41bcdc4194aacc5c7a54f4) (cherry picked from commit f041aec17265ca34943f961d9640e60c96c0f24d) (cherry picked from commit 60463e3bab9dfa6aae7f379fc77309ab52b1d250) (cherry picked from commit b2fc2a7c1338ad2a9bd0a6887443e604dccda53f) (cherry picked from commit fb2759b6afc213d5e2db8f6e523391927c611b23) (cherry picked from commit 37cfc3b2275afd523f53e149781d69a2914fd83b) (cherry picked from commit 832607500a576a40ee1e35897e58ef319dba9788) (cherry picked from commit 143d4007b1b60ad1880b64ce1d4998e77854a05a) (cherry picked from commit a17e803fbfb7908729c2d92fd58360806cce6fbe) (cherry picked from commit 72ffd49bc3c218acf8c87f7464f2c028860adba8) (cherry picked from commit 9b92a5fd725b34d7ed18b5f1ebc78e1a45de07eb) (cherry picked from commit 0a334d0a9b02617360aaa76d55dbb0f6fd5547f4) (cherry picked from commit 3add683c9411018f735c0a19827354870310b9c8) (cherry picked from commit 8ed3f3f86b0ddab8fc6adc0557b3f5d5e3b8ed4a) (cherry picked from commit 97dc65852b215d85c8608033bd4180d91aef1a54) (cherry picked from commit e0503cbfead8e5f7f48d0eb0f32bd6881437bb9b) (cherry picked from commit f25e6a196b32dcb500fa7df65f21124e5e6a10c4) (cherry picked from commit cca19c7a4e35982c5fec4de4fcb53690f22d1212) (cherry picked from commit 2c89d7f18c9493e720e0d06c4b2cd072445eff47) (cherry picked from commit 4eb8695c40e5d1c64113ce19bb3deae1eafe64c9) (cherry picked from commit e4c42a68ffccb83fcd7379baa402daae15d6f57b) (cherry picked from commit f76d5e0a820fbce8c7be968d8318a5a543e43e85) (cherry picked from commit c6e5a2962cae3eee0b39469bcf39396ee6efddcd) (cherry picked from commit f9927a2253263110b60c4a49771bf72f660b3ad5)
2023-08-09 11:00:36 +00:00
filename = fmt.Sprintf("tests/integration/migration-test/forgejo-v%s.%s.sql.gz", version, setting.Database.Type)
if _, err := os.Stat(filename); os.IsNotExist(err) {
return "", nil
}
}
file, err := os.Open(filename)
if err != nil {
return "", err
}
defer file.Close()
gr, err := gzip.NewReader(file)
if err != nil {
return "", err
}
defer gr.Close()
bytes, err := io.ReadAll(gr)
if err != nil {
return "", err
}
return string(charset.MaybeRemoveBOM(bytes, charset.ConvertOpts{})), nil
}
func restoreOldDB(t *testing.T, version string) bool {
data, err := readSQLFromFile(version)
require.NoError(t, err)
if len(data) == 0 {
tests.Printf("No db found to restore for %s version: %s\n", setting.Database.Type, version)
return false
}
switch {
case setting.Database.Type.IsSQLite3():
util.Remove(setting.Database.Path)
err := os.MkdirAll(path.Dir(setting.Database.Path), os.ModePerm)
require.NoError(t, err)
db, err := sql.Open("sqlite3", fmt.Sprintf("file:%s?cache=shared&mode=rwc&_busy_timeout=%d&_txlock=immediate", setting.Database.Path, setting.Database.Timeout))
require.NoError(t, err)
defer db.Close()
_, err = db.Exec(data)
require.NoError(t, err)
db.Close()
case setting.Database.Type.IsMySQL():
db, err := sql.Open("mysql", fmt.Sprintf("%s:%s@tcp(%s)/",
setting.Database.User, setting.Database.Passwd, setting.Database.Host))
require.NoError(t, err)
defer db.Close()
2024-07-15 15:32:36 +00:00
databaseName := strings.SplitN(setting.Database.Name, "?", 2)[0]
_, err = db.Exec(fmt.Sprintf("DROP DATABASE IF EXISTS %s", databaseName))
require.NoError(t, err)
2024-07-15 15:32:36 +00:00
_, err = db.Exec(fmt.Sprintf("CREATE DATABASE IF NOT EXISTS %s", databaseName))
require.NoError(t, err)
db.Close()
2024-07-15 15:32:36 +00:00
db, err = sql.Open("mysql", fmt.Sprintf("%s:%s@tcp(%s)/%s",
setting.Database.User, setting.Database.Passwd, setting.Database.Host, setting.Database.Name))
require.NoError(t, err)
defer db.Close()
_, err = db.Exec(data)
require.NoError(t, err)
db.Close()
case setting.Database.Type.IsPostgreSQL():
var db *sql.DB
var err error
if setting.Database.Host[0] == '/' {
db, err = sql.Open("postgres", fmt.Sprintf("postgres://%s:%s@/?sslmode=%s&host=%s",
setting.Database.User, setting.Database.Passwd, setting.Database.SSLMode, setting.Database.Host))
require.NoError(t, err)
} else {
db, err = sql.Open("postgres", fmt.Sprintf("postgres://%s:%s@%s/?sslmode=%s",
setting.Database.User, setting.Database.Passwd, setting.Database.Host, setting.Database.SSLMode))
require.NoError(t, err)
}
defer db.Close()
_, err = db.Exec(fmt.Sprintf("DROP DATABASE IF EXISTS %s", setting.Database.Name))
require.NoError(t, err)
_, err = db.Exec(fmt.Sprintf("CREATE DATABASE %s", setting.Database.Name))
require.NoError(t, err)
db.Close()
// Check if we need to setup a specific schema
if len(setting.Database.Schema) != 0 {
if setting.Database.Host[0] == '/' {
db, err = sql.Open("postgres", fmt.Sprintf("postgres://%s:%s@/%s?sslmode=%s&host=%s",
setting.Database.User, setting.Database.Passwd, setting.Database.Name, setting.Database.SSLMode, setting.Database.Host))
} else {
db, err = sql.Open("postgres", fmt.Sprintf("postgres://%s:%s@%s/%s?sslmode=%s",
setting.Database.User, setting.Database.Passwd, setting.Database.Host, setting.Database.Name, setting.Database.SSLMode))
}
require.NoError(t, err)
defer db.Close()
schrows, err := db.Query(fmt.Sprintf("SELECT 1 FROM information_schema.schemata WHERE schema_name = '%s'", setting.Database.Schema))
require.NoError(t, err)
if !assert.NotEmpty(t, schrows) {
return false
}
if !schrows.Next() {
// Create and setup a DB schema
_, err = db.Exec(fmt.Sprintf("CREATE SCHEMA %s", setting.Database.Schema))
require.NoError(t, err)
}
schrows.Close()
// Make the user's default search path the created schema; this will affect new connections
_, err = db.Exec(fmt.Sprintf(`ALTER USER "%s" SET search_path = %s`, setting.Database.User, setting.Database.Schema))
require.NoError(t, err)
db.Close()
}
if setting.Database.Host[0] == '/' {
db, err = sql.Open("postgres", fmt.Sprintf("postgres://%s:%s@/%s?sslmode=%s&host=%s",
setting.Database.User, setting.Database.Passwd, setting.Database.Name, setting.Database.SSLMode, setting.Database.Host))
} else {
db, err = sql.Open("postgres", fmt.Sprintf("postgres://%s:%s@%s/%s?sslmode=%s",
setting.Database.User, setting.Database.Passwd, setting.Database.Host, setting.Database.Name, setting.Database.SSLMode))
}
require.NoError(t, err)
defer db.Close()
_, err = db.Exec(data)
require.NoError(t, err)
db.Close()
}
return true
}
func wrappedMigrate(x *xorm.Engine) error {
currentEngine = x
return migrations.Migrate(x)
}
func doMigrationTest(t *testing.T, version string) {
defer tests.PrintCurrentTest(t)()
tests.Printf("Performing migration test for %s version: %s\n", setting.Database.Type, version)
if !restoreOldDB(t, version) {
return
}
Rewrite logger system (#24726) ## ⚠️ Breaking The `log.<mode>.<logger>` style config has been dropped. If you used it, please check the new config manual & app.example.ini to make your instance output logs as expected. Although many legacy options still work, it's encouraged to upgrade to the new options. The SMTP logger is deleted because SMTP is not suitable to collect logs. If you have manually configured Gitea log options, please confirm the logger system works as expected after upgrading. ## Description Close #12082 and maybe more log-related issues, resolve some related FIXMEs in old code (which seems unfixable before) Just like rewriting queue #24505 : make code maintainable, clear legacy bugs, and add the ability to support more writers (eg: JSON, structured log) There is a new document (with examples): `logging-config.en-us.md` This PR is safer than the queue rewriting, because it's just for logging, it won't break other logic. ## The old problems The logging system is quite old and difficult to maintain: * Unclear concepts: Logger, NamedLogger, MultiChannelledLogger, SubLogger, EventLogger, WriterLogger etc * Some code is diffuclt to konw whether it is right: `log.DelNamedLogger("console")` vs `log.DelNamedLogger(log.DEFAULT)` vs `log.DelLogger("console")` * The old system heavily depends on ini config system, it's difficult to create new logger for different purpose, and it's very fragile. * The "color" trick is difficult to use and read, many colors are unnecessary, and in the future structured log could help * It's difficult to add other log formats, eg: JSON format * The log outputer doesn't have full control of its goroutine, it's difficult to make outputer have advanced behaviors * The logs could be lost in some cases: eg: no Fatal error when using CLI. * Config options are passed by JSON, which is quite fragile. * INI package makes the KEY in `[log]` section visible in `[log.sub1]` and `[log.sub1.subA]`, this behavior is quite fragile and would cause more unclear problems, and there is no strong requirement to support `log.<mode>.<logger>` syntax. ## The new design See `logger.go` for documents. ## Screenshot <details> ![image](https://github.com/go-gitea/gitea/assets/2114189/4462d713-ba39-41f5-bb08-de912e67e1ff) ![image](https://github.com/go-gitea/gitea/assets/2114189/b188035e-f691-428b-8b2d-ff7b2199b2f9) ![image](https://github.com/go-gitea/gitea/assets/2114189/132e9745-1c3b-4e00-9e0d-15eaea495dee) </details> ## TODO * [x] add some new tests * [x] fix some tests * [x] test some sub-commands (manually ....) --------- Co-authored-by: Jason Song <i@wolfogre.com> Co-authored-by: delvh <dev.lh@web.de> Co-authored-by: Giteabot <teabot@gitea.io>
2023-05-21 22:35:11 +00:00
setting.InitSQLLoggersForCli(log.INFO)
err := db.InitEngineWithMigration(context.Background(), wrappedMigrate)
require.NoError(t, err)
currentEngine.Close()
beans, _ := db.NamesToBean()
err = db.InitEngineWithMigration(context.Background(), func(x *xorm.Engine) error {
currentEngine = x
return migrate_base.RecreateTables(beans...)(x)
})
require.NoError(t, err)
currentEngine.Close()
// We do this a second time to ensure that there is not a problem with retained indices
err = db.InitEngineWithMigration(context.Background(), func(x *xorm.Engine) error {
currentEngine = x
return migrate_base.RecreateTables(beans...)(x)
})
require.NoError(t, err)
currentEngine.Close()
}
func TestMigrations(t *testing.T) {
defer initMigrationTest(t)()
dialect := setting.Database.Type
versions, err := availableVersions()
require.NoError(t, err)
if len(versions) == 0 {
tests.Printf("No old database versions available to migration test for %s\n", dialect)
return
}
tests.Printf("Preparing to test %d migrations for %s\n", len(versions), dialect)
for _, version := range versions {
t.Run(fmt.Sprintf("Migrate-%s-%s", dialect, version), func(t *testing.T) {
doMigrationTest(t, version)
})
}
}