From 782169da765baf1d651c1b71e5f974f762c92d8e Mon Sep 17 00:00:00 2001 From: tobi <31960611+tsmethurst@users.noreply.github.com> Date: Thu, 26 Jan 2023 15:12:48 +0100 Subject: [PATCH] [chore] set max open / idle conns + conn max lifetime for both postgres and sqlite (#1369) * [chore] set max open / idle conns + conn max lifetime for both postgres and sqlite * reduce cache size default to 8MiB, reduce connections to 2 * cpu * introduce max open conns multiplier, tune sqlite and pg separately * go fmt --- docs/configuration/database.md | 32 ++++++- example/config.yaml | 32 ++++++- internal/config/config.go | 25 ++--- internal/config/defaults.go | 25 ++--- internal/config/flags.go | 1 + internal/config/helpers.gen.go | 25 +++++ internal/db/bundb/bundb.go | 167 ++++++++++++++++++--------------- test/envparsing.sh | 3 +- testrig/config.go | 23 +++-- 9 files changed, 212 insertions(+), 121 deletions(-) diff --git a/docs/configuration/database.md b/docs/configuration/database.md index 38514e036..4fe8836fd 100644 --- a/docs/configuration/database.md +++ b/docs/configuration/database.md @@ -108,6 +108,30 @@ db-tls-mode: "disable" # Default: "" db-tls-ca-cert: "" +# Int. Number to multiply by CPU count to set permitted total of open database connections (in-use and idle). +# You can use this setting to tune your database connection behavior, though most admins won't need to touch it. +# +# Example values for multiplier 8: +# +# 1 cpu = 08 open connections +# 2 cpu = 16 open connections +# 4 cpu = 32 open connections +# +# Example values for multiplier 4: +# +# 1 cpu = 04 open connections +# 2 cpu = 08 open connections +# 4 cpu = 16 open connections +# +# A multiplier of 8 is a sensible default, but you may wish to increase this for instances +# running on very performant hardware, or decrease it for instances using v. slow CPUs. +# +# If you set this to 0 or less, it will be adjusted to 1. +# +# Examples: [16, 8, 10, 2] +# Default: 8 +db-max-open-conns-multiplier: 8 + # String. SQLite journaling mode. # SQLite only -- unused otherwise. # If set to empty string, the sqlite default will be used. @@ -126,11 +150,11 @@ db-sqlite-synchronous: "NORMAL" # Byte size. SQlite cache size. # SQLite only -- unused otherwise. -# If set to empty string or zero, the sqlite default will be used. +# If set to empty string or zero, the sqlite default (2MiB) will be used. # See: https://www.sqlite.org/pragma.html#pragma_cache_size -# Examples: ["32MiB", "0", "64MiB"] -# Default: "64MiB" -db-sqlite-cache-size: "64MiB" +# Examples: ["0", "2MiB", "8MiB", "64MiB"] +# Default: "8MiB" +db-sqlite-cache-size: "8MiB" # Duration. SQlite busy timeout. # SQLite only -- unused otherwise. diff --git a/example/config.yaml b/example/config.yaml index 030ad5275..2f99fd326 100644 --- a/example/config.yaml +++ b/example/config.yaml @@ -164,6 +164,30 @@ db-tls-mode: "disable" # Default: "" db-tls-ca-cert: "" +# Int. Number to multiply by CPU count to set permitted total of open database connections (in-use and idle). +# You can use this setting to tune your database connection behavior, though most admins won't need to touch it. +# +# Example values for multiplier 8: +# +# 1 cpu = 08 open connections +# 2 cpu = 16 open connections +# 4 cpu = 32 open connections +# +# Example values for multiplier 4: +# +# 1 cpu = 04 open connections +# 2 cpu = 08 open connections +# 4 cpu = 16 open connections +# +# A multiplier of 8 is a sensible default, but you may wish to increase this for instances +# running on very performant hardware, or decrease it for instances using v. slow CPUs. +# +# If you set this to 0 or less, it will be adjusted to 1. +# +# Examples: [16, 8, 10, 2] +# Default: 8 +db-max-open-conns-multiplier: 8 + # String. SQLite journaling mode. # SQLite only -- unused otherwise. # If set to empty string, the sqlite default will be used. @@ -182,11 +206,11 @@ db-sqlite-synchronous: "NORMAL" # Byte size. SQlite cache size. # SQLite only -- unused otherwise. -# If set to empty string or zero, the sqlite default will be used. +# If set to empty string or zero, the sqlite default (2MiB) will be used. # See: https://www.sqlite.org/pragma.html#pragma_cache_size -# Examples: ["32MiB", "0", "64MiB"] -# Default: "64MiB" -db-sqlite-cache-size: "64MiB" +# Examples: ["0", "2MiB", "8MiB", "64MiB"] +# Default: "8MiB" +db-sqlite-cache-size: "8MiB" # Duration. SQlite busy timeout. # SQLite only -- unused otherwise. diff --git a/internal/config/config.go b/internal/config/config.go index 472e7b93d..7025ab63a 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -58,18 +58,19 @@ type Configuration struct { TrustedProxies []string `name:"trusted-proxies" usage:"Proxies to trust when parsing x-forwarded headers into real IPs."` SoftwareVersion string `name:"software-version" usage:""` - DbType string `name:"db-type" usage:"Database type: eg., postgres"` - DbAddress string `name:"db-address" usage:"Database ipv4 address, hostname, or filename"` - DbPort int `name:"db-port" usage:"Database port"` - DbUser string `name:"db-user" usage:"Database username"` - DbPassword string `name:"db-password" usage:"Database password"` - DbDatabase string `name:"db-database" usage:"Database name"` - DbTLSMode string `name:"db-tls-mode" usage:"Database tls mode"` - DbTLSCACert string `name:"db-tls-ca-cert" usage:"Path to CA cert for db tls connection"` - DbSqliteJournalMode string `name:"db-sqlite-journal-mode" usage:"Sqlite only: see https://www.sqlite.org/pragma.html#pragma_journal_mode"` - DbSqliteSynchronous string `name:"db-sqlite-synchronous" usage:"Sqlite only: see https://www.sqlite.org/pragma.html#pragma_synchronous"` - DbSqliteCacheSize bytesize.Size `name:"db-sqlite-cache-size" usage:"Sqlite only: see https://www.sqlite.org/pragma.html#pragma_cache_size"` - DbSqliteBusyTimeout time.Duration `name:"db-sqlite-busy-timeout" usage:"Sqlite only: see https://www.sqlite.org/pragma.html#pragma_busy_timeout"` + DbType string `name:"db-type" usage:"Database type: eg., postgres"` + DbAddress string `name:"db-address" usage:"Database ipv4 address, hostname, or filename"` + DbPort int `name:"db-port" usage:"Database port"` + DbUser string `name:"db-user" usage:"Database username"` + DbPassword string `name:"db-password" usage:"Database password"` + DbDatabase string `name:"db-database" usage:"Database name"` + DbTLSMode string `name:"db-tls-mode" usage:"Database tls mode"` + DbTLSCACert string `name:"db-tls-ca-cert" usage:"Path to CA cert for db tls connection"` + DbMaxOpenConnsMultiplier int `name:"db-max-open-conns-multiplier" usage:"Multiplier to use per cpu for max open database connections. 0 or less is normalized to 1."` + DbSqliteJournalMode string `name:"db-sqlite-journal-mode" usage:"Sqlite only: see https://www.sqlite.org/pragma.html#pragma_journal_mode"` + DbSqliteSynchronous string `name:"db-sqlite-synchronous" usage:"Sqlite only: see https://www.sqlite.org/pragma.html#pragma_synchronous"` + DbSqliteCacheSize bytesize.Size `name:"db-sqlite-cache-size" usage:"Sqlite only: see https://www.sqlite.org/pragma.html#pragma_cache_size"` + DbSqliteBusyTimeout time.Duration `name:"db-sqlite-busy-timeout" usage:"Sqlite only: see https://www.sqlite.org/pragma.html#pragma_busy_timeout"` WebTemplateBaseDir string `name:"web-template-base-dir" usage:"Basedir for html templating files for rendering pages and composing emails."` WebAssetBaseDir string `name:"web-asset-base-dir" usage:"Directory to serve static assets from, accessible at example.org/assets/"` diff --git a/internal/config/defaults.go b/internal/config/defaults.go index ecd32be2f..6b4c5db17 100644 --- a/internal/config/defaults.go +++ b/internal/config/defaults.go @@ -40,18 +40,19 @@ var Defaults = Configuration{ Port: 8080, TrustedProxies: []string{"127.0.0.1/32", "::1"}, // localhost - DbType: "postgres", - DbAddress: "", - DbPort: 5432, - DbUser: "", - DbPassword: "", - DbDatabase: "gotosocial", - DbTLSMode: "disable", - DbTLSCACert: "", - DbSqliteJournalMode: "WAL", - DbSqliteSynchronous: "NORMAL", - DbSqliteCacheSize: 64 * bytesize.MiB, - DbSqliteBusyTimeout: time.Minute * 5, + DbType: "postgres", + DbAddress: "", + DbPort: 5432, + DbUser: "", + DbPassword: "", + DbDatabase: "gotosocial", + DbTLSMode: "disable", + DbTLSCACert: "", + DbMaxOpenConnsMultiplier: 8, + DbSqliteJournalMode: "WAL", + DbSqliteSynchronous: "NORMAL", + DbSqliteCacheSize: 8 * bytesize.MiB, + DbSqliteBusyTimeout: time.Minute * 5, WebTemplateBaseDir: "./web/template/", WebAssetBaseDir: "./web/assets/", diff --git a/internal/config/flags.go b/internal/config/flags.go index a21f17f53..e594615d1 100644 --- a/internal/config/flags.go +++ b/internal/config/flags.go @@ -51,6 +51,7 @@ func (s *ConfigState) AddGlobalFlags(cmd *cobra.Command) { cmd.PersistentFlags().String(DbDatabaseFlag(), cfg.DbDatabase, fieldtag("DbDatabase", "usage")) cmd.PersistentFlags().String(DbTLSModeFlag(), cfg.DbTLSMode, fieldtag("DbTLSMode", "usage")) cmd.PersistentFlags().String(DbTLSCACertFlag(), cfg.DbTLSCACert, fieldtag("DbTLSCACert", "usage")) + cmd.PersistentFlags().Int(DbMaxOpenConnsMultiplierFlag(), cfg.DbMaxOpenConnsMultiplier, fieldtag("DbMaxOpenConnsMultiplier", "usage")) cmd.PersistentFlags().String(DbSqliteJournalModeFlag(), cfg.DbSqliteJournalMode, fieldtag("DbSqliteJournalMode", "usage")) cmd.PersistentFlags().String(DbSqliteSynchronousFlag(), cfg.DbSqliteSynchronous, fieldtag("DbSqliteSynchronous", "usage")) cmd.PersistentFlags().Uint64(DbSqliteCacheSizeFlag(), uint64(cfg.DbSqliteCacheSize), fieldtag("DbSqliteCacheSize", "usage")) diff --git a/internal/config/helpers.gen.go b/internal/config/helpers.gen.go index 3560c566a..3bf62efc6 100644 --- a/internal/config/helpers.gen.go +++ b/internal/config/helpers.gen.go @@ -524,6 +524,31 @@ func GetDbTLSCACert() string { return global.GetDbTLSCACert() } // SetDbTLSCACert safely sets the value for global configuration 'DbTLSCACert' field func SetDbTLSCACert(v string) { global.SetDbTLSCACert(v) } +// GetDbMaxOpenConnsMultiplier safely fetches the Configuration value for state's 'DbMaxOpenConnsMultiplier' field +func (st *ConfigState) GetDbMaxOpenConnsMultiplier() (v int) { + st.mutex.Lock() + v = st.config.DbMaxOpenConnsMultiplier + st.mutex.Unlock() + return +} + +// SetDbMaxOpenConnsMultiplier safely sets the Configuration value for state's 'DbMaxOpenConnsMultiplier' field +func (st *ConfigState) SetDbMaxOpenConnsMultiplier(v int) { + st.mutex.Lock() + defer st.mutex.Unlock() + st.config.DbMaxOpenConnsMultiplier = v + st.reloadToViper() +} + +// DbMaxOpenConnsMultiplierFlag returns the flag name for the 'DbMaxOpenConnsMultiplier' field +func DbMaxOpenConnsMultiplierFlag() string { return "db-max-open-conns-multiplier" } + +// GetDbMaxOpenConnsMultiplier safely fetches the value for global configuration 'DbMaxOpenConnsMultiplier' field +func GetDbMaxOpenConnsMultiplier() int { return global.GetDbMaxOpenConnsMultiplier() } + +// SetDbMaxOpenConnsMultiplier safely sets the value for global configuration 'DbMaxOpenConnsMultiplier' field +func SetDbMaxOpenConnsMultiplier(v int) { global.SetDbMaxOpenConnsMultiplier(v) } + // GetDbSqliteJournalMode safely fetches the Configuration value for state's 'DbSqliteJournalMode' field func (st *ConfigState) GetDbSqliteJournalMode() (v string) { st.mutex.Lock() diff --git a/internal/db/bundb/bundb.go b/internal/db/bundb/bundb.go index b6a07bdc6..385b0ca1f 100644 --- a/internal/db/bundb/bundb.go +++ b/internal/db/bundb/bundb.go @@ -222,6 +222,32 @@ func NewBunDBService(ctx context.Context, state *state.State) (db.DB, error) { return ps, nil } +func pgConn(ctx context.Context) (*DBConn, error) { + opts, err := deriveBunDBPGOptions() //nolint:contextcheck + if err != nil { + return nil, fmt.Errorf("could not create bundb postgres options: %s", err) + } + + sqldb := stdlib.OpenDB(*opts) + + // Tune db connections for postgres, see: + // - https://bun.uptrace.dev/guide/running-bun-in-production.html#database-sql + // - https://www.alexedwards.net/blog/configuring-sqldb + sqldb.SetMaxOpenConns(maxOpenConns()) // x number of conns per CPU + sqldb.SetMaxIdleConns(2) // assume default 2; if max idle is less than max open, it will be automatically adjusted + sqldb.SetConnMaxLifetime(5 * time.Minute) // fine to kill old connections + + conn := WrapDBConn(bun.NewDB(sqldb, pgdialect.New())) + + // ping to check the db is there and listening + if err := conn.PingContext(ctx); err != nil { + return nil, fmt.Errorf("postgres ping: %s", err) + } + + log.Info("connected to POSTGRES database") + return conn, nil +} + func sqliteConn(ctx context.Context) (*DBConn, error) { // validate db address has actually been set address := config.GetDbAddress() @@ -236,13 +262,10 @@ func sqliteConn(ctx context.Context) (*DBConn, error) { // Append our own SQLite preferences address = "file:" + address - var inMem bool - if address == "file::memory:" { address = fmt.Sprintf("file:%s?mode=memory&cache=shared", uuid.NewString()) log.Infof("using in-memory database address " + address) log.Warn("sqlite in-memory database should only be used for debugging") - inMem = true } // Open new DB instance @@ -254,12 +277,12 @@ func sqliteConn(ctx context.Context) (*DBConn, error) { return nil, fmt.Errorf("could not open sqlite db: %s", err) } - if inMem { - // don't close connections on disconnect -- otherwise - // the SQLite database will be deleted when there - // are no active connections - sqldb.SetConnMaxLifetime(0) - } + // Tune db connections for sqlite, see: + // - https://bun.uptrace.dev/guide/running-bun-in-production.html#database-sql + // - https://www.alexedwards.net/blog/configuring-sqldb + sqldb.SetMaxOpenConns(maxOpenConns()) // x number of conns per cpu + sqldb.SetMaxIdleConns(1) // only keep max 1 idle connection around + sqldb.SetConnMaxLifetime(0) // don't kill connections due to age // Wrap Bun database conn in our own wrapper conn := WrapDBConn(bun.NewDB(sqldb, sqlitedialect.New())) @@ -276,79 +299,20 @@ func sqliteConn(ctx context.Context) (*DBConn, error) { return conn, nil } -func sqlitePragmas(ctx context.Context, conn *DBConn) error { - var pragmas [][]string - if mode := config.GetDbSqliteJournalMode(); mode != "" { - // Set the user provided SQLite journal mode - pragmas = append(pragmas, []string{"journal_mode", mode}) - } - - if mode := config.GetDbSqliteSynchronous(); mode != "" { - // Set the user provided SQLite synchronous mode - pragmas = append(pragmas, []string{"synchronous", mode}) - } - - if size := config.GetDbSqliteCacheSize(); size > 0 { - // Set the user provided SQLite cache size (in kibibytes) - // Prepend a '-' character to this to indicate to sqlite - // that we're giving kibibytes rather than num pages. - // https://www.sqlite.org/pragma.html#pragma_cache_size - s := "-" + strconv.FormatUint(uint64(size/bytesize.KiB), 10) - pragmas = append(pragmas, []string{"cache_size", s}) - } - - if timeout := config.GetDbSqliteBusyTimeout(); timeout > 0 { - t := strconv.FormatInt(timeout.Milliseconds(), 10) - pragmas = append(pragmas, []string{"busy_timeout", t}) - } - - for _, p := range pragmas { - pk := p[0] - pv := p[1] - - if _, err := conn.DB.ExecContext(ctx, "PRAGMA ?=?", bun.Ident(pk), bun.Safe(pv)); err != nil { - return fmt.Errorf("error executing sqlite pragma %s: %w", pk, err) - } - - var res string - if err := conn.DB.NewRaw("PRAGMA ?", bun.Ident(pk)).Scan(ctx, &res); err != nil { - return fmt.Errorf("error scanning sqlite pragma %s: %w", pv, err) - } - - log.Infof("sqlite pragma %s set to %s", pk, res) - } - - return nil -} - -func pgConn(ctx context.Context) (*DBConn, error) { - opts, err := deriveBunDBPGOptions() //nolint:contextcheck - if err != nil { - return nil, fmt.Errorf("could not create bundb postgres options: %s", err) - } - - sqldb := stdlib.OpenDB(*opts) - - // https://bun.uptrace.dev/postgres/running-bun-in-production.html#database-sql - maxOpenConns := 4 * runtime.GOMAXPROCS(0) - sqldb.SetMaxOpenConns(maxOpenConns) - sqldb.SetMaxIdleConns(maxOpenConns) - - conn := WrapDBConn(bun.NewDB(sqldb, pgdialect.New())) - - // ping to check the db is there and listening - if err := conn.PingContext(ctx); err != nil { - return nil, fmt.Errorf("postgres ping: %s", err) - } - - log.Info("connected to POSTGRES database") - return conn, nil -} - /* HANDY STUFF */ +// maxOpenConns returns multiplier * GOMAXPROCS, +// clamping multiplier to 1 if it was below 1. +func maxOpenConns() int { + multiplier := config.GetDbMaxOpenConnsMultiplier() + if multiplier < 1 { + multiplier = 1 + } + return multiplier * runtime.GOMAXPROCS(0) +} + // deriveBunDBPGOptions takes an application config and returns either a ready-to-use set of options // with sensible defaults, or an error if it's not satisfied by the provided config. func deriveBunDBPGOptions() (*pgx.ConnConfig, error) { @@ -434,6 +398,53 @@ func deriveBunDBPGOptions() (*pgx.ConnConfig, error) { return cfg, nil } +// sqlitePragmas sets desired sqlite pragmas based on configured values, and +// logs the results of the pragma queries. Errors if something goes wrong. +func sqlitePragmas(ctx context.Context, conn *DBConn) error { + var pragmas [][]string + if mode := config.GetDbSqliteJournalMode(); mode != "" { + // Set the user provided SQLite journal mode + pragmas = append(pragmas, []string{"journal_mode", mode}) + } + + if mode := config.GetDbSqliteSynchronous(); mode != "" { + // Set the user provided SQLite synchronous mode + pragmas = append(pragmas, []string{"synchronous", mode}) + } + + if size := config.GetDbSqliteCacheSize(); size > 0 { + // Set the user provided SQLite cache size (in kibibytes) + // Prepend a '-' character to this to indicate to sqlite + // that we're giving kibibytes rather than num pages. + // https://www.sqlite.org/pragma.html#pragma_cache_size + s := "-" + strconv.FormatUint(uint64(size/bytesize.KiB), 10) + pragmas = append(pragmas, []string{"cache_size", s}) + } + + if timeout := config.GetDbSqliteBusyTimeout(); timeout > 0 { + t := strconv.FormatInt(timeout.Milliseconds(), 10) + pragmas = append(pragmas, []string{"busy_timeout", t}) + } + + for _, p := range pragmas { + pk := p[0] + pv := p[1] + + if _, err := conn.DB.ExecContext(ctx, "PRAGMA ?=?", bun.Ident(pk), bun.Safe(pv)); err != nil { + return fmt.Errorf("error executing sqlite pragma %s: %w", pk, err) + } + + var res string + if err := conn.DB.NewRaw("PRAGMA ?", bun.Ident(pk)).Scan(ctx, &res); err != nil { + return fmt.Errorf("error scanning sqlite pragma %s: %w", pv, err) + } + + log.Infof("sqlite pragma %s set to %s", pk, res) + } + + return nil +} + /* CONVERSION FUNCTIONS */ diff --git a/test/envparsing.sh b/test/envparsing.sh index 02f3aa63a..c50fa99ea 100755 --- a/test/envparsing.sh +++ b/test/envparsing.sh @@ -2,7 +2,7 @@ set -eu -EXPECT='{"account-domain":"peepee","accounts-allow-custom-css":true,"accounts-approval-required":false,"accounts-reason-required":false,"accounts-registration-open":true,"advanced-cookies-samesite":"strict","advanced-rate-limit-requests":6969,"advanced-throttling-multiplier":-1,"application-name":"gts","bind-address":"127.0.0.1","cache":{"gts":{"account-max-size":99,"account-sweep-freq":1000000000,"account-ttl":10800000000000,"block-max-size":100,"block-sweep-freq":10000000000,"block-ttl":300000000000,"domain-block-max-size":1000,"domain-block-sweep-freq":60000000000,"domain-block-ttl":86400000000000,"emoji-category-max-size":100,"emoji-category-sweep-freq":10000000000,"emoji-category-ttl":300000000000,"emoji-max-size":500,"emoji-sweep-freq":10000000000,"emoji-ttl":300000000000,"mention-max-size":500,"mention-sweep-freq":10000000000,"mention-ttl":300000000000,"notification-max-size":500,"notification-sweep-freq":10000000000,"notification-ttl":300000000000,"report-max-size":100,"report-sweep-freq":10000000000,"report-ttl":300000000000,"status-max-size":500,"status-sweep-freq":10000000000,"status-ttl":300000000000,"tombstone-max-size":100,"tombstone-sweep-freq":10000000000,"tombstone-ttl":300000000000,"user-max-size":100,"user-sweep-freq":10000000000,"user-ttl":300000000000}},"config-path":"internal/config/testdata/test.yaml","db-address":":memory:","db-database":"gotosocial_prod","db-password":"hunter2","db-port":6969,"db-sqlite-busy-timeout":1000000000,"db-sqlite-cache-size":0,"db-sqlite-journal-mode":"DELETE","db-sqlite-synchronous":"FULL","db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"sqlite","db-user":"sex-haver","dry-run":false,"email":"","host":"example.com","instance-deliver-to-shared-inboxes":false,"instance-expose-peers":true,"instance-expose-public-timeline":true,"instance-expose-suspended":true,"instance-expose-suspended-web":true,"landing-page-user":"admin","letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":true,"letsencrypt-port":80,"log-db-queries":true,"log-level":"info","media-description-max-chars":5000,"media-description-min-chars":69,"media-emoji-local-max-size":420,"media-emoji-remote-max-size":420,"media-image-max-size":420,"media-remote-cache-days":30,"media-video-max-size":420,"oidc-client-id":"1234","oidc-client-secret":"shhhh its a secret","oidc-enabled":true,"oidc-idp-name":"sex-haver","oidc-issuer":"whoknows","oidc-link-existing":true,"oidc-scopes":["read","write"],"oidc-skip-verification":true,"password":"","path":"","port":6969,"protocol":"http","smtp-from":"queen.rip.in.piss@terfisland.org","smtp-host":"example.com","smtp-password":"hunter2","smtp-port":4269,"smtp-username":"sex-haver","software-version":"","statuses-cw-max-chars":420,"statuses-max-chars":69,"statuses-media-max-files":1,"statuses-poll-max-options":1,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/root/store","storage-s3-access-key":"minio","storage-s3-bucket":"gts","storage-s3-endpoint":"localhost:9000","storage-s3-proxy":true,"storage-s3-secret-key":"miniostorage","storage-s3-use-ssl":false,"syslog-address":"127.0.0.1:6969","syslog-enabled":true,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32","docker.host.local"],"username":"","web-asset-base-dir":"/root","web-template-base-dir":"/root"}' +EXPECT='{"account-domain":"peepee","accounts-allow-custom-css":true,"accounts-approval-required":false,"accounts-reason-required":false,"accounts-registration-open":true,"advanced-cookies-samesite":"strict","advanced-rate-limit-requests":6969,"advanced-throttling-multiplier":-1,"application-name":"gts","bind-address":"127.0.0.1","cache":{"gts":{"account-max-size":99,"account-sweep-freq":1000000000,"account-ttl":10800000000000,"block-max-size":100,"block-sweep-freq":10000000000,"block-ttl":300000000000,"domain-block-max-size":1000,"domain-block-sweep-freq":60000000000,"domain-block-ttl":86400000000000,"emoji-category-max-size":100,"emoji-category-sweep-freq":10000000000,"emoji-category-ttl":300000000000,"emoji-max-size":500,"emoji-sweep-freq":10000000000,"emoji-ttl":300000000000,"mention-max-size":500,"mention-sweep-freq":10000000000,"mention-ttl":300000000000,"notification-max-size":500,"notification-sweep-freq":10000000000,"notification-ttl":300000000000,"report-max-size":100,"report-sweep-freq":10000000000,"report-ttl":300000000000,"status-max-size":500,"status-sweep-freq":10000000000,"status-ttl":300000000000,"tombstone-max-size":100,"tombstone-sweep-freq":10000000000,"tombstone-ttl":300000000000,"user-max-size":100,"user-sweep-freq":10000000000,"user-ttl":300000000000}},"config-path":"internal/config/testdata/test.yaml","db-address":":memory:","db-database":"gotosocial_prod","db-max-open-conns-multiplier":3,"db-password":"hunter2","db-port":6969,"db-sqlite-busy-timeout":1000000000,"db-sqlite-cache-size":0,"db-sqlite-journal-mode":"DELETE","db-sqlite-synchronous":"FULL","db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"sqlite","db-user":"sex-haver","dry-run":false,"email":"","host":"example.com","instance-deliver-to-shared-inboxes":false,"instance-expose-peers":true,"instance-expose-public-timeline":true,"instance-expose-suspended":true,"instance-expose-suspended-web":true,"landing-page-user":"admin","letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":true,"letsencrypt-port":80,"log-db-queries":true,"log-level":"info","media-description-max-chars":5000,"media-description-min-chars":69,"media-emoji-local-max-size":420,"media-emoji-remote-max-size":420,"media-image-max-size":420,"media-remote-cache-days":30,"media-video-max-size":420,"oidc-client-id":"1234","oidc-client-secret":"shhhh its a secret","oidc-enabled":true,"oidc-idp-name":"sex-haver","oidc-issuer":"whoknows","oidc-link-existing":true,"oidc-scopes":["read","write"],"oidc-skip-verification":true,"password":"","path":"","port":6969,"protocol":"http","smtp-from":"queen.rip.in.piss@terfisland.org","smtp-host":"example.com","smtp-password":"hunter2","smtp-port":4269,"smtp-username":"sex-haver","software-version":"","statuses-cw-max-chars":420,"statuses-max-chars":69,"statuses-media-max-files":1,"statuses-poll-max-options":1,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/root/store","storage-s3-access-key":"minio","storage-s3-bucket":"gts","storage-s3-endpoint":"localhost:9000","storage-s3-proxy":true,"storage-s3-secret-key":"miniostorage","storage-s3-use-ssl":false,"syslog-address":"127.0.0.1:6969","syslog-enabled":true,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32","docker.host.local"],"username":"","web-asset-base-dir":"/root","web-template-base-dir":"/root"}' # Set all the environment variables to # ensure that these are parsed without panic @@ -22,6 +22,7 @@ GTS_DB_PORT=6969 \ GTS_DB_USER='sex-haver' \ GTS_DB_PASSWORD='hunter2' \ GTS_DB_DATABASE='gotosocial_prod' \ +GTS_DB_MAX_OPEN_CONNS_MULTIPLIER=3 \ GTS_DB_SQLITE_JOURNAL_MODE='DELETE' \ GTS_DB_SQLITE_SYNCHRONOUS='FULL' \ GTS_DB_SQLITE_CACHE_SIZE=0 \ diff --git a/testrig/config.go b/testrig/config.go index b6a7b4c83..140d73e6a 100644 --- a/testrig/config.go +++ b/testrig/config.go @@ -46,16 +46,19 @@ var testDefaults = config.Configuration{ Port: 8080, TrustedProxies: []string{"127.0.0.1/32", "::1"}, - DbType: "sqlite", - DbAddress: ":memory:", - DbPort: 5432, - DbUser: "postgres", - DbPassword: "postgres", - DbDatabase: "postgres", - DbSqliteJournalMode: "WAL", - DbSqliteSynchronous: "NORMAL", - DbSqliteCacheSize: 64 * bytesize.MiB, - DbSqliteBusyTimeout: time.Minute * 5, + DbType: "sqlite", + DbAddress: ":memory:", + DbPort: 5432, + DbUser: "postgres", + DbPassword: "postgres", + DbDatabase: "postgres", + DbTLSMode: "disable", + DbTLSCACert: "", + DbMaxOpenConnsMultiplier: 8, + DbSqliteJournalMode: "WAL", + DbSqliteSynchronous: "NORMAL", + DbSqliteCacheSize: 8 * bytesize.MiB, + DbSqliteBusyTimeout: time.Minute * 5, WebTemplateBaseDir: "./web/template/", WebAssetBaseDir: "./web/assets/",