2021-03-02 17:26:30 +00:00
/ *
GoToSocial
Copyright ( C ) 2021 GoToSocial Authors admin @ gotosocial . org
This program is free software : you can redistribute it and / or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation , either version 3 of the License , or
( at your option ) any later version .
This program is distributed in the hope that it will be useful ,
but WITHOUT ANY WARRANTY ; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE . See the
GNU Affero General Public License for more details .
You should have received a copy of the GNU Affero General Public License
along with this program . If not , see < http : //www.gnu.org/licenses/>.
* /
2021-08-25 13:34:33 +00:00
package bundb
2021-03-02 17:26:30 +00:00
import (
"context"
2021-07-19 16:03:07 +00:00
"crypto/tls"
"crypto/x509"
2021-08-25 13:34:33 +00:00
"database/sql"
2021-07-19 16:03:07 +00:00
"encoding/pem"
2021-03-02 17:26:30 +00:00
"errors"
"fmt"
2021-07-19 16:03:07 +00:00
"os"
2021-09-20 16:20:21 +00:00
"runtime"
2021-03-03 17:12:02 +00:00
"strings"
2021-03-02 21:52:31 +00:00
"time"
2021-03-02 17:26:30 +00:00
2021-08-29 14:41:41 +00:00
"github.com/ReneKroon/ttlcache"
2021-08-25 13:34:33 +00:00
"github.com/jackc/pgx/v4"
"github.com/jackc/pgx/v4/stdlib"
2021-03-02 21:52:31 +00:00
"github.com/sirupsen/logrus"
2021-08-29 14:41:41 +00:00
"github.com/superseriousbusiness/gotosocial/internal/cache"
2021-04-01 18:46:45 +00:00
"github.com/superseriousbusiness/gotosocial/internal/config"
2021-05-15 09:58:11 +00:00
"github.com/superseriousbusiness/gotosocial/internal/db"
2021-08-31 17:27:02 +00:00
"github.com/superseriousbusiness/gotosocial/internal/db/bundb/migrations"
2021-05-08 12:25:55 +00:00
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
2021-06-13 16:42:28 +00:00
"github.com/superseriousbusiness/gotosocial/internal/id"
2021-08-25 13:34:33 +00:00
"github.com/uptrace/bun"
"github.com/uptrace/bun/dialect/pgdialect"
2021-08-29 14:41:41 +00:00
"github.com/uptrace/bun/dialect/sqlitedialect"
2021-08-31 17:27:02 +00:00
"github.com/uptrace/bun/migrate"
2021-09-30 09:16:23 +00:00
// blank import for the sqlite driver for bun
2021-08-29 14:41:41 +00:00
_ "modernc.org/sqlite"
2021-08-25 13:34:33 +00:00
)
const (
dbTypePostgres = "postgres"
dbTypeSqlite = "sqlite"
2021-03-02 17:26:30 +00:00
)
2021-08-20 10:26:56 +00:00
var registerTables [ ] interface { } = [ ] interface { } {
& gtsmodel . StatusToEmoji { } ,
& gtsmodel . StatusToTag { } ,
}
2021-08-25 13:34:33 +00:00
// bunDBService satisfies the DB interface
type bunDBService struct {
2021-08-20 10:26:56 +00:00
db . Account
db . Admin
db . Basic
db . Domain
db . Instance
db . Media
db . Mention
db . Notification
db . Relationship
2021-08-25 13:34:33 +00:00
db . Session
2021-08-20 10:26:56 +00:00
db . Status
db . Timeline
2021-05-21 13:48:26 +00:00
config * config . Config
2021-08-29 14:41:41 +00:00
conn * DBConn
2021-03-02 17:26:30 +00:00
}
2021-10-11 12:37:33 +00:00
func doMigration ( ctx context . Context , db * bun . DB ) error {
l := logrus . WithField ( "func" , "doMigration" )
2021-08-31 17:27:02 +00:00
migrator := migrate . NewMigrator ( db , migrations . Migrations )
if err := migrator . Init ( ctx ) ; err != nil {
return err
}
group , err := migrator . Migrate ( ctx )
if err != nil {
2021-09-01 16:29:25 +00:00
if err . Error ( ) == "migrate: there are no any migrations" {
return nil
}
2021-08-31 17:27:02 +00:00
return err
}
if group . ID == 0 {
l . Info ( "there are no new migrations to run" )
return nil
}
l . Infof ( "MIGRATED DATABASE TO %s" , group )
return nil
}
2021-08-25 13:34:33 +00:00
// NewBunDBService returns a bunDB derived from the provided config, which implements the go-fed DB interface.
// Under the hood, it uses https://github.com/uptrace/bun to create and maintain a database connection.
2021-10-11 12:37:33 +00:00
func NewBunDBService ( ctx context . Context , c * config . Config ) ( db . DB , error ) {
2021-08-25 13:34:33 +00:00
var sqldb * sql . DB
2021-08-29 14:41:41 +00:00
var conn * DBConn
2021-08-25 13:34:33 +00:00
// depending on the database type we're trying to create, we need to use a different driver...
switch strings . ToLower ( c . DBConfig . Type ) {
case dbTypePostgres :
// POSTGRES
opts , err := deriveBunDBPGOptions ( c )
if err != nil {
return nil , fmt . Errorf ( "could not create bundb postgres options: %s" , err )
}
sqldb = stdlib . OpenDB ( * opts )
2021-09-20 16:20:21 +00:00
tweakConnectionValues ( sqldb )
2021-10-11 12:37:33 +00:00
conn = WrapDBConn ( bun . NewDB ( sqldb , pgdialect . New ( ) ) )
2021-08-25 13:34:33 +00:00
case dbTypeSqlite :
// SQLITE
2021-09-01 09:08:21 +00:00
// Drop anything fancy from DB address
c . DBConfig . Address = strings . Split ( c . DBConfig . Address , "?" ) [ 0 ]
c . DBConfig . Address = strings . TrimPrefix ( c . DBConfig . Address , "file:" )
// Append our own SQLite preferences
c . DBConfig . Address = "file:" + c . DBConfig . Address + "?cache=shared"
// Open new DB instance
2021-08-29 14:41:41 +00:00
var err error
sqldb , err = sql . Open ( "sqlite" , c . DBConfig . Address )
if err != nil {
return nil , fmt . Errorf ( "could not open sqlite db: %s" , err )
}
2021-09-20 16:20:21 +00:00
tweakConnectionValues ( sqldb )
2021-10-11 12:37:33 +00:00
conn = WrapDBConn ( bun . NewDB ( sqldb , sqlitedialect . New ( ) ) )
2021-08-29 14:41:41 +00:00
2021-09-01 09:08:21 +00:00
if c . DBConfig . Address == "file::memory:?cache=shared" {
2021-10-11 12:37:33 +00:00
logrus . Warn ( "sqlite in-memory database should only be used for debugging" )
2021-08-29 14:41:41 +00:00
// don't close connections on disconnect -- otherwise
// the SQLite database will be deleted when there
// are no active connections
sqldb . SetConnMaxLifetime ( 0 )
}
2021-08-25 13:34:33 +00:00
default :
return nil , fmt . Errorf ( "database type %s not supported for bundb" , strings . ToLower ( c . DBConfig . Type ) )
2021-03-05 17:31:12 +00:00
}
2021-10-11 12:37:33 +00:00
if logrus . GetLevel ( ) >= logrus . TraceLevel {
2021-09-11 11:19:06 +00:00
// add a hook to just log queries and the time they take
2021-10-11 12:37:33 +00:00
conn . DB . AddQueryHook ( newDebugQueryHook ( ) )
2021-09-11 11:19:06 +00:00
}
2021-04-19 17:42:19 +00:00
// actually *begin* the connection so that we can tell if the db is there and listening
2021-08-25 13:34:33 +00:00
if err := conn . Ping ( ) ; err != nil {
2021-03-02 21:52:31 +00:00
return nil , fmt . Errorf ( "db connection error: %s" , err )
}
2021-10-11 12:37:33 +00:00
logrus . Info ( "connected to database" )
2021-03-02 21:52:31 +00:00
2021-08-25 13:34:33 +00:00
for _ , t := range registerTables {
// https://bun.uptrace.dev/orm/many-to-many-relation/
conn . RegisterModel ( t )
2021-03-02 21:52:31 +00:00
}
2021-10-11 12:37:33 +00:00
if err := doMigration ( ctx , conn . DB ) ; err != nil {
2021-08-31 17:27:02 +00:00
return nil , fmt . Errorf ( "db migration error: %s" , err )
}
2021-09-01 09:08:21 +00:00
2021-09-01 09:13:01 +00:00
accounts := & accountDB { config : c , conn : conn , cache : cache . NewAccountCache ( ) }
2021-08-25 13:34:33 +00:00
ps := & bunDBService {
2021-09-01 09:08:21 +00:00
Account : accounts ,
2021-08-20 10:26:56 +00:00
Admin : & adminDB {
config : c ,
conn : conn ,
} ,
Basic : & basicDB {
config : c ,
conn : conn ,
} ,
Domain : & domainDB {
config : c ,
conn : conn ,
} ,
Instance : & instanceDB {
config : c ,
conn : conn ,
} ,
Media : & mediaDB {
config : c ,
conn : conn ,
} ,
Mention : & mentionDB {
config : c ,
conn : conn ,
2021-08-29 14:41:41 +00:00
cache : ttlcache . NewCache ( ) ,
2021-08-20 10:26:56 +00:00
} ,
Notification : & notificationDB {
config : c ,
conn : conn ,
2021-08-29 14:41:41 +00:00
cache : ttlcache . NewCache ( ) ,
2021-08-20 10:26:56 +00:00
} ,
Relationship : & relationshipDB {
config : c ,
conn : conn ,
2021-08-25 13:34:33 +00:00
} ,
Session : & sessionDB {
config : c ,
conn : conn ,
2021-08-20 10:26:56 +00:00
} ,
Status : & statusDB {
2021-09-01 09:08:21 +00:00
config : c ,
conn : conn ,
cache : cache . NewStatusCache ( ) ,
accounts : accounts ,
2021-08-20 10:26:56 +00:00
} ,
Timeline : & timelineDB {
config : c ,
conn : conn ,
} ,
2021-04-01 18:46:45 +00:00
config : c ,
conn : conn ,
}
2021-03-02 17:26:30 +00:00
2021-08-25 13:34:33 +00:00
// we can confidently return this useable service now
2021-04-01 18:46:45 +00:00
return ps , nil
2021-03-22 21:26:54 +00:00
}
2021-03-02 17:26:30 +00:00
/ *
HANDY STUFF
* /
2021-08-25 13:34:33 +00:00
// deriveBunDBPGOptions takes an application config and returns either a ready-to-use set of options
2021-03-02 17:26:30 +00:00
// with sensible defaults, or an error if it's not satisfied by the provided config.
2021-08-25 13:34:33 +00:00
func deriveBunDBPGOptions ( c * config . Config ) ( * pgx . ConnConfig , error ) {
2021-05-15 09:58:11 +00:00
if strings . ToUpper ( c . DBConfig . Type ) != db . DBTypePostgres {
return nil , fmt . Errorf ( "expected db type of %s but got %s" , db . DBTypePostgres , c . DBConfig . Type )
2021-03-02 17:26:30 +00:00
}
2021-03-04 11:07:24 +00:00
// validate port
2021-03-04 13:38:18 +00:00
if c . DBConfig . Port == 0 {
2021-03-04 11:07:24 +00:00
return nil , errors . New ( "no port set" )
2021-03-02 17:26:30 +00:00
}
// validate address
2021-03-04 13:38:18 +00:00
if c . DBConfig . Address == "" {
2021-03-04 11:07:24 +00:00
return nil , errors . New ( "no address set" )
2021-03-02 21:52:31 +00:00
}
2021-03-04 13:38:18 +00:00
2021-03-02 21:52:31 +00:00
// validate username
2021-03-04 13:38:18 +00:00
if c . DBConfig . User == "" {
2021-03-04 11:07:24 +00:00
return nil , errors . New ( "no user set" )
2021-03-02 17:26:30 +00:00
}
2021-03-02 21:52:31 +00:00
// validate that there's a password
2021-03-04 13:38:18 +00:00
if c . DBConfig . Password == "" {
2021-03-02 21:52:31 +00:00
return nil , errors . New ( "no password set" )
}
// validate database
2021-03-04 13:38:18 +00:00
if c . DBConfig . Database == "" {
2021-03-04 11:07:24 +00:00
return nil , errors . New ( "no database set" )
2021-03-02 17:26:30 +00:00
}
2021-07-19 16:03:07 +00:00
var tlsConfig * tls . Config
switch c . DBConfig . TLSMode {
case config . DBTLSModeDisable , config . DBTLSModeUnset :
break // nothing to do
case config . DBTLSModeEnable :
tlsConfig = & tls . Config {
InsecureSkipVerify : true ,
}
case config . DBTLSModeRequire :
tlsConfig = & tls . Config {
InsecureSkipVerify : false ,
2021-07-19 17:31:47 +00:00
ServerName : c . DBConfig . Address ,
2021-07-19 16:03:07 +00:00
}
}
if tlsConfig != nil && c . DBConfig . TLSCACert != "" {
// load the system cert pool first -- we'll append the given CA cert to this
certPool , err := x509 . SystemCertPool ( )
if err != nil {
return nil , fmt . Errorf ( "error fetching system CA cert pool: %s" , err )
}
// open the file itself and make sure there's something in it
caCertBytes , err := os . ReadFile ( c . DBConfig . TLSCACert )
if err != nil {
return nil , fmt . Errorf ( "error opening CA certificate at %s: %s" , c . DBConfig . TLSCACert , err )
}
if len ( caCertBytes ) == 0 {
return nil , fmt . Errorf ( "ca cert at %s was empty" , c . DBConfig . TLSCACert )
}
// make sure we have a PEM block
caPem , _ := pem . Decode ( caCertBytes )
if caPem == nil {
return nil , fmt . Errorf ( "could not parse cert at %s into PEM" , c . DBConfig . TLSCACert )
}
// parse the PEM block into the certificate
caCert , err := x509 . ParseCertificate ( caPem . Bytes )
if err != nil {
return nil , fmt . Errorf ( "could not parse cert at %s into x509 certificate: %s" , c . DBConfig . TLSCACert , err )
}
// we're happy, add it to the existing pool and then use this pool in our tls config
certPool . AddCert ( caCert )
tlsConfig . RootCAs = certPool
}
2021-08-25 13:34:33 +00:00
cfg , _ := pgx . ParseConfig ( "" )
cfg . Host = c . DBConfig . Address
cfg . Port = uint16 ( c . DBConfig . Port )
cfg . User = c . DBConfig . User
cfg . Password = c . DBConfig . Password
cfg . TLSConfig = tlsConfig
cfg . Database = c . DBConfig . Database
cfg . PreferSimpleProtocol = true
2021-09-20 16:20:21 +00:00
cfg . RuntimeParams [ "application_name" ] = c . ApplicationName
2021-03-02 17:26:30 +00:00
2021-08-25 13:34:33 +00:00
return cfg , nil
2021-03-02 17:26:30 +00:00
}
2021-09-20 16:20:21 +00:00
// https://bun.uptrace.dev/postgres/running-bun-in-production.html#database-sql
func tweakConnectionValues ( sqldb * sql . DB ) {
maxOpenConns := 4 * runtime . GOMAXPROCS ( 0 )
sqldb . SetMaxOpenConns ( maxOpenConns )
sqldb . SetMaxIdleConns ( maxOpenConns )
}
2021-04-19 17:42:19 +00:00
/ *
CONVERSION FUNCTIONS
* /
2021-04-01 18:46:45 +00:00
2021-06-13 16:42:28 +00:00
// TODO: move these to the type converter, it's bananas that they're here and not there
2021-08-25 13:34:33 +00:00
func ( ps * bunDBService ) MentionStringsToMentions ( ctx context . Context , targetAccounts [ ] string , originAccountID string , statusID string ) ( [ ] * gtsmodel . Mention , error ) {
2021-05-21 13:48:26 +00:00
ogAccount := & gtsmodel . Account { }
2021-08-25 13:34:33 +00:00
if err := ps . conn . NewSelect ( ) . Model ( ogAccount ) . Where ( "id = ?" , originAccountID ) . Scan ( ctx ) ; err != nil {
2021-05-21 13:48:26 +00:00
return nil , err
}
2021-04-19 17:42:19 +00:00
menchies := [ ] * gtsmodel . Mention { }
for _ , a := range targetAccounts {
// A mentioned account looks like "@test@example.org" or just "@test" for a local account
// -- we can guarantee this from the regex that targetAccounts should have been derived from.
// But we still need to do a bit of fiddling to get what we need here -- the username and domain (if given).
// 1. trim off the first @
t := strings . TrimPrefix ( a , "@" )
// 2. split the username and domain
s := strings . Split ( t , "@" )
// 3. if it's length 1 it's a local account, length 2 means remote, anything else means something is wrong
var local bool
switch len ( s ) {
case 1 :
local = true
case 2 :
local = false
default :
return nil , fmt . Errorf ( "mentioned account format '%s' was not valid" , a )
2021-04-01 18:46:45 +00:00
}
2021-04-19 17:42:19 +00:00
var username , domain string
username = s [ 0 ]
if ! local {
domain = s [ 1 ]
}
// 4. check we now have a proper username and domain
if username == "" || ( ! local && domain == "" ) {
return nil , fmt . Errorf ( "username or domain for '%s' was nil" , a )
2021-04-01 18:46:45 +00:00
}
2021-04-19 17:42:19 +00:00
// okay we're good now, we can start pulling accounts out of the database
mentionedAccount := & gtsmodel . Account { }
var err error
2021-06-13 16:42:28 +00:00
// match username + account, case insensitive
2021-04-19 17:42:19 +00:00
if local {
// local user -- should have a null domain
2021-08-25 13:34:33 +00:00
err = ps . conn . NewSelect ( ) . Model ( mentionedAccount ) . Where ( "LOWER(?) = LOWER(?)" , bun . Ident ( "username" ) , username ) . Where ( "? IS NULL" , bun . Ident ( "domain" ) ) . Scan ( ctx )
2021-04-19 17:42:19 +00:00
} else {
// remote user -- should have domain defined
2021-08-25 13:34:33 +00:00
err = ps . conn . NewSelect ( ) . Model ( mentionedAccount ) . Where ( "LOWER(?) = LOWER(?)" , bun . Ident ( "username" ) , username ) . Where ( "LOWER(?) = LOWER(?)" , bun . Ident ( "domain" ) , domain ) . Scan ( ctx )
2021-04-19 17:42:19 +00:00
}
if err != nil {
2021-08-25 13:34:33 +00:00
if err == sql . ErrNoRows {
2021-04-19 17:42:19 +00:00
// no result found for this username/domain so just don't include it as a mencho and carry on about our business
2021-10-11 12:37:33 +00:00
logrus . Debugf ( "no account found with username '%s' and domain '%s', skipping it" , username , domain )
2021-04-19 17:42:19 +00:00
continue
}
// a serious error has happened so bail
return nil , fmt . Errorf ( "error getting account with username '%s' and domain '%s': %s" , username , domain , err )
}
// id, createdAt and updatedAt will be populated by the db, so we have everything we need!
menchies = append ( menchies , & gtsmodel . Mention {
2021-08-20 10:26:56 +00:00
StatusID : statusID ,
OriginAccountID : ogAccount . ID ,
OriginAccountURI : ogAccount . URI ,
TargetAccountID : mentionedAccount . ID ,
NameString : a ,
TargetAccountURI : mentionedAccount . URI ,
TargetAccountURL : mentionedAccount . URL ,
OriginAccount : mentionedAccount ,
2021-04-19 17:42:19 +00:00
} )
2021-04-01 18:46:45 +00:00
}
2021-04-19 17:42:19 +00:00
return menchies , nil
}
2021-04-01 18:46:45 +00:00
2021-09-11 11:19:06 +00:00
func ( ps * bunDBService ) TagStringsToTags ( ctx context . Context , tags [ ] string , originAccountID string ) ( [ ] * gtsmodel . Tag , error ) {
2021-04-19 17:42:19 +00:00
newTags := [ ] * gtsmodel . Tag { }
for _ , t := range tags {
tag := & gtsmodel . Tag { }
// we can use selectorinsert here to create the new tag if it doesn't exist already
// inserted will be true if this is a new tag we just created
2021-08-25 13:34:33 +00:00
if err := ps . conn . NewSelect ( ) . Model ( tag ) . Where ( "LOWER(?) = LOWER(?)" , bun . Ident ( "name" ) , t ) . Scan ( ctx ) ; err != nil {
if err == sql . ErrNoRows {
2021-04-19 17:42:19 +00:00
// tag doesn't exist yet so populate it
2021-06-13 16:42:28 +00:00
newID , err := id . NewRandomULID ( )
if err != nil {
return nil , err
}
tag . ID = newID
tag . URL = fmt . Sprintf ( "%s://%s/tags/%s" , ps . config . Protocol , ps . config . Host , t )
2021-04-19 17:42:19 +00:00
tag . Name = t
tag . FirstSeenFromAccountID = originAccountID
tag . CreatedAt = time . Now ( )
tag . UpdatedAt = time . Now ( )
tag . Useable = true
tag . Listable = true
} else {
return nil , fmt . Errorf ( "error getting tag with name %s: %s" , t , err )
}
}
// bail already if the tag isn't useable
if ! tag . Useable {
continue
}
tag . LastStatusAt = time . Now ( )
newTags = append ( newTags , tag )
}
return newTags , nil
}
2021-09-11 11:19:06 +00:00
func ( ps * bunDBService ) EmojiStringsToEmojis ( ctx context . Context , emojis [ ] string ) ( [ ] * gtsmodel . Emoji , error ) {
2021-04-19 17:42:19 +00:00
newEmojis := [ ] * gtsmodel . Emoji { }
for _ , e := range emojis {
emoji := & gtsmodel . Emoji { }
2021-08-25 13:34:33 +00:00
err := ps . conn . NewSelect ( ) . Model ( emoji ) . Where ( "shortcode = ?" , e ) . Where ( "visible_in_picker = true" ) . Where ( "disabled = false" ) . Scan ( ctx )
2021-04-19 17:42:19 +00:00
if err != nil {
2021-08-25 13:34:33 +00:00
if err == sql . ErrNoRows {
2021-04-19 17:42:19 +00:00
// no result found for this username/domain so just don't include it as an emoji and carry on about our business
2021-10-11 12:37:33 +00:00
logrus . Debugf ( "no emoji found with shortcode %s, skipping it" , e )
2021-04-19 17:42:19 +00:00
continue
}
// a serious error has happened so bail
return nil , fmt . Errorf ( "error getting emoji with shortcode %s: %s" , e , err )
}
newEmojis = append ( newEmojis , emoji )
}
return newEmojis , nil
2021-04-01 18:46:45 +00:00
}