[performance] Don't fetch avatar + header if uri hasn't changed (#1463)

This commit is contained in:
tobi 2023-02-09 09:27:07 +01:00 committed by GitHub
parent 65b19411a4
commit 95715f9251
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 74 additions and 164 deletions

View file

@ -36,6 +36,7 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/id" "github.com/superseriousbusiness/gotosocial/internal/id"
"github.com/superseriousbusiness/gotosocial/internal/log" "github.com/superseriousbusiness/gotosocial/internal/log"
"github.com/superseriousbusiness/gotosocial/internal/media" "github.com/superseriousbusiness/gotosocial/internal/media"
"github.com/superseriousbusiness/gotosocial/internal/transport"
) )
func (d *deref) GetAccountByURI(ctx context.Context, requestUser string, uri *url.URL, block bool) (*gtsmodel.Account, error) { func (d *deref) GetAccountByURI(ctx context.Context, requestUser string, uri *url.URL, block bool) (*gtsmodel.Account, error) {
@ -145,9 +146,14 @@ func (d *deref) enrichAccount(ctx context.Context, requestUser string, uri *url.
} }
} }
transport, err := d.transportController.NewTransportForUsername(ctx, requestUser)
if err != nil {
return nil, fmt.Errorf("enrichAccount: couldn't create transport: %w", err)
}
if account.Username != "" { if account.Username != "" {
// A username was provided so we can attempt a webfinger, this ensures up-to-date accountdomain info. // A username was provided so we can attempt a webfinger, this ensures up-to-date accountdomain info.
accDomain, accURI, err := d.fingerRemoteAccount(ctx, requestUser, account.Username, account.Domain) accDomain, accURI, err := d.fingerRemoteAccount(ctx, transport, account.Username, account.Domain)
if err != nil && account.URI == "" { if err != nil && account.URI == "" {
// this is a new account (to us) with username@domain but failed // this is a new account (to us) with username@domain but failed
@ -185,7 +191,7 @@ func (d *deref) enrichAccount(ctx context.Context, requestUser string, uri *url.
defer d.stopHandshake(requestUser, uri) defer d.stopHandshake(requestUser, uri)
// Dereference this account to get the latest available. // Dereference this account to get the latest available.
apubAcc, err := d.dereferenceAccountable(ctx, requestUser, uri) apubAcc, err := d.dereferenceAccountable(ctx, transport, uri)
if err != nil { if err != nil {
return nil, fmt.Errorf("enrichAccount: error dereferencing account %s: %w", uri, err) return nil, fmt.Errorf("enrichAccount: error dereferencing account %s: %w", uri, err)
} }
@ -202,7 +208,7 @@ func (d *deref) enrichAccount(ctx context.Context, requestUser string, uri *url.
// No username was provided, so no webfinger was attempted earlier. // No username was provided, so no webfinger was attempted earlier.
// //
// Now we have a username we can attempt it now, this ensures up-to-date accountdomain info. // Now we have a username we can attempt it now, this ensures up-to-date accountdomain info.
accDomain, _, err := d.fingerRemoteAccount(ctx, requestUser, latestAcc.Username, uri.Host) accDomain, _, err := d.fingerRemoteAccount(ctx, transport, latestAcc.Username, uri.Host)
if err == nil { if err == nil {
// Update account with latest info. // Update account with latest info.
@ -214,9 +220,32 @@ func (d *deref) enrichAccount(ctx context.Context, requestUser string, uri *url.
latestAcc.ID = account.ID latestAcc.ID = account.ID
latestAcc.FetchedAt = time.Now() latestAcc.FetchedAt = time.Now()
// Fetch latest account media (TODO: check for changed URI to previous). // Fetch latest account avatar only if remote URI has changed
if err = d.fetchRemoteAccountMedia(ctx, latestAcc, requestUser, block); err != nil { if latestAcc.AvatarRemoteURL != "" && latestAcc.AvatarRemoteURL != account.AvatarRemoteURL {
log.Errorf("error fetching remote media for account %s: %v", uri, err) d.dereferencingAvatarsLock.Lock()
newAvatarID, err := d.fetchRemoteAccountMedia(ctx, transport, latestAcc.AvatarRemoteURL, latestAcc.ID, d.dereferencingAvatars, true, false)
d.dereferencingAvatarsLock.Unlock()
if err != nil {
log.Errorf("error fetching remote avatar for account %s: %v", uri, err)
} else {
latestAcc.AvatarMediaAttachmentID = newAvatarID
}
} else {
latestAcc.AvatarMediaAttachmentID = account.AvatarMediaAttachmentID // no change / empty url
}
// Fetch latest account header only if remote URI has changed
if latestAcc.AvatarRemoteURL != "" && latestAcc.AvatarRemoteURL != account.AvatarRemoteURL {
d.dereferencingHeadersLock.Lock()
newHeaderID, err := d.fetchRemoteAccountMedia(ctx, transport, latestAcc.HeaderRemoteURL, latestAcc.ID, d.dereferencingHeaders, false, true)
d.dereferencingHeadersLock.Unlock()
if err != nil {
log.Errorf("error fetching remote header for account %s: %v", uri, err)
} else {
latestAcc.HeaderMediaAttachmentID = newHeaderID
}
} else {
latestAcc.HeaderMediaAttachmentID = account.HeaderMediaAttachmentID // no change / empty url
} }
// Fetch the latest remote account emoji IDs used in account display name/bio. // Fetch the latest remote account emoji IDs used in account display name/bio.
@ -258,12 +287,7 @@ func (d *deref) enrichAccount(ctx context.Context, requestUser string, uri *url.
// it finds as something that an account model can be constructed out of. // it finds as something that an account model can be constructed out of.
// //
// Will work for Person, Application, or Service models. // Will work for Person, Application, or Service models.
func (d *deref) dereferenceAccountable(ctx context.Context, username string, remoteAccountID *url.URL) (ap.Accountable, error) { func (d *deref) dereferenceAccountable(ctx context.Context, transport transport.Transport, remoteAccountID *url.URL) (ap.Accountable, error) {
transport, err := d.transportController.NewTransportForUsername(ctx, username)
if err != nil {
return nil, fmt.Errorf("DereferenceAccountable: transport err: %w", err)
}
b, err := transport.Dereference(ctx, remoteAccountID) b, err := transport.Dereference(ctx, remoteAccountID)
if err != nil { if err != nil {
return nil, fmt.Errorf("DereferenceAccountable: error deferencing %s: %w", remoteAccountID.String(), err) return nil, fmt.Errorf("DereferenceAccountable: error deferencing %s: %w", remoteAccountID.String(), err)
@ -279,7 +303,7 @@ func (d *deref) dereferenceAccountable(ctx context.Context, username string, rem
return nil, fmt.Errorf("DereferenceAccountable: error resolving json into ap vocab type: %w", err) return nil, fmt.Errorf("DereferenceAccountable: error resolving json into ap vocab type: %w", err)
} }
//nolint shutup linter //nolint:forcetypeassert
switch t.GetTypeName() { switch t.GetTypeName() {
case ap.ActorApplication: case ap.ActorApplication:
return t.(vocab.ActivityStreamsApplication), nil return t.(vocab.ActivityStreamsApplication), nil
@ -296,156 +320,47 @@ func (d *deref) dereferenceAccountable(ctx context.Context, username string, rem
return nil, newErrWrongType(fmt.Errorf("DereferenceAccountable: type name %s not supported as Accountable", t.GetTypeName())) return nil, newErrWrongType(fmt.Errorf("DereferenceAccountable: type name %s not supported as Accountable", t.GetTypeName()))
} }
// fetchRemoteAccountMedia fetches and stores the header and avatar for a remote account, func (d *deref) fetchRemoteAccountMedia(
// using a transport on behalf of requestingUsername. ctx context.Context,
// transport transport.Transport,
// The returned boolean indicates whether anything changed -- in other words, whether the mediaRemoteURL string,
// account should be updated in the database. targetAccountID string,
// dereferencingMap map[string]*media.ProcessingMedia,
// targetAccount's AvatarMediaAttachmentID and HeaderMediaAttachmentID will be updated as necessary. avatar bool,
// header bool,
// If refresh is true, then the media will be fetched again even if it's already been fetched before. ) (string, error) {
// // first check if we're already processing this media
// If blocking is true, then the calls to the media manager made by this function will be blocking: if alreadyProcessing, ok := dereferencingMap[targetAccountID]; ok {
// in other words, the function won't return until the header and the avatar have been fully processed. // we're already on it, nothing else to do
func (d *deref) fetchRemoteAccountMedia(ctx context.Context, targetAccount *gtsmodel.Account, requestingUsername string, blocking bool) error { return alreadyProcessing.AttachmentID(), nil
// Fetch a transport beforehand for either(or both) avatar / header dereferencing. }
tsport, err := d.transportController.NewTransportForUsername(ctx, requestingUsername)
avatarIRI, err := url.Parse(mediaRemoteURL)
if err != nil { if err != nil {
return fmt.Errorf("fetchRemoteAccountMedia: error getting transport for user: %s", err) return "", err
} }
if targetAccount.AvatarRemoteURL != "" { data := func(innerCtx context.Context) (io.ReadCloser, int64, error) {
var processingMedia *media.ProcessingMedia return transport.DereferenceMedia(innerCtx, avatarIRI)
// Parse the target account's avatar URL into URL object.
avatarIRI, err := url.Parse(targetAccount.AvatarRemoteURL)
if err != nil {
return err
}
d.dereferencingAvatarsLock.Lock() // LOCK HERE
// first check if we're already processing this media
if alreadyProcessing, ok := d.dereferencingAvatars[targetAccount.ID]; ok {
// we're already on it, no worries
processingMedia = alreadyProcessing
} else {
data := func(innerCtx context.Context) (io.ReadCloser, int64, error) {
return tsport.DereferenceMedia(innerCtx, avatarIRI)
}
avatar := true
newProcessing, err := d.mediaManager.ProcessMedia(ctx, data, nil, targetAccount.ID, &media.AdditionalMediaInfo{
RemoteURL: &targetAccount.AvatarRemoteURL,
Avatar: &avatar,
})
if err != nil {
d.dereferencingAvatarsLock.Unlock()
return err
}
// store it in our map to indicate it's in process
d.dereferencingAvatars[targetAccount.ID] = newProcessing
processingMedia = newProcessing
}
d.dereferencingAvatarsLock.Unlock() // UNLOCK HERE
load := func(innerCtx context.Context) error {
_, err := processingMedia.LoadAttachment(innerCtx)
return err
}
cleanup := func() {
d.dereferencingAvatarsLock.Lock()
delete(d.dereferencingAvatars, targetAccount.ID)
d.dereferencingAvatarsLock.Unlock()
}
// block until loaded if required...
if blocking {
if err := loadAndCleanup(ctx, load, cleanup); err != nil {
return err
}
} else {
// ...otherwise do it async
go func() {
dlCtx, done := context.WithDeadline(context.Background(), time.Now().Add(1*time.Minute))
if err := loadAndCleanup(dlCtx, load, cleanup); err != nil {
log.Errorf("fetchRemoteAccountMedia: error during async lock and load of avatar: %s", err)
}
done()
}()
}
targetAccount.AvatarMediaAttachmentID = processingMedia.AttachmentID()
} }
if targetAccount.HeaderRemoteURL != "" { processingMedia, err := d.mediaManager.ProcessMedia(ctx, data, nil, targetAccountID, &media.AdditionalMediaInfo{
var processingMedia *media.ProcessingMedia RemoteURL: &mediaRemoteURL,
Avatar: &avatar,
// Parse the target account's header URL into URL object. Header: &header,
headerIRI, err := url.Parse(targetAccount.HeaderRemoteURL) })
if err != nil { if err != nil {
return err return "", err
}
d.dereferencingHeadersLock.Lock() // LOCK HERE
// first check if we're already processing this media
if alreadyProcessing, ok := d.dereferencingHeaders[targetAccount.ID]; ok {
// we're already on it, no worries
processingMedia = alreadyProcessing
} else {
data := func(innerCtx context.Context) (io.ReadCloser, int64, error) {
return tsport.DereferenceMedia(innerCtx, headerIRI)
}
header := true
newProcessing, err := d.mediaManager.ProcessMedia(ctx, data, nil, targetAccount.ID, &media.AdditionalMediaInfo{
RemoteURL: &targetAccount.HeaderRemoteURL,
Header: &header,
})
if err != nil {
d.dereferencingAvatarsLock.Unlock()
return err
}
// store it in our map to indicate it's in process
d.dereferencingHeaders[targetAccount.ID] = newProcessing
processingMedia = newProcessing
}
d.dereferencingHeadersLock.Unlock() // UNLOCK HERE
load := func(innerCtx context.Context) error {
_, err := processingMedia.LoadAttachment(innerCtx)
return err
}
cleanup := func() {
d.dereferencingHeadersLock.Lock()
delete(d.dereferencingHeaders, targetAccount.ID)
d.dereferencingHeadersLock.Unlock()
}
// block until loaded if required...
if blocking {
if err := loadAndCleanup(ctx, load, cleanup); err != nil {
return err
}
} else {
// ...otherwise do it async
go func() {
dlCtx, done := context.WithDeadline(context.Background(), time.Now().Add(1*time.Minute))
if err := loadAndCleanup(dlCtx, load, cleanup); err != nil {
log.Errorf("fetchRemoteAccountMedia: error during async lock and load of header: %s", err)
}
done()
}()
}
targetAccount.HeaderMediaAttachmentID = processingMedia.AttachmentID()
} }
return nil // store it in our map to indicate it's in process
dereferencingMap[targetAccountID] = processingMedia
defer delete(dereferencingMap, targetAccountID)
if _, err := processingMedia.LoadAttachment(ctx); err != nil {
return "", err
}
return processingMedia.AttachmentID(), nil
} }
func (d *deref) fetchRemoteAccountEmojis(ctx context.Context, targetAccount *gtsmodel.Account, requestingUsername string) (bool, error) { func (d *deref) fetchRemoteAccountEmojis(ctx context.Context, targetAccount *gtsmodel.Account, requestingUsername string) (bool, error) {

View file

@ -27,17 +27,12 @@ import (
"strings" "strings"
apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
"github.com/superseriousbusiness/gotosocial/internal/transport"
"github.com/superseriousbusiness/gotosocial/internal/util" "github.com/superseriousbusiness/gotosocial/internal/util"
) )
func (d *deref) fingerRemoteAccount(ctx context.Context, username string, targetUsername string, targetHost string) (accountDomain string, accountURI *url.URL, err error) { func (d *deref) fingerRemoteAccount(ctx context.Context, transport transport.Transport, targetUsername string, targetHost string) (accountDomain string, accountURI *url.URL, err error) {
t, err := d.transportController.NewTransportForUsername(ctx, username) b, err := transport.Finger(ctx, targetUsername, targetHost)
if err != nil {
err = fmt.Errorf("fingerRemoteAccount: error getting transport for %s: %s", username, err)
return
}
b, err := t.Finger(ctx, targetUsername, targetHost)
if err != nil { if err != nil {
err = fmt.Errorf("fingerRemoteAccount: error fingering @%s@%s: %s", targetUsername, targetHost, err) err = fmt.Errorf("fingerRemoteAccount: error fingering @%s@%s: %s", targetUsername, targetHost, err)
return return