gotosocial/internal/federation/dereferencing/account.go

477 lines
16 KiB
Go
Raw Normal View History

/*
GoToSocial
Copyright (C) 2021-2023 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package dereferencing
import (
"context"
"encoding/json"
"errors"
"fmt"
2022-01-16 17:52:55 +00:00
"io"
"net/url"
2022-01-24 17:12:04 +00:00
"time"
2021-11-13 16:29:43 +00:00
"github.com/superseriousbusiness/activity/streams"
"github.com/superseriousbusiness/activity/streams/vocab"
"github.com/superseriousbusiness/gotosocial/internal/ap"
"github.com/superseriousbusiness/gotosocial/internal/config"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/id"
[chore] use our own logging implementation (#716) * first commit Signed-off-by: kim <grufwub@gmail.com> * replace logging with our own log library Signed-off-by: kim <grufwub@gmail.com> * fix imports Signed-off-by: kim <grufwub@gmail.com> * fix log imports Signed-off-by: kim <grufwub@gmail.com> * add license text Signed-off-by: kim <grufwub@gmail.com> * fix package import cycle between config and log package Signed-off-by: kim <grufwub@gmail.com> * fix empty kv.Fields{} being passed to WithFields() Signed-off-by: kim <grufwub@gmail.com> * fix uses of log.WithFields() with whitespace issues and empty slices Signed-off-by: kim <grufwub@gmail.com> * *linter related grumbling* Signed-off-by: kim <grufwub@gmail.com> * gofmt the codebase! also fix more log.WithFields() formatting issues Signed-off-by: kim <grufwub@gmail.com> * update testrig code to match new changes Signed-off-by: kim <grufwub@gmail.com> * fix error wrapping in non fmt.Errorf function Signed-off-by: kim <grufwub@gmail.com> * add benchmarking of log.Caller() vs non-cached Signed-off-by: kim <grufwub@gmail.com> * fix syslog tests, add standard build tags to test runner to ensure consistency Signed-off-by: kim <grufwub@gmail.com> * make syslog tests more robust Signed-off-by: kim <grufwub@gmail.com> * fix caller depth arithmatic (is that how you spell it?) Signed-off-by: kim <grufwub@gmail.com> * update to use unkeyed fields in kv.Field{} instances Signed-off-by: kim <grufwub@gmail.com> * update go-kv library Signed-off-by: kim <grufwub@gmail.com> * update libraries list Signed-off-by: kim <grufwub@gmail.com> * fuck you linter get nerfed Signed-off-by: kim <grufwub@gmail.com> Co-authored-by: tobi <31960611+tsmethurst@users.noreply.github.com>
2022-07-19 08:47:55 +00:00
"github.com/superseriousbusiness/gotosocial/internal/log"
"github.com/superseriousbusiness/gotosocial/internal/media"
"github.com/superseriousbusiness/gotosocial/internal/transport"
)
func (d *deref) GetAccountByURI(ctx context.Context, requestUser string, uri *url.URL, block bool) (*gtsmodel.Account, error) {
var (
account *gtsmodel.Account
uriStr = uri.String()
err error
)
// Search the database for existing account with ID URI.
account, err = d.db.GetAccountByURI(ctx, uriStr)
if err != nil && !errors.Is(err, db.ErrNoEntries) {
return nil, fmt.Errorf("GetAccountByURI: error checking database for account %s by uri: %w", uriStr, err)
}
if account == nil {
// Else, search the database for existing by ID URL.
account, err = d.db.GetAccountByURL(ctx, uriStr)
if err != nil && !errors.Is(err, db.ErrNoEntries) {
return nil, fmt.Errorf("GetAccountByURI: error checking database for account %s by url: %w", uriStr, err)
}
}
if account == nil {
// Ensure that this is isn't a search for a local account.
if uri.Host == config.GetHost() || uri.Host == config.GetAccountDomain() {
return nil, NewErrNotRetrievable(err) // this will be db.ErrNoEntries
}
// Create and pass-through a new bare-bones model for dereferencing.
return d.enrichAccount(ctx, requestUser, uri, &gtsmodel.Account{
ID: id.NewULID(),
Domain: uri.Host,
URI: uriStr,
}, false, true)
}
// Try to update existing account model
enriched, err := d.enrichAccount(ctx, requestUser, uri, account, false, block)
if err != nil {
log.Errorf("error enriching remote account: %v", err)
return account, nil // fall back to returning existing
}
return enriched, nil
}
func (d *deref) GetAccountByUsernameDomain(ctx context.Context, requestUser string, username string, domain string, block bool) (*gtsmodel.Account, error) {
if domain == config.GetHost() || domain == config.GetAccountDomain() {
// We do local lookups using an empty domain,
// else it will fail the db search below.
domain = ""
}
// Search the database for existing account with USERNAME@DOMAIN
account, err := d.db.GetAccountByUsernameDomain(ctx, username, domain)
if err != nil && !errors.Is(err, db.ErrNoEntries) {
return nil, fmt.Errorf("GetAccountByUsernameDomain: error checking database for account %s@%s: %w", username, domain, err)
}
2022-01-24 17:12:04 +00:00
if account == nil {
// Check for failed local lookup.
if domain == "" {
return nil, NewErrNotRetrievable(err) // will be db.ErrNoEntries
}
// Create and pass-through a new bare-bones model for dereferencing.
return d.enrichAccount(ctx, requestUser, nil, &gtsmodel.Account{
ID: id.NewULID(),
Username: username,
Domain: domain,
}, false, true)
}
// Try to update existing account model
enriched, err := d.enrichAccount(ctx, requestUser, nil, account, false, block)
if err != nil {
log.Errorf("GetAccountByUsernameDomain: error enriching account from remote: %v", err)
return account, nil // fall back to returning unchanged existing account model
}
return enriched, nil
}
func (d *deref) UpdateAccount(ctx context.Context, requestUser string, account *gtsmodel.Account, force bool) (*gtsmodel.Account, error) {
return d.enrichAccount(ctx, requestUser, nil, account, force, false)
}
// enrichAccount will ensure the given account is the most up-to-date model of the account, re-webfingering and re-dereferencing if necessary.
func (d *deref) enrichAccount(ctx context.Context, requestUser string, uri *url.URL, account *gtsmodel.Account, force, block bool) (*gtsmodel.Account, error) {
if account.IsLocal() {
// Can't update local accounts.
return account, nil
}
if !account.CreatedAt.IsZero() && account.IsInstance() {
// Existing instance account. No need for update.
return account, nil
}
if !force {
const interval = time.Hour * 48
// If this account was updated recently (last interval), we return as-is.
if next := account.FetchedAt.Add(interval); time.Now().Before(next) {
return account, nil
}
}
transport, err := d.transportController.NewTransportForUsername(ctx, requestUser)
if err != nil {
return nil, fmt.Errorf("enrichAccount: couldn't create transport: %w", err)
}
if account.Username != "" {
// A username was provided so we can attempt a webfinger, this ensures up-to-date accountdomain info.
accDomain, accURI, err := d.fingerRemoteAccount(ctx, transport, account.Username, account.Domain)
if err != nil && account.URI == "" {
// this is a new account (to us) with username@domain but failed
// webfinger, there is nothing more we can do in this situation.
return nil, fmt.Errorf("enrichAccount: error webfingering account: %w", err)
}
if err == nil {
// Update account with latest info.
account.URI = accURI.String()
account.Domain = accDomain
uri = accURI
}
}
if uri == nil {
var err error
2022-01-24 12:12:17 +00:00
// No URI provided / found, must parse from account.
uri, err = url.Parse(account.URI)
if err != nil {
return nil, fmt.Errorf("enrichAccount: invalid uri %q: %w", account.URI, err)
}
}
// Check whether this account URI is a blocked domain / subdomain
if blocked, err := d.db.IsDomainBlocked(ctx, uri.Host); err != nil {
return nil, newErrDB(fmt.Errorf("enrichAccount: error checking blocked domain: %w", err))
} else if blocked {
return nil, fmt.Errorf("enrichAccount: %s is blocked", uri.Host)
}
// Mark deref+update handshake start
d.startHandshake(requestUser, uri)
defer d.stopHandshake(requestUser, uri)
2022-01-24 12:12:17 +00:00
// Dereference this account to get the latest available.
apubAcc, err := d.dereferenceAccountable(ctx, transport, uri)
if err != nil {
return nil, fmt.Errorf("enrichAccount: error dereferencing account %s: %w", uri, err)
2022-01-24 12:12:17 +00:00
}
// Convert the dereferenced AP account object to our GTS model.
latestAcc, err := d.typeConverter.ASRepresentationToAccount(
ctx, apubAcc, account.Domain,
)
if err != nil {
return nil, fmt.Errorf("enrichAccount: error converting accountable to gts model for account %s: %w", uri, err)
}
if account.Username == "" {
// No username was provided, so no webfinger was attempted earlier.
//
// Now we have a username we can attempt it now, this ensures up-to-date accountdomain info.
accDomain, _, err := d.fingerRemoteAccount(ctx, transport, latestAcc.Username, uri.Host)
if err == nil {
// Update account with latest info.
latestAcc.Domain = accDomain
}
}
// Ensure ID is set and update fetch time.
latestAcc.ID = account.ID
latestAcc.FetchedAt = time.Now()
// Fetch latest account avatar only if remote URI has changed
if latestAcc.AvatarRemoteURL != "" && latestAcc.AvatarRemoteURL != account.AvatarRemoteURL {
d.dereferencingAvatarsLock.Lock()
newAvatarID, err := d.fetchRemoteAccountMedia(ctx, transport, latestAcc.AvatarRemoteURL, latestAcc.ID, d.dereferencingAvatars, true, false)
d.dereferencingAvatarsLock.Unlock()
if err != nil {
log.Errorf("error fetching remote avatar for account %s: %v", uri, err)
} else {
latestAcc.AvatarMediaAttachmentID = newAvatarID
}
} else {
latestAcc.AvatarMediaAttachmentID = account.AvatarMediaAttachmentID // no change / empty url
}
// Fetch latest account header only if remote URI has changed
if latestAcc.AvatarRemoteURL != "" && latestAcc.AvatarRemoteURL != account.AvatarRemoteURL {
d.dereferencingHeadersLock.Lock()
newHeaderID, err := d.fetchRemoteAccountMedia(ctx, transport, latestAcc.HeaderRemoteURL, latestAcc.ID, d.dereferencingHeaders, false, true)
d.dereferencingHeadersLock.Unlock()
if err != nil {
log.Errorf("error fetching remote header for account %s: %v", uri, err)
} else {
latestAcc.HeaderMediaAttachmentID = newHeaderID
}
} else {
latestAcc.HeaderMediaAttachmentID = account.HeaderMediaAttachmentID // no change / empty url
2022-01-24 12:12:17 +00:00
}
// Fetch the latest remote account emoji IDs used in account display name/bio.
_, err = d.fetchRemoteAccountEmojis(ctx, latestAcc, requestUser)
if err != nil {
log.Errorf("error fetching remote emojis for account %s: %v", uri, err)
2022-01-24 12:12:17 +00:00
}
if account.CreatedAt.IsZero() {
// CreatedAt will be zero if no local copy was
// found in one of the GetAccountBy___() functions.
//
// Set time of creation from the last-fetched date.
latestAcc.CreatedAt = latestAcc.FetchedAt
latestAcc.UpdatedAt = latestAcc.FetchedAt
// This is a new account, we need to place it in the database.
if err := d.db.PutAccount(ctx, latestAcc); err != nil {
return nil, fmt.Errorf("enrichAccount: error putting in database: %w", err)
}
} else {
// Set time of update from the last-fetched date.
latestAcc.UpdatedAt = latestAcc.FetchedAt
// Use existing account values.
latestAcc.CreatedAt = account.CreatedAt
latestAcc.Language = account.Language
// This is an existing account, update the model in the database.
if err := d.db.UpdateAccount(ctx, latestAcc); err != nil {
return nil, fmt.Errorf("enrichAccount: error updating database: %w", err)
2022-01-25 10:21:22 +00:00
}
2022-01-24 12:12:17 +00:00
}
return latestAcc, nil
}
// dereferenceAccountable calls remoteAccountID with a GET request, and tries to parse whatever
// it finds as something that an account model can be constructed out of.
//
// Will work for Person, Application, or Service models.
func (d *deref) dereferenceAccountable(ctx context.Context, transport transport.Transport, remoteAccountID *url.URL) (ap.Accountable, error) {
b, err := transport.Dereference(ctx, remoteAccountID)
if err != nil {
return nil, fmt.Errorf("DereferenceAccountable: error deferencing %s: %w", remoteAccountID.String(), err)
}
m := make(map[string]interface{})
if err := json.Unmarshal(b, &m); err != nil {
return nil, fmt.Errorf("DereferenceAccountable: error unmarshalling bytes into json: %w", err)
}
t, err := streams.ToType(ctx, m)
if err != nil {
return nil, fmt.Errorf("DereferenceAccountable: error resolving json into ap vocab type: %w", err)
}
//nolint:forcetypeassert
switch t.GetTypeName() {
2021-09-03 08:30:40 +00:00
case ap.ActorApplication:
return t.(vocab.ActivityStreamsApplication), nil
2021-09-30 10:27:42 +00:00
case ap.ActorGroup:
return t.(vocab.ActivityStreamsGroup), nil
2021-09-30 10:27:42 +00:00
case ap.ActorOrganization:
return t.(vocab.ActivityStreamsOrganization), nil
2021-09-30 10:27:42 +00:00
case ap.ActorPerson:
return t.(vocab.ActivityStreamsPerson), nil
2021-09-03 08:30:40 +00:00
case ap.ActorService:
return t.(vocab.ActivityStreamsService), nil
}
return nil, newErrWrongType(fmt.Errorf("DereferenceAccountable: type name %s not supported as Accountable", t.GetTypeName()))
}
func (d *deref) fetchRemoteAccountMedia(
ctx context.Context,
transport transport.Transport,
mediaRemoteURL string,
targetAccountID string,
dereferencingMap map[string]*media.ProcessingMedia,
avatar bool,
header bool,
) (string, error) {
// first check if we're already processing this media
if alreadyProcessing, ok := dereferencingMap[targetAccountID]; ok {
// we're already on it, nothing else to do
return alreadyProcessing.AttachmentID(), nil
}
avatarIRI, err := url.Parse(mediaRemoteURL)
if err != nil {
return "", err
}
data := func(innerCtx context.Context) (io.ReadCloser, int64, error) {
return transport.DereferenceMedia(innerCtx, avatarIRI)
}
processingMedia, err := d.mediaManager.ProcessMedia(ctx, data, nil, targetAccountID, &media.AdditionalMediaInfo{
RemoteURL: &mediaRemoteURL,
Avatar: &avatar,
Header: &header,
})
if err != nil {
return "", err
}
2022-01-24 17:12:04 +00:00
// store it in our map to indicate it's in process
dereferencingMap[targetAccountID] = processingMedia
defer delete(dereferencingMap, targetAccountID)
if _, err := processingMedia.LoadAttachment(ctx); err != nil {
return "", err
}
2022-01-24 12:12:17 +00:00
return processingMedia.AttachmentID(), nil
}
2022-01-24 12:12:17 +00:00
func (d *deref) fetchRemoteAccountEmojis(ctx context.Context, targetAccount *gtsmodel.Account, requestingUsername string) (bool, error) {
maybeEmojis := targetAccount.Emojis
maybeEmojiIDs := targetAccount.EmojiIDs
// It's possible that the account had emoji IDs set on it, but not Emojis
// themselves, depending on how it was fetched before being passed to us.
//
// If we only have IDs, fetch the emojis from the db. We know they're in
// there or else they wouldn't have IDs.
if len(maybeEmojiIDs) > len(maybeEmojis) {
maybeEmojis = make([]*gtsmodel.Emoji, 0, len(maybeEmojiIDs))
for _, emojiID := range maybeEmojiIDs {
maybeEmoji, err := d.db.GetEmojiByID(ctx, emojiID)
if err != nil {
return false, err
}
maybeEmojis = append(maybeEmojis, maybeEmoji)
}
}
// For all the maybe emojis we have, we either fetch them from the database
// (if we haven't already), or dereference them from the remote instance.
gotEmojis, err := d.populateEmojis(ctx, maybeEmojis, requestingUsername)
if err != nil {
return false, err
}
// Extract the ID of each fetched or dereferenced emoji, so we can attach
// this to the account if necessary.
gotEmojiIDs := make([]string, 0, len(gotEmojis))
for _, e := range gotEmojis {
gotEmojiIDs = append(gotEmojiIDs, e.ID)
}
var (
changed = false // have the emojis for this account changed?
maybeLen = len(maybeEmojis)
gotLen = len(gotEmojis)
)
// if the length of everything is zero, this is simple:
// nothing has changed and there's nothing to do
if maybeLen == 0 && gotLen == 0 {
return changed, nil
}
// if the *amount* of emojis on the account has changed, then the got emojis
// are definitely different from the previous ones (if there were any) --
// the account has either more or fewer emojis set on it now, so take the
// discovered emojis as the new correct ones.
if maybeLen != gotLen {
changed = true
targetAccount.Emojis = gotEmojis
targetAccount.EmojiIDs = gotEmojiIDs
return changed, nil
}
// if the lengths are the same but not all of the slices are
// zero, something *might* have changed, so we have to check
// 1. did we have emojis before that we don't have now?
for _, maybeEmoji := range maybeEmojis {
var stillPresent bool
for _, gotEmoji := range gotEmojis {
if maybeEmoji.URI == gotEmoji.URI {
// the emoji we maybe had is still present now,
// so we can stop checking gotEmojis
stillPresent = true
break
}
}
if !stillPresent {
// at least one maybeEmoji is no longer present in
// the got emojis, so we can stop checking now
changed = true
targetAccount.Emojis = gotEmojis
targetAccount.EmojiIDs = gotEmojiIDs
return changed, nil
}
}
// 2. do we have emojis now that we didn't have before?
for _, gotEmoji := range gotEmojis {
var wasPresent bool
for _, maybeEmoji := range maybeEmojis {
// check emoji IDs here as well, because unreferenced
// maybe emojis we didn't already have would not have
// had IDs set on them yet
if gotEmoji.URI == maybeEmoji.URI && gotEmoji.ID == maybeEmoji.ID {
// this got emoji was present already in the maybeEmoji,
// so we can stop checking through maybeEmojis
wasPresent = true
break
}
}
if !wasPresent {
// at least one gotEmojis was not present in
// the maybeEmojis, so we can stop checking now
changed = true
targetAccount.Emojis = gotEmojis
targetAccount.EmojiIDs = gotEmojiIDs
return changed, nil
}
}
return changed, nil
}