// GoToSocial // Copyright (C) GoToSocial Authors admin@gotosocial.org // SPDX-License-Identifier: AGPL-3.0-or-later // // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU Affero General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // This program is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU Affero General Public License for more details. // // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . package dereferencing import ( "context" "errors" "net/http" "net/url" "slices" "time" "github.com/superseriousbusiness/gotosocial/internal/ap" "github.com/superseriousbusiness/gotosocial/internal/config" "github.com/superseriousbusiness/gotosocial/internal/db" "github.com/superseriousbusiness/gotosocial/internal/gtscontext" "github.com/superseriousbusiness/gotosocial/internal/gtserror" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/id" "github.com/superseriousbusiness/gotosocial/internal/log" "github.com/superseriousbusiness/gotosocial/internal/media" "github.com/superseriousbusiness/gotosocial/internal/util" ) // statusFresh returns true if the given status is still // considered "fresh" according to the desired freshness // window (falls back to default status freshness if nil). // // Local statuses will always be considered fresh, // because there's no remote state that may have changed. // // Return value of false indicates that the status // is not fresh and should be refreshed from remote. func statusFresh( status *gtsmodel.Status, window *FreshnessWindow, ) bool { // Take default if no // freshness window preferred. if window == nil { window = DefaultStatusFreshness } if status.IsLocal() { // Can't refresh // local statuses. return true } // Moment when the status is // considered stale according to // desired freshness window. staleAt := status.FetchedAt.Add( time.Duration(*window), ) // It's still fresh if the time now // is not past the point of staleness. return !time.Now().After(staleAt) } // GetStatusByURI will attempt to fetch a status by its URI, first checking the database. In the case of a newly-met remote model, or a remote model whose 'last_fetched' date // is beyond a certain interval, the status will be dereferenced. In the case of dereferencing, some low-priority status information may be enqueued for asynchronous fetching, // e.g. dereferencing the status thread. An ActivityPub object indicates the status was dereferenced. func (d *Dereferencer) GetStatusByURI(ctx context.Context, requestUser string, uri *url.URL) (*gtsmodel.Status, ap.Statusable, error) { // Fetch and dereference / update status if necessary. status, statusable, isNew, err := d.getStatusByURI(ctx, requestUser, uri, ) if err != nil { if status == nil { // err with no existing // status for fallback. return nil, nil, err } log.Errorf(ctx, "error updating status %s: %v", uri, err) } else if statusable != nil { // Deref parents + children. d.dereferenceThread(ctx, requestUser, uri, status, statusable, isNew, ) } return status, statusable, nil } // getStatusByURI is a package internal form of .GetStatusByURI() that doesn't dereference thread on update, and may return an existing status with error on failed re-fetch. func (d *Dereferencer) getStatusByURI(ctx context.Context, requestUser string, uri *url.URL) (*gtsmodel.Status, ap.Statusable, bool, error) { var ( status *gtsmodel.Status uriStr = uri.String() err error ) // Search the database for existing by URI. status, err = d.state.DB.GetStatusByURI( // request a barebones object, it may be in the // db but with related models not yet dereferenced. gtscontext.SetBarebones(ctx), uriStr, ) if err != nil && !errors.Is(err, db.ErrNoEntries) { return nil, nil, false, gtserror.Newf("error checking database for status %s by uri: %w", uriStr, err) } if status == nil { // Else, search database for existing by URL. status, err = d.state.DB.GetStatusByURL( gtscontext.SetBarebones(ctx), uriStr, ) if err != nil && !errors.Is(err, db.ErrNoEntries) { return nil, nil, false, gtserror.Newf("error checking database for status %s by url: %w", uriStr, err) } } if status == nil { // Ensure not a failed search for a local // status, if so we know it doesn't exist. if uri.Host == config.GetHost() || uri.Host == config.GetAccountDomain() { return nil, nil, false, gtserror.SetUnretrievable(err) } // Create and pass-through a new bare-bones model for deref. return d.enrichStatusSafely(ctx, requestUser, uri, >smodel.Status{ Local: util.Ptr(false), URI: uriStr, }, nil) } if statusFresh(status, DefaultStatusFreshness) { // This is an existing status that is up-to-date, // before returning ensure it is fully populated. if err := d.state.DB.PopulateStatus(ctx, status); err != nil { log.Errorf(ctx, "error populating existing status: %v", err) } return status, nil, false, nil } // Try to deref and update existing. return d.enrichStatusSafely(ctx, requestUser, uri, status, nil, ) } // RefreshStatus is functionally equivalent to GetStatusByURI(), except that it requires a pre // populated status model (with AT LEAST uri set), and ALL thread dereferencing is asynchronous. func (d *Dereferencer) RefreshStatus( ctx context.Context, requestUser string, status *gtsmodel.Status, statusable ap.Statusable, window *FreshnessWindow, ) (*gtsmodel.Status, ap.Statusable, error) { // If no incoming data is provided, // check whether status needs update. if statusable == nil && statusFresh(status, window) { return status, nil, nil } // Parse the URI from status. uri, err := url.Parse(status.URI) if err != nil { return nil, nil, gtserror.Newf("invalid status uri %q: %w", status.URI, err) } // Try to update + dereference the passed status model. latest, statusable, isNew, err := d.enrichStatusSafely(ctx, requestUser, uri, status, statusable, ) if statusable != nil { // Deref parents + children. d.dereferenceThread(ctx, requestUser, uri, latest, statusable, isNew, ) } return latest, statusable, err } // RefreshStatusAsync is functionally equivalent to RefreshStatus(), except that ALL // dereferencing is queued for asynchronous processing, (both thread AND status). func (d *Dereferencer) RefreshStatusAsync( ctx context.Context, requestUser string, status *gtsmodel.Status, statusable ap.Statusable, window *FreshnessWindow, ) { // If no incoming data is provided, // check whether status needs update. if statusable == nil && statusFresh(status, window) { return } // Parse the URI from status. uri, err := url.Parse(status.URI) if err != nil { log.Errorf(ctx, "invalid status uri %q: %v", status.URI, err) return } // Enqueue a worker function to re-fetch this status entirely async. d.state.Workers.Dereference.Queue.Push(func(ctx context.Context) { latest, statusable, _, err := d.enrichStatusSafely(ctx, requestUser, uri, status, statusable, ) if err != nil { log.Errorf(ctx, "error enriching remote status: %v", err) return } if statusable != nil { if err := d.DereferenceStatusAncestors(ctx, requestUser, latest); err != nil { log.Error(ctx, err) } if err := d.DereferenceStatusDescendants(ctx, requestUser, uri, statusable); err != nil { log.Error(ctx, err) } } }) } // enrichStatusSafely wraps enrichStatus() to perform it within // a State{}.FedLocks mutexmap, which protects it within per-URI // mutex locks. This also handles necessary delete of now-deleted // statuses, and updating fetched_at on returned HTTP errors. func (d *Dereferencer) enrichStatusSafely( ctx context.Context, requestUser string, uri *url.URL, status *gtsmodel.Status, statusable ap.Statusable, ) (*gtsmodel.Status, ap.Statusable, bool, error) { uriStr := status.URI var isNew bool // Check if this is a new status (to us). if isNew = (status.ID == ""); !isNew { // This is an existing status, first try to populate it. This // is required by the checks below for existing tags, media etc. if err := d.state.DB.PopulateStatus(ctx, status); err != nil { log.Errorf(ctx, "error populating existing status %s: %v", uriStr, err) } } // Acquire per-URI deref lock, wraping unlock // to safely defer in case of panic, while still // performing more granular unlocks when needed. unlock := d.state.FedLocks.Lock(uriStr) unlock = util.DoOnce(unlock) defer unlock() // Perform status enrichment with passed vars. latest, statusable, err := d.enrichStatus(ctx, requestUser, uri, status, statusable, ) // Check for a returned HTTP code via error. switch code := gtserror.StatusCode(err); { // Gone (410) definitely indicates deletion. // Remove status if it was an existing one. case code == http.StatusGone && !isNew: if err := d.state.DB.DeleteStatusByID(ctx, status.ID); err != nil { log.Error(ctx, "error deleting gone status %s: %v", uriStr, err) } // Don't return any status. return nil, nil, false, err // Any other HTTP error mesg // code, with existing status. case code >= 400 && !isNew: // Update fetched_at to slow re-attempts // but don't return early. We can still // return the model we had stored already. status.FetchedAt = time.Now() if err := d.state.DB.UpdateStatus(ctx, status, "fetched_at"); err != nil { log.Error(ctx, "error updating %s fetched_at: %v", uriStr, err) } // See below. fallthrough // In case of error with an existing // status in the database, return error // but still return existing status. case err != nil && !isNew: latest = status statusable = nil } // Unlock now // we're done. unlock() if errors.Is(err, db.ErrAlreadyExists) { // We leave 'isNew' set so that caller // still dereferences parents, otherwise // the version we pass back may not have // these attached as inReplyTos yet (since // those happen OUTSIDE federator lock). // // TODO: performance-wise, this won't be // great. should improve this if we can! // DATA RACE! We likely lost out to another goroutine // in a call to db.Put(Status). Look again in DB by URI. latest, err = d.state.DB.GetStatusByURI(ctx, status.URI) if err != nil { err = gtserror.Newf("error getting status %s from database after race: %w", uriStr, err) } } return latest, statusable, isNew, err } // enrichStatus will enrich the given status, whether a new // barebones model, or existing model from the database. // It handles necessary dereferencing, database updates, etc. func (d *Dereferencer) enrichStatus( ctx context.Context, requestUser string, uri *url.URL, status *gtsmodel.Status, statusable ap.Statusable, ) ( *gtsmodel.Status, ap.Statusable, error, ) { // Pre-fetch a transport for requesting username, used by later dereferencing. tsport, err := d.transportController.NewTransportForUsername(ctx, requestUser) if err != nil { return nil, nil, gtserror.Newf("couldn't create transport: %w", err) } // Check whether this account URI is a blocked domain / subdomain. if blocked, err := d.state.DB.IsDomainBlocked(ctx, uri.Host); err != nil { return nil, nil, gtserror.Newf("error checking blocked domain: %w", err) } else if blocked { err := gtserror.Newf("%s is blocked", uri.Host) return nil, nil, gtserror.SetUnretrievable(err) } if statusable == nil { // Dereference latest version of the status. rsp, err := tsport.Dereference(ctx, uri) if err != nil { err := gtserror.Newf("error dereferencing %s: %w", uri, err) return nil, nil, gtserror.SetUnretrievable(err) } // Attempt to resolve ActivityPub status from response. statusable, err = ap.ResolveStatusable(ctx, rsp.Body) // Tidy up now done. _ = rsp.Body.Close() if err != nil { // ResolveStatusable will set gtserror.WrongType // on the returned error, so we don't need to do it here. err := gtserror.Newf("error resolving statusable %s: %w", uri, err) return nil, nil, err } // Check whether input URI and final returned URI // have changed (i.e. we followed some redirects). if finalURIStr := rsp.Request.URL.String(); // finalURIStr != uri.String() { // NOTE: this URI check + database call is performed // AFTER reading and closing response body, for performance. // // Check whether we have this status stored under *final* URI. alreadyStatus, err := d.state.DB.GetStatusByURI(ctx, finalURIStr) if err != nil && !errors.Is(err, db.ErrNoEntries) { return nil, nil, gtserror.Newf("db error getting status after redirects: %w", err) } if alreadyStatus != nil { // We had this status stored // under discovered final URI. // // Proceed with this status. status = alreadyStatus } // Update the input URI to // the final determined URI // for later URI checks. uri = rsp.Request.URL } } // Get the attributed-to account in order to fetch profile. attributedTo, err := ap.ExtractAttributedToURI(statusable) if err != nil { return nil, nil, gtserror.New("attributedTo was empty") } // Ensure we have the author account of the status dereferenced (+ up-to-date). If this is a new status // (i.e. status.AccountID == "") then any error here is irrecoverable. status.AccountID must ALWAYS be set. if _, _, err := d.getAccountByURI(ctx, requestUser, attributedTo); err != nil && status.AccountID == "" { // Note that we specifically DO NOT wrap the error, instead collapsing it as string. // Errors fetching an account do not necessarily relate to dereferencing the status. return nil, nil, gtserror.Newf("failed to dereference status author %s: %v", uri, err) } // ActivityPub model was recently dereferenced, so assume passed status // may contain out-of-date information. Convert AP model to our GTS model. latestStatus, err := d.converter.ASStatusToStatus(ctx, statusable) if err != nil { return nil, nil, gtserror.Newf("error converting statusable to gts model for status %s: %w", uri, err) } // Ensure final status isn't attempting // to claim being authored by local user. if latestStatus.Account.IsLocal() { return nil, nil, gtserror.Newf( "dereferenced status %s claiming to be local", latestStatus.URI, ) } // Ensure the final parsed status URI or URL matches // the input URI we fetched (or received) it as. matches, err := util.URIMatches( uri, append( ap.GetURL(statusable), // status URL(s) ap.GetJSONLDId(statusable), // status URI )..., ) if err != nil { return nil, nil, gtserror.Newf( "error checking dereferenced status uri %s: %w", latestStatus.URI, err, ) } if !matches { return nil, nil, gtserror.Newf( "dereferenced status uri %s does not match %s", latestStatus.URI, uri.String(), ) } var isNew bool // Based on the original provided // status model, determine whether // this is a new insert / update. if isNew = (status.ID == ""); isNew { // Generate new status ID from the provided creation date. latestStatus.ID, err = id.NewULIDFromTime(latestStatus.CreatedAt) if err != nil { log.Errorf(ctx, "invalid created at date (falling back to 'now'): %v", err) latestStatus.ID = id.NewULID() // just use "now" } } else { // Reuse existing status ID. latestStatus.ID = status.ID } // Carry-over values and set fetch time. latestStatus.UpdatedAt = status.UpdatedAt latestStatus.FetchedAt = time.Now() latestStatus.Local = status.Local // Carry-over approvals. Remote instances might not yet // serve statuses with the `approved_by` field, but we // might have marked a status as pre-approved on our side // based on the author's inclusion in a followers/following // collection. By carrying over previously-set values we // can avoid marking such statuses as "pending" again. // // If a remote has in the meantime retracted its approval, // the next call to 'isPermittedStatus' will catch that. if latestStatus.ApprovedByURI == "" && status.ApprovedByURI != "" { latestStatus.ApprovedByURI = status.ApprovedByURI } // Check if this is a permitted status we should accept. // Function also sets "PendingApproval" bool as necessary. permit, err := d.isPermittedStatus(ctx, requestUser, status, latestStatus) if err != nil { return nil, nil, gtserror.Newf("error checking permissibility for status %s: %w", uri, err) } if !permit { // Return a checkable error type that can be ignored. err := gtserror.Newf("dropping unpermitted status: %s", uri) return nil, nil, gtserror.SetNotPermitted(err) } // Ensure the status' mentions are populated, and pass in existing to check for changes. if err := d.fetchStatusMentions(ctx, requestUser, status, latestStatus); err != nil { return nil, nil, gtserror.Newf("error populating mentions for status %s: %w", uri, err) } // Ensure the status' poll remains consistent, else reset the poll. if err := d.fetchStatusPoll(ctx, status, latestStatus); err != nil { return nil, nil, gtserror.Newf("error populating poll for status %s: %w", uri, err) } // Now that we know who this status replies to (handled by ASStatusToStatus) // and who it mentions, we can add a ThreadID to it if necessary. if err := d.threadStatus(ctx, latestStatus); err != nil { return nil, nil, gtserror.Newf("error checking / creating threadID for status %s: %w", uri, err) } // Ensure the status' tags are populated, (changes are expected / okay). if err := d.fetchStatusTags(ctx, status, latestStatus); err != nil { return nil, nil, gtserror.Newf("error populating tags for status %s: %w", uri, err) } // Ensure the status' media attachments are populated, passing in existing to check for changes. if err := d.fetchStatusAttachments(ctx, requestUser, status, latestStatus); err != nil { return nil, nil, gtserror.Newf("error populating attachments for status %s: %w", uri, err) } // Ensure the status' emoji attachments are populated, passing in existing to check for changes. if err := d.fetchStatusEmojis(ctx, status, latestStatus); err != nil { return nil, nil, gtserror.Newf("error populating emojis for status %s: %w", uri, err) } if isNew { // This is new, put the status in the database. err := d.state.DB.PutStatus(ctx, latestStatus) if err != nil { return nil, nil, gtserror.Newf("error putting in database: %w", err) } } else { // This is an existing status, update the model in the database. if err := d.state.DB.UpdateStatus(ctx, latestStatus); err != nil { return nil, nil, gtserror.Newf("error updating database: %w", err) } } return latestStatus, statusable, nil } func (d *Dereferencer) fetchStatusMentions( ctx context.Context, requestUser string, existing *gtsmodel.Status, status *gtsmodel.Status, ) error { // Allocate new slice to take the yet-to-be created mention IDs. status.MentionIDs = make([]string, len(status.Mentions)) for i := range status.Mentions { var ( mention = status.Mentions[i] alreadyExists bool err error ) // Search existing status for a mention already stored, // else ensure new mention's target account is populated. mention, alreadyExists, err = d.getPopulatedMention(ctx, requestUser, existing, mention, ) if err != nil { log.Errorf(ctx, "failed to derive mention: %v", err) continue } if alreadyExists { // This mention was already attached // to the status, use it and continue. status.Mentions[i] = mention status.MentionIDs[i] = mention.ID continue } // This mention didn't exist yet. // Generate new ID according to status creation. // TODO: update this to use "edited_at" when we add // support for edited status revision history. mention.ID, err = id.NewULIDFromTime(status.CreatedAt) if err != nil { log.Errorf(ctx, "invalid created at date (falling back to 'now'): %v", err) mention.ID = id.NewULID() // just use "now" } // Set known further mention details. mention.CreatedAt = status.CreatedAt mention.UpdatedAt = status.UpdatedAt mention.OriginAccount = status.Account mention.OriginAccountID = status.AccountID mention.OriginAccountURI = status.AccountURI mention.TargetAccountID = mention.TargetAccount.ID mention.TargetAccountURI = mention.TargetAccount.URI mention.TargetAccountURL = mention.TargetAccount.URL mention.StatusID = status.ID mention.Status = status // Place the new mention into the database. if err := d.state.DB.PutMention(ctx, mention); err != nil { return gtserror.Newf("error putting mention in database: %w", err) } // Set the *new* mention and ID. status.Mentions[i] = mention status.MentionIDs[i] = mention.ID } for i := 0; i < len(status.MentionIDs); { if status.MentionIDs[i] == "" { // This is a failed mention population, likely due // to invalid incoming data / now-deleted accounts. copy(status.Mentions[i:], status.Mentions[i+1:]) copy(status.MentionIDs[i:], status.MentionIDs[i+1:]) status.Mentions = status.Mentions[:len(status.Mentions)-1] status.MentionIDs = status.MentionIDs[:len(status.MentionIDs)-1] continue } i++ } return nil } func (d *Dereferencer) threadStatus(ctx context.Context, status *gtsmodel.Status) error { if status.InReplyTo != nil { if parentThreadID := status.InReplyTo.ThreadID; parentThreadID != "" { // Simplest case: parent status // is threaded, so inherit threadID. status.ThreadID = parentThreadID return nil } } // Parent wasn't threaded. If this // status mentions a local account, // we should thread it so that local // account can mute it if they want. mentionsLocal := slices.ContainsFunc( status.Mentions, func(m *gtsmodel.Mention) bool { // If TargetAccount couldn't // be deref'd, we know it's not // a local account, so only // check for non-nil accounts. return m.TargetAccount != nil && m.TargetAccount.IsLocal() }, ) if !mentionsLocal { // Status doesn't mention a // local account, so we don't // need to thread it. return nil } // Status mentions a local account. // Create a new thread and assign // it to the status. threadID := id.NewULID() if err := d.state.DB.PutThread( ctx, >smodel.Thread{ ID: threadID, }, ); err != nil { return gtserror.Newf("error inserting new thread in db: %w", err) } status.ThreadID = threadID return nil } func (d *Dereferencer) fetchStatusTags( ctx context.Context, existing *gtsmodel.Status, status *gtsmodel.Status, ) error { // Allocate new slice to take the yet-to-be determined tag IDs. status.TagIDs = make([]string, len(status.Tags)) for i := range status.Tags { tag := status.Tags[i] // Look for tag in existing status with name. existing, ok := existing.GetTagByName(tag.Name) if ok && existing.ID != "" { status.Tags[i] = existing status.TagIDs[i] = existing.ID continue } // Look for existing tag with name in the database. existing, err := d.state.DB.GetTagByName(ctx, tag.Name) if err != nil && !errors.Is(err, db.ErrNoEntries) { return gtserror.Newf("db error getting tag %s: %w", tag.Name, err) } else if existing != nil { status.Tags[i] = existing status.TagIDs[i] = existing.ID continue } // Create new ID for tag. tag.ID = id.NewULID() // Insert this tag with new name into the database. if err := d.state.DB.PutTag(ctx, tag); err != nil { log.Errorf(ctx, "db error putting tag %s: %v", tag.Name, err) continue } // Set new tag ID in slice. status.TagIDs[i] = tag.ID } // Remove any tag we couldn't get or create. for i := 0; i < len(status.TagIDs); { if status.TagIDs[i] == "" { // This is a failed tag population, likely due // to some database peculiarity / race condition. copy(status.Tags[i:], status.Tags[i+1:]) copy(status.TagIDs[i:], status.TagIDs[i+1:]) status.Tags = status.Tags[:len(status.Tags)-1] status.TagIDs = status.TagIDs[:len(status.TagIDs)-1] continue } i++ } return nil } func (d *Dereferencer) fetchStatusPoll( ctx context.Context, existing *gtsmodel.Status, status *gtsmodel.Status, ) error { var ( // insertStatusPoll generates ID and inserts the poll attached to status into the database. insertStatusPoll = func(ctx context.Context, status *gtsmodel.Status) error { var err error // Generate new ID for poll from the status CreatedAt. // TODO: update this to use "edited_at" when we add // support for edited status revision history. status.Poll.ID, err = id.NewULIDFromTime(status.CreatedAt) if err != nil { log.Errorf(ctx, "invalid created at date (falling back to 'now'): %v", err) status.Poll.ID = id.NewULID() // just use "now" } // Update the status<->poll links. status.PollID = status.Poll.ID status.Poll.StatusID = status.ID status.Poll.Status = status // Insert this latest poll into the database. err = d.state.DB.PutPoll(ctx, status.Poll) if err != nil { return gtserror.Newf("error putting in database: %w", err) } return nil } // deleteStatusPoll deletes the poll with ID, and all attached votes, from the database. deleteStatusPoll = func(ctx context.Context, pollID string) error { if err := d.state.DB.DeletePollByID(ctx, pollID); err != nil { return gtserror.Newf("error deleting existing poll from database: %w", err) } if err := d.state.DB.DeletePollVotes(ctx, pollID); err != nil { return gtserror.Newf("error deleting existing votes from database: %w", err) } return nil } ) switch { case existing.Poll == nil && status.Poll == nil: // no poll before or after, nothing to do. return nil case existing.Poll == nil && status.Poll != nil: // no previous poll, insert new poll! return insertStatusPoll(ctx, status) case status.Poll == nil: // existing poll has been deleted, remove this. return deleteStatusPoll(ctx, existing.PollID) case pollChanged(existing.Poll, status.Poll): // poll has changed since original, delete and reinsert new. if err := deleteStatusPoll(ctx, existing.PollID); err != nil { return err } return insertStatusPoll(ctx, status) case pollUpdated(existing.Poll, status.Poll): // Since we last saw it, the poll has updated! // Whether that be stats, or close time. poll := existing.Poll poll.Closing = pollJustClosed(existing.Poll, status.Poll) poll.ClosedAt = status.Poll.ClosedAt poll.Voters = status.Poll.Voters poll.Votes = status.Poll.Votes // Update poll model in the database (specifically only the possible changed columns). if err := d.state.DB.UpdatePoll(ctx, poll, "closed_at", "voters", "votes"); err != nil { return gtserror.Newf("error updating poll: %w", err) } // Update poll on status. status.PollID = poll.ID status.Poll = poll return nil default: // latest and existing // polls are up to date. poll := existing.Poll status.PollID = poll.ID status.Poll = poll return nil } } func (d *Dereferencer) fetchStatusAttachments( ctx context.Context, requestUser string, existing *gtsmodel.Status, status *gtsmodel.Status, ) error { // Allocate new slice to take the yet-to-be fetched attachment IDs. status.AttachmentIDs = make([]string, len(status.Attachments)) for i := range status.Attachments { placeholder := status.Attachments[i] // Look for existing media attachment with remote URL first. existing, ok := existing.GetAttachmentByRemoteURL(placeholder.RemoteURL) if ok && existing.ID != "" { // Ensure the existing media attachment is up-to-date and cached. existing, err := d.updateAttachment(ctx, requestUser, existing, placeholder) if err != nil { log.Errorf(ctx, "error updating existing attachment: %v", err) // specifically do NOT continue here, // we already have a model, we don't // want to drop it from the status, just // log that an update for it failed. } // Set the existing attachment. status.Attachments[i] = existing status.AttachmentIDs[i] = existing.ID continue } // Load this new media attachment. attachment, err := d.GetMedia( ctx, requestUser, status.AccountID, placeholder.RemoteURL, media.AdditionalMediaInfo{ StatusID: &status.ID, RemoteURL: &placeholder.RemoteURL, Description: &placeholder.Description, Blurhash: &placeholder.Blurhash, }, ) if err != nil { if attachment == nil { log.Errorf(ctx, "error loading attachment %s: %v", placeholder.RemoteURL, err) continue } // non-fatal error occurred during loading, still use it. log.Warnf(ctx, "partially loaded attachment: %v", err) } // Set the *new* attachment and ID. status.Attachments[i] = attachment status.AttachmentIDs[i] = attachment.ID } for i := 0; i < len(status.AttachmentIDs); { if status.AttachmentIDs[i] == "" { // Remove totally failed attachment populations copy(status.Attachments[i:], status.Attachments[i+1:]) copy(status.AttachmentIDs[i:], status.AttachmentIDs[i+1:]) status.Attachments = status.Attachments[:len(status.Attachments)-1] status.AttachmentIDs = status.AttachmentIDs[:len(status.AttachmentIDs)-1] continue } i++ } return nil } func (d *Dereferencer) fetchStatusEmojis( ctx context.Context, existing *gtsmodel.Status, status *gtsmodel.Status, ) error { // Fetch the updated emojis for our status. emojis, changed, err := d.fetchEmojis(ctx, existing.Emojis, status.Emojis, ) if err != nil { return gtserror.Newf("error fetching emojis: %w", err) } if !changed { // Use existing status emoji objects. status.EmojiIDs = existing.EmojiIDs status.Emojis = existing.Emojis return nil } // Set latest emojis. status.Emojis = emojis // Iterate over and set changed emoji IDs. status.EmojiIDs = make([]string, len(emojis)) for i, emoji := range emojis { status.EmojiIDs[i] = emoji.ID } return nil } // getPopulatedMention tries to populate the given // mention with the correct TargetAccount and (if not // yet set) TargetAccountURI, returning the populated // mention. // // Will check on the existing status if the mention // is already there and populated; if so, existing // mention will be returned along with `true`. // // Otherwise, this function will try to parse first // the Href of the mention, and then the namestring, // to see who it targets, and go fetch that account. func (d *Dereferencer) getPopulatedMention( ctx context.Context, requestUser string, existing *gtsmodel.Status, mention *gtsmodel.Mention, ) ( *gtsmodel.Mention, bool, // True if mention already exists in the DB. error, ) { // Mentions can be created using Name or Href. // Prefer Href (TargetAccountURI), fall back to Name. if mention.TargetAccountURI != "" { // Look for existing mention with target account's URI, if so use this. existingMention, ok := existing.GetMentionByTargetURI(mention.TargetAccountURI) if ok && existingMention.ID != "" { return existingMention, true, nil } // Ensure that mention account URI is parseable. accountURI, err := url.Parse(mention.TargetAccountURI) if err != nil { err := gtserror.Newf("invalid account uri %q: %w", mention.TargetAccountURI, err) return nil, false, err } // Ensure we have account of the mention target dereferenced. mention.TargetAccount, _, err = d.getAccountByURI(ctx, requestUser, accountURI, ) if err != nil { err := gtserror.Newf("failed to dereference account %s: %w", accountURI, err) return nil, false, err } } else { // Href wasn't set, extract the username and domain parts from namestring. username, domain, err := util.ExtractNamestringParts(mention.NameString) if err != nil { err := gtserror.Newf("failed to parse namestring %s: %w", mention.NameString, err) return nil, false, err } // Look for existing mention with username domain target, if so use this. existingMention, ok := existing.GetMentionByUsernameDomain(username, domain) if ok && existingMention.ID != "" { return existingMention, true, nil } // Ensure we have the account of the mention target dereferenced. mention.TargetAccount, _, err = d.getAccountByUsernameDomain(ctx, requestUser, username, domain, ) if err != nil { err := gtserror.Newf("failed to dereference account %s: %w", mention.NameString, err) return nil, false, err } // Look for existing mention with target account's URI, if so use this. existingMention, ok = existing.GetMentionByTargetURI(mention.TargetAccountURI) if ok && existingMention.ID != "" { return existingMention, true, nil } } // At this point, mention.TargetAccountURI // and mention.TargetAccount must be set. return mention, false, nil }