gotosocial/internal/federation/dereferencing/dereferencer.go
kim 3b7faac604
[bugfix] concurrent map writes in dereferencer media processing maps (#2964)
* removes the avatar / header deref maps as we now have per-uri status / account locks, adds retries on data-races, adds separate emoji map mutex

* work with a copy of account / status for each retry loop

* revert to old data race behaviour, it gets too complicated otherwise

---------

Co-authored-by: tobi <tobi.smethurst@protonmail.com>
2024-06-06 10:50:14 +02:00

125 lines
4.3 KiB
Go

// GoToSocial
// Copyright (C) GoToSocial Authors admin@gotosocial.org
// SPDX-License-Identifier: AGPL-3.0-or-later
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package dereferencing
import (
"net/url"
"sync"
"time"
"github.com/superseriousbusiness/gotosocial/internal/filter/visibility"
"github.com/superseriousbusiness/gotosocial/internal/media"
"github.com/superseriousbusiness/gotosocial/internal/state"
"github.com/superseriousbusiness/gotosocial/internal/transport"
"github.com/superseriousbusiness/gotosocial/internal/typeutils"
"github.com/superseriousbusiness/gotosocial/internal/util"
)
// FreshnessWindow represents a duration in which a
// Status or Account is still considered to be "fresh"
// (ie., not in need of a refresh from remote), if its
// last FetchedAt value falls within the window.
//
// For example, if an Account was FetchedAt 09:00, and it
// is now 12:00, then it would be considered "fresh"
// according to DefaultAccountFreshness, but not according
// to Fresh, which would indicate that the Account requires
// refreshing from remote.
type FreshnessWindow time.Duration
var (
// 6 hours.
//
// Default window for doing a
// fresh dereference of an Account.
DefaultAccountFreshness = util.Ptr(FreshnessWindow(6 * time.Hour))
// 2 hours.
//
// Default window for doing a
// fresh dereference of a Status.
DefaultStatusFreshness = util.Ptr(FreshnessWindow(2 * time.Hour))
// 5 minutes.
//
// Fresh is useful when you're wanting
// a more up-to-date model of something
// that exceeds default freshness windows.
//
// This is tuned to be quite fresh without
// causing loads of dereferencing calls.
Fresh = util.Ptr(FreshnessWindow(5 * time.Minute))
// 10 seconds.
//
// Freshest is useful when you want an
// immediately up to date model of something
// that's even fresher than Fresh.
//
// Be careful using this one; it can cause
// lots of unnecessary traffic if used unwisely.
Freshest = util.Ptr(FreshnessWindow(10 * time.Second))
)
// Dereferencer wraps logic and functionality for doing dereferencing
// of remote accounts, statuses, etc, from federated instances.
type Dereferencer struct {
state *state.State
converter *typeutils.Converter
transportController transport.Controller
mediaManager *media.Manager
visibility *visibility.Filter
// in-progress dereferencing emoji. we already perform
// locks per-status and per-account so we don't need
// processing maps for other media which won't often
// end up being repeated. worst case we run into an
// db.ErrAlreadyExists error which then gets handled
// appropriately by enrich{Account,Status}Safely().
derefEmojis map[string]*media.ProcessingEmoji
derefEmojisMu sync.Mutex
// handshakes marks current in-progress handshakes
// occurring, useful to prevent a deadlock between
// gotosocial instances attempting to dereference
// accounts for the first time. when a handshake is
// currently ongoing we know not to block waiting
// on certain data and instead return an in-progress
// form of the data as we currently see it.
handshakes map[string][]*url.URL
handshakesMu sync.Mutex
}
// NewDereferencer returns a Dereferencer initialized with the given parameters.
func NewDereferencer(
state *state.State,
converter *typeutils.Converter,
transportController transport.Controller,
visFilter *visibility.Filter,
mediaManager *media.Manager,
) Dereferencer {
return Dereferencer{
state: state,
converter: converter,
transportController: transportController,
mediaManager: mediaManager,
visibility: visFilter,
derefEmojis: make(map[string]*media.ProcessingEmoji),
handshakes: make(map[string][]*url.URL),
}
}