[feature] Rework timeline code to make it useful for more than just statuses (#373)

* add preparable and timelineable interfaces

* initialize timeline manager within the processor

* generic renaming

* move status-specific timeline logic into the processor

* refactor timeline to make it useful for more than statuses
This commit is contained in:
tobi 2022-02-05 12:47:38 +01:00 committed by GitHub
parent 98341a1d4d
commit 1b36e85840
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
26 changed files with 801 additions and 566 deletions

View file

@ -65,7 +65,6 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/oidc"
"github.com/superseriousbusiness/gotosocial/internal/processing"
"github.com/superseriousbusiness/gotosocial/internal/router"
timelineprocessing "github.com/superseriousbusiness/gotosocial/internal/timeline"
"github.com/superseriousbusiness/gotosocial/internal/transport"
"github.com/superseriousbusiness/gotosocial/internal/typeutils"
"github.com/superseriousbusiness/gotosocial/internal/web"
@ -95,7 +94,6 @@ var Start action.GTSAction = func(ctx context.Context) error {
// build converters and util
typeConverter := typeutils.NewConverter(dbService)
timelineManager := timelineprocessing.NewManager(dbService, typeConverter)
// Open the storage backend
storageBasePath := viper.GetString(config.Keys.StorageLocalBasePath)
@ -128,7 +126,7 @@ var Start action.GTSAction = func(ctx context.Context) error {
}
// create and start the message processor using the other services we've created so far
processor := processing.NewProcessor(typeConverter, federator, oauthServer, mediaHandler, storage, timelineManager, dbService, emailSender)
processor := processing.NewProcessor(typeConverter, federator, oauthServer, mediaHandler, storage, dbService, emailSender)
if err := processor.Start(ctx); err != nil {
return fmt.Errorf("error starting processor: %s", err)
}

View file

@ -96,6 +96,36 @@ type Status struct {
Text string `json:"text"`
}
/*
** The below functions are added onto the API model status so that it satisfies
** the Preparable interface in internal/timeline.
*/
func (s *Status) GetID() string {
return s.ID
}
func (s *Status) GetAccountID() string {
if s.Account != nil {
return s.Account.ID
}
return ""
}
func (s *Status) GetBoostOfID() string {
if s.Reblog != nil {
return s.Reblog.ID
}
return ""
}
func (s *Status) GetBoostOfAccountID() string {
if s.Reblog != nil && s.Reblog.Account != nil {
return s.Reblog.Account.ID
}
return ""
}
// StatusReblogged represents a reblogged status.
//
// swagger:model statusReblogged

View file

@ -69,7 +69,7 @@ func (suite *WebfingerGetTestSuite) TestFingerUser() {
func (suite *WebfingerGetTestSuite) TestFingerUserWithDifferentAccountDomainByHost() {
viper.Set(config.Keys.Host, "gts.example.org")
viper.Set(config.Keys.AccountDomain, "example.org")
suite.processor = processing.NewProcessor(suite.tc, suite.federator, testrig.NewTestOauthServer(suite.db), testrig.NewTestMediaHandler(suite.db, suite.storage), suite.storage, testrig.NewTestTimelineManager(suite.db), suite.db, suite.emailSender)
suite.processor = processing.NewProcessor(suite.tc, suite.federator, testrig.NewTestOauthServer(suite.db), testrig.NewTestMediaHandler(suite.db, suite.storage), suite.storage, suite.db, suite.emailSender)
suite.webfingerModule = webfinger.New(suite.processor).(*webfinger.Module)
targetAccount := accountDomainAccount()
@ -103,7 +103,7 @@ func (suite *WebfingerGetTestSuite) TestFingerUserWithDifferentAccountDomainByHo
func (suite *WebfingerGetTestSuite) TestFingerUserWithDifferentAccountDomainByAccountDomain() {
viper.Set(config.Keys.Host, "gts.example.org")
viper.Set(config.Keys.AccountDomain, "example.org")
suite.processor = processing.NewProcessor(suite.tc, suite.federator, testrig.NewTestOauthServer(suite.db), testrig.NewTestMediaHandler(suite.db, suite.storage), suite.storage, testrig.NewTestTimelineManager(suite.db), suite.db, suite.emailSender)
suite.processor = processing.NewProcessor(suite.tc, suite.federator, testrig.NewTestOauthServer(suite.db), testrig.NewTestMediaHandler(suite.db, suite.storage), suite.storage, suite.db, suite.emailSender)
suite.webfingerModule = webfinger.New(suite.processor).(*webfinger.Module)
targetAccount := accountDomainAccount()

View file

@ -66,6 +66,27 @@ type Status struct {
Likeable bool `validate:"-" bun:",notnull"` // This status can be liked/faved
}
/*
The below functions are added onto the gtsmodel status so that it satisfies
the Timelineable interface in internal/timeline.
*/
func (s *Status) GetID() string {
return s.ID
}
func (s *Status) GetAccountID() string {
return s.AccountID
}
func (s *Status) GetBoostOfID() string {
return s.BoostOfID
}
func (s *Status) GetBoostOfAccountID() string {
return s.BoostOfAccountID
}
// StatusToTag is an intermediate struct to facilitate the many2many relationship between a status and one or more tags.
type StatusToTag struct {
StatusID string `validate:"ulid,required" bun:"type:CHAR(26),unique:statustag,nullzero,notnull"`

View file

@ -192,10 +192,10 @@ func (p *processor) processCreateBlockFromClientAPI(ctx context.Context, clientM
}
// remove any of the blocking account's statuses from the blocked account's timeline, and vice versa
if err := p.timelineManager.WipeStatusesFromAccountID(ctx, block.AccountID, block.TargetAccountID); err != nil {
if err := p.statusTimelines.WipeItemsFromAccountID(ctx, block.AccountID, block.TargetAccountID); err != nil {
return err
}
if err := p.timelineManager.WipeStatusesFromAccountID(ctx, block.TargetAccountID, block.AccountID); err != nil {
if err := p.statusTimelines.WipeItemsFromAccountID(ctx, block.TargetAccountID, block.AccountID); err != nil {
return err
}

View file

@ -413,7 +413,7 @@ func (p *processor) timelineStatusForAccount(ctx context.Context, status *gtsmod
}
// stick the status in the timeline for the account and then immediately prepare it so they can see it right away
inserted, err := p.timelineManager.IngestAndPrepare(ctx, status, timelineAccount.ID)
inserted, err := p.statusTimelines.IngestAndPrepare(ctx, status, timelineAccount.ID)
if err != nil {
errors <- fmt.Errorf("timelineStatusForAccount: error ingesting status %s: %s", status.ID, err)
return
@ -436,7 +436,7 @@ func (p *processor) timelineStatusForAccount(ctx context.Context, status *gtsmod
// deleteStatusFromTimelines completely removes the given status from all timelines.
// It will also stream deletion of the status to all open streams.
func (p *processor) deleteStatusFromTimelines(ctx context.Context, status *gtsmodel.Status) error {
if err := p.timelineManager.WipeStatusFromAllTimelines(ctx, status.ID); err != nil {
if err := p.statusTimelines.WipeItemFromAllTimelines(ctx, status.ID); err != nil {
return err
}

View file

@ -213,10 +213,10 @@ func (p *processor) processCreateBlockFromFederator(ctx context.Context, federat
}
// remove any of the blocking account's statuses from the blocked account's timeline, and vice versa
if err := p.timelineManager.WipeStatusesFromAccountID(ctx, block.AccountID, block.TargetAccountID); err != nil {
if err := p.statusTimelines.WipeItemsFromAccountID(ctx, block.AccountID, block.TargetAccountID); err != nil {
return err
}
if err := p.timelineManager.WipeStatusesFromAccountID(ctx, block.TargetAccountID, block.AccountID); err != nil {
if err := p.statusTimelines.WipeItemsFromAccountID(ctx, block.TargetAccountID, block.AccountID); err != nil {
return err
}
// TODO: same with notifications

View file

@ -237,7 +237,7 @@ type processor struct {
oauthServer oauth.Server
mediaHandler media.Handler
storage *kv.KVStore
timelineManager timeline.Manager
statusTimelines timeline.Manager
db db.DB
filter visibility.Filter
@ -261,7 +261,6 @@ func NewProcessor(
oauthServer oauth.Server,
mediaHandler media.Handler,
storage *kv.KVStore,
timelineManager timeline.Manager,
db db.DB,
emailSender email.Sender) Processor {
fromClientAPI := make(chan messages.FromClientAPI, 1000)
@ -274,6 +273,7 @@ func NewProcessor(
mediaProcessor := mediaProcessor.New(db, tc, mediaHandler, storage)
userProcessor := user.New(db, emailSender)
federationProcessor := federationProcessor.New(db, tc, federator, fromFederator)
filter := visibility.NewFilter(db)
return &processor{
fromClientAPI: fromClientAPI,
@ -284,7 +284,7 @@ func NewProcessor(
oauthServer: oauthServer,
mediaHandler: mediaHandler,
storage: storage,
timelineManager: timelineManager,
statusTimelines: timeline.NewManager(StatusGrabFunction(db), StatusFilterFunction(db, filter), StatusPrepareFunction(db, tc), StatusSkipInsertFunction()),
db: db,
filter: visibility.NewFilter(db),

View file

@ -219,10 +219,9 @@ func (suite *ProcessingStandardTestSuite) SetupTest() {
suite.federator = testrig.NewTestFederator(suite.db, suite.transportController, suite.storage)
suite.oauthServer = testrig.NewTestOauthServer(suite.db)
suite.mediaHandler = testrig.NewTestMediaHandler(suite.db, suite.storage)
suite.timelineManager = testrig.NewTestTimelineManager(suite.db)
suite.emailSender = testrig.NewEmailSender("../../web/template/", nil)
suite.processor = processing.NewProcessor(suite.typeconverter, suite.federator, suite.oauthServer, suite.mediaHandler, suite.storage, suite.timelineManager, suite.db, suite.emailSender)
suite.processor = processing.NewProcessor(suite.typeconverter, suite.federator, suite.oauthServer, suite.mediaHandler, suite.storage, suite.db, suite.emailSender)
testrig.StandardDBSetup(suite.db, suite.testAccounts)
testrig.StandardStorageSetup(suite.storage, "../../testrig/media")

View file

@ -20,6 +20,7 @@ package processing
import (
"context"
"errors"
"fmt"
"net/url"
@ -32,8 +33,113 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/oauth"
"github.com/superseriousbusiness/gotosocial/internal/timeline"
"github.com/superseriousbusiness/gotosocial/internal/typeutils"
"github.com/superseriousbusiness/gotosocial/internal/visibility"
)
const boostReinsertionDepth = 50
// StatusGrabFunction returns a function that satisfies the GrabFunction interface in internal/timeline.
func StatusGrabFunction(database db.DB) timeline.GrabFunction {
return func(ctx context.Context, timelineAccountID string, maxID string, sinceID string, minID string, limit int) ([]timeline.Timelineable, bool, error) {
statuses, err := database.GetHomeTimeline(ctx, timelineAccountID, maxID, sinceID, minID, limit, false)
if err != nil {
if err == db.ErrNoEntries {
return nil, true, nil // we just don't have enough statuses left in the db so return stop = true
}
return nil, false, fmt.Errorf("statusGrabFunction: error getting statuses from db: %s", err)
}
items := []timeline.Timelineable{}
for _, s := range statuses {
items = append(items, s)
}
return items, false, nil
}
}
// StatusFilterFunction returns a function that satisfies the FilterFunction interface in internal/timeline.
func StatusFilterFunction(database db.DB, filter visibility.Filter) timeline.FilterFunction {
return func(ctx context.Context, timelineAccountID string, item timeline.Timelineable) (shouldIndex bool, err error) {
status, ok := item.(*gtsmodel.Status)
if !ok {
return false, errors.New("statusFilterFunction: could not convert item to *gtsmodel.Status")
}
requestingAccount, err := database.GetAccountByID(ctx, timelineAccountID)
if err != nil {
return false, fmt.Errorf("statusFilterFunction: error getting account with id %s", timelineAccountID)
}
timelineable, err := filter.StatusHometimelineable(ctx, status, requestingAccount)
if err != nil {
logrus.Warnf("error checking hometimelineability of status %s for account %s: %s", status.ID, timelineAccountID, err)
}
return timelineable, nil // we don't return the error here because we want to just skip this item if something goes wrong
}
}
// StatusPrepareFunction returns a function that satisfies the PrepareFunction interface in internal/timeline.
func StatusPrepareFunction(database db.DB, tc typeutils.TypeConverter) timeline.PrepareFunction {
return func(ctx context.Context, timelineAccountID string, itemID string) (timeline.Preparable, error) {
status, err := database.GetStatusByID(ctx, itemID)
if err != nil {
return nil, fmt.Errorf("statusPrepareFunction: error getting status with id %s", itemID)
}
requestingAccount, err := database.GetAccountByID(ctx, timelineAccountID)
if err != nil {
return nil, fmt.Errorf("statusPrepareFunction: error getting account with id %s", timelineAccountID)
}
return tc.StatusToAPIStatus(ctx, status, requestingAccount)
}
}
// StatusSkipInsertFunction returns a function that satisifes the SkipInsertFunction interface in internal/timeline.
func StatusSkipInsertFunction() timeline.SkipInsertFunction {
return func(
ctx context.Context,
newItemID string,
newItemAccountID string,
newItemBoostOfID string,
newItemBoostOfAccountID string,
nextItemID string,
nextItemAccountID string,
nextItemBoostOfID string,
nextItemBoostOfAccountID string,
depth int) (bool, error) {
// make sure we don't insert a duplicate
if newItemID == nextItemID {
return true, nil
}
// check if it's a boost
if newItemBoostOfID != "" {
// skip if we've recently put another boost of this status in the timeline
if newItemBoostOfID == nextItemBoostOfID {
if depth < boostReinsertionDepth {
return true, nil
}
}
// skip if we've recently put the original status in the timeline
if newItemBoostOfID == nextItemID {
if depth < boostReinsertionDepth {
return true, nil
}
}
}
// insert the item
return false, nil
}
}
func (p *processor) packageStatusResponse(statuses []*apimodel.Status, path string, nextMaxID string, prevMinID string, limit int) (*apimodel.StatusTimelineResponse, gtserror.WithCode) {
resp := &apimodel.StatusTimelineResponse{
Statuses: []*apimodel.Status{},
@ -67,18 +173,27 @@ func (p *processor) packageStatusResponse(statuses []*apimodel.Status, path stri
}
func (p *processor) HomeTimelineGet(ctx context.Context, authed *oauth.Auth, maxID string, sinceID string, minID string, limit int, local bool) (*apimodel.StatusTimelineResponse, gtserror.WithCode) {
statuses, err := p.timelineManager.HomeTimeline(ctx, authed.Account.ID, maxID, sinceID, minID, limit, local)
preparedItems, err := p.statusTimelines.GetTimeline(ctx, authed.Account.ID, maxID, sinceID, minID, limit, local)
if err != nil {
return nil, gtserror.NewErrorInternalError(err)
}
if len(statuses) == 0 {
if len(preparedItems) == 0 {
return &apimodel.StatusTimelineResponse{
Statuses: []*apimodel.Status{},
}, nil
}
return p.packageStatusResponse(statuses, "api/v1/timelines/home", statuses[len(statuses)-1].ID, statuses[0].ID, limit)
statuses := []*apimodel.Status{}
for _, i := range preparedItems {
status, ok := i.(*apimodel.Status)
if !ok {
return nil, gtserror.NewErrorInternalError(errors.New("error converting prepared timeline entry to api status"))
}
statuses = append(statuses, status)
}
return p.packageStatusResponse(statuses, "api/v1/timelines/home", statuses[len(preparedItems)-1].ID, statuses[0].ID, limit)
}
func (p *processor) PublicTimelineGet(ctx context.Context, authed *oauth.Auth, maxID string, sinceID string, minID string, limit int, local bool) (*apimodel.StatusTimelineResponse, gtserror.WithCode) {

View file

@ -25,12 +25,11 @@ import (
"fmt"
"github.com/sirupsen/logrus"
apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
)
const retries = 5
func (t *timeline) Get(ctx context.Context, amount int, maxID string, sinceID string, minID string, prepareNext bool) ([]*apimodel.Status, error) {
func (t *timeline) Get(ctx context.Context, amount int, maxID string, sinceID string, minID string, prepareNext bool) ([]Preparable, error) {
l := logrus.WithFields(logrus.Fields{
"func": "Get",
"accountID": t.accountID,
@ -41,16 +40,16 @@ func (t *timeline) Get(ctx context.Context, amount int, maxID string, sinceID st
})
l.Debug("entering get")
var statuses []*apimodel.Status
var items []Preparable
var err error
// no params are defined to just fetch from the top
// this is equivalent to a user asking for the top x posts from their timeline
// this is equivalent to a user asking for the top x items from their timeline
if maxID == "" && sinceID == "" && minID == "" {
statuses, err = t.GetXFromTop(ctx, amount)
items, err = t.GetXFromTop(ctx, amount)
// aysnchronously prepare the next predicted query so it's ready when the user asks for it
if len(statuses) != 0 {
nextMaxID := statuses[len(statuses)-1].ID
if len(items) != 0 {
nextMaxID := items[len(items)-1].GetID()
if prepareNext {
// already cache the next query to speed up scrolling
go func() {
@ -64,13 +63,13 @@ func (t *timeline) Get(ctx context.Context, amount int, maxID string, sinceID st
}
// maxID is defined but sinceID isn't so take from behind
// this is equivalent to a user asking for the next x posts from their timeline, starting from maxID
// this is equivalent to a user asking for the next x items from their timeline, starting from maxID
if maxID != "" && sinceID == "" {
attempts := 0
statuses, err = t.GetXBehindID(ctx, amount, maxID, &attempts)
items, err = t.GetXBehindID(ctx, amount, maxID, &attempts)
// aysnchronously prepare the next predicted query so it's ready when the user asks for it
if len(statuses) != 0 {
nextMaxID := statuses[len(statuses)-1].ID
if len(items) != 0 {
nextMaxID := items[len(items)-1].GetID()
if prepareNext {
// already cache the next query to speed up scrolling
go func() {
@ -84,59 +83,59 @@ func (t *timeline) Get(ctx context.Context, amount int, maxID string, sinceID st
}
// maxID is defined and sinceID || minID are as well, so take a slice between them
// this is equivalent to a user asking for posts older than x but newer than y
// this is equivalent to a user asking for items older than x but newer than y
if maxID != "" && sinceID != "" {
statuses, err = t.GetXBetweenID(ctx, amount, maxID, minID)
items, err = t.GetXBetweenID(ctx, amount, maxID, minID)
}
if maxID != "" && minID != "" {
statuses, err = t.GetXBetweenID(ctx, amount, maxID, minID)
items, err = t.GetXBetweenID(ctx, amount, maxID, minID)
}
// maxID isn't defined, but sinceID || minID are, so take x before
// this is equivalent to a user asking for posts newer than x (eg., refreshing the top of their timeline)
// this is equivalent to a user asking for items newer than x (eg., refreshing the top of their timeline)
if maxID == "" && sinceID != "" {
statuses, err = t.GetXBeforeID(ctx, amount, sinceID, true)
items, err = t.GetXBeforeID(ctx, amount, sinceID, true)
}
if maxID == "" && minID != "" {
statuses, err = t.GetXBeforeID(ctx, amount, minID, true)
items, err = t.GetXBeforeID(ctx, amount, minID, true)
}
return statuses, err
return items, err
}
func (t *timeline) GetXFromTop(ctx context.Context, amount int) ([]*apimodel.Status, error) {
// make a slice of statuses with the length we need to return
statuses := make([]*apimodel.Status, 0, amount)
func (t *timeline) GetXFromTop(ctx context.Context, amount int) ([]Preparable, error) {
// make a slice of preparedItems with the length we need to return
preparedItems := make([]Preparable, 0, amount)
if t.preparedPosts.data == nil {
t.preparedPosts.data = &list.List{}
if t.preparedItems.data == nil {
t.preparedItems.data = &list.List{}
}
// make sure we have enough posts prepared to return
if t.preparedPosts.data.Len() < amount {
// make sure we have enough items prepared to return
if t.preparedItems.data.Len() < amount {
if err := t.PrepareFromTop(ctx, amount); err != nil {
return nil, err
}
}
// work through the prepared posts from the top and return
// work through the prepared items from the top and return
var served int
for e := t.preparedPosts.data.Front(); e != nil; e = e.Next() {
entry, ok := e.Value.(*preparedPostsEntry)
for e := t.preparedItems.data.Front(); e != nil; e = e.Next() {
entry, ok := e.Value.(*preparedItemsEntry)
if !ok {
return nil, errors.New("GetXFromTop: could not parse e as a preparedPostsEntry")
return nil, errors.New("GetXFromTop: could not parse e as a preparedItemsEntry")
}
statuses = append(statuses, entry.prepared)
preparedItems = append(preparedItems, entry.prepared)
served++
if served >= amount {
break
}
}
return statuses, nil
return preparedItems, nil
}
func (t *timeline) GetXBehindID(ctx context.Context, amount int, behindID string, attempts *int) ([]*apimodel.Status, error) {
func (t *timeline) GetXBehindID(ctx context.Context, amount int, behindID string, attempts *int) ([]Preparable, error) {
l := logrus.WithFields(logrus.Fields{
"func": "GetXBehindID",
"amount": amount,
@ -148,11 +147,11 @@ func (t *timeline) GetXBehindID(ctx context.Context, amount int, behindID string
newAttempts++
attempts = &newAttempts
// make a slice of statuses with the length we need to return
statuses := make([]*apimodel.Status, 0, amount)
// make a slice of items with the length we need to return
items := make([]Preparable, 0, amount)
if t.preparedPosts.data == nil {
t.preparedPosts.data = &list.List{}
if t.preparedItems.data == nil {
t.preparedItems.data = &list.List{}
}
// iterate through the modified list until we hit the mark we're looking for
@ -160,14 +159,14 @@ func (t *timeline) GetXBehindID(ctx context.Context, amount int, behindID string
var behindIDMark *list.Element
findMarkLoop:
for e := t.preparedPosts.data.Front(); e != nil; e = e.Next() {
for e := t.preparedItems.data.Front(); e != nil; e = e.Next() {
position++
entry, ok := e.Value.(*preparedPostsEntry)
entry, ok := e.Value.(*preparedItemsEntry)
if !ok {
return nil, errors.New("GetXBehindID: could not parse e as a preparedPostsEntry")
}
if entry.statusID <= behindID {
if entry.itemID <= behindID {
l.Trace("found behindID mark")
behindIDMark = e
break findMarkLoop
@ -175,33 +174,33 @@ findMarkLoop:
}
// we didn't find it, so we need to make sure it's indexed and prepared and then try again
// this can happen when a user asks for really old posts
// this can happen when a user asks for really old items
if behindIDMark == nil {
if err := t.PrepareBehind(ctx, behindID, amount); err != nil {
return nil, fmt.Errorf("GetXBehindID: error preparing behind and including ID %s", behindID)
}
oldestID, err := t.OldestPreparedPostID(ctx)
oldestID, err := t.OldestPreparedItemID(ctx)
if err != nil {
return nil, err
}
if oldestID == "" {
l.Tracef("oldestID is empty so we can't return behindID %s", behindID)
return statuses, nil
return items, nil
}
if oldestID == behindID {
l.Tracef("given behindID %s is the same as oldestID %s so there's nothing to return behind it", behindID, oldestID)
return statuses, nil
return items, nil
}
if *attempts > retries {
l.Tracef("exceeded retries looking for behindID %s", behindID)
return statuses, nil
return items, nil
}
l.Trace("trying GetXBehindID again")
return t.GetXBehindID(ctx, amount, behindID, attempts)
}
// make sure we have enough posts prepared behind it to return what we're being asked for
if t.preparedPosts.data.Len() < amount+position {
// make sure we have enough items prepared behind it to return what we're being asked for
if t.preparedItems.data.Len() < amount+position {
if err := t.PrepareBehind(ctx, behindID, amount); err != nil {
return nil, err
}
@ -211,40 +210,40 @@ findMarkLoop:
var served int
serveloop:
for e := behindIDMark.Next(); e != nil; e = e.Next() {
entry, ok := e.Value.(*preparedPostsEntry)
entry, ok := e.Value.(*preparedItemsEntry)
if !ok {
return nil, errors.New("GetXBehindID: could not parse e as a preparedPostsEntry")
}
// serve up to the amount requested
statuses = append(statuses, entry.prepared)
items = append(items, entry.prepared)
served++
if served >= amount {
break serveloop
}
}
return statuses, nil
return items, nil
}
func (t *timeline) GetXBeforeID(ctx context.Context, amount int, beforeID string, startFromTop bool) ([]*apimodel.Status, error) {
// make a slice of statuses with the length we need to return
statuses := make([]*apimodel.Status, 0, amount)
func (t *timeline) GetXBeforeID(ctx context.Context, amount int, beforeID string, startFromTop bool) ([]Preparable, error) {
// make a slice of items with the length we need to return
items := make([]Preparable, 0, amount)
if t.preparedPosts.data == nil {
t.preparedPosts.data = &list.List{}
if t.preparedItems.data == nil {
t.preparedItems.data = &list.List{}
}
// iterate through the modified list until we hit the mark we're looking for, or as close as possible to it
var beforeIDMark *list.Element
findMarkLoop:
for e := t.preparedPosts.data.Front(); e != nil; e = e.Next() {
entry, ok := e.Value.(*preparedPostsEntry)
for e := t.preparedItems.data.Front(); e != nil; e = e.Next() {
entry, ok := e.Value.(*preparedItemsEntry)
if !ok {
return nil, errors.New("GetXBeforeID: could not parse e as a preparedPostsEntry")
}
if entry.statusID >= beforeID {
if entry.itemID >= beforeID {
beforeIDMark = e
} else {
break findMarkLoop
@ -252,26 +251,26 @@ findMarkLoop:
}
if beforeIDMark == nil {
return statuses, nil
return items, nil
}
var served int
if startFromTop {
// start serving from the front/top and keep going until we hit mark or get x amount statuses
// start serving from the front/top and keep going until we hit mark or get x amount items
serveloopFromTop:
for e := t.preparedPosts.data.Front(); e != nil; e = e.Next() {
entry, ok := e.Value.(*preparedPostsEntry)
for e := t.preparedItems.data.Front(); e != nil; e = e.Next() {
entry, ok := e.Value.(*preparedItemsEntry)
if !ok {
return nil, errors.New("GetXBeforeID: could not parse e as a preparedPostsEntry")
}
if entry.statusID == beforeID {
if entry.itemID == beforeID {
break serveloopFromTop
}
// serve up to the amount requested
statuses = append(statuses, entry.prepared)
items = append(items, entry.prepared)
served++
if served >= amount {
break serveloopFromTop
@ -281,13 +280,13 @@ findMarkLoop:
// start serving from the entry right before the mark
serveloopFromBottom:
for e := beforeIDMark.Prev(); e != nil; e = e.Prev() {
entry, ok := e.Value.(*preparedPostsEntry)
entry, ok := e.Value.(*preparedItemsEntry)
if !ok {
return nil, errors.New("GetXBeforeID: could not parse e as a preparedPostsEntry")
}
// serve up to the amount requested
statuses = append(statuses, entry.prepared)
items = append(items, entry.prepared)
served++
if served >= amount {
break serveloopFromBottom
@ -295,29 +294,29 @@ findMarkLoop:
}
}
return statuses, nil
return items, nil
}
func (t *timeline) GetXBetweenID(ctx context.Context, amount int, behindID string, beforeID string) ([]*apimodel.Status, error) {
// make a slice of statuses with the length we need to return
statuses := make([]*apimodel.Status, 0, amount)
func (t *timeline) GetXBetweenID(ctx context.Context, amount int, behindID string, beforeID string) ([]Preparable, error) {
// make a slice of items with the length we need to return
items := make([]Preparable, 0, amount)
if t.preparedPosts.data == nil {
t.preparedPosts.data = &list.List{}
if t.preparedItems.data == nil {
t.preparedItems.data = &list.List{}
}
// iterate through the modified list until we hit the mark we're looking for
var position int
var behindIDMark *list.Element
findMarkLoop:
for e := t.preparedPosts.data.Front(); e != nil; e = e.Next() {
for e := t.preparedItems.data.Front(); e != nil; e = e.Next() {
position++
entry, ok := e.Value.(*preparedPostsEntry)
entry, ok := e.Value.(*preparedItemsEntry)
if !ok {
return nil, errors.New("GetXBetweenID: could not parse e as a preparedPostsEntry")
}
if entry.statusID == behindID {
if entry.itemID == behindID {
behindIDMark = e
break findMarkLoop
}
@ -325,11 +324,11 @@ findMarkLoop:
// we didn't find it
if behindIDMark == nil {
return nil, fmt.Errorf("GetXBetweenID: couldn't find status with ID %s", behindID)
return nil, fmt.Errorf("GetXBetweenID: couldn't find item with ID %s", behindID)
}
// make sure we have enough posts prepared behind it to return what we're being asked for
if t.preparedPosts.data.Len() < amount+position {
// make sure we have enough items prepared behind it to return what we're being asked for
if t.preparedItems.data.Len() < amount+position {
if err := t.PrepareBehind(ctx, behindID, amount); err != nil {
return nil, err
}
@ -339,22 +338,22 @@ findMarkLoop:
var served int
serveloop:
for e := behindIDMark.Next(); e != nil; e = e.Next() {
entry, ok := e.Value.(*preparedPostsEntry)
entry, ok := e.Value.(*preparedItemsEntry)
if !ok {
return nil, errors.New("GetXBetweenID: could not parse e as a preparedPostsEntry")
}
if entry.statusID == beforeID {
if entry.itemID == beforeID {
break serveloop
}
// serve up to the amount requested
statuses = append(statuses, entry.prepared)
items = append(items, entry.prepared)
served++
if served >= amount {
break serveloop
}
}
return statuses, nil
return items, nil
}

View file

@ -24,7 +24,9 @@ import (
"time"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/processing"
"github.com/superseriousbusiness/gotosocial/internal/timeline"
"github.com/superseriousbusiness/gotosocial/internal/visibility"
"github.com/superseriousbusiness/gotosocial/testrig"
)
@ -43,18 +45,26 @@ func (suite *GetTestSuite) SetupTest() {
suite.db = testrig.NewTestDB()
suite.tc = testrig.NewTestTypeConverter(suite.db)
suite.filter = visibility.NewFilter(suite.db)
testrig.StandardDBSetup(suite.db, nil)
// let's take local_account_1 as the timeline owner
tl, err := timeline.NewTimeline(context.Background(), suite.testAccounts["local_account_1"].ID, suite.db, suite.tc)
tl, err := timeline.NewTimeline(
context.Background(),
suite.testAccounts["local_account_1"].ID,
processing.StatusGrabFunction(suite.db),
processing.StatusFilterFunction(suite.db, suite.filter),
processing.StatusPrepareFunction(suite.db, suite.tc),
processing.StatusSkipInsertFunction(),
)
if err != nil {
suite.FailNow(err.Error())
}
// prepare the timeline by just shoving all test statuses in it -- let's not be fussy about who sees what
for _, s := range suite.testStatuses {
_, err := tl.IndexAndPrepareOne(context.Background(), s.CreatedAt, s.ID, s.BoostOfID, s.AccountID, s.BoostOfAccountID)
_, err := tl.IndexAndPrepareOne(context.Background(), s.GetID(), s.BoostOfID, s.AccountID, s.BoostOfAccountID)
if err != nil {
suite.FailNow(err.Error())
}
@ -81,10 +91,10 @@ func (suite *GetTestSuite) TestGetDefault() {
var highest string
for i, s := range statuses {
if i == 0 {
highest = s.ID
highest = s.GetID()
} else {
suite.Less(s.ID, highest)
highest = s.ID
suite.Less(s.GetID(), highest)
highest = s.GetID()
}
}
}
@ -102,10 +112,10 @@ func (suite *GetTestSuite) TestGetDefaultPrepareNext() {
var highest string
for i, s := range statuses {
if i == 0 {
highest = s.ID
highest = s.GetID()
} else {
suite.Less(s.ID, highest)
highest = s.ID
suite.Less(s.GetID(), highest)
highest = s.GetID()
}
}
@ -127,10 +137,10 @@ func (suite *GetTestSuite) TestGetMaxID() {
var highest string
for i, s := range statuses {
if i == 0 {
highest = s.ID
highest = s.GetID()
} else {
suite.Less(s.ID, highest)
highest = s.ID
suite.Less(s.GetID(), highest)
highest = s.GetID()
}
}
}
@ -149,10 +159,10 @@ func (suite *GetTestSuite) TestGetMaxIDPrepareNext() {
var highest string
for i, s := range statuses {
if i == 0 {
highest = s.ID
highest = s.GetID()
} else {
suite.Less(s.ID, highest)
highest = s.ID
suite.Less(s.GetID(), highest)
highest = s.GetID()
}
}
@ -174,10 +184,10 @@ func (suite *GetTestSuite) TestGetMinID() {
var highest string
for i, s := range statuses {
if i == 0 {
highest = s.ID
highest = s.GetID()
} else {
suite.Less(s.ID, highest)
highest = s.ID
suite.Less(s.GetID(), highest)
highest = s.GetID()
}
}
}
@ -196,10 +206,10 @@ func (suite *GetTestSuite) TestGetSinceID() {
var highest string
for i, s := range statuses {
if i == 0 {
highest = s.ID
highest = s.GetID()
} else {
suite.Less(s.ID, highest)
highest = s.ID
suite.Less(s.GetID(), highest)
highest = s.GetID()
}
}
}
@ -218,10 +228,10 @@ func (suite *GetTestSuite) TestGetSinceIDPrepareNext() {
var highest string
for i, s := range statuses {
if i == 0 {
highest = s.ID
highest = s.GetID()
} else {
suite.Less(s.ID, highest)
highest = s.ID
suite.Less(s.GetID(), highest)
highest = s.GetID()
}
}
@ -243,10 +253,10 @@ func (suite *GetTestSuite) TestGetBetweenID() {
var highest string
for i, s := range statuses {
if i == 0 {
highest = s.ID
highest = s.GetID()
} else {
suite.Less(s.ID, highest)
highest = s.ID
suite.Less(s.GetID(), highest)
highest = s.GetID()
}
}
}
@ -265,10 +275,10 @@ func (suite *GetTestSuite) TestGetBetweenIDPrepareNext() {
var highest string
for i, s := range statuses {
if i == 0 {
highest = s.ID
highest = s.GetID()
} else {
suite.Less(s.ID, highest)
highest = s.ID
suite.Less(s.GetID(), highest)
highest = s.GetID()
}
}
@ -289,10 +299,10 @@ func (suite *GetTestSuite) TestGetXFromTop() {
var highest string
for i, s := range statuses {
if i == 0 {
highest = s.ID
highest = s.GetID()
} else {
suite.Less(s.ID, highest)
highest = s.ID
suite.Less(s.GetID(), highest)
highest = s.GetID()
}
}
}
@ -314,12 +324,12 @@ func (suite *GetTestSuite) TestGetXBehindID() {
var highest string
for i, s := range statuses {
if i == 0 {
highest = s.ID
highest = s.GetID()
} else {
suite.Less(s.ID, highest)
highest = s.ID
suite.Less(s.GetID(), highest)
highest = s.GetID()
}
suite.Less(s.ID, "01F8MHBQCBTDKN6X5VHGMMN4MA")
suite.Less(s.GetID(), "01F8MHBQCBTDKN6X5VHGMMN4MA")
}
}
@ -353,12 +363,12 @@ func (suite *GetTestSuite) TestGetXBehindNonexistentReasonableID() {
var highest string
for i, s := range statuses {
if i == 0 {
highest = s.ID
highest = s.GetID()
} else {
suite.Less(s.ID, highest)
highest = s.ID
suite.Less(s.GetID(), highest)
highest = s.GetID()
}
suite.Less(s.ID, "01F8MHBCN8120SYH7D5S050MGK")
suite.Less(s.GetID(), "01F8MHBCN8120SYH7D5S050MGK")
}
}
@ -380,12 +390,12 @@ func (suite *GetTestSuite) TestGetXBehindVeryHighID() {
var highest string
for i, s := range statuses {
if i == 0 {
highest = s.ID
highest = s.GetID()
} else {
suite.Less(s.ID, highest)
highest = s.ID
suite.Less(s.GetID(), highest)
highest = s.GetID()
}
suite.Less(s.ID, "9998MHBQCBTDKN6X5VHGMMN4MA")
suite.Less(s.GetID(), "9998MHBQCBTDKN6X5VHGMMN4MA")
}
}
@ -403,12 +413,12 @@ func (suite *GetTestSuite) TestGetXBeforeID() {
var highest string
for i, s := range statuses {
if i == 0 {
highest = s.ID
highest = s.GetID()
} else {
suite.Less(s.ID, highest)
highest = s.ID
suite.Less(s.GetID(), highest)
highest = s.GetID()
}
suite.Greater(s.ID, "01F8MHBQCBTDKN6X5VHGMMN4MA")
suite.Greater(s.GetID(), "01F8MHBQCBTDKN6X5VHGMMN4MA")
}
}
@ -426,12 +436,12 @@ func (suite *GetTestSuite) TestGetXBeforeIDNoStartFromTop() {
var lowest string
for i, s := range statuses {
if i == 0 {
lowest = s.ID
lowest = s.GetID()
} else {
suite.Greater(s.ID, lowest)
lowest = s.ID
suite.Greater(s.GetID(), lowest)
lowest = s.GetID()
}
suite.Greater(s.ID, "01F8MHBQCBTDKN6X5VHGMMN4MA")
suite.Greater(s.GetID(), "01F8MHBQCBTDKN6X5VHGMMN4MA")
}
}

View file

@ -23,173 +23,166 @@ import (
"context"
"errors"
"fmt"
"time"
"github.com/sirupsen/logrus"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
)
func (t *timeline) IndexBefore(ctx context.Context, statusID string, include bool, amount int) error {
func (t *timeline) IndexBefore(ctx context.Context, itemID string, amount int) error {
l := logrus.WithFields(logrus.Fields{
"func": "IndexBefore",
"amount": amount,
})
// lazily initialize index if it hasn't been done already
if t.postIndex.data == nil {
t.postIndex.data = &list.List{}
t.postIndex.data.Init()
if t.itemIndex.data == nil {
t.itemIndex.data = &list.List{}
t.itemIndex.data.Init()
}
filtered := []*gtsmodel.Status{}
offsetStatus := statusID
toIndex := []Timelineable{}
offsetID := itemID
if include {
// if we have the status with given statusID in the database, include it in the results set as well
s := &gtsmodel.Status{}
if err := t.db.GetByID(ctx, statusID, s); err == nil {
filtered = append(filtered, s)
}
}
i := 0
l.Trace("entering grabloop")
grabloop:
for ; len(filtered) < amount && i < 5; i++ { // try the grabloop 5 times only
statuses, err := t.db.GetHomeTimeline(ctx, t.accountID, "", "", offsetStatus, amount, false)
for i := 0; len(toIndex) < amount && i < 5; i++ { // try the grabloop 5 times only
// first grab items using the caller-provided grab function
l.Trace("grabbing...")
items, stop, err := t.grabFunction(ctx, t.accountID, "", "", offsetID, amount)
if err != nil {
if err == db.ErrNoEntries {
break grabloop // we just don't have enough statuses left in the db so index what we've got and then bail
}
return fmt.Errorf("IndexBefore: error getting statuses from db: %s", err)
return err
}
if stop {
break grabloop
}
for _, s := range statuses {
timelineable, err := t.filter.StatusHometimelineable(ctx, s, t.account)
l.Trace("filtering...")
// now filter each item using the caller-provided filter function
for _, item := range items {
shouldIndex, err := t.filterFunction(ctx, t.accountID, item)
if err != nil {
continue
return err
}
if timelineable {
filtered = append(filtered, s)
if shouldIndex {
toIndex = append(toIndex, item)
}
offsetStatus = s.ID
offsetID = item.GetID()
}
}
l.Trace("left grabloop")
for _, s := range filtered {
if _, err := t.IndexOne(ctx, s.CreatedAt, s.ID, s.BoostOfID, s.AccountID, s.BoostOfAccountID); err != nil {
return fmt.Errorf("IndexBefore: error indexing status with id %s: %s", s.ID, err)
// index the items we got
for _, s := range toIndex {
if _, err := t.IndexOne(ctx, s.GetID(), s.GetBoostOfID(), s.GetAccountID(), s.GetBoostOfAccountID()); err != nil {
return fmt.Errorf("IndexBehind: error indexing item with id %s: %s", s.GetID(), err)
}
}
return nil
}
func (t *timeline) IndexBehind(ctx context.Context, statusID string, include bool, amount int) error {
func (t *timeline) IndexBehind(ctx context.Context, itemID string, amount int) error {
l := logrus.WithFields(logrus.Fields{
"func": "IndexBehind",
"include": include,
"amount": amount,
"func": "IndexBehind",
"amount": amount,
})
// lazily initialize index if it hasn't been done already
if t.postIndex.data == nil {
t.postIndex.data = &list.List{}
t.postIndex.data.Init()
if t.itemIndex.data == nil {
t.itemIndex.data = &list.List{}
t.itemIndex.data.Init()
}
// If we're already indexedBehind given statusID by the required amount, we can return nil.
// First find position of statusID (or as near as possible).
// If we're already indexedBehind given itemID by the required amount, we can return nil.
// First find position of itemID (or as near as possible).
var position int
positionLoop:
for e := t.postIndex.data.Front(); e != nil; e = e.Next() {
entry, ok := e.Value.(*postIndexEntry)
for e := t.itemIndex.data.Front(); e != nil; e = e.Next() {
entry, ok := e.Value.(*itemIndexEntry)
if !ok {
return errors.New("IndexBehind: could not parse e as a postIndexEntry")
return errors.New("IndexBehind: could not parse e as an itemIndexEntry")
}
if entry.statusID <= statusID {
if entry.itemID <= itemID {
// we've found it
break positionLoop
}
position++
}
// now check if the length of indexed posts exceeds the amount of posts required (position of statusID, plus amount of posts requested after that)
if t.postIndex.data.Len() > position+amount {
// now check if the length of indexed items exceeds the amount of items required (position of itemID, plus amount of posts requested after that)
if t.itemIndex.data.Len() > position+amount {
// we have enough indexed behind already to satisfy amount, so don't need to make db calls
l.Trace("returning nil since we already have enough posts indexed")
l.Trace("returning nil since we already have enough items indexed")
return nil
}
filtered := []*gtsmodel.Status{}
offsetStatus := statusID
toIndex := []Timelineable{}
offsetID := itemID
if include {
// if we have the status with given statusID in the database, include it in the results set as well
s := &gtsmodel.Status{}
if err := t.db.GetByID(ctx, statusID, s); err == nil {
filtered = append(filtered, s)
}
}
i := 0
l.Trace("entering grabloop")
grabloop:
for ; len(filtered) < amount && i < 5; i++ { // try the grabloop 5 times only
l.Tracef("entering grabloop; i is %d; len(filtered) is %d", i, len(filtered))
statuses, err := t.db.GetHomeTimeline(ctx, t.accountID, offsetStatus, "", "", amount, false)
for i := 0; len(toIndex) < amount && i < 5; i++ { // try the grabloop 5 times only
// first grab items using the caller-provided grab function
l.Trace("grabbing...")
items, stop, err := t.grabFunction(ctx, t.accountID, offsetID, "", "", amount)
if err != nil {
if err == db.ErrNoEntries {
break grabloop // we just don't have enough statuses left in the db so index what we've got and then bail
}
return fmt.Errorf("IndexBehind: error getting statuses from db: %s", err)
return err
}
if stop {
break grabloop
}
l.Tracef("got %d statuses", len(statuses))
for _, s := range statuses {
timelineable, err := t.filter.StatusHometimelineable(ctx, s, t.account)
l.Trace("filtering...")
// now filter each item using the caller-provided filter function
for _, item := range items {
shouldIndex, err := t.filterFunction(ctx, t.accountID, item)
if err != nil {
l.Tracef("status was not hometimelineable: %s", err)
continue
return err
}
if timelineable {
filtered = append(filtered, s)
if shouldIndex {
toIndex = append(toIndex, item)
}
offsetStatus = s.ID
offsetID = item.GetID()
}
}
l.Trace("left grabloop")
for _, s := range filtered {
if _, err := t.IndexOne(ctx, s.CreatedAt, s.ID, s.BoostOfID, s.AccountID, s.BoostOfAccountID); err != nil {
return fmt.Errorf("IndexBehind: error indexing status with id %s: %s", s.ID, err)
// index the items we got
for _, s := range toIndex {
if _, err := t.IndexOne(ctx, s.GetID(), s.GetBoostOfID(), s.GetAccountID(), s.GetBoostOfAccountID()); err != nil {
return fmt.Errorf("IndexBehind: error indexing item with id %s: %s", s.GetID(), err)
}
}
l.Trace("exiting function")
return nil
}
func (t *timeline) IndexOne(ctx context.Context, statusCreatedAt time.Time, statusID string, boostOfID string, accountID string, boostOfAccountID string) (bool, error) {
func (t *timeline) IndexOne(ctx context.Context, itemID string, boostOfID string, accountID string, boostOfAccountID string) (bool, error) {
t.Lock()
defer t.Unlock()
postIndexEntry := &postIndexEntry{
statusID: statusID,
postIndexEntry := &itemIndexEntry{
itemID: itemID,
boostOfID: boostOfID,
accountID: accountID,
boostOfAccountID: boostOfAccountID,
}
return t.postIndex.insertIndexed(postIndexEntry)
return t.itemIndex.insertIndexed(ctx, postIndexEntry)
}
func (t *timeline) IndexAndPrepareOne(ctx context.Context, statusCreatedAt time.Time, statusID string, boostOfID string, accountID string, boostOfAccountID string) (bool, error) {
func (t *timeline) IndexAndPrepareOne(ctx context.Context, statusID string, boostOfID string, accountID string, boostOfAccountID string) (bool, error) {
t.Lock()
defer t.Unlock()
postIndexEntry := &postIndexEntry{
statusID: statusID,
postIndexEntry := &itemIndexEntry{
itemID: statusID,
boostOfID: boostOfID,
accountID: accountID,
boostOfAccountID: boostOfAccountID,
}
inserted, err := t.postIndex.insertIndexed(postIndexEntry)
inserted, err := t.itemIndex.insertIndexed(ctx, postIndexEntry)
if err != nil {
return inserted, fmt.Errorf("IndexAndPrepareOne: error inserting indexed: %s", err)
}
@ -203,32 +196,32 @@ func (t *timeline) IndexAndPrepareOne(ctx context.Context, statusCreatedAt time.
return inserted, nil
}
func (t *timeline) OldestIndexedPostID(ctx context.Context) (string, error) {
func (t *timeline) OldestIndexedItemID(ctx context.Context) (string, error) {
var id string
if t.postIndex == nil || t.postIndex.data == nil || t.postIndex.data.Back() == nil {
if t.itemIndex == nil || t.itemIndex.data == nil || t.itemIndex.data.Back() == nil {
// return an empty string if postindex hasn't been initialized yet
return id, nil
}
e := t.postIndex.data.Back()
entry, ok := e.Value.(*postIndexEntry)
e := t.itemIndex.data.Back()
entry, ok := e.Value.(*itemIndexEntry)
if !ok {
return id, errors.New("OldestIndexedPostID: could not parse e as a postIndexEntry")
return id, errors.New("OldestIndexedItemID: could not parse e as itemIndexEntry")
}
return entry.statusID, nil
return entry.itemID, nil
}
func (t *timeline) NewestIndexedPostID(ctx context.Context) (string, error) {
func (t *timeline) NewestIndexedItemID(ctx context.Context) (string, error) {
var id string
if t.postIndex == nil || t.postIndex.data == nil || t.postIndex.data.Front() == nil {
if t.itemIndex == nil || t.itemIndex.data == nil || t.itemIndex.data.Front() == nil {
// return an empty string if postindex hasn't been initialized yet
return id, nil
}
e := t.postIndex.data.Front()
entry, ok := e.Value.(*postIndexEntry)
e := t.itemIndex.data.Front()
entry, ok := e.Value.(*itemIndexEntry)
if !ok {
return id, errors.New("NewestIndexedPostID: could not parse e as a postIndexEntry")
return id, errors.New("NewestIndexedItemID: could not parse e as itemIndexEntry")
}
return entry.statusID, nil
return entry.itemID, nil
}

View file

@ -25,7 +25,9 @@ import (
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/processing"
"github.com/superseriousbusiness/gotosocial/internal/timeline"
"github.com/superseriousbusiness/gotosocial/internal/visibility"
"github.com/superseriousbusiness/gotosocial/testrig"
)
@ -44,11 +46,19 @@ func (suite *IndexTestSuite) SetupTest() {
suite.db = testrig.NewTestDB()
suite.tc = testrig.NewTestTypeConverter(suite.db)
suite.filter = visibility.NewFilter(suite.db)
testrig.StandardDBSetup(suite.db, nil)
// let's take local_account_1 as the timeline owner, and start with an empty timeline
tl, err := timeline.NewTimeline(context.Background(), suite.testAccounts["local_account_1"].ID, suite.db, suite.tc)
tl, err := timeline.NewTimeline(
context.Background(),
suite.testAccounts["local_account_1"].ID,
processing.StatusGrabFunction(suite.db),
processing.StatusFilterFunction(suite.db, suite.filter),
processing.StatusPrepareFunction(suite.db, suite.tc),
processing.StatusSkipInsertFunction(),
)
if err != nil {
suite.FailNow(err.Error())
}
@ -61,82 +71,82 @@ func (suite *IndexTestSuite) TearDownTest() {
func (suite *IndexTestSuite) TestIndexBeforeLowID() {
// index 10 before the lowest status ID possible
err := suite.timeline.IndexBefore(context.Background(), "00000000000000000000000000", true, 10)
err := suite.timeline.IndexBefore(context.Background(), "00000000000000000000000000", 10)
suite.NoError(err)
// the oldest indexed post should be the lowest one we have in our testrig
postID, err := suite.timeline.OldestIndexedPostID(context.Background())
postID, err := suite.timeline.OldestIndexedItemID(context.Background())
suite.NoError(err)
suite.Equal("01F8MHAYFKS4KMXF8K5Y1C0KRN", postID)
indexLength := suite.timeline.PostIndexLength(context.Background())
indexLength := suite.timeline.ItemIndexLength(context.Background())
suite.Equal(10, indexLength)
}
func (suite *IndexTestSuite) TestIndexBeforeHighID() {
// index 10 before the highest status ID possible
err := suite.timeline.IndexBefore(context.Background(), "ZZZZZZZZZZZZZZZZZZZZZZZZZZ", true, 10)
err := suite.timeline.IndexBefore(context.Background(), "ZZZZZZZZZZZZZZZZZZZZZZZZZZ", 10)
suite.NoError(err)
// the oldest indexed post should be empty
postID, err := suite.timeline.OldestIndexedPostID(context.Background())
postID, err := suite.timeline.OldestIndexedItemID(context.Background())
suite.NoError(err)
suite.Empty(postID)
// indexLength should be 0
indexLength := suite.timeline.PostIndexLength(context.Background())
indexLength := suite.timeline.ItemIndexLength(context.Background())
suite.Equal(0, indexLength)
}
func (suite *IndexTestSuite) TestIndexBehindHighID() {
// index 10 behind the highest status ID possible
err := suite.timeline.IndexBehind(context.Background(), "ZZZZZZZZZZZZZZZZZZZZZZZZZZ", true, 10)
err := suite.timeline.IndexBehind(context.Background(), "ZZZZZZZZZZZZZZZZZZZZZZZZZZ", 10)
suite.NoError(err)
// the newest indexed post should be the highest one we have in our testrig
postID, err := suite.timeline.NewestIndexedPostID(context.Background())
postID, err := suite.timeline.NewestIndexedItemID(context.Background())
suite.NoError(err)
suite.Equal("01FN3VJGFH10KR7S2PB0GFJZYG", postID)
// indexLength should be 10 because that's all this user has hometimelineable
indexLength := suite.timeline.PostIndexLength(context.Background())
indexLength := suite.timeline.ItemIndexLength(context.Background())
suite.Equal(10, indexLength)
}
func (suite *IndexTestSuite) TestIndexBehindLowID() {
// index 10 behind the lowest status ID possible
err := suite.timeline.IndexBehind(context.Background(), "00000000000000000000000000", true, 10)
err := suite.timeline.IndexBehind(context.Background(), "00000000000000000000000000", 10)
suite.NoError(err)
// the newest indexed post should be empty
postID, err := suite.timeline.NewestIndexedPostID(context.Background())
postID, err := suite.timeline.NewestIndexedItemID(context.Background())
suite.NoError(err)
suite.Empty(postID)
// indexLength should be 0
indexLength := suite.timeline.PostIndexLength(context.Background())
indexLength := suite.timeline.ItemIndexLength(context.Background())
suite.Equal(0, indexLength)
}
func (suite *IndexTestSuite) TestOldestIndexedPostIDEmpty() {
func (suite *IndexTestSuite) TestOldestIndexedItemIDEmpty() {
// the oldest indexed post should be an empty string since there's nothing indexed yet
postID, err := suite.timeline.OldestIndexedPostID(context.Background())
postID, err := suite.timeline.OldestIndexedItemID(context.Background())
suite.NoError(err)
suite.Empty(postID)
// indexLength should be 0
indexLength := suite.timeline.PostIndexLength(context.Background())
indexLength := suite.timeline.ItemIndexLength(context.Background())
suite.Equal(0, indexLength)
}
func (suite *IndexTestSuite) TestNewestIndexedPostIDEmpty() {
func (suite *IndexTestSuite) TestNewestIndexedItemIDEmpty() {
// the newest indexed post should be an empty string since there's nothing indexed yet
postID, err := suite.timeline.NewestIndexedPostID(context.Background())
postID, err := suite.timeline.NewestIndexedItemID(context.Background())
suite.NoError(err)
suite.Empty(postID)
// indexLength should be 0
indexLength := suite.timeline.PostIndexLength(context.Background())
indexLength := suite.timeline.ItemIndexLength(context.Background())
suite.Equal(0, indexLength)
}
@ -144,12 +154,12 @@ func (suite *IndexTestSuite) TestIndexAlreadyIndexed() {
testStatus := suite.testStatuses["local_account_1_status_1"]
// index one post -- it should be indexed
indexed, err := suite.timeline.IndexOne(context.Background(), testStatus.CreatedAt, testStatus.ID, testStatus.BoostOfID, testStatus.AccountID, testStatus.BoostOfAccountID)
indexed, err := suite.timeline.IndexOne(context.Background(), testStatus.ID, testStatus.BoostOfID, testStatus.AccountID, testStatus.BoostOfAccountID)
suite.NoError(err)
suite.True(indexed)
// try to index the same post again -- it should not be indexed
indexed, err = suite.timeline.IndexOne(context.Background(), testStatus.CreatedAt, testStatus.ID, testStatus.BoostOfID, testStatus.AccountID, testStatus.BoostOfAccountID)
indexed, err = suite.timeline.IndexOne(context.Background(), testStatus.ID, testStatus.BoostOfID, testStatus.AccountID, testStatus.BoostOfAccountID)
suite.NoError(err)
suite.False(indexed)
}
@ -158,12 +168,12 @@ func (suite *IndexTestSuite) TestIndexAndPrepareAlreadyIndexedAndPrepared() {
testStatus := suite.testStatuses["local_account_1_status_1"]
// index and prepare one post -- it should be indexed
indexed, err := suite.timeline.IndexAndPrepareOne(context.Background(), testStatus.CreatedAt, testStatus.ID, testStatus.BoostOfID, testStatus.AccountID, testStatus.BoostOfAccountID)
indexed, err := suite.timeline.IndexAndPrepareOne(context.Background(), testStatus.ID, testStatus.BoostOfID, testStatus.AccountID, testStatus.BoostOfAccountID)
suite.NoError(err)
suite.True(indexed)
// try to index and prepare the same post again -- it should not be indexed
indexed, err = suite.timeline.IndexAndPrepareOne(context.Background(), testStatus.CreatedAt, testStatus.ID, testStatus.BoostOfID, testStatus.AccountID, testStatus.BoostOfAccountID)
indexed, err = suite.timeline.IndexAndPrepareOne(context.Background(), testStatus.ID, testStatus.BoostOfID, testStatus.AccountID, testStatus.BoostOfAccountID)
suite.NoError(err)
suite.False(indexed)
}
@ -179,12 +189,12 @@ func (suite *IndexTestSuite) TestIndexBoostOfAlreadyIndexed() {
}
// index one post -- it should be indexed
indexed, err := suite.timeline.IndexOne(context.Background(), testStatus.CreatedAt, testStatus.ID, testStatus.BoostOfID, testStatus.AccountID, testStatus.BoostOfAccountID)
indexed, err := suite.timeline.IndexOne(context.Background(), testStatus.ID, testStatus.BoostOfID, testStatus.AccountID, testStatus.BoostOfAccountID)
suite.NoError(err)
suite.True(indexed)
// try to index the a boost of that post -- it should not be indexed
indexed, err = suite.timeline.IndexOne(context.Background(), boostOfTestStatus.CreatedAt, boostOfTestStatus.ID, boostOfTestStatus.BoostOfID, boostOfTestStatus.AccountID, boostOfTestStatus.BoostOfAccountID)
indexed, err = suite.timeline.IndexOne(context.Background(), boostOfTestStatus.ID, boostOfTestStatus.BoostOfID, boostOfTestStatus.AccountID, boostOfTestStatus.BoostOfAccountID)
suite.NoError(err)
suite.False(indexed)
}

View file

@ -20,21 +20,23 @@ package timeline
import (
"container/list"
"context"
"errors"
)
type postIndex struct {
data *list.List
type itemIndex struct {
data *list.List
skipInsert SkipInsertFunction
}
type postIndexEntry struct {
statusID string
type itemIndexEntry struct {
itemID string
boostOfID string
accountID string
boostOfAccountID string
}
func (p *postIndex) insertIndexed(i *postIndexEntry) (bool, error) {
func (p *itemIndex) insertIndexed(ctx context.Context, i *itemIndexEntry) (bool, error) {
if p.data == nil {
p.data = &list.List{}
}
@ -47,36 +49,30 @@ func (p *postIndex) insertIndexed(i *postIndexEntry) (bool, error) {
var insertMark *list.Element
var position int
// We need to iterate through the index to make sure we put this post in the appropriate place according to when it was created.
// We also need to make sure we're not inserting a duplicate post -- this can happen sometimes and it's not nice UX (*shudder*).
// We need to iterate through the index to make sure we put this item in the appropriate place according to when it was created.
// We also need to make sure we're not inserting a duplicate item -- this can happen sometimes and it's not nice UX (*shudder*).
for e := p.data.Front(); e != nil; e = e.Next() {
position++
entry, ok := e.Value.(*postIndexEntry)
entry, ok := e.Value.(*itemIndexEntry)
if !ok {
return false, errors.New("index: could not parse e as a postIndexEntry")
return false, errors.New("index: could not parse e as an itemIndexEntry")
}
// don't insert this if it's a boost of a status we've seen recently
if i.boostOfID != "" {
if i.boostOfID == entry.boostOfID || i.boostOfID == entry.statusID {
if position < boostReinsertionDepth {
return false, nil
}
}
skip, err := p.skipInsert(ctx, i.itemID, i.accountID, i.boostOfID, i.boostOfAccountID, entry.itemID, entry.accountID, entry.boostOfID, entry.boostOfAccountID, position)
if err != nil {
return false, err
}
if skip {
return false, nil
}
// if the post to index is newer than e, insert it before e in the list
// if the item to index is newer than e, insert it before e in the list
if insertMark == nil {
if i.statusID > entry.statusID {
if i.itemID > entry.itemID {
insertMark = e
}
}
// make sure we don't insert a duplicate
if entry.statusID == i.statusID {
return false, nil
}
}
if insertMark != nil {
@ -84,7 +80,7 @@ func (p *postIndex) insertIndexed(i *postIndexEntry) (bool, error) {
return true, nil
}
// if we reach this point it's the oldest post we've seen so put it at the back
// if we reach this point it's the oldest item we've seen so put it at the back
p.data.PushBack(i)
return true, nil
}

View file

@ -25,10 +25,6 @@ import (
"sync"
"github.com/sirupsen/logrus"
apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/typeutils"
)
const (
@ -37,71 +33,75 @@ const (
// Manager abstracts functions for creating timelines for multiple accounts, and adding, removing, and fetching entries from those timelines.
//
// By the time a status hits the manager interface, it should already have been filtered and it should be established that the status indeed
// belongs in the home timeline of the given account ID.
// By the time a timelineable hits the manager interface, it should already have been filtered and it should be established that the item indeed
// belongs in the timeline of the given account ID.
//
// The manager makes a distinction between *indexed* posts and *prepared* posts.
// The manager makes a distinction between *indexed* items and *prepared* items.
//
// Indexed posts consist of just that post's ID (in the database) and the time it was created. An indexed post takes up very little memory, so
// it's not a huge priority to keep trimming the indexed posts list.
// Indexed items consist of just that item's ID (in the database) and the time it was created. An indexed item takes up very little memory, so
// it's not a huge priority to keep trimming the indexed items list.
//
// Prepared posts consist of the post's database ID, the time it was created, AND the apimodel representation of that post, for quick serialization.
// Prepared posts of course take up more memory than indexed posts, so they should be regularly pruned if they're not being actively served.
// Prepared items consist of the item's database ID, the time it was created, AND the apimodel representation of that item, for quick serialization.
// Prepared items of course take up more memory than indexed items, so they should be regularly pruned if they're not being actively served.
type Manager interface {
// Ingest takes one status and indexes it into the timeline for the given account ID.
// Ingest takes one item and indexes it into the timeline for the given account ID.
//
// It should already be established before calling this function that the status/post actually belongs in the timeline!
// It should already be established before calling this function that the item actually belongs in the timeline!
//
// The returned bool indicates whether the status was actually put in the timeline. This could be false in cases where
// the status is a boost, but a boost of the original post or the post itself already exists recently in the timeline.
Ingest(ctx context.Context, status *gtsmodel.Status, timelineAccountID string) (bool, error)
// IngestAndPrepare takes one status and indexes it into the timeline for the given account ID, and then immediately prepares it for serving.
// This is useful in cases where we know the status will need to be shown at the top of a user's timeline immediately (eg., a new status is created).
// The returned bool indicates whether the item was actually put in the timeline. This could be false in cases where
// the item is a boosted status, but a boost of the original status or the status itself already exists recently in the timeline.
Ingest(ctx context.Context, item Timelineable, timelineAccountID string) (bool, error)
// IngestAndPrepare takes one timelineable and indexes it into the timeline for the given account ID, and then immediately prepares it for serving.
// This is useful in cases where we know the item will need to be shown at the top of a user's timeline immediately (eg., a new status is created).
//
// It should already be established before calling this function that the status/post actually belongs in the timeline!
// It should already be established before calling this function that the item actually belongs in the timeline!
//
// The returned bool indicates whether the status was actually put in the timeline. This could be false in cases where
// the status is a boost, but a boost of the original post or the post itself already exists recently in the timeline.
IngestAndPrepare(ctx context.Context, status *gtsmodel.Status, timelineAccountID string) (bool, error)
// HomeTimeline returns limit n amount of entries from the home timeline of the given account ID, in descending chronological order.
// If maxID is provided, it will return entries from that maxID onwards, inclusive.
HomeTimeline(ctx context.Context, accountID string, maxID string, sinceID string, minID string, limit int, local bool) ([]*apimodel.Status, error)
// GetIndexedLength returns the amount of posts/statuses that have been *indexed* for the given account ID.
// The returned bool indicates whether the item was actually put in the timeline. This could be false in cases where
// a status is a boost, but a boost of the original status or the status itself already exists recently in the timeline.
IngestAndPrepare(ctx context.Context, item Timelineable, timelineAccountID string) (bool, error)
// GetTimeline returns limit n amount of prepared entries from the timeline of the given account ID, in descending chronological order.
// If maxID is provided, it will return prepared entries from that maxID onwards, inclusive.
GetTimeline(ctx context.Context, accountID string, maxID string, sinceID string, minID string, limit int, local bool) ([]Preparable, error)
// GetIndexedLength returns the amount of items that have been *indexed* for the given account ID.
GetIndexedLength(ctx context.Context, timelineAccountID string) int
// GetDesiredIndexLength returns the amount of posts that we, ideally, index for each user.
// GetDesiredIndexLength returns the amount of items that we, ideally, index for each user.
GetDesiredIndexLength(ctx context.Context) int
// GetOldestIndexedID returns the status ID for the oldest post that we have indexed for the given account.
// GetOldestIndexedID returns the id ID for the oldest item that we have indexed for the given account.
GetOldestIndexedID(ctx context.Context, timelineAccountID string) (string, error)
// PrepareXFromTop prepares limit n amount of posts, based on their indexed representations, from the top of the index.
// PrepareXFromTop prepares limit n amount of items, based on their indexed representations, from the top of the index.
PrepareXFromTop(ctx context.Context, timelineAccountID string, limit int) error
// Remove removes one status from the timeline of the given timelineAccountID
Remove(ctx context.Context, timelineAccountID string, statusID string) (int, error)
// WipeStatusFromAllTimelines removes one status from the index and prepared posts of all timelines
WipeStatusFromAllTimelines(ctx context.Context, statusID string) error
// WipeStatusesFromAccountID removes all statuses by the given accountID from the timelineAccountID's timelines.
WipeStatusesFromAccountID(ctx context.Context, timelineAccountID string, accountID string) error
// Remove removes one item from the timeline of the given timelineAccountID
Remove(ctx context.Context, timelineAccountID string, itemID string) (int, error)
// WipeItemFromAllTimelines removes one item from the index and prepared items of all timelines
WipeItemFromAllTimelines(ctx context.Context, itemID string) error
// WipeStatusesFromAccountID removes all items by the given accountID from the timelineAccountID's timelines.
WipeItemsFromAccountID(ctx context.Context, timelineAccountID string, accountID string) error
}
// NewManager returns a new timeline manager with the given database, typeconverter, config, and log.
func NewManager(db db.DB, tc typeutils.TypeConverter) Manager {
// NewManager returns a new timeline manager.
func NewManager(grabFunction GrabFunction, filterFunction FilterFunction, prepareFunction PrepareFunction, skipInsertFunction SkipInsertFunction) Manager {
return &manager{
accountTimelines: sync.Map{},
db: db,
tc: tc,
accountTimelines: sync.Map{},
grabFunction: grabFunction,
filterFunction: filterFunction,
prepareFunction: prepareFunction,
skipInsertFunction: skipInsertFunction,
}
}
type manager struct {
accountTimelines sync.Map
db db.DB
tc typeutils.TypeConverter
accountTimelines sync.Map
grabFunction GrabFunction
filterFunction FilterFunction
prepareFunction PrepareFunction
skipInsertFunction SkipInsertFunction
}
func (m *manager) Ingest(ctx context.Context, status *gtsmodel.Status, timelineAccountID string) (bool, error) {
func (m *manager) Ingest(ctx context.Context, item Timelineable, timelineAccountID string) (bool, error) {
l := logrus.WithFields(logrus.Fields{
"func": "Ingest",
"timelineAccountID": timelineAccountID,
"statusID": status.ID,
"itemID": item.GetID(),
})
t, err := m.getOrCreateTimeline(ctx, timelineAccountID)
@ -109,15 +109,15 @@ func (m *manager) Ingest(ctx context.Context, status *gtsmodel.Status, timelineA
return false, err
}
l.Trace("ingesting status")
return t.IndexOne(ctx, status.CreatedAt, status.ID, status.BoostOfID, status.AccountID, status.BoostOfAccountID)
l.Trace("ingesting item")
return t.IndexOne(ctx, item.GetID(), item.GetBoostOfID(), item.GetAccountID(), item.GetBoostOfAccountID())
}
func (m *manager) IngestAndPrepare(ctx context.Context, status *gtsmodel.Status, timelineAccountID string) (bool, error) {
func (m *manager) IngestAndPrepare(ctx context.Context, item Timelineable, timelineAccountID string) (bool, error) {
l := logrus.WithFields(logrus.Fields{
"func": "IngestAndPrepare",
"timelineAccountID": timelineAccountID,
"statusID": status.ID,
"itemID": item.GetID(),
})
t, err := m.getOrCreateTimeline(ctx, timelineAccountID)
@ -125,15 +125,15 @@ func (m *manager) IngestAndPrepare(ctx context.Context, status *gtsmodel.Status,
return false, err
}
l.Trace("ingesting status")
return t.IndexAndPrepareOne(ctx, status.CreatedAt, status.ID, status.BoostOfID, status.AccountID, status.BoostOfAccountID)
l.Trace("ingesting item")
return t.IndexAndPrepareOne(ctx, item.GetID(), item.GetBoostOfID(), item.GetAccountID(), item.GetBoostOfAccountID())
}
func (m *manager) Remove(ctx context.Context, timelineAccountID string, statusID string) (int, error) {
func (m *manager) Remove(ctx context.Context, timelineAccountID string, itemID string) (int, error) {
l := logrus.WithFields(logrus.Fields{
"func": "Remove",
"timelineAccountID": timelineAccountID,
"statusID": statusID,
"itemID": itemID,
})
t, err := m.getOrCreateTimeline(ctx, timelineAccountID)
@ -141,13 +141,13 @@ func (m *manager) Remove(ctx context.Context, timelineAccountID string, statusID
return 0, err
}
l.Trace("removing status")
return t.Remove(ctx, statusID)
l.Trace("removing item")
return t.Remove(ctx, itemID)
}
func (m *manager) HomeTimeline(ctx context.Context, timelineAccountID string, maxID string, sinceID string, minID string, limit int, local bool) ([]*apimodel.Status, error) {
func (m *manager) GetTimeline(ctx context.Context, timelineAccountID string, maxID string, sinceID string, minID string, limit int, local bool) ([]Preparable, error) {
l := logrus.WithFields(logrus.Fields{
"func": "HomeTimelineGet",
"func": "GetTimeline",
"timelineAccountID": timelineAccountID,
})
@ -156,11 +156,11 @@ func (m *manager) HomeTimeline(ctx context.Context, timelineAccountID string, ma
return nil, err
}
statuses, err := t.Get(ctx, limit, maxID, sinceID, minID, true)
items, err := t.Get(ctx, limit, maxID, sinceID, minID, true)
if err != nil {
l.Errorf("error getting statuses: %s", err)
}
return statuses, nil
return items, nil
}
func (m *manager) GetIndexedLength(ctx context.Context, timelineAccountID string) int {
@ -169,7 +169,7 @@ func (m *manager) GetIndexedLength(ctx context.Context, timelineAccountID string
return 0
}
return t.PostIndexLength(ctx)
return t.ItemIndexLength(ctx)
}
func (m *manager) GetDesiredIndexLength(ctx context.Context) int {
@ -182,7 +182,7 @@ func (m *manager) GetOldestIndexedID(ctx context.Context, timelineAccountID stri
return "", err
}
return t.OldestIndexedPostID(ctx)
return t.OldestIndexedItemID(ctx)
}
func (m *manager) PrepareXFromTop(ctx context.Context, timelineAccountID string, limit int) error {
@ -194,7 +194,7 @@ func (m *manager) PrepareXFromTop(ctx context.Context, timelineAccountID string,
return t.PrepareFromTop(ctx, limit)
}
func (m *manager) WipeStatusFromAllTimelines(ctx context.Context, statusID string) error {
func (m *manager) WipeItemFromAllTimelines(ctx context.Context, statusID string) error {
errors := []string{}
m.accountTimelines.Range(func(k interface{}, i interface{}) bool {
t, ok := i.(Timeline)
@ -217,7 +217,7 @@ func (m *manager) WipeStatusFromAllTimelines(ctx context.Context, statusID strin
return err
}
func (m *manager) WipeStatusesFromAccountID(ctx context.Context, timelineAccountID string, accountID string) error {
func (m *manager) WipeItemsFromAccountID(ctx context.Context, timelineAccountID string, accountID string) error {
t, err := m.getOrCreateTimeline(ctx, timelineAccountID)
if err != nil {
return err
@ -232,7 +232,7 @@ func (m *manager) getOrCreateTimeline(ctx context.Context, timelineAccountID str
i, ok := m.accountTimelines.Load(timelineAccountID)
if !ok {
var err error
t, err = NewTimeline(ctx, timelineAccountID, m.db, m.tc)
t, err = NewTimeline(ctx, timelineAccountID, m.grabFunction, m.filterFunction, m.prepareFunction, m.skipInsertFunction)
if err != nil {
return nil, err
}

View file

@ -23,6 +23,9 @@ import (
"testing"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/processing"
"github.com/superseriousbusiness/gotosocial/internal/timeline"
"github.com/superseriousbusiness/gotosocial/internal/visibility"
"github.com/superseriousbusiness/gotosocial/testrig"
)
@ -41,10 +44,16 @@ func (suite *ManagerTestSuite) SetupTest() {
suite.db = testrig.NewTestDB()
suite.tc = testrig.NewTestTypeConverter(suite.db)
suite.filter = visibility.NewFilter(suite.db)
testrig.StandardDBSetup(suite.db, nil)
manager := testrig.NewTestTimelineManager(suite.db)
manager := timeline.NewManager(
processing.StatusGrabFunction(suite.db),
processing.StatusFilterFunction(suite.db, suite.filter),
processing.StatusPrepareFunction(suite.db, suite.tc),
processing.StatusSkipInsertFunction(),
)
suite.manager = manager
}
@ -78,12 +87,12 @@ func (suite *ManagerTestSuite) TestManagerIntegration() {
suite.Equal("01F8MH75CBF9JFX4ZAD54N0W0R", oldestIndexed)
// get hometimeline
statuses, err := suite.manager.HomeTimeline(context.Background(), testAccount.ID, "", "", "", 20, false)
statuses, err := suite.manager.GetTimeline(context.Background(), testAccount.ID, "", "", "", 20, false)
suite.NoError(err)
suite.Len(statuses, 14)
// now wipe the last status from all timelines, as though it had been deleted by the owner
err = suite.manager.WipeStatusFromAllTimelines(context.Background(), "01F8MH75CBF9JFX4ZAD54N0W0R")
err = suite.manager.WipeItemFromAllTimelines(context.Background(), "01F8MH75CBF9JFX4ZAD54N0W0R")
suite.NoError(err)
// timeline should be shorter
@ -110,7 +119,7 @@ func (suite *ManagerTestSuite) TestManagerIntegration() {
suite.Equal("01F8MHAAY43M6RJ473VQFCVH37", oldestIndexed)
// now remove all entries by local_account_2 from the timeline
err = suite.manager.WipeStatusesFromAccountID(context.Background(), testAccount.ID, suite.testAccounts["local_account_2"].ID)
err = suite.manager.WipeItemsFromAccountID(context.Background(), testAccount.ID, suite.testAccounts["local_account_2"].ID)
suite.NoError(err)
// timeline should be shorter

View file

@ -0,0 +1,26 @@
/*
GoToSocial
Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package timeline
type Preparable interface {
GetID() string
GetAccountID() string
GetBoostOfID() string
GetBoostOfAccountID() string
}

View file

@ -26,7 +26,6 @@ import (
"github.com/sirupsen/logrus"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
)
func (t *timeline) prepareNextQuery(ctx context.Context, amount int, maxID string, sinceID string, minID string) error {
@ -59,19 +58,19 @@ func (t *timeline) prepareNextQuery(ctx context.Context, amount int, maxID strin
return err
}
func (t *timeline) PrepareBehind(ctx context.Context, statusID string, amount int) error {
// lazily initialize prepared posts if it hasn't been done already
if t.preparedPosts.data == nil {
t.preparedPosts.data = &list.List{}
t.preparedPosts.data.Init()
func (t *timeline) PrepareBehind(ctx context.Context, itemID string, amount int) error {
// lazily initialize prepared items if it hasn't been done already
if t.preparedItems.data == nil {
t.preparedItems.data = &list.List{}
t.preparedItems.data.Init()
}
if err := t.IndexBehind(ctx, statusID, true, amount); err != nil {
return fmt.Errorf("PrepareBehind: error indexing behind id %s: %s", statusID, err)
if err := t.IndexBehind(ctx, itemID, amount); err != nil {
return fmt.Errorf("PrepareBehind: error indexing behind id %s: %s", itemID, err)
}
// if the postindex is nil, nothing has been indexed yet so there's nothing to prepare
if t.postIndex.data == nil {
// if the itemindex is nil, nothing has been indexed yet so there's nothing to prepare
if t.itemIndex.data == nil {
return nil
}
@ -80,25 +79,25 @@ func (t *timeline) PrepareBehind(ctx context.Context, statusID string, amount in
t.Lock()
defer t.Unlock()
prepareloop:
for e := t.postIndex.data.Front(); e != nil; e = e.Next() {
entry, ok := e.Value.(*postIndexEntry)
for e := t.itemIndex.data.Front(); e != nil; e = e.Next() {
entry, ok := e.Value.(*itemIndexEntry)
if !ok {
return errors.New("PrepareBehind: could not parse e as a postIndexEntry")
return errors.New("PrepareBehind: could not parse e as itemIndexEntry")
}
if !preparing {
// we haven't hit the position we need to prepare from yet
if entry.statusID == statusID {
if entry.itemID == itemID {
preparing = true
}
}
if preparing {
if err := t.prepare(ctx, entry.statusID); err != nil {
if err := t.prepare(ctx, entry.itemID); err != nil {
// there's been an error
if err != db.ErrNoEntries {
// it's a real error
return fmt.Errorf("PrepareBehind: error preparing status with id %s: %s", entry.statusID, err)
return fmt.Errorf("PrepareBehind: error preparing item with id %s: %s", entry.itemID, err)
}
// the status just doesn't exist (anymore) so continue to the next one
continue
@ -119,28 +118,28 @@ func (t *timeline) PrepareBefore(ctx context.Context, statusID string, include b
defer t.Unlock()
// lazily initialize prepared posts if it hasn't been done already
if t.preparedPosts.data == nil {
t.preparedPosts.data = &list.List{}
t.preparedPosts.data.Init()
if t.preparedItems.data == nil {
t.preparedItems.data = &list.List{}
t.preparedItems.data.Init()
}
// if the postindex is nil, nothing has been indexed yet so there's nothing to prepare
if t.postIndex.data == nil {
if t.itemIndex.data == nil {
return nil
}
var prepared int
var preparing bool
prepareloop:
for e := t.postIndex.data.Back(); e != nil; e = e.Prev() {
entry, ok := e.Value.(*postIndexEntry)
for e := t.itemIndex.data.Back(); e != nil; e = e.Prev() {
entry, ok := e.Value.(*itemIndexEntry)
if !ok {
return errors.New("PrepareBefore: could not parse e as a postIndexEntry")
}
if !preparing {
// we haven't hit the position we need to prepare from yet
if entry.statusID == statusID {
if entry.itemID == statusID {
preparing = true
if !include {
continue
@ -149,11 +148,11 @@ prepareloop:
}
if preparing {
if err := t.prepare(ctx, entry.statusID); err != nil {
if err := t.prepare(ctx, entry.itemID); err != nil {
// there's been an error
if err != db.ErrNoEntries {
// it's a real error
return fmt.Errorf("PrepareBefore: error preparing status with id %s: %s", entry.statusID, err)
return fmt.Errorf("PrepareBefore: error preparing status with id %s: %s", entry.itemID, err)
}
// the status just doesn't exist (anymore) so continue to the next one
continue
@ -176,15 +175,15 @@ func (t *timeline) PrepareFromTop(ctx context.Context, amount int) error {
})
// lazily initialize prepared posts if it hasn't been done already
if t.preparedPosts.data == nil {
t.preparedPosts.data = &list.List{}
t.preparedPosts.data.Init()
if t.preparedItems.data == nil {
t.preparedItems.data = &list.List{}
t.preparedItems.data.Init()
}
// if the postindex is nil, nothing has been indexed yet so index from the highest ID possible
if t.postIndex.data == nil {
if t.itemIndex.data == nil {
l.Debug("postindex.data was nil, indexing behind highest possible ID")
if err := t.IndexBehind(ctx, "ZZZZZZZZZZZZZZZZZZZZZZZZZZ", false, amount); err != nil {
if err := t.IndexBehind(ctx, "ZZZZZZZZZZZZZZZZZZZZZZZZZZ", amount); err != nil {
return fmt.Errorf("PrepareFromTop: error indexing behind id %s: %s", "ZZZZZZZZZZZZZZZZZZZZZZZZZZ", err)
}
}
@ -194,21 +193,21 @@ func (t *timeline) PrepareFromTop(ctx context.Context, amount int) error {
defer t.Unlock()
var prepared int
prepareloop:
for e := t.postIndex.data.Front(); e != nil; e = e.Next() {
for e := t.itemIndex.data.Front(); e != nil; e = e.Next() {
if e == nil {
continue
}
entry, ok := e.Value.(*postIndexEntry)
entry, ok := e.Value.(*itemIndexEntry)
if !ok {
return errors.New("PrepareFromTop: could not parse e as a postIndexEntry")
}
if err := t.prepare(ctx, entry.statusID); err != nil {
if err := t.prepare(ctx, entry.itemID); err != nil {
// there's been an error
if err != db.ErrNoEntries {
// it's a real error
return fmt.Errorf("PrepareFromTop: error preparing status with id %s: %s", entry.statusID, err)
return fmt.Errorf("PrepareFromTop: error preparing status with id %s: %s", entry.itemID, err)
}
// the status just doesn't exist (anymore) so continue to the next one
continue
@ -226,57 +225,42 @@ prepareloop:
return nil
}
func (t *timeline) prepare(ctx context.Context, statusID string) error {
// start by getting the status out of the database according to its indexed ID
gtsStatus := &gtsmodel.Status{}
if err := t.db.GetByID(ctx, statusID, gtsStatus); err != nil {
return err
}
// if the account pointer hasn't been set on this timeline already, set it lazily here
if t.account == nil {
timelineOwnerAccount := &gtsmodel.Account{}
if err := t.db.GetByID(ctx, t.accountID, timelineOwnerAccount); err != nil {
return err
}
t.account = timelineOwnerAccount
}
// serialize the status (or, at least, convert it to a form that's ready to be serialized)
apiModelStatus, err := t.tc.StatusToAPIStatus(ctx, gtsStatus, t.account)
func (t *timeline) prepare(ctx context.Context, itemID string) error {
// trigger the caller-provided prepare function
prepared, err := t.prepareFunction(ctx, t.accountID, itemID)
if err != nil {
return err
}
// shove it in prepared posts as a prepared posts entry
preparedPostsEntry := &preparedPostsEntry{
statusID: gtsStatus.ID,
boostOfID: gtsStatus.BoostOfID,
accountID: gtsStatus.AccountID,
boostOfAccountID: gtsStatus.BoostOfAccountID,
prepared: apiModelStatus,
// shove it in prepared items as a prepared items entry
preparedItemsEntry := &preparedItemsEntry{
itemID: prepared.GetID(),
boostOfID: prepared.GetBoostOfID(),
accountID: prepared.GetAccountID(),
boostOfAccountID: prepared.GetBoostOfAccountID(),
prepared: prepared,
}
return t.preparedPosts.insertPrepared(preparedPostsEntry)
return t.preparedItems.insertPrepared(ctx, preparedItemsEntry)
}
func (t *timeline) OldestPreparedPostID(ctx context.Context) (string, error) {
func (t *timeline) OldestPreparedItemID(ctx context.Context) (string, error) {
var id string
if t.preparedPosts == nil || t.preparedPosts.data == nil {
// return an empty string if prepared posts hasn't been initialized yet
if t.preparedItems == nil || t.preparedItems.data == nil {
// return an empty string if prepared items hasn't been initialized yet
return id, nil
}
e := t.preparedPosts.data.Back()
e := t.preparedItems.data.Back()
if e == nil {
// return an empty string if there's no back entry (ie., the index list hasn't been initialized yet)
return id, nil
}
entry, ok := e.Value.(*preparedPostsEntry)
entry, ok := e.Value.(*preparedItemsEntry)
if !ok {
return id, errors.New("OldestPreparedPostID: could not parse e as a preparedPostsEntry")
return id, errors.New("OldestPreparedItemID: could not parse e as a preparedItemsEntry")
}
return entry.statusID, nil
return entry.itemID, nil
}

View file

@ -20,24 +20,24 @@ package timeline
import (
"container/list"
"context"
"errors"
apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
)
type preparedPosts struct {
data *list.List
type preparedItems struct {
data *list.List
skipInsert SkipInsertFunction
}
type preparedPostsEntry struct {
statusID string
type preparedItemsEntry struct {
itemID string
boostOfID string
accountID string
boostOfAccountID string
prepared *apimodel.Status
prepared Preparable
}
func (p *preparedPosts) insertPrepared(i *preparedPostsEntry) error {
func (p *preparedItems) insertPrepared(ctx context.Context, i *preparedItemsEntry) error {
if p.data == nil {
p.data = &list.List{}
}
@ -55,35 +55,28 @@ func (p *preparedPosts) insertPrepared(i *preparedPostsEntry) error {
for e := p.data.Front(); e != nil; e = e.Next() {
position++
entry, ok := e.Value.(*preparedPostsEntry)
entry, ok := e.Value.(*preparedItemsEntry)
if !ok {
return errors.New("index: could not parse e as a preparedPostsEntry")
}
// don't insert this if it's a boost of a status we've seen recently
if i.prepared.Reblog != nil {
if entry.prepared.Reblog != nil && i.prepared.Reblog.ID == entry.prepared.Reblog.ID {
if position < boostReinsertionDepth {
return nil
}
}
if i.prepared.Reblog.ID == entry.statusID {
if position < boostReinsertionDepth {
return nil
}
}
skip, err := p.skipInsert(ctx, i.itemID, i.accountID, i.boostOfID, i.boostOfAccountID, entry.itemID, entry.accountID, entry.boostOfID, entry.boostOfAccountID, position)
if err != nil {
return err
}
if skip {
return nil
}
// if the post to index is newer than e, insert it before e in the list
if insertMark == nil {
if i.statusID > entry.statusID {
if i.itemID > entry.itemID {
insertMark = e
}
}
// make sure we don't insert a duplicate
if entry.statusID == i.statusID {
if entry.itemID == i.itemID {
return nil
}
}

View file

@ -38,39 +38,39 @@ func (t *timeline) Remove(ctx context.Context, statusID string) (int, error) {
// remove entr(ies) from the post index
removeIndexes := []*list.Element{}
if t.postIndex != nil && t.postIndex.data != nil {
for e := t.postIndex.data.Front(); e != nil; e = e.Next() {
entry, ok := e.Value.(*postIndexEntry)
if t.itemIndex != nil && t.itemIndex.data != nil {
for e := t.itemIndex.data.Front(); e != nil; e = e.Next() {
entry, ok := e.Value.(*itemIndexEntry)
if !ok {
return removed, errors.New("Remove: could not parse e as a postIndexEntry")
}
if entry.statusID == statusID {
if entry.itemID == statusID {
l.Debug("found status in postIndex")
removeIndexes = append(removeIndexes, e)
}
}
}
for _, e := range removeIndexes {
t.postIndex.data.Remove(e)
t.itemIndex.data.Remove(e)
removed++
}
// remove entr(ies) from prepared posts
removePrepared := []*list.Element{}
if t.preparedPosts != nil && t.preparedPosts.data != nil {
for e := t.preparedPosts.data.Front(); e != nil; e = e.Next() {
entry, ok := e.Value.(*preparedPostsEntry)
if t.preparedItems != nil && t.preparedItems.data != nil {
for e := t.preparedItems.data.Front(); e != nil; e = e.Next() {
entry, ok := e.Value.(*preparedItemsEntry)
if !ok {
return removed, errors.New("Remove: could not parse e as a preparedPostsEntry")
}
if entry.statusID == statusID {
if entry.itemID == statusID {
l.Debug("found status in preparedPosts")
removePrepared = append(removePrepared, e)
}
}
}
for _, e := range removePrepared {
t.preparedPosts.data.Remove(e)
t.preparedItems.data.Remove(e)
removed++
}
@ -90,9 +90,9 @@ func (t *timeline) RemoveAllBy(ctx context.Context, accountID string) (int, erro
// remove entr(ies) from the post index
removeIndexes := []*list.Element{}
if t.postIndex != nil && t.postIndex.data != nil {
for e := t.postIndex.data.Front(); e != nil; e = e.Next() {
entry, ok := e.Value.(*postIndexEntry)
if t.itemIndex != nil && t.itemIndex.data != nil {
for e := t.itemIndex.data.Front(); e != nil; e = e.Next() {
entry, ok := e.Value.(*itemIndexEntry)
if !ok {
return removed, errors.New("Remove: could not parse e as a postIndexEntry")
}
@ -103,15 +103,15 @@ func (t *timeline) RemoveAllBy(ctx context.Context, accountID string) (int, erro
}
}
for _, e := range removeIndexes {
t.postIndex.data.Remove(e)
t.itemIndex.data.Remove(e)
removed++
}
// remove entr(ies) from prepared posts
removePrepared := []*list.Element{}
if t.preparedPosts != nil && t.preparedPosts.data != nil {
for e := t.preparedPosts.data.Front(); e != nil; e = e.Next() {
entry, ok := e.Value.(*preparedPostsEntry)
if t.preparedItems != nil && t.preparedItems.data != nil {
for e := t.preparedItems.data.Front(); e != nil; e = e.Next() {
entry, ok := e.Value.(*preparedItemsEntry)
if !ok {
return removed, errors.New("Remove: could not parse e as a preparedPostsEntry")
}
@ -122,7 +122,7 @@ func (t *timeline) RemoveAllBy(ctx context.Context, accountID string) (int, erro
}
}
for _, e := range removePrepared {
t.preparedPosts.data.Remove(e)
t.preparedItems.data.Remove(e)
removed++
}

View file

@ -21,104 +21,135 @@ package timeline
import (
"context"
"sync"
"time"
apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/typeutils"
"github.com/superseriousbusiness/gotosocial/internal/visibility"
)
const boostReinsertionDepth = 50
// GrabFunction is used by a Timeline to grab more items to index.
//
// It should be provided to NewTimeline when the caller is creating a timeline
// (of statuses, notifications, etc).
//
// timelineAccountID: the owner of the timeline
// maxID: the maximum item ID desired.
// sinceID: the minimum item ID desired.
// minID: see sinceID
// limit: the maximum amount of items to be returned
//
// If an error is returned, the timeline will stop processing whatever request called GrabFunction,
// and return the error. If no error is returned, but stop = true, this indicates to the caller of GrabFunction
// that there are no more items to return, and processing should continue with the items already grabbed.
type GrabFunction func(ctx context.Context, timelineAccountID string, maxID string, sinceID string, minID string, limit int) (items []Timelineable, stop bool, err error)
// Timeline represents a timeline for one account, and contains indexed and prepared posts.
// FilterFunction is used by a Timeline to filter whether or not a grabbed item should be indexed.
type FilterFunction func(ctx context.Context, timelineAccountID string, item Timelineable) (shouldIndex bool, err error)
// PrepareFunction converts a Timelineable into a Preparable.
//
// For example, this might result in the converstion of a *gtsmodel.Status with the given itemID into a serializable *apimodel.Status.
type PrepareFunction func(ctx context.Context, timelineAccountID string, itemID string) (Preparable, error)
// SkipInsertFunction indicates whether a new item about to be inserted in the prepared list should be skipped,
// based on the item itself, the next item in the timeline, and the depth at which nextItem has been found in the list.
//
// This will be called for every item found while iterating through a timeline, so callers should be very careful
// not to do anything expensive here.
type SkipInsertFunction func(ctx context.Context,
newItemID string,
newItemAccountID string,
newItemBoostOfID string,
newItemBoostOfAccountID string,
nextItemID string,
nextItemAccountID string,
nextItemBoostOfID string,
nextItemBoostOfAccountID string,
depth int) (bool, error)
// Timeline represents a timeline for one account, and contains indexed and prepared items.
type Timeline interface {
/*
RETRIEVAL FUNCTIONS
*/
// Get returns an amount of statuses with the given parameters.
// Get returns an amount of prepared items with the given parameters.
// If prepareNext is true, then the next predicted query will be prepared already in a goroutine,
// to make the next call to Get faster.
Get(ctx context.Context, amount int, maxID string, sinceID string, minID string, prepareNext bool) ([]*apimodel.Status, error)
// GetXFromTop returns x amount of posts from the top of the timeline, from newest to oldest.
GetXFromTop(ctx context.Context, amount int) ([]*apimodel.Status, error)
// GetXBehindID returns x amount of posts from the given id onwards, from newest to oldest.
// This will NOT include the status with the given ID.
Get(ctx context.Context, amount int, maxID string, sinceID string, minID string, prepareNext bool) ([]Preparable, error)
// GetXFromTop returns x amount of items from the top of the timeline, from newest to oldest.
GetXFromTop(ctx context.Context, amount int) ([]Preparable, error)
// GetXBehindID returns x amount of items from the given id onwards, from newest to oldest.
// This will NOT include the item with the given ID.
//
// This corresponds to an api call to /timelines/home?max_id=WHATEVER
GetXBehindID(ctx context.Context, amount int, fromID string, attempts *int) ([]*apimodel.Status, error)
// GetXBeforeID returns x amount of posts up to the given id, from newest to oldest.
// This will NOT include the status with the given ID.
GetXBehindID(ctx context.Context, amount int, fromID string, attempts *int) ([]Preparable, error)
// GetXBeforeID returns x amount of items up to the given id, from newest to oldest.
// This will NOT include the item with the given ID.
//
// This corresponds to an api call to /timelines/home?since_id=WHATEVER
GetXBeforeID(ctx context.Context, amount int, sinceID string, startFromTop bool) ([]*apimodel.Status, error)
// GetXBetweenID returns x amount of posts from the given maxID, up to the given id, from newest to oldest.
// This will NOT include the status with the given IDs.
GetXBeforeID(ctx context.Context, amount int, sinceID string, startFromTop bool) ([]Preparable, error)
// GetXBetweenID returns x amount of items from the given maxID, up to the given id, from newest to oldest.
// This will NOT include the item with the given IDs.
//
// This corresponds to an api call to /timelines/home?since_id=WHATEVER&max_id=WHATEVER_ELSE
GetXBetweenID(ctx context.Context, amount int, maxID string, sinceID string) ([]*apimodel.Status, error)
GetXBetweenID(ctx context.Context, amount int, maxID string, sinceID string) ([]Preparable, error)
/*
INDEXING FUNCTIONS
*/
// IndexOne puts a status into the timeline at the appropriate place according to its 'createdAt' property.
// IndexOne puts a item into the timeline at the appropriate place according to its 'createdAt' property.
//
// The returned bool indicates whether or not the status was actually inserted into the timeline. This will be false
// if the status is a boost and the original post or another boost of it already exists < boostReinsertionDepth back in the timeline.
IndexOne(ctx context.Context, statusCreatedAt time.Time, statusID string, boostOfID string, accountID string, boostOfAccountID string) (bool, error)
// The returned bool indicates whether or not the item was actually inserted into the timeline. This will be false
// if the item is a boost and the original item or another boost of it already exists < boostReinsertionDepth back in the timeline.
IndexOne(ctx context.Context, itemID string, boostOfID string, accountID string, boostOfAccountID string) (bool, error)
// OldestIndexedPostID returns the id of the rearmost (ie., the oldest) indexed post, or an error if something goes wrong.
// If nothing goes wrong but there's no oldest post, an empty string will be returned so make sure to check for this.
OldestIndexedPostID(ctx context.Context) (string, error)
// NewestIndexedPostID returns the id of the frontmost (ie., the newest) indexed post, or an error if something goes wrong.
// If nothing goes wrong but there's no newest post, an empty string will be returned so make sure to check for this.
NewestIndexedPostID(ctx context.Context) (string, error)
// OldestIndexedItemID returns the id of the rearmost (ie., the oldest) indexed item, or an error if something goes wrong.
// If nothing goes wrong but there's no oldest item, an empty string will be returned so make sure to check for this.
OldestIndexedItemID(ctx context.Context) (string, error)
// NewestIndexedItemID returns the id of the frontmost (ie., the newest) indexed item, or an error if something goes wrong.
// If nothing goes wrong but there's no newest item, an empty string will be returned so make sure to check for this.
NewestIndexedItemID(ctx context.Context) (string, error)
IndexBefore(ctx context.Context, statusID string, include bool, amount int) error
IndexBehind(ctx context.Context, statusID string, include bool, amount int) error
IndexBefore(ctx context.Context, itemID string, amount int) error
IndexBehind(ctx context.Context, itemID string, amount int) error
/*
PREPARATION FUNCTIONS
*/
// PrepareXFromTop instructs the timeline to prepare x amount of posts from the top of the timeline.
// PrepareXFromTop instructs the timeline to prepare x amount of items from the top of the timeline.
PrepareFromTop(ctx context.Context, amount int) error
// PrepareBehind instructs the timeline to prepare the next amount of entries for serialization, from position onwards.
// If include is true, then the given status ID will also be prepared, otherwise only entries behind it will be prepared.
PrepareBehind(ctx context.Context, statusID string, amount int) error
// IndexOne puts a status into the timeline at the appropriate place according to its 'createdAt' property,
// If include is true, then the given item ID will also be prepared, otherwise only entries behind it will be prepared.
PrepareBehind(ctx context.Context, itemID string, amount int) error
// IndexOne puts a item into the timeline at the appropriate place according to its 'createdAt' property,
// and then immediately prepares it.
//
// The returned bool indicates whether or not the status was actually inserted into the timeline. This will be false
// if the status is a boost and the original post or another boost of it already exists < boostReinsertionDepth back in the timeline.
IndexAndPrepareOne(ctx context.Context, statusCreatedAt time.Time, statusID string, boostOfID string, accountID string, boostOfAccountID string) (bool, error)
// OldestPreparedPostID returns the id of the rearmost (ie., the oldest) prepared post, or an error if something goes wrong.
// If nothing goes wrong but there's no oldest post, an empty string will be returned so make sure to check for this.
OldestPreparedPostID(ctx context.Context) (string, error)
// The returned bool indicates whether or not the item was actually inserted into the timeline. This will be false
// if the item is a boost and the original item or another boost of it already exists < boostReinsertionDepth back in the timeline.
IndexAndPrepareOne(ctx context.Context, itemID string, boostOfID string, accountID string, boostOfAccountID string) (bool, error)
// OldestPreparedItemID returns the id of the rearmost (ie., the oldest) prepared item, or an error if something goes wrong.
// If nothing goes wrong but there's no oldest item, an empty string will be returned so make sure to check for this.
OldestPreparedItemID(ctx context.Context) (string, error)
/*
INFO FUNCTIONS
*/
// ActualPostIndexLength returns the actual length of the post index at this point in time.
PostIndexLength(ctx context.Context) int
// ActualPostIndexLength returns the actual length of the item index at this point in time.
ItemIndexLength(ctx context.Context) int
/*
UTILITY FUNCTIONS
*/
// Reset instructs the timeline to reset to its base state -- cache only the minimum amount of posts.
// Reset instructs the timeline to reset to its base state -- cache only the minimum amount of items.
Reset() error
// Remove removes a status from both the index and prepared posts.
// Remove removes a item from both the index and prepared items.
//
// If a status has multiple entries in a timeline, they will all be removed.
// If a item has multiple entries in a timeline, they will all be removed.
//
// The returned int indicates the amount of entries that were removed.
Remove(ctx context.Context, statusID string) (int, error)
// RemoveAllBy removes all statuses by the given accountID, from both the index and prepared posts.
Remove(ctx context.Context, itemID string) (int, error)
// RemoveAllBy removes all items by the given accountID, from both the index and prepared items.
//
// The returned int indicates the amount of entries that were removed.
RemoveAllBy(ctx context.Context, accountID string) (int, error)
@ -126,31 +157,34 @@ type Timeline interface {
// timeline fulfils the Timeline interface
type timeline struct {
postIndex *postIndex
preparedPosts *preparedPosts
accountID string
account *gtsmodel.Account
db db.DB
filter visibility.Filter
tc typeutils.TypeConverter
itemIndex *itemIndex
preparedItems *preparedItems
grabFunction GrabFunction
filterFunction FilterFunction
prepareFunction PrepareFunction
accountID string
sync.Mutex
}
// NewTimeline returns a new Timeline for the given account ID
func NewTimeline(ctx context.Context, accountID string, db db.DB, typeConverter typeutils.TypeConverter) (Timeline, error) {
timelineOwnerAccount := &gtsmodel.Account{}
if err := db.GetByID(ctx, accountID, timelineOwnerAccount); err != nil {
return nil, err
}
func NewTimeline(
ctx context.Context,
timelineAccountID string,
grabFunction GrabFunction,
filterFunction FilterFunction,
prepareFunction PrepareFunction,
skipInsertFunction SkipInsertFunction) (Timeline, error) {
return &timeline{
postIndex: &postIndex{},
preparedPosts: &preparedPosts{},
accountID: accountID,
account: timelineOwnerAccount,
db: db,
filter: visibility.NewFilter(db),
tc: typeConverter,
itemIndex: &itemIndex{
skipInsert: skipInsertFunction,
},
preparedItems: &preparedItems{
skipInsert: skipInsertFunction,
},
grabFunction: grabFunction,
filterFunction: filterFunction,
prepareFunction: prepareFunction,
accountID: timelineAccountID,
}, nil
}
@ -158,10 +192,10 @@ func (t *timeline) Reset() error {
return nil
}
func (t *timeline) PostIndexLength(ctx context.Context) int {
if t.postIndex == nil || t.postIndex.data == nil {
func (t *timeline) ItemIndexLength(ctx context.Context) int {
if t.itemIndex == nil || t.itemIndex.data == nil {
return 0
}
return t.postIndex.data.Len()
return t.itemIndex.data.Len()
}

View file

@ -24,12 +24,14 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/timeline"
"github.com/superseriousbusiness/gotosocial/internal/typeutils"
"github.com/superseriousbusiness/gotosocial/internal/visibility"
)
type TimelineStandardTestSuite struct {
suite.Suite
db db.DB
tc typeutils.TypeConverter
db db.DB
tc typeutils.TypeConverter
filter visibility.Filter
testAccounts map[string]*gtsmodel.Account
testStatuses map[string]*gtsmodel.Status

View file

@ -0,0 +1,27 @@
/*
GoToSocial
Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package timeline
// Timelineable represents any item that can be put in a timeline.
type Timelineable interface {
GetID() string
GetAccountID() string
GetBoostOfID() string
GetBoostOfAccountID() string
}

View file

@ -28,5 +28,5 @@ import (
// NewTestProcessor returns a Processor suitable for testing purposes
func NewTestProcessor(db db.DB, storage *kv.KVStore, federator federation.Federator, emailSender email.Sender) processing.Processor {
return processing.NewProcessor(NewTestTypeConverter(db), federator, NewTestOauthServer(db), NewTestMediaHandler(db, storage), storage, NewTestTimelineManager(db), db, emailSender)
return processing.NewProcessor(NewTestTypeConverter(db), federator, NewTestOauthServer(db), NewTestMediaHandler(db, storage), storage, db, emailSender)
}

View file

@ -1,11 +0,0 @@
package testrig
import (
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/timeline"
)
// NewTestTimelineManager retuts a new timeline.Manager, suitable for testing, using the given db.
func NewTestTimelineManager(db db.DB) timeline.Manager {
return timeline.NewManager(db, NewTestTypeConverter(db))
}