From fccb0bc102f2a54a21eed343cda64f9a5221b677 Mon Sep 17 00:00:00 2001 From: Vyr Cossont Date: Wed, 12 Feb 2025 09:49:33 -0800 Subject: [PATCH] [feature] Implement backfilling statuses thru scheduled_at (#3685) * Implement backfilling statuses thru scheduled_at * Forbid mentioning others in backfills * Update error messages & codes * Add new tests for backfilled statuses * Test that backfilling doesn't timeline or notify * Fix check for absence of notification * Test that backfills do not cause federation * Fix type of apimodel.StatusCreateRequest.ScheduledAt in tests * Add config file switch and min date check --- docs/api/swagger.yaml | 10 +- docs/configuration/instance.md | 13 ++ example/config.yaml | 13 ++ internal/api/client/statuses/statuscreate.go | 16 +- .../api/client/statuses/statuscreate_test.go | 132 ++++++++++++++- internal/api/model/status.go | 15 +- internal/config/config.go | 1 + internal/config/defaults.go | 1 + internal/config/flags.go | 1 + internal/config/helpers.gen.go | 25 +++ .../federation/dereferencing/instance_test.go | 6 +- internal/gtsmodel/status.go | 9 +- internal/processing/status/create.go | 118 ++++++++++++- internal/processing/status/create_test.go | 14 +- internal/processing/workers/fromclientapi.go | 25 ++- .../processing/workers/fromclientapi_test.go | 156 ++++++++++++++++++ test/envparsing.sh | 1 + testrig/config.go | 1 + 18 files changed, 515 insertions(+), 42 deletions(-) diff --git a/docs/api/swagger.yaml b/docs/api/swagger.yaml index df8f09321..836df83e8 100644 --- a/docs/api/swagger.yaml +++ b/docs/api/swagger.yaml @@ -10397,10 +10397,14 @@ paths: x-go-name: Federated - description: |- ISO 8601 Datetime at which to schedule a status. - Providing this parameter will cause ScheduledStatus to be returned instead of Status. - Must be at least 5 minutes in the future. - This feature isn't implemented yet; attemping to set it will return 501 Not Implemented. + Providing this parameter with a *future* time will cause ScheduledStatus to be returned instead of Status. + Must be at least 5 minutes in the future. + This feature isn't implemented yet. + + Providing this parameter with a *past* time will cause the status to be backdated, + and will not push it to the user's followers. This is intended for importing old statuses. + format: date-time in: formData name: scheduled_at type: string diff --git a/docs/configuration/instance.md b/docs/configuration/instance.md index bffec8f70..2a945eed2 100644 --- a/docs/configuration/instance.md +++ b/docs/configuration/instance.md @@ -171,4 +171,17 @@ instance-subscriptions-process-every: "24h" # Options: ["", "zero", "serve", "baffle"] # Default: "" instance-stats-mode: "" + +# Bool. This flag controls whether local accounts may backdate statuses +# using past dates with the scheduled_at param to /api/v1/statuses. +# This flag does not affect scheduling posts in the future +# (which is currently not implemented anyway), +# nor can it prevent remote accounts from backdating their own statuses. +# +# If true, all local accounts may backdate statuses. +# If false, status backdating will be disabled and an error will be returned if it's used. +# +# Options: [true, false] +# Default: true +instance-allow-backdating-statuses: true ``` diff --git a/example/config.yaml b/example/config.yaml index b618ad7ba..2b3a873fb 100644 --- a/example/config.yaml +++ b/example/config.yaml @@ -458,6 +458,19 @@ instance-subscriptions-process-every: "24h" # Default: "" instance-stats-mode: "" +# Bool. This flag controls whether local accounts may backdate statuses +# using past dates with the scheduled_at param to /api/v1/statuses. +# This flag does not affect scheduling posts in the future +# (which is currently not implemented anyway), +# nor can it prevent remote accounts from backdating their own statuses. +# +# If true, all local accounts may backdate statuses. +# If false, status backdating will be disabled and an error will be returned if it's used. +# +# Options: [true, false] +# Default: true +instance-allow-backdating-statuses: true + ########################### ##### ACCOUNTS CONFIG ##### ########################### diff --git a/internal/api/client/statuses/statuscreate.go b/internal/api/client/statuses/statuscreate.go index d187e823f..bfb1c486d 100644 --- a/internal/api/client/statuses/statuscreate.go +++ b/internal/api/client/statuses/statuscreate.go @@ -179,11 +179,15 @@ import ( // x-go-name: ScheduledAt // description: |- // ISO 8601 Datetime at which to schedule a status. -// Providing this parameter will cause ScheduledStatus to be returned instead of Status. -// Must be at least 5 minutes in the future. // -// This feature isn't implemented yet; attemping to set it will return 501 Not Implemented. +// Providing this parameter with a *future* time will cause ScheduledStatus to be returned instead of Status. +// Must be at least 5 minutes in the future. +// This feature isn't implemented yet. +// +// Providing this parameter with a *past* time will cause the status to be backdated, +// and will not push it to the user's followers. This is intended for importing old statuses. // type: string +// format: date-time // in: formData // - // name: language @@ -384,12 +388,6 @@ func parseStatusCreateForm(c *gin.Context) (*apimodel.StatusCreateRequest, gtser return nil, gtserror.NewErrorNotAcceptable(errors.New(text), text) } - // Check not scheduled status. - if form.ScheduledAt != "" { - const text = "scheduled_at is not yet implemented" - return nil, gtserror.NewErrorNotImplemented(errors.New(text), text) - } - // Check if the deprecated "federated" field was // set in lieu of "local_only", and use it if so. if form.LocalOnly == nil && form.Federated != nil { // nolint:staticcheck diff --git a/internal/api/client/statuses/statuscreate_test.go b/internal/api/client/statuses/statuscreate_test.go index 227e7d83e..53e517a6e 100644 --- a/internal/api/client/statuses/statuscreate_test.go +++ b/internal/api/client/statuses/statuscreate_test.go @@ -20,14 +20,18 @@ package statuses_test import ( "bytes" "context" + "encoding/json" "fmt" + "io" "net/http" "net/http/httptest" "testing" + "time" "github.com/stretchr/testify/suite" "github.com/superseriousbusiness/gotosocial/internal/api/client/statuses" apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" + "github.com/superseriousbusiness/gotosocial/internal/id" "github.com/superseriousbusiness/gotosocial/internal/oauth" "github.com/superseriousbusiness/gotosocial/testrig" ) @@ -41,10 +45,11 @@ const ( statusMarkdown = "# Title\n\n## Smaller title\n\nThis is a post written in [markdown](https://www.markdownguide.org/)\n\n" ) -func (suite *StatusCreateTestSuite) postStatus( +// Post a status. +func (suite *StatusCreateTestSuite) postStatusCore( formData map[string][]string, jsonData string, -) (string, *httptest.ResponseRecorder) { +) *httptest.ResponseRecorder { recorder := httptest.NewRecorder() ctx, _ := testrig.CreateGinTestContext(recorder, nil) ctx.Set(oauth.SessionAuthorizedApplication, suite.testApplications["application_1"]) @@ -77,9 +82,42 @@ func (suite *StatusCreateTestSuite) postStatus( // Trigger handler. suite.statusModule.StatusCreatePOSTHandler(ctx) + + return recorder +} + +// Post a status and return the result as deterministic JSON. +func (suite *StatusCreateTestSuite) postStatus( + formData map[string][]string, + jsonData string, +) (string, *httptest.ResponseRecorder) { + recorder := suite.postStatusCore(formData, jsonData) return suite.parseStatusResponse(recorder) } +// Post a status and return the result as a non-deterministic API structure. +func (suite *StatusCreateTestSuite) postStatusStruct( + formData map[string][]string, + jsonData string, +) (*apimodel.Status, *httptest.ResponseRecorder) { + recorder := suite.postStatusCore(formData, jsonData) + + result := recorder.Result() + defer result.Body.Close() + + data, err := io.ReadAll(result.Body) + if err != nil { + suite.FailNow(err.Error()) + } + + apiStatus := apimodel.Status{} + if err := json.Unmarshal(data, &apiStatus); err != nil { + suite.FailNow(err.Error()) + } + + return &apiStatus, recorder +} + // Post a new status with some custom visibility settings func (suite *StatusCreateTestSuite) TestPostNewStatus() { out, recorder := suite.postStatus(map[string][]string{ @@ -383,10 +421,98 @@ func (suite *StatusCreateTestSuite) TestPostNewScheduledStatus() { // We should have a helpful error message. suite.Equal(`{ - "error": "Not Implemented: scheduled_at is not yet implemented" + "error": "Not Implemented: scheduled statuses are not yet supported" }`, out) } +func (suite *StatusCreateTestSuite) TestPostNewBackfilledStatus() { + // A time in the past. + scheduledAtStr := "2020-10-04T15:32:02.018Z" + scheduledAt, err := time.Parse(time.RFC3339Nano, scheduledAtStr) + if err != nil { + suite.FailNow(err.Error()) + } + + status, recorder := suite.postStatusStruct(map[string][]string{ + "status": {"this is a recycled status from the past!"}, + "scheduled_at": {scheduledAtStr}, + }, "") + + // Creating a status in the past should succeed. + suite.Equal(http.StatusOK, recorder.Code) + + // The status should be backdated. + createdAt, err := time.Parse(time.RFC3339Nano, status.CreatedAt) + if err != nil { + suite.FailNow(err.Error()) + return + } + suite.Equal(scheduledAt, createdAt.UTC()) + + // The status's ULID should be backdated. + timeFromULID, err := id.TimeFromULID(status.ID) + if err != nil { + suite.FailNow(err.Error()) + return + } + suite.Equal(scheduledAt, timeFromULID.UTC()) +} + +func (suite *StatusCreateTestSuite) TestPostNewBackfilledStatusWithSelfMention() { + _, recorder := suite.postStatus(map[string][]string{ + "status": {"@the_mighty_zork this is a recycled mention from the past!"}, + "scheduled_at": {"2020-10-04T15:32:02.018Z"}, + }, "") + + // Mentioning yourself is allowed in backfilled statuses. + suite.Equal(http.StatusOK, recorder.Code) +} + +func (suite *StatusCreateTestSuite) TestPostNewBackfilledStatusWithMention() { + _, recorder := suite.postStatus(map[string][]string{ + "status": {"@admin this is a recycled mention from the past!"}, + "scheduled_at": {"2020-10-04T15:32:02.018Z"}, + }, "") + + // Mentioning others is forbidden in backfilled statuses. + suite.Equal(http.StatusForbidden, recorder.Code) +} + +func (suite *StatusCreateTestSuite) TestPostNewBackfilledStatusWithSelfReply() { + _, recorder := suite.postStatus(map[string][]string{ + "status": {"this is a recycled reply from the past!"}, + "scheduled_at": {"2020-10-04T15:32:02.018Z"}, + "in_reply_to_id": {suite.testStatuses["local_account_1_status_1"].ID}, + }, "") + + // Replying to yourself is allowed in backfilled statuses. + suite.Equal(http.StatusOK, recorder.Code) +} + +func (suite *StatusCreateTestSuite) TestPostNewBackfilledStatusWithReply() { + _, recorder := suite.postStatus(map[string][]string{ + "status": {"this is a recycled reply from the past!"}, + "scheduled_at": {"2020-10-04T15:32:02.018Z"}, + "in_reply_to_id": {suite.testStatuses["admin_account_status_1"].ID}, + }, "") + + // Replying to others is forbidden in backfilled statuses. + suite.Equal(http.StatusForbidden, recorder.Code) +} + +func (suite *StatusCreateTestSuite) TestPostNewBackfilledStatusWithPoll() { + _, recorder := suite.postStatus(map[string][]string{ + "status": {"this is a recycled poll from the past!"}, + "scheduled_at": {"2020-10-04T15:32:02.018Z"}, + "poll[options][]": {"first option", "second option"}, + "poll[expires_in]": {"3600"}, + "poll[multiple]": {"true"}, + }, "") + + // Polls are forbidden in backfilled statuses. + suite.Equal(http.StatusForbidden, recorder.Code) +} + func (suite *StatusCreateTestSuite) TestPostNewStatusMarkdown() { out, recorder := suite.postStatus(map[string][]string{ "status": {statusMarkdown}, diff --git a/internal/api/model/status.go b/internal/api/model/status.go index ea9fbaa35..2ee3123e6 100644 --- a/internal/api/model/status.go +++ b/internal/api/model/status.go @@ -17,7 +17,11 @@ package model -import "github.com/superseriousbusiness/gotosocial/internal/language" +import ( + "time" + + "github.com/superseriousbusiness/gotosocial/internal/language" +) // Status models a status or post. // @@ -231,9 +235,14 @@ type StatusCreateRequest struct { Federated *bool `form:"federated" json:"federated"` // ISO 8601 Datetime at which to schedule a status. - // Providing this parameter will cause ScheduledStatus to be returned instead of Status. + // + // Providing this parameter with a *future* time will cause ScheduledStatus to be returned instead of Status. // Must be at least 5 minutes in the future. - ScheduledAt string `form:"scheduled_at" json:"scheduled_at"` + // This feature isn't implemented yet. + // + // Providing this parameter with a *past* time will cause the status to be backdated, + // and will not push it to the user's followers. This is intended for importing old statuses. + ScheduledAt *time.Time `form:"scheduled_at" json:"scheduled_at"` // ISO 639 language code for this status. Language string `form:"language" json:"language"` diff --git a/internal/config/config.go b/internal/config/config.go index 33003d0f9..8ce2105b4 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -91,6 +91,7 @@ type Configuration struct { InstanceSubscriptionsProcessFrom string `name:"instance-subscriptions-process-from" usage:"Time of day from which to start running instance subscriptions processing jobs. Should be in the format 'hh:mm:ss', eg., '15:04:05'."` InstanceSubscriptionsProcessEvery time.Duration `name:"instance-subscriptions-process-every" usage:"Period to elapse between instance subscriptions processing jobs, starting from instance-subscriptions-process-from."` InstanceStatsMode string `name:"instance-stats-mode" usage:"Allows you to customize the way stats are served to crawlers: one of '', 'serve', 'zero', 'baffle'. Home page stats remain unchanged."` + InstanceAllowBackdatingStatuses bool `name:"instance-allow-backdating-statuses" usage:"Allow local accounts to backdate statuses using the scheduled_at param to /api/v1/statuses"` AccountsRegistrationOpen bool `name:"accounts-registration-open" usage:"Allow anyone to submit an account signup request. If false, server will be invite-only."` AccountsReasonRequired bool `name:"accounts-reason-required" usage:"Do new account signups require a reason to be submitted on registration?"` diff --git a/internal/config/defaults.go b/internal/config/defaults.go index 7f66e4209..78a8230d5 100644 --- a/internal/config/defaults.go +++ b/internal/config/defaults.go @@ -67,6 +67,7 @@ var Defaults = Configuration{ InstanceLanguages: make(language.Languages, 0), InstanceSubscriptionsProcessFrom: "23:00", // 11pm, InstanceSubscriptionsProcessEvery: 24 * time.Hour, // 1/day. + InstanceAllowBackdatingStatuses: true, AccountsRegistrationOpen: false, AccountsReasonRequired: true, diff --git a/internal/config/flags.go b/internal/config/flags.go index d67085d6d..3a2564c94 100644 --- a/internal/config/flags.go +++ b/internal/config/flags.go @@ -93,6 +93,7 @@ func (s *ConfigState) AddServerFlags(cmd *cobra.Command) { cmd.Flags().String(InstanceSubscriptionsProcessFromFlag(), cfg.InstanceSubscriptionsProcessFrom, fieldtag("InstanceSubscriptionsProcessFrom", "usage")) cmd.Flags().Duration(InstanceSubscriptionsProcessEveryFlag(), cfg.InstanceSubscriptionsProcessEvery, fieldtag("InstanceSubscriptionsProcessEvery", "usage")) cmd.Flags().String(InstanceStatsModeFlag(), cfg.InstanceStatsMode, fieldtag("InstanceStatsMode", "usage")) + cmd.Flags().Bool(InstanceAllowBackdatingStatusesFlag(), cfg.InstanceAllowBackdatingStatuses, fieldtag("InstanceAllowBackdatingStatuses", "usage")) // Accounts cmd.Flags().Bool(AccountsRegistrationOpenFlag(), cfg.AccountsRegistrationOpen, fieldtag("AccountsRegistrationOpen", "usage")) diff --git a/internal/config/helpers.gen.go b/internal/config/helpers.gen.go index d3ccf16ea..156c19fd5 100644 --- a/internal/config/helpers.gen.go +++ b/internal/config/helpers.gen.go @@ -1082,6 +1082,31 @@ func GetInstanceStatsMode() string { return global.GetInstanceStatsMode() } // SetInstanceStatsMode safely sets the value for global configuration 'InstanceStatsMode' field func SetInstanceStatsMode(v string) { global.SetInstanceStatsMode(v) } +// GetInstanceAllowBackdatingStatuses safely fetches the Configuration value for state's 'InstanceAllowBackdatingStatuses' field +func (st *ConfigState) GetInstanceAllowBackdatingStatuses() (v bool) { + st.mutex.RLock() + v = st.config.InstanceAllowBackdatingStatuses + st.mutex.RUnlock() + return +} + +// SetInstanceAllowBackdatingStatuses safely sets the Configuration value for state's 'InstanceAllowBackdatingStatuses' field +func (st *ConfigState) SetInstanceAllowBackdatingStatuses(v bool) { + st.mutex.Lock() + defer st.mutex.Unlock() + st.config.InstanceAllowBackdatingStatuses = v + st.reloadToViper() +} + +// InstanceAllowBackdatingStatusesFlag returns the flag name for the 'InstanceAllowBackdatingStatuses' field +func InstanceAllowBackdatingStatusesFlag() string { return "instance-allow-backdating-statuses" } + +// GetInstanceAllowBackdatingStatuses safely fetches the value for global configuration 'InstanceAllowBackdatingStatuses' field +func GetInstanceAllowBackdatingStatuses() bool { return global.GetInstanceAllowBackdatingStatuses() } + +// SetInstanceAllowBackdatingStatuses safely sets the value for global configuration 'InstanceAllowBackdatingStatuses' field +func SetInstanceAllowBackdatingStatuses(v bool) { global.SetInstanceAllowBackdatingStatuses(v) } + // GetAccountsRegistrationOpen safely fetches the Configuration value for state's 'AccountsRegistrationOpen' field func (st *ConfigState) GetAccountsRegistrationOpen() (v bool) { st.mutex.RLock() diff --git a/internal/federation/dereferencing/instance_test.go b/internal/federation/dereferencing/instance_test.go index 15f075479..c07490d4b 100644 --- a/internal/federation/dereferencing/instance_test.go +++ b/internal/federation/dereferencing/instance_test.go @@ -50,7 +50,7 @@ func (suite *InstanceTestSuite) TestDerefInstance() { // // Debug-level logs should show something like: // - // - "can't fetch /nodeinfo/2.1: robots.txt disallows it" + // - "can't fetch /nodeinfo/2.1: robots.txt disallows it" instanceIRI: testrig.URLMustParse("https://furtive-nerds.example.org"), expectedSoftware: "", }, @@ -60,7 +60,7 @@ func (suite *InstanceTestSuite) TestDerefInstance() { // // Debug-level logs should show something like: // - // - "can't fetch api/v1/instance: robots.txt disallows it" + // - "can't fetch api/v1/instance: robots.txt disallows it" // - "can't fetch .well-known/nodeinfo: robots.txt disallows it" instanceIRI: testrig.URLMustParse("https://robotic.furtive-nerds.example.org"), expectedSoftware: "", @@ -71,7 +71,7 @@ func (suite *InstanceTestSuite) TestDerefInstance() { // // Debug-level logs should show something like: // - // - "can't use fetched .well-known/nodeinfo: robots tags disallows it" + // - "can't use fetched .well-known/nodeinfo: robots tags disallows it" instanceIRI: testrig.URLMustParse("https://really.furtive-nerds.example.org"), expectedSoftware: "", }, diff --git a/internal/gtsmodel/status.go b/internal/gtsmodel/status.go index d28898ed1..e170e7464 100644 --- a/internal/gtsmodel/status.go +++ b/internal/gtsmodel/status.go @@ -86,7 +86,7 @@ func (s *Status) GetAccountID() string { return s.AccountID } -// GetBoostID implements timeline.Timelineable{}. +// GetBoostOfID implements timeline.Timelineable{}. func (s *Status) GetBoostOfID() string { return s.BoostOfID } @@ -171,7 +171,7 @@ func (s *Status) EditsPopulated() bool { return true } -// EmojissUpToDate returns whether status emoji attachments of receiving status are up-to-date +// EmojisUpToDate returns whether status emoji attachments of receiving status are up-to-date // according to emoji attachments of the passed status, by comparing their emoji URIs. We don't // use IDs as this is used to determine whether there are new emojis to fetch. func (s *Status) EmojisUpToDate(other *Status) bool { @@ -386,3 +386,8 @@ type Content struct { Content string ContentMap map[string]string } + +// BackfillStatus is a wrapper for creating a status without pushing notifications to followers. +type BackfillStatus struct { + *Status +} diff --git a/internal/processing/status/create.go b/internal/processing/status/create.go index b77d0af9c..46052d0aa 100644 --- a/internal/processing/status/create.go +++ b/internal/processing/status/create.go @@ -19,10 +19,14 @@ package status import ( "context" + "errors" "time" "github.com/superseriousbusiness/gotosocial/internal/ap" apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model" + "github.com/superseriousbusiness/gotosocial/internal/config" + "github.com/superseriousbusiness/gotosocial/internal/db" + "github.com/superseriousbusiness/gotosocial/internal/gtscontext" "github.com/superseriousbusiness/gotosocial/internal/gtserror" "github.com/superseriousbusiness/gotosocial/internal/gtsmodel" "github.com/superseriousbusiness/gotosocial/internal/id" @@ -92,11 +96,54 @@ func (p *Processor) Create( // Get current time. now := time.Now() + // Default to current time as creation time. + createdAt := now + + // Handle backfilled/scheduled statuses. + backfill := false + if form.ScheduledAt != nil { + scheduledAt := *form.ScheduledAt + + // Statuses may only be scheduled a minimum time into the future. + if now.Before(scheduledAt) { + const errText = "scheduled statuses are not yet supported" + err := gtserror.New(errText) + return nil, gtserror.NewErrorNotImplemented(err, errText) + } + + // If not scheduled into the future, this status is being backfilled. + if !config.GetInstanceAllowBackdatingStatuses() { + const errText = "backdating statuses has been disabled on this instance" + err := gtserror.New(errText) + return nil, gtserror.NewErrorForbidden(err) + } + + // Statuses can't be backdated to or before the UNIX epoch + // since this would prevent generating a ULID. + // If backdated even further to the Go epoch, + // this would also cause issues with time.Time.IsZero() checks + // that normally signify an absent optional time, + // but this check covers both cases. + if scheduledAt.Compare(time.UnixMilli(0)) <= 0 { + const errText = "statuses can't be backdated to or before the UNIX epoch" + err := gtserror.New(errText) + return nil, gtserror.NewErrorNotAcceptable(err, errText) + } + + // Allow the backfill and generate an appropriate ID for the creation time. + backfill = true + createdAt = scheduledAt + var err error + if statusID, err = p.backfilledStatusID(ctx, createdAt); err != nil { + return nil, gtserror.NewErrorInternalError(err) + } + } + status := >smodel.Status{ ID: statusID, URI: accountURIs.StatusesURI + "/" + statusID, URL: accountURIs.StatusesURL + "/" + statusID, - CreatedAt: now, + CreatedAt: createdAt, Local: util.Ptr(true), Account: requester, AccountID: requester.ID, @@ -134,11 +181,24 @@ func (p *Processor) Create( PendingApproval: util.Ptr(false), } + if backfill { + log.Infof(ctx, "%d mentions", len(status.Mentions)) + for _, mention := range status.Mentions { + log.Infof(ctx, "mention: target account ID = %s, requester ID = %s", mention.TargetAccountID, requester.ID) + if mention.TargetAccountID != requester.ID { + const errText = "statuses mentioning others can't be backfilled" + err := gtserror.New(errText) + return nil, gtserror.NewErrorForbidden(err, errText) + } + } + } + // Check + attach in-reply-to status. if errWithCode := p.processInReplyTo(ctx, requester, status, form.InReplyToID, + backfill, ); errWithCode != nil { return nil, errWithCode } @@ -165,11 +225,17 @@ func (p *Processor) Create( } if form.Poll != nil { + if backfill { + const errText = "statuses with polls can't be backfilled" + err := gtserror.New(errText) + return nil, gtserror.NewErrorForbidden(err, errText) + } + // Process poll, inserting into database. poll, errWithCode := p.processPoll(ctx, statusID, form.Poll, - now, + createdAt, ) if errWithCode != nil { return nil, errWithCode @@ -200,10 +266,14 @@ func (p *Processor) Create( } // Send it to the client API worker for async side-effects. + var model any = status + if backfill { + model = >smodel.BackfillStatus{Status: status} + } p.state.Workers.Client.Queue.Push(&messages.FromClientAPI{ APObjectType: ap.ObjectNote, APActivityType: ap.ActivityCreate, - GTSModel: status, + GTSModel: model, Origin: requester, }) @@ -227,7 +297,40 @@ func (p *Processor) Create( return p.c.GetAPIStatus(ctx, requester, status) } -func (p *Processor) processInReplyTo(ctx context.Context, requester *gtsmodel.Account, status *gtsmodel.Status, inReplyToID string) gtserror.WithCode { +// backfilledStatusID tries to find an unused ULID for a backfilled status. +func (p *Processor) backfilledStatusID(ctx context.Context, createdAt time.Time) (string, error) { + // backfilledStatusIDRetries should be more than enough attempts. + const backfilledStatusIDRetries = 100 + + for try := 0; try < backfilledStatusIDRetries; try++ { + var err error + + // Generate a ULID based on the backfilled status's original creation time. + statusID := id.NewULIDFromTime(createdAt) + + // Check for an existing status with that ID. + _, err = p.state.DB.GetStatusByID(gtscontext.SetBarebones(ctx), statusID) + if errors.Is(err, db.ErrNoEntries) { + // We found an unused one. + return statusID, nil + } else if err != nil { + err := gtserror.Newf("DB error checking if a status ID was in use: %w", err) + return "", err + } + // That status ID is in use. Try again. + } + + err := gtserror.Newf("failed to find an unused ID after %d tries", backfilledStatusIDRetries) + return "", err +} + +func (p *Processor) processInReplyTo( + ctx context.Context, + requester *gtsmodel.Account, + status *gtsmodel.Status, + inReplyToID string, + backfill bool, +) gtserror.WithCode { if inReplyToID == "" { // Not a reply. // Nothing to do. @@ -269,6 +372,13 @@ func (p *Processor) processInReplyTo(ctx context.Context, requester *gtsmodel.Ac return gtserror.NewErrorForbidden(err, errText) } + // When backfilling, only self-replies are allowed. + if backfill && requester.ID != inReplyTo.AccountID { + const errText = "replies to others can't be backfilled" + err := gtserror.New(errText) + return gtserror.NewErrorForbidden(err, errText) + } + // Derive pendingApproval status. var pendingApproval bool switch { diff --git a/internal/processing/status/create_test.go b/internal/processing/status/create_test.go index d0a5c7f92..16cefcebf 100644 --- a/internal/processing/status/create_test.go +++ b/internal/processing/status/create_test.go @@ -48,7 +48,7 @@ func (suite *StatusCreateTestSuite) TestProcessContentWarningWithQuotationMarks( SpoilerText: "\"test\"", // these should not be html-escaped when the final text is rendered Visibility: apimodel.VisibilityPublic, LocalOnly: util.Ptr(false), - ScheduledAt: "", + ScheduledAt: nil, Language: "en", ContentType: apimodel.StatusContentTypePlain, } @@ -75,7 +75,7 @@ func (suite *StatusCreateTestSuite) TestProcessContentWarningWithHTMLEscapedQuot SpoilerText: ""test"", // the html-escaped quotation marks should appear as normal quotation marks in the finished text Visibility: apimodel.VisibilityPublic, LocalOnly: util.Ptr(false), - ScheduledAt: "", + ScheduledAt: nil, Language: "en", ContentType: apimodel.StatusContentTypePlain, } @@ -106,7 +106,7 @@ func (suite *StatusCreateTestSuite) TestProcessStatusMarkdownWithUnderscoreEmoji Sensitive: false, Visibility: apimodel.VisibilityPublic, LocalOnly: util.Ptr(false), - ScheduledAt: "", + ScheduledAt: nil, Language: "en", ContentType: apimodel.StatusContentTypeMarkdown, } @@ -133,7 +133,7 @@ func (suite *StatusCreateTestSuite) TestProcessStatusMarkdownWithSpoilerTextEmoj Sensitive: false, Visibility: apimodel.VisibilityPublic, LocalOnly: util.Ptr(false), - ScheduledAt: "", + ScheduledAt: nil, Language: "en", ContentType: apimodel.StatusContentTypeMarkdown, } @@ -164,7 +164,7 @@ func (suite *StatusCreateTestSuite) TestProcessMediaDescriptionTooShort() { SpoilerText: "", Visibility: apimodel.VisibilityPublic, LocalOnly: util.Ptr(false), - ScheduledAt: "", + ScheduledAt: nil, Language: "en", ContentType: apimodel.StatusContentTypePlain, } @@ -189,7 +189,7 @@ func (suite *StatusCreateTestSuite) TestProcessLanguageWithScriptPart() { SpoilerText: "", Visibility: apimodel.VisibilityPublic, LocalOnly: util.Ptr(false), - ScheduledAt: "", + ScheduledAt: nil, Language: "zh-Hans", ContentType: apimodel.StatusContentTypePlain, } @@ -219,7 +219,7 @@ func (suite *StatusCreateTestSuite) TestProcessReplyToUnthreadedRemoteStatus() { SpoilerText: "this is a reply", Visibility: apimodel.VisibilityPublic, LocalOnly: util.Ptr(false), - ScheduledAt: "", + ScheduledAt: nil, Language: "en", ContentType: apimodel.StatusContentTypePlain, } diff --git a/internal/processing/workers/fromclientapi.go b/internal/processing/workers/fromclientapi.go index c5dfc157d..a208d97b0 100644 --- a/internal/processing/workers/fromclientapi.go +++ b/internal/processing/workers/fromclientapi.go @@ -260,9 +260,16 @@ func (p *clientAPI) CreateUser(ctx context.Context, cMsg *messages.FromClientAPI } func (p *clientAPI) CreateStatus(ctx context.Context, cMsg *messages.FromClientAPI) error { - status, ok := cMsg.GTSModel.(*gtsmodel.Status) - if !ok { - return gtserror.Newf("%T not parseable as *gtsmodel.Status", cMsg.GTSModel) + var status *gtsmodel.Status + backfill := false + if backfillStatus, ok := cMsg.GTSModel.(*gtsmodel.BackfillStatus); ok { + status = backfillStatus.Status + backfill = true + } else { + status, ok = cMsg.GTSModel.(*gtsmodel.Status) + if !ok { + return gtserror.Newf("%T not parseable as *gtsmodel.Status or *gtsmodel.BackfillStatus", cMsg.GTSModel) + } } // If pending approval is true then status must @@ -344,12 +351,14 @@ func (p *clientAPI) CreateStatus(ctx context.Context, cMsg *messages.FromClientA log.Errorf(ctx, "error updating account stats: %v", err) } - if err := p.surface.timelineAndNotifyStatus(ctx, status); err != nil { - log.Errorf(ctx, "error timelining and notifying status: %v", err) - } + if !backfill { + if err := p.surface.timelineAndNotifyStatus(ctx, status); err != nil { + log.Errorf(ctx, "error timelining and notifying status: %v", err) + } - if err := p.federate.CreateStatus(ctx, status); err != nil { - log.Errorf(ctx, "error federating status: %v", err) + if err := p.federate.CreateStatus(ctx, status); err != nil { + log.Errorf(ctx, "error federating status: %v", err) + } } if status.InReplyToID != "" { diff --git a/internal/processing/workers/fromclientapi_test.go b/internal/processing/workers/fromclientapi_test.go index acb25673d..1d70eb96c 100644 --- a/internal/processing/workers/fromclientapi_test.go +++ b/internal/processing/workers/fromclientapi_test.go @@ -368,6 +368,162 @@ func (suite *FromClientAPITestSuite) TestProcessCreateStatusWithNotification() { suite.checkWebPushed(testStructs.WebPushSender, receivingAccount.ID, gtsmodel.NotificationStatus) } +// Even with notifications on for a user, backfilling a status should not notify or timeline it. +func (suite *FromClientAPITestSuite) TestProcessCreateBackfilledStatusWithNotification() { + testStructs := testrig.SetupTestStructs(rMediaPath, rTemplatePath) + defer testrig.TearDownTestStructs(testStructs) + + var ( + ctx = context.Background() + postingAccount = suite.testAccounts["admin_account"] + receivingAccount = suite.testAccounts["local_account_1"] + testList = suite.testLists["local_account_1_list_1"] + streams = suite.openStreams(ctx, + testStructs.Processor, + receivingAccount, + []string{testList.ID}, + ) + homeStream = streams[stream.TimelineHome] + listStream = streams[stream.TimelineList+":"+testList.ID] + notifStream = streams[stream.TimelineNotifications] + + // Admin account posts a new top-level status. + status = suite.newStatus( + ctx, + testStructs.State, + postingAccount, + gtsmodel.VisibilityPublic, + nil, + nil, + nil, + false, + nil, + ) + ) + + // Update the follow from receiving account -> posting account so + // that receiving account wants notifs when posting account posts. + follow := new(gtsmodel.Follow) + *follow = *suite.testFollows["local_account_1_admin_account"] + + follow.Notify = util.Ptr(true) + if err := testStructs.State.DB.UpdateFollow(ctx, follow); err != nil { + suite.FailNow(err.Error()) + } + + // Process the new status as a backfill. + if err := testStructs.Processor.Workers().ProcessFromClientAPI( + ctx, + &messages.FromClientAPI{ + APObjectType: ap.ObjectNote, + APActivityType: ap.ActivityCreate, + GTSModel: >smodel.BackfillStatus{Status: status}, + Origin: postingAccount, + }, + ); err != nil { + suite.FailNow(err.Error()) + } + + // There should be no message in the home stream. + suite.checkStreamed( + homeStream, + false, + "", + "", + ) + + // There should be no message in the list stream. + suite.checkStreamed( + listStream, + false, + "", + "", + ) + + // No notification should appear for the status. + if testrig.WaitFor(func() bool { + var err error + _, err = testStructs.State.DB.GetNotification( + ctx, + gtsmodel.NotificationStatus, + receivingAccount.ID, + postingAccount.ID, + status.ID, + ) + return err == nil + }) { + suite.FailNow("a status notification was created, but should not have been") + } + + // There should be no message in the notification stream. + suite.checkStreamed( + notifStream, + false, + "", + "", + ) + + // There should be no Web Push status notification. + suite.checkNotWebPushed(testStructs.WebPushSender, receivingAccount.ID) +} + +// Backfilled statuses should not federate when created. +func (suite *FromClientAPITestSuite) TestProcessCreateBackfilledStatusWithRemoteFollower() { + testStructs := testrig.SetupTestStructs(rMediaPath, rTemplatePath) + defer testrig.TearDownTestStructs(testStructs) + + var ( + ctx = context.Background() + postingAccount = suite.testAccounts["local_account_1"] + receivingAccount = suite.testAccounts["remote_account_1"] + + // Local account posts a new top-level status. + status = suite.newStatus( + ctx, + testStructs.State, + postingAccount, + gtsmodel.VisibilityPublic, + nil, + nil, + nil, + false, + nil, + ) + ) + + // Follow the local account from the remote account. + follow := >smodel.Follow{ + ID: "01JJHW9RW28SC1NEPZ0WBJQ4ZK", + CreatedAt: testrig.TimeMustParse("2022-05-14T13:21:09+02:00"), + UpdatedAt: testrig.TimeMustParse("2022-05-14T13:21:09+02:00"), + AccountID: receivingAccount.ID, + TargetAccountID: postingAccount.ID, + ShowReblogs: util.Ptr(true), + URI: "http://fossbros-anonymous.io/users/foss_satan/follow/01JJHWEVC7F8W2JDW1136K431K", + Notify: util.Ptr(false), + } + + if err := testStructs.State.DB.PutFollow(ctx, follow); err != nil { + suite.FailNow(err.Error()) + } + + // Process the new status as a backfill. + if err := testStructs.Processor.Workers().ProcessFromClientAPI( + ctx, + &messages.FromClientAPI{ + APObjectType: ap.ObjectNote, + APActivityType: ap.ActivityCreate, + GTSModel: >smodel.BackfillStatus{Status: status}, + Origin: postingAccount, + }, + ); err != nil { + suite.FailNow(err.Error()) + } + + // No deliveries should be queued. + suite.Zero(testStructs.State.Workers.Delivery.Queue.Len()) +} + func (suite *FromClientAPITestSuite) TestProcessCreateStatusReply() { testStructs := testrig.SetupTestStructs(rMediaPath, rTemplatePath) defer testrig.TearDownTestStructs(testStructs) diff --git a/test/envparsing.sh b/test/envparsing.sh index 0e7c0db20..fc6afbcc2 100755 --- a/test/envparsing.sh +++ b/test/envparsing.sh @@ -108,6 +108,7 @@ EXPECT=$(cat << "EOF" "timeout": 30000000000, "tls-insecure-skip-verify": false }, + "instance-allow-backdating-statuses": true, "instance-deliver-to-shared-inboxes": false, "instance-expose-peers": true, "instance-expose-public-timeline": true, diff --git a/testrig/config.go b/testrig/config.go index f68a8ffb7..9f17530c4 100644 --- a/testrig/config.go +++ b/testrig/config.go @@ -101,6 +101,7 @@ func testDefaults() config.Configuration { }, InstanceSubscriptionsProcessFrom: "23:00", // 11pm, InstanceSubscriptionsProcessEvery: 24 * time.Hour, // 1/day. + InstanceAllowBackdatingStatuses: true, AccountsRegistrationOpen: true, AccountsReasonRequired: true,