Announce/boost (#35)

Remote boosts incoming/outgoing now working.

internal/api/client/status/status.go

@@ -96,6 +96,8 @@ func (m *Module) Route(r router.Router) error {
 	r.AttachHandler(http.MethodPost, FavouritePath, m.StatusFavePOSTHandler)
 	r.AttachHandler(http.MethodPost, UnfavouritePath, m.StatusUnfavePOSTHandler)
 
+	r.AttachHandler(http.MethodPost, ReblogPath, m.StatusBoostPOSTHandler)
+
 	r.AttachHandler(http.MethodGet, BasePathWithID, m.muxHandler)
 	return nil
 }

internal/db/pg/pg.go

@@ -963,7 +963,7 @@ func (ps *postgresService) PullRelevantAccountsFromStatus(targetStatus *gtsmodel
 	if targetStatus.InReplyToAccountID != "" {
 		repliedToAccount := &gtsmodel.Account{}
 		if err := ps.conn.Model(repliedToAccount).Where("id = ?", targetStatus.InReplyToAccountID).Select(); err != nil {
-			return accounts, err
+			return accounts, fmt.Errorf("PullRelevantAccountsFromStatus: error getting repliedToAcount with id %s: %s", targetStatus.InReplyToAccountID, err)
 		}
 		accounts.ReplyToAccount = repliedToAccount
 	}
@@ -973,11 +973,11 @@ func (ps *postgresService) PullRelevantAccountsFromStatus(targetStatus *gtsmodel
 		// retrieve the boosted status first
 		boostedStatus := &gtsmodel.Status{}
 		if err := ps.conn.Model(boostedStatus).Where("id = ?", targetStatus.BoostOfID).Select(); err != nil {
-			return accounts, err
+			return accounts, fmt.Errorf("PullRelevantAccountsFromStatus: error getting boostedStatus with id %s: %s", targetStatus.BoostOfID, err)
 		}
 		boostedAccount := &gtsmodel.Account{}
 		if err := ps.conn.Model(boostedAccount).Where("id = ?", boostedStatus.AccountID).Select(); err != nil {
-			return accounts, err
+			return accounts, fmt.Errorf("PullRelevantAccountsFromStatus: error getting boostedAccount with id %s: %s", boostedStatus.AccountID, err)
 		}
 		accounts.BoostedAccount = boostedAccount
 
@@ -985,7 +985,7 @@ func (ps *postgresService) PullRelevantAccountsFromStatus(targetStatus *gtsmodel
 		if boostedStatus.InReplyToAccountID != "" {
 			boostedStatusRepliedToAccount := &gtsmodel.Account{}
 			if err := ps.conn.Model(boostedStatusRepliedToAccount).Where("id = ?", boostedStatus.InReplyToAccountID).Select(); err != nil {
-				return accounts, err
+				return accounts, fmt.Errorf("PullRelevantAccountsFromStatus: error getting boostedStatusRepliedToAccount with id %s: %s", boostedStatus.InReplyToAccountID, err)
 			}
 			accounts.BoostedReplyToAccount = boostedStatusRepliedToAccount
 		}
@@ -996,12 +996,12 @@ func (ps *postgresService) PullRelevantAccountsFromStatus(targetStatus *gtsmodel
 
 		mention := &gtsmodel.Mention{}
 		if err := ps.conn.Model(mention).Where("id = ?", mentionID).Select(); err != nil {
-			return accounts, fmt.Errorf("error getting mention with id %s: %s", mentionID, err)
+			return accounts, fmt.Errorf("PullRelevantAccountsFromStatus: error getting mention with id %s: %s", mentionID, err)
 		}
 
 		mentionedAccount := &gtsmodel.Account{}
 		if err := ps.conn.Model(mentionedAccount).Where("id = ?", mention.TargetAccountID).Select(); err != nil {
-			return accounts, fmt.Errorf("error getting mentioned account: %s", err)
+			return accounts, fmt.Errorf("PullRelevantAccountsFromStatus: error getting mentioned account: %s", err)
 		}
 		accounts.MentionedAccounts = append(accounts.MentionedAccounts, mentionedAccount)
 	}

internal/federation/federatingdb/announce.go

@@ -0,0 +1,73 @@
+package federatingdb
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+
+	"github.com/go-fed/activity/streams"
+	"github.com/go-fed/activity/streams/vocab"
+	"github.com/sirupsen/logrus"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+	"github.com/superseriousbusiness/gotosocial/internal/util"
+)
+
+func (f *federatingDB) Announce(ctx context.Context, announce vocab.ActivityStreamsAnnounce) error {
+	l := f.log.WithFields(
+		logrus.Fields{
+			"func": "Announce",
+		},
+	)
+	m, err := streams.Serialize(announce)
+	if err != nil {
+		return err
+	}
+	b, err := json.Marshal(m)
+	if err != nil {
+		return err
+	}
+
+	l.Debugf("received ANNOUNCE %s", string(b))
+
+	targetAcctI := ctx.Value(util.APAccount)
+	if targetAcctI == nil {
+		l.Error("target account wasn't set on context")
+		return nil
+	}
+	targetAcct, ok := targetAcctI.(*gtsmodel.Account)
+	if !ok {
+		l.Error("target account was set on context but couldn't be parsed")
+		return nil
+	}
+
+	fromFederatorChanI := ctx.Value(util.APFromFederatorChanKey)
+	if fromFederatorChanI == nil {
+		l.Error("from federator channel wasn't set on context")
+		return nil
+	}
+	fromFederatorChan, ok := fromFederatorChanI.(chan gtsmodel.FromFederator)
+	if !ok {
+		l.Error("from federator channel was set on context but couldn't be parsed")
+		return nil
+	}
+
+	boost, isNew, err := f.typeConverter.ASAnnounceToStatus(announce)
+	if err != nil {
+		return fmt.Errorf("Announce: error converting announce to boost: %s", err)
+	}
+
+	if !isNew {
+		// nothing to do here if this isn't a new announce
+		return nil
+	}
+
+	// it's a new announce so pass it back to the processor async for dereferencing etc
+	fromFederatorChan <- gtsmodel.FromFederator{
+		APObjectType:     gtsmodel.ActivityStreamsAnnounce,
+		APActivityType:   gtsmodel.ActivityStreamsCreate,
+		GTSModel:         boost,
+		ReceivingAccount: targetAcct,
+	}
+
+	return nil
+}

internal/federation/federatingdb/db.go

@@ -35,6 +35,7 @@ type DB interface {
 	pub.Database
 	Undo(ctx context.Context, undo vocab.ActivityStreamsUndo) error
 	Accept(ctx context.Context, accept vocab.ActivityStreamsAccept) error
+	Announce(ctx context.Context, announce vocab.ActivityStreamsAnnounce) error
 }
 
 // FederatingDB uses the underlying DB interface to implement the go-fed pub.Database interface.

internal/federation/federatingdb/federating_db_test.go

@@ -16,6 +16,6 @@
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
-package federation
+package federatingdb_test
 
 // TODO: write tests for pgfed

internal/federation/federatingdb/util.go

@@ -130,6 +130,19 @@ func (f *federatingDB) NewID(c context.Context, t vocab.Type) (id *url.URL, err
 				return idProp.GetIRI(), nil
 			}
 		}
+	case gtsmodel.ActivityStreamsAnnounce:
+		// ANNOUNCE aka BOOST
+		// ID might already be set on an announce we've created, so check it here and return it if it is
+		announce, ok := t.(vocab.ActivityStreamsAnnounce)
+		if !ok {
+			return nil, errors.New("newid: fave couldn't be parsed into vocab.ActivityStreamsAnnounce")
+		}
+		idProp := announce.GetJSONLDId()
+		if idProp != nil {
+			if idProp.IsIRI() {
+				return idProp.GetIRI(), nil
+			}
+		}
 	}
 
 	// fallback default behavior: just return a random UUID after our protocol and host

internal/federation/federatingprotocol.go

@@ -257,6 +257,10 @@ func (f *federator) FederatingCallbacks(ctx context.Context) (wrapped pub.Federa
 		func(ctx context.Context, accept vocab.ActivityStreamsAccept) error {
 			return f.FederatingDB().Accept(ctx, accept)
 		},
+		// override default announce behavior and trigger our own side effects
+		func(ctx context.Context, announce vocab.ActivityStreamsAnnounce) error {
+			return f.FederatingDB().Announce(ctx, announce)
+		},
 	}
 
 	return

internal/federation/federator.go

@@ -43,6 +43,9 @@ type Federator interface {
 	// DereferenceRemoteAccount can be used to get the representation of a remote account, based on the account ID (which is a URI).
 	// The given username will be used to create a transport for making outgoing requests. See the implementation for more detailed comments.
 	DereferenceRemoteAccount(username string, remoteAccountID *url.URL) (typeutils.Accountable, error)
+	// DereferenceRemoteStatus can be used to get the representation of a remote status, based on its ID (which is a URI).
+	// The given username will be used to create a transport for making outgoing requests. See the implementation for more detailed comments.
+	DereferenceRemoteStatus(username string, remoteStatusID *url.URL) (typeutils.Statusable, error)
 	// GetTransportForUser returns a new transport initialized with the key credentials belonging to the given username.
 	// This can be used for making signed http requests.
 	//

internal/federation/util.go

@@ -258,6 +258,88 @@ func (f *federator) DereferenceRemoteAccount(username string, remoteAccountID *u
 	return nil, fmt.Errorf("type name %s not supported", t.GetTypeName())
 }
 
+func (f *federator) DereferenceRemoteStatus(username string, remoteStatusID *url.URL) (typeutils.Statusable, error) {
+	transport, err := f.GetTransportForUser(username)
+	if err != nil {
+		return nil, fmt.Errorf("transport err: %s", err)
+	}
+
+	b, err := transport.Dereference(context.Background(), remoteStatusID)
+	if err != nil {
+		return nil, fmt.Errorf("error deferencing %s: %s", remoteStatusID.String(), err)
+	}
+
+	m := make(map[string]interface{})
+	if err := json.Unmarshal(b, &m); err != nil {
+		return nil, fmt.Errorf("error unmarshalling bytes into json: %s", err)
+	}
+
+	t, err := streams.ToType(context.Background(), m)
+	if err != nil {
+		return nil, fmt.Errorf("error resolving json into ap vocab type: %s", err)
+	}
+
+	// Article, Document, Image, Video, Note, Page, Event, Place, Mention, Profile
+	switch t.GetTypeName() {
+	case gtsmodel.ActivityStreamsArticle:
+		p, ok := t.(vocab.ActivityStreamsArticle)
+		if !ok {
+			return nil, errors.New("error resolving type as ActivityStreamsArticle")
+		}
+		return p, nil
+	case gtsmodel.ActivityStreamsDocument:
+		p, ok := t.(vocab.ActivityStreamsDocument)
+		if !ok {
+			return nil, errors.New("error resolving type as ActivityStreamsDocument")
+		}
+		return p, nil
+	case gtsmodel.ActivityStreamsImage:
+		p, ok := t.(vocab.ActivityStreamsImage)
+		if !ok {
+			return nil, errors.New("error resolving type as ActivityStreamsImage")
+		}
+		return p, nil
+	case gtsmodel.ActivityStreamsVideo:
+		p, ok := t.(vocab.ActivityStreamsVideo)
+		if !ok {
+			return nil, errors.New("error resolving type as ActivityStreamsVideo")
+		}
+		return p, nil
+	case gtsmodel.ActivityStreamsNote:
+		p, ok := t.(vocab.ActivityStreamsNote)
+		if !ok {
+			return nil, errors.New("error resolving type as ActivityStreamsNote")
+		}
+		return p, nil
+	case gtsmodel.ActivityStreamsPage:
+		p, ok := t.(vocab.ActivityStreamsPage)
+		if !ok {
+			return nil, errors.New("error resolving type as ActivityStreamsPage")
+		}
+		return p, nil
+	case gtsmodel.ActivityStreamsEvent:
+		p, ok := t.(vocab.ActivityStreamsEvent)
+		if !ok {
+			return nil, errors.New("error resolving type as ActivityStreamsEvent")
+		}
+		return p, nil
+	case gtsmodel.ActivityStreamsPlace:
+		p, ok := t.(vocab.ActivityStreamsPlace)
+		if !ok {
+			return nil, errors.New("error resolving type as ActivityStreamsPlace")
+		}
+		return p, nil
+	case gtsmodel.ActivityStreamsProfile:
+		p, ok := t.(vocab.ActivityStreamsProfile)
+		if !ok {
+			return nil, errors.New("error resolving type as ActivityStreamsProfile")
+		}
+		return p, nil
+	}
+
+	return nil, fmt.Errorf("type name %s not supported", t.GetTypeName())
+}
+
 func (f *federator) GetTransportForUser(username string) (transport.Transport, error) {
 	// We need an account to use to create a transport for dereferecing the signature.
 	// If a username has been given, we can fetch the account with that username and use it.
@@ -279,5 +361,3 @@ func (f *federator) GetTransportForUser(username string) (transport.Transport, e
 	}
 	return transport, nil
 }
-
-

internal/gtsmodel/account.go

@@ -76,13 +76,13 @@ type Account struct {
 	*/
 
 	// Does this account need an approval for new followers?
-	Locked bool
+	Locked bool `pg:",default:false"`
 	// Should this account be shown in the instance's profile directory?
-	Discoverable bool
+	Discoverable bool `pg:",default:false"`
 	// Default post privacy for this account
-	Privacy Visibility
+	Privacy Visibility `pg:",default:'public'"`
 	// Set posts from this account to sensitive by default?
-	Sensitive bool
+	Sensitive bool `pg:",default:false"`
 	// What language does this account post in?
 	Language string `pg:",default:'en'"`
 

internal/gtsmodel/activitystreams.go

@@ -24,7 +24,7 @@ const (
 	// ActivityStreamsAudio https://www.w3.org/TR/activitystreams-vocabulary/#dfn-audio
 	ActivityStreamsAudio = "Audio"
 	// ActivityStreamsDocument https://www.w3.org/TR/activitystreams-vocabulary/#dfn-document
-	ActivityStreamsDocument = "Event"
+	ActivityStreamsDocument = "Document"
 	// ActivityStreamsEvent https://www.w3.org/TR/activitystreams-vocabulary/#dfn-event
 	ActivityStreamsEvent = "Event"
 	// ActivityStreamsImage https://www.w3.org/TR/activitystreams-vocabulary/#dfn-image

internal/message/accountprocess.go

@@ -454,13 +454,9 @@ func (p *processor) AccountFollowCreate(authed *oauth.Auth, form *apimodel.Accou
 	p.fromClientAPI <- gtsmodel.FromClientAPI{
 		APObjectType:   gtsmodel.ActivityStreamsFollow,
 		APActivityType: gtsmodel.ActivityStreamsCreate,
-		GTSModel: &gtsmodel.Follow{
-			AccountID:       authed.Account.ID,
-			TargetAccountID: form.TargetAccountID,
-			URI:             fr.URI,
-		},
-		OriginAccount: authed.Account,
-		TargetAccount: targetAcct,
+		GTSModel:       fr,
+		OriginAccount:  authed.Account,
+		TargetAccount:  targetAcct,
 	}
 
 	// return whatever relationship results from this

internal/message/fromclientapiprocess.go

@@ -72,6 +72,19 @@ func (p *processor) processFromClientAPI(clientMsg gtsmodel.FromClientAPI) error
 			}
 
 			return p.federateFave(fave, clientMsg.OriginAccount, clientMsg.TargetAccount)
+
+		case gtsmodel.ActivityStreamsAnnounce:
+			// CREATE BOOST/ANNOUNCE
+			boostWrapperStatus, ok := clientMsg.GTSModel.(*gtsmodel.Status)
+			if !ok {
+				return errors.New("boost was not parseable as *gtsmodel.Status")
+			}
+
+			if err := p.notifyAnnounce(boostWrapperStatus); err != nil {
+				return err
+			}
+
+			return p.federateAnnounce(boostWrapperStatus, clientMsg.OriginAccount, clientMsg.TargetAccount)
 		}
 	case gtsmodel.ActivityStreamsUpdate:
 		// UPDATE
@@ -253,3 +266,18 @@ func (p *processor) federateFave(fave *gtsmodel.StatusFave, originAccount *gtsmo
 	_, err = p.federator.FederatingActor().Send(context.Background(), outboxIRI, asFave)
 	return err
 }
+
+func (p *processor) federateAnnounce(boostWrapperStatus *gtsmodel.Status, boostingAccount *gtsmodel.Account, boostedAccount *gtsmodel.Account) error {
+	announce, err := p.tc.BoostToAS(boostWrapperStatus, boostingAccount, boostedAccount)
+	if err != nil {
+		return fmt.Errorf("federateAnnounce: error converting status to announce: %s", err)
+	}
+
+	outboxIRI, err := url.Parse(boostingAccount.OutboxURI)
+	if err != nil {
+		return fmt.Errorf("federateAnnounce: error parsing outboxURI %s: %s", boostingAccount.OutboxURI, err)
+	}
+	
+	_, err = p.federator.FederatingActor().Send(context.Background(), outboxIRI, announce)
+	return err
+}

internal/message/fromcommonprocess.go

@@ -158,3 +158,7 @@ func (p *processor) notifyFave(fave *gtsmodel.StatusFave, receivingAccount *gtsm
 
 	return nil
 }
+
+func (p *processor) notifyAnnounce(status *gtsmodel.Status) error {
+	return nil
+}

internal/message/fromfederatorprocess.go

@@ -27,7 +27,6 @@ import (
 	"github.com/sirupsen/logrus"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
-	"github.com/superseriousbusiness/gotosocial/internal/transport"
 )
 
 func (p *processor) processFromFederator(federatorMsg gtsmodel.FromFederator) error {
@@ -50,7 +49,7 @@ func (p *processor) processFromFederator(federatorMsg gtsmodel.FromFederator) er
 			}
 
 			l.Debug("will now derefence incoming status")
-			if err := p.dereferenceStatusFields(incomingStatus); err != nil {
+			if err := p.dereferenceStatusFields(incomingStatus, federatorMsg.ReceivingAccount.Username); err != nil {
 				return fmt.Errorf("error dereferencing status from federator: %s", err)
 			}
 			if err := p.db.UpdateByID(incomingStatus.ID, incomingStatus); err != nil {
@@ -88,12 +87,30 @@ func (p *processor) processFromFederator(federatorMsg gtsmodel.FromFederator) er
 			// CREATE A FOLLOW REQUEST
 			incomingFollowRequest, ok := federatorMsg.GTSModel.(*gtsmodel.FollowRequest)
 			if !ok {
-				return errors.New("like was not parseable as *gtsmodel.FollowRequest")
+				return errors.New("incomingFollowRequest was not parseable as *gtsmodel.FollowRequest")
 			}
 
 			if err := p.notifyFollowRequest(incomingFollowRequest, federatorMsg.ReceivingAccount); err != nil {
 				return err
 			}
+		case gtsmodel.ActivityStreamsAnnounce:
+			// CREATE AN ANNOUNCE
+			incomingAnnounce, ok := federatorMsg.GTSModel.(*gtsmodel.Status)
+			if !ok {
+				return errors.New("announce was not parseable as *gtsmodel.Status")
+			}
+
+			if err := p.dereferenceAnnounce(incomingAnnounce, federatorMsg.ReceivingAccount.Username); err != nil {
+				return fmt.Errorf("error dereferencing announce from federator: %s", err)
+			}
+
+			if err := p.db.Put(incomingAnnounce); err != nil {
+				return fmt.Errorf("error adding dereferenced announce to the db: %s", err)
+			}
+
+			if err := p.notifyAnnounce(incomingAnnounce); err != nil {
+				return err
+			}
 		}
 	case gtsmodel.ActivityStreamsUpdate:
 		// UPDATE
@@ -168,18 +185,14 @@ func (p *processor) processFromFederator(federatorMsg gtsmodel.FromFederator) er
 // This function will deference all of the above, insert them in the database as necessary,
 // and attach them to the status. The status itself will not be added to the database yet,
 // that's up the caller to do.
-func (p *processor) dereferenceStatusFields(status *gtsmodel.Status) error {
+func (p *processor) dereferenceStatusFields(status *gtsmodel.Status, requestingUsername string) error {
 	l := p.log.WithFields(logrus.Fields{
 		"func":   "dereferenceStatusFields",
 		"status": fmt.Sprintf("%+v", status),
 	})
 	l.Debug("entering function")
 
-	var t transport.Transport
-	var err error
-	var username string
-	// TODO: dereference with a user that's addressed by the status
-	t, err = p.federator.GetTransportForUser(username)
+	t, err := p.federator.GetTransportForUser(requestingUsername)
 	if err != nil {
 		return fmt.Errorf("error creating transport: %s", err)
 	}
@@ -224,10 +237,10 @@ func (p *processor) dereferenceStatusFields(status *gtsmodel.Status) error {
 		}
 		l.Debugf("dereferenced attachment: %+v", deferencedAttachment)
 		deferencedAttachment.StatusID = status.ID
+		deferencedAttachment.Description = a.Description
 		if err := p.db.Put(deferencedAttachment); err != nil {
 			return fmt.Errorf("error inserting dereferenced attachment with remote url %s: %s", a.RemoteURL, err)
 		}
-		deferencedAttachment.Description = a.Description
 		attachmentIDs = append(attachmentIDs, deferencedAttachment.ID)
 	}
 	status.Attachments = attachmentIDs
@@ -260,7 +273,7 @@ func (p *processor) dereferenceStatusFields(status *gtsmodel.Status) error {
 			}
 
 			// we just don't have it yet, so we should go get it....
-			accountable, err := p.federator.DereferenceRemoteAccount(username, uri)
+			accountable, err := p.federator.DereferenceRemoteAccount(requestingUsername, uri)
 			if err != nil {
 				// we can't dereference it so just skip it
 				l.Debugf("error dereferencing remote account with uri %s: %s", uri.String(), err)
@@ -313,3 +326,106 @@ func (p *processor) dereferenceAccountFields(account *gtsmodel.Account, requesti
 
 	return nil
 }
+
+func (p *processor) dereferenceAnnounce(announce *gtsmodel.Status, requestingUsername string) error {
+	if announce.GTSBoostedStatus == nil || announce.GTSBoostedStatus.URI == "" {
+		// we can't do anything unfortunately
+		return errors.New("dereferenceAnnounce: no URI to dereference")
+	}
+
+	// check if we already have the boosted status in the database
+	boostedStatus := &gtsmodel.Status{}
+	err := p.db.GetWhere([]db.Where{{Key: "uri", Value: announce.GTSBoostedStatus.URI}}, boostedStatus)
+	if err == nil {
+		// nice, we already have it so we don't actually need to dereference it from remote
+		announce.Content = boostedStatus.Content
+		announce.ContentWarning = boostedStatus.ContentWarning
+		announce.ActivityStreamsType = boostedStatus.ActivityStreamsType
+		announce.Sensitive = boostedStatus.Sensitive
+		announce.Language = boostedStatus.Language
+		announce.Text = boostedStatus.Text
+		announce.BoostOfID = boostedStatus.ID
+		announce.Visibility = boostedStatus.Visibility
+		announce.VisibilityAdvanced = boostedStatus.VisibilityAdvanced
+		announce.GTSBoostedStatus = boostedStatus
+		return nil
+	}
+
+	// we don't have it so we need to dereference it
+	remoteStatusID, err := url.Parse(announce.GTSBoostedStatus.URI)
+	if err != nil {
+		return fmt.Errorf("dereferenceAnnounce: error parsing url %s: %s", announce.GTSBoostedStatus.URI, err)
+	}
+
+	statusable, err := p.federator.DereferenceRemoteStatus(requestingUsername, remoteStatusID)
+	if err != nil {
+		return fmt.Errorf("dereferenceAnnounce: error dereferencing remote status with id %s: %s", announce.GTSBoostedStatus.URI, err)
+	}
+
+	// make sure we have the author account in the db
+	attributedToProp := statusable.GetActivityStreamsAttributedTo()
+	for iter := attributedToProp.Begin(); iter != attributedToProp.End(); iter = iter.Next() {
+		accountURI := iter.GetIRI()
+		if accountURI == nil {
+			continue
+		}
+
+		if err := p.db.GetWhere([]db.Where{{Key: "uri", Value: accountURI.String()}}, &gtsmodel.Account{}); err == nil {
+			// we already have it, fine
+			continue
+		}
+
+		// we don't have the boosted status author account yet so dereference it
+		accountable, err := p.federator.DereferenceRemoteAccount(requestingUsername, accountURI)
+		if err != nil {
+			return fmt.Errorf("dereferenceAnnounce: error dereferencing remote account with id %s: %s", accountURI.String(), err)
+		}
+		account, err := p.tc.ASRepresentationToAccount(accountable, false)
+		if err != nil {
+			return fmt.Errorf("dereferenceAnnounce: error converting dereferenced account with id %s into account : %s", accountURI.String(), err)
+		}
+
+		// insert the dereferenced account so it gets an ID etc
+		if err := p.db.Put(account); err != nil {
+			return fmt.Errorf("dereferenceAnnounce: error putting dereferenced account with id %s into database : %s", accountURI.String(), err)
+		}
+
+		if err := p.dereferenceAccountFields(account, requestingUsername, false); err != nil {
+			return fmt.Errorf("dereferenceAnnounce: error dereferencing fields on account with id %s : %s", accountURI.String(), err)
+		}
+	}
+
+	// now convert the statusable into something we can understand
+	boostedStatus, err = p.tc.ASStatusToStatus(statusable)
+	if err != nil {
+		return fmt.Errorf("dereferenceAnnounce: error converting dereferenced statusable with id %s into status : %s", announce.GTSBoostedStatus.URI, err)
+	}
+
+	// put it in the db already so it gets an ID generated for it
+	if err := p.db.Put(boostedStatus); err != nil {
+		return fmt.Errorf("dereferenceAnnounce: error putting dereferenced status with id %s into the db: %s", announce.GTSBoostedStatus.URI, err)
+	}
+
+	// now dereference additional fields straight away (we're already async here so we have time)
+	if err := p.dereferenceStatusFields(boostedStatus, requestingUsername); err !=  nil {
+		return fmt.Errorf("dereferenceAnnounce: error dereferencing status fields for status with id %s: %s", announce.GTSBoostedStatus.URI, err)
+	}
+
+	// update with the newly dereferenced fields
+	if err := p.db.UpdateByID(boostedStatus.ID, boostedStatus); err != nil {
+		return fmt.Errorf("dereferenceAnnounce: error updating dereferenced status in the db: %s", err)
+	}
+
+	// we have everything we need!
+	announce.Content = boostedStatus.Content
+	announce.ContentWarning = boostedStatus.ContentWarning
+	announce.ActivityStreamsType = boostedStatus.ActivityStreamsType
+	announce.Sensitive = boostedStatus.Sensitive
+	announce.Language = boostedStatus.Language
+	announce.Text = boostedStatus.Text
+	announce.BoostOfID = boostedStatus.ID
+	announce.Visibility = boostedStatus.Visibility
+	announce.VisibilityAdvanced = boostedStatus.VisibilityAdvanced
+	announce.GTSBoostedStatus = boostedStatus
+	return nil
+}

internal/message/statusprocess.go

@@ -291,6 +291,15 @@ func (p *processor) StatusBoost(authed *oauth.Auth, targetStatusID string) (*api
 		return nil, NewErrorInternalError(err)
 	}
 
+	// send it to the processor for async processing
+	p.fromClientAPI <- gtsmodel.FromClientAPI{
+		APObjectType:   gtsmodel.ActivityStreamsAnnounce,
+		APActivityType: gtsmodel.ActivityStreamsCreate,
+		GTSModel:       boostWrapperStatus,
+		OriginAccount:  authed.Account,
+		TargetAccount:  targetAccount,
+	}
+
 	// return the frontend representation of the new status to the submitter
 	mastoStatus, err := p.tc.StatusToMasto(boostWrapperStatus, authed.Account, authed.Account, targetAccount, nil, targetStatus)
 	if err != nil {

internal/message/timelineprocess.go

@@ -18,17 +18,17 @@ func (p *processor) HomeTimelineGet(authed *oauth.Auth, maxID string, sinceID st
 	for _, s := range statuses {
 		targetAccount := &gtsmodel.Account{}
 		if err := p.db.GetByID(s.AccountID, targetAccount); err != nil {
-			return nil, NewErrorInternalError(fmt.Errorf("error getting status author: %s", err))
+			return nil, NewErrorInternalError(fmt.Errorf("HomeTimelineGet: error getting status author: %s", err))
 		}
 
 		relevantAccounts, err := p.db.PullRelevantAccountsFromStatus(s)
 		if err != nil {
-			return nil, NewErrorInternalError(fmt.Errorf("error getting relevant statuses: %s", err))
+			return nil, NewErrorInternalError(fmt.Errorf("HomeTimelineGet: error getting relevant statuses for status with id %s and uri %s: %s", s.ID, s.URI, err))
 		}
 
 		visible, err := p.db.StatusVisible(s, targetAccount, authed.Account, relevantAccounts)
 		if err != nil {
-			return nil, NewErrorInternalError(fmt.Errorf("error checking status visibility: %s", err))
+			return nil, NewErrorInternalError(fmt.Errorf("HomeTimelineGet: error checking status visibility: %s", err))
 		}
 		if !visible {
 			continue
@@ -38,16 +38,16 @@ func (p *processor) HomeTimelineGet(authed *oauth.Auth, maxID string, sinceID st
 		if s.BoostOfID != "" {
 			bs := &gtsmodel.Status{}
 			if err := p.db.GetByID(s.BoostOfID, bs); err != nil {
-				return nil, NewErrorInternalError(fmt.Errorf("error getting boosted status: %s", err))
+				return nil, NewErrorInternalError(fmt.Errorf("HomeTimelineGet: error getting boosted status: %s", err))
 			}
 			boostedRelevantAccounts, err := p.db.PullRelevantAccountsFromStatus(bs)
 			if err != nil {
-				return nil, NewErrorInternalError(fmt.Errorf("error getting relevant accounts from boosted status: %s", err))
+				return nil, NewErrorInternalError(fmt.Errorf("HomeTimelineGet: error getting relevant accounts from boosted status: %s", err))
 			}
 
 			boostedVisible, err := p.db.StatusVisible(bs, relevantAccounts.BoostedAccount, authed.Account, boostedRelevantAccounts)
 			if err != nil {
-				return nil, NewErrorInternalError(fmt.Errorf("error checking boosted status visibility: %s", err))
+				return nil, NewErrorInternalError(fmt.Errorf("HomeTimelineGet: error checking boosted status visibility: %s", err))
 			}
 
 			if boostedVisible {
@@ -57,7 +57,7 @@ func (p *processor) HomeTimelineGet(authed *oauth.Auth, maxID string, sinceID st
 
 		apiStatus, err := p.tc.StatusToMasto(s, targetAccount, authed.Account, relevantAccounts.BoostedAccount, relevantAccounts.ReplyToAccount, boostedStatus)
 		if err != nil {
-			return nil, NewErrorInternalError(fmt.Errorf("error converting status to masto: %s", err))
+			return nil, NewErrorInternalError(fmt.Errorf("HomeTimelineGet: error converting status to masto: %s", err))
 		}
 
 		apiStatuses = append(apiStatuses, *apiStatus)

internal/typeutils/asinterfaces.go

@@ -111,6 +111,18 @@ type Likeable interface {
 	withObject
 }
 
+// Announceable represents the minimum interface for an activitystreams 'announce' activity.
+type Announceable interface {
+	withJSONLDId
+	withTypeName
+
+	withActor
+	withObject
+	withPublished
+	withTo
+	withCC
+}
+
 type withJSONLDId interface {
 	GetJSONLDId() vocab.JSONLDIdProperty
 }

internal/typeutils/astointernal.go

@@ -422,6 +422,99 @@ func (c *converter) ASLikeToFave(likeable Likeable) (*gtsmodel.StatusFave, error
 	}, nil
 }
 
+func (c *converter) ASAnnounceToStatus(announceable Announceable) (*gtsmodel.Status, bool, error) {
+	status := &gtsmodel.Status{}
+	isNew := true
+
+	// check if we already have the boost in the database
+	idProp := announceable.GetJSONLDId()
+	if idProp == nil || !idProp.IsIRI() {
+		return nil, isNew, errors.New("no id property set on announce, or was not an iri")
+	}
+	uri := idProp.GetIRI().String()
+
+	if err := c.db.GetWhere([]db.Where{{Key: "uri", Value: uri}}, status); err == nil {
+		// we already have it, great, just return it as-is :)
+		isNew = false
+		return status, isNew, nil
+	}
+	status.URI = uri
+
+	// get the URI of the announced/boosted status
+	boostedStatusURI, err := extractObject(announceable)
+	if err != nil {
+		return nil, isNew, fmt.Errorf("ASAnnounceToStatus: error getting object from announce: %s", err)
+	}
+
+	// set the URI on the new status for dereferencing later
+	status.GTSBoostedStatus = &gtsmodel.Status{
+		URI: boostedStatusURI.String(),
+	}
+
+	// get the published time for the announce
+	published, err := extractPublished(announceable)
+	if err != nil {
+		return nil, isNew, fmt.Errorf("ASAnnounceToStatus: error extracting published time: %s", err)
+	}
+	status.CreatedAt = published
+	status.UpdatedAt = published
+
+	// get the actor's IRI (ie., the person who boosted the status)
+	actor, err := extractActor(announceable)
+	if err != nil {
+		return nil, isNew, fmt.Errorf("ASAnnounceToStatus: error extracting actor: %s", err)
+	}
+
+	// get the boosting account based on the URI
+	// this should have been dereferenced already before we hit this point so we can confidently error out if we don't have it
+	boostingAccount := &gtsmodel.Account{}
+	if err := c.db.GetWhere([]db.Where{{Key: "uri", Value: actor.String()}}, boostingAccount); err != nil {
+		return nil, isNew, fmt.Errorf("ASAnnounceToStatus: error in db fetching account with uri %s: %s", actor.String(), err)
+	}
+	status.AccountID = boostingAccount.ID
+
+	// these will all be wrapped in the boosted status so set them empty here
+	status.Attachments = []string{}
+	status.Tags = []string{}
+	status.Mentions = []string{}
+	status.Emojis = []string{}
+
+	// parse the visibility from the To and CC entries
+	var visibility gtsmodel.Visibility
+
+	to, err := extractTos(announceable)
+	if err != nil {
+		return nil, isNew, fmt.Errorf("error extracting TO values: %s", err)
+	}
+
+	cc, err := extractCCs(announceable)
+	if err != nil {
+		return nil, isNew, fmt.Errorf("error extracting CC values: %s", err)
+	}
+
+	if len(to) == 0 && len(cc) == 0 {
+		return nil, isNew, errors.New("message wasn't TO or CC anyone")
+	}
+
+	// if it's CC'ed to public, it's public or unlocked
+	if isPublic(cc) {
+		visibility = gtsmodel.VisibilityUnlocked
+	}
+	if isPublic(to) {
+		visibility = gtsmodel.VisibilityPublic
+	}
+
+	// we should have a visibility by now
+	if visibility == "" {
+		return nil, isNew, errors.New("couldn't derive visibility")
+	}
+	status.Visibility = visibility
+
+	// the rest of the fields will be taken from the target status, but it's not our job to do the dereferencing here
+
+	return status, isNew, nil
+}
+
 func isPublic(tos []*url.URL) bool {
 	for _, entry := range tos {
 		if strings.EqualFold(entry.String(), "https://www.w3.org/ns/activitystreams#Public") {

internal/typeutils/converter.go

@@ -100,6 +100,19 @@ type TypeConverter interface {
 	ASFollowToFollow(followable Followable) (*gtsmodel.Follow, error)
 	// ASLikeToFave converts a remote activitystreams 'like' representation into a gts model status fave.
 	ASLikeToFave(likeable Likeable) (*gtsmodel.StatusFave, error)
+	// ASAnnounceToStatus converts an activitystreams 'announce' into a status.
+	//
+	// The returned bool indicates whether this status is new (true) or not new (false).
+	//
+	// In other words, if the status is already in the database with the ID set on the announceable, then that will be returned,
+	// the returned bool will be false, and no further processing is necessary. If the returned bool is true, indicating
+	// that this is a new announce, then further processing will be necessary, because the returned status will be bareboned and
+	// require further dereferencing.
+	//
+	// This is useful when multiple users on an instance might receive the same boost, and we only want to process the boost once.
+	//
+	// NOTE -- this is different from one status being boosted multiple times! In this case, new boosts should indeed be created.
+	ASAnnounceToStatus(announceable Announceable) (status *gtsmodel.Status, new bool, err error)
 
 	/*
 		INTERNAL (gts) MODEL TO ACTIVITYSTREAMS MODEL
@@ -117,6 +130,8 @@ type TypeConverter interface {
 	AttachmentToAS(a *gtsmodel.MediaAttachment) (vocab.ActivityStreamsDocument, error)
 	// FaveToAS converts a gts model status fave into an activityStreams LIKE, suitable for federation.
 	FaveToAS(f *gtsmodel.StatusFave) (vocab.ActivityStreamsLike, error)
+	// BoostToAS converts a gts model boost into an activityStreams ANNOUNCE, suitable for federation
+	BoostToAS(boostWrapperStatus *gtsmodel.Status, boostingAccount *gtsmodel.Account, boostedAccount *gtsmodel.Account) (vocab.ActivityStreamsAnnounce, error)
 
 	/*
 		INTERNAL (gts) MODEL TO INTERNAL MODEL

internal/typeutils/internaltoas.go

@@ -640,3 +640,76 @@ func (c *converter) FaveToAS(f *gtsmodel.StatusFave) (vocab.ActivityStreamsLike,
 
 	return like, nil
 }
+
+func (c *converter) BoostToAS(boostWrapperStatus *gtsmodel.Status, boostingAccount *gtsmodel.Account, boostedAccount *gtsmodel.Account) (vocab.ActivityStreamsAnnounce, error) {
+	// the boosted status is probably pinned to the boostWrapperStatus but double check to make sure
+	if boostWrapperStatus.GTSBoostedStatus == nil {
+		b := &gtsmodel.Status{}
+		if err := c.db.GetByID(boostWrapperStatus.BoostOfID, b); err != nil {
+			return nil, fmt.Errorf("BoostToAS: error getting status with ID %s from the db: %s", boostWrapperStatus.BoostOfID, err)
+		}
+		boostWrapperStatus = b
+	}
+
+	// create the announce
+	announce := streams.NewActivityStreamsAnnounce()
+
+	// set the actor
+	boosterURI, err := url.Parse(boostingAccount.URI)
+	if err != nil {
+		return nil, fmt.Errorf("BoostToAS: error parsing uri %s: %s", boostingAccount.URI, err)
+	}
+	actorProp := streams.NewActivityStreamsActorProperty()
+	actorProp.AppendIRI(boosterURI)
+	announce.SetActivityStreamsActor(actorProp)
+
+	// set the ID
+	boostIDURI, err := url.Parse(boostWrapperStatus.URI)
+	if err != nil {
+		return nil, fmt.Errorf("BoostToAS: error parsing uri %s: %s", boostWrapperStatus.URI, err)
+	}
+	idProp := streams.NewJSONLDIdProperty()
+	idProp.SetIRI(boostIDURI)
+	announce.SetJSONLDId(idProp)
+
+	// set the object
+	boostedStatusURI, err := url.Parse(boostWrapperStatus.GTSBoostedStatus.URI)
+	if err != nil {
+		return nil, fmt.Errorf("BoostToAS: error parsing uri %s: %s", boostWrapperStatus.GTSBoostedStatus.URI, err)
+	}
+	objectProp := streams.NewActivityStreamsObjectProperty()
+	objectProp.AppendIRI(boostedStatusURI)
+	announce.SetActivityStreamsObject(objectProp)
+
+	// set the published time
+	publishedProp := streams.NewActivityStreamsPublishedProperty()
+	publishedProp.Set(boostWrapperStatus.CreatedAt)
+	announce.SetActivityStreamsPublished(publishedProp)
+
+	// set the to
+	followersURI, err := url.Parse(boostingAccount.FollowersURI)
+	if err != nil {
+		return nil, fmt.Errorf("BoostToAS: error parsing uri %s: %s", boostingAccount.FollowersURI, err)
+	}
+	toProp := streams.NewActivityStreamsToProperty()
+	toProp.AppendIRI(followersURI)
+	announce.SetActivityStreamsTo(toProp)
+
+	// set the cc
+	boostedURI, err := url.Parse(boostedAccount.URI)
+	if err != nil {
+		return nil, fmt.Errorf("BoostToAS: error parsing uri %s: %s", boostedAccount.URI, err)
+	}
+
+	publicURI, err := url.Parse(asPublicURI)
+	if err != nil {
+		return nil, fmt.Errorf("BoostToAS: error parsing uri %s: %s", asPublicURI, err)
+	}
+
+	ccProp := streams.NewActivityStreamsCcProperty()
+	ccProp.AppendIRI(boostedURI)
+	ccProp.AppendIRI(publicURI)
+	announce.SetActivityStreamsCc(ccProp)
+
+	return announce, nil
+}