Dereference remote replies (#132)

* decided where to put reply dereferencing

* fiddling with dereferencing threads

* further adventures

* tidy up some stuff

* move dereferencing functionality

* a bunch of refactoring

* go fmt

* more refactoring

* bleep bloop

* docs and linting

* start implementing replies collection on gts side

* fiddling around

* allow dereferencing our replies

* lint, fmt

docs/api/swagger.yaml

@@ -1562,6 +1562,64 @@ definitions:
     type: string
     x-go-name: Visibility
     x-go-package: github.com/superseriousbusiness/gotosocial/internal/api/model
+  swaggerStatusRepliesCollection:
+    properties:
+      '@context':
+        description: ActivityStreams context.
+        example: https://www.w3.org/ns/activitystreams
+        type: string
+        x-go-name: Context
+      first:
+        $ref: '#/definitions/swaggerStatusRepliesCollectionPage'
+      id:
+        description: ActivityStreams ID.
+        example: https://example.org/users/some_user/statuses/106717595988259568/replies
+        type: string
+        x-go-name: ID
+      type:
+        description: ActivityStreams type.
+        example: Collection
+        type: string
+        x-go-name: Type
+    title: SwaggerStatusRepliesCollection represents a response to GET /users/{username}/statuses/{status}/replies.
+    type: object
+    x-go-name: SwaggerStatusRepliesCollection
+    x-go-package: github.com/superseriousbusiness/gotosocial/internal/api/s2s/user
+  swaggerStatusRepliesCollectionPage:
+    properties:
+      id:
+        description: ActivityStreams ID.
+        example: https://example.org/users/some_user/statuses/106717595988259568/replies?page=true
+        type: string
+        x-go-name: ID
+      items:
+        description: Items on this page.
+        example:
+        - https://example.org/users/some_other_user/statuses/086417595981111564
+        - https://another.example.com/users/another_user/statuses/01FCN8XDV3YG7B4R42QA6YQZ9R
+        items:
+          type: string
+        type: array
+        x-go-name: Items
+      next:
+        description: Link to the next page.
+        example: https://example.org/users/some_user/statuses/106717595988259568/replies?only_other_accounts=true&page=true
+        type: string
+        x-go-name: Next
+      partOf:
+        description: Collection this page belongs to.
+        example: https://example.org/users/some_user/statuses/106717595988259568/replies
+        type: string
+        x-go-name: PartOf
+      type:
+        description: ActivityStreams type.
+        example: CollectionPage
+        type: string
+        x-go-name: Type
+    title: SwaggerStatusRepliesCollectionPage represents one page of a collection.
+    type: object
+    x-go-name: SwaggerStatusRepliesCollectionPage
+    x-go-package: github.com/superseriousbusiness/gotosocial/internal/api/s2s/user
   tag:
     properties:
       name:
@@ -1621,7 +1679,7 @@ info:
     name: AGPL3
     url: https://www.gnu.org/licenses/agpl-3.0.en.html
   title: GoToSocial
-  version: 0.1.0-SNAPSHOT
+  version: 0.1.0-SNAPSHOT-dereference_remote_replies
 paths:
   /api/v1/accounts:
     post:
@@ -2395,11 +2453,10 @@ paths:
       - blocks
   /api/v1/instance:
     get:
-      description: "This is mostly provided for Mastodon application compatibility,
+      description: |-
-        since many apps that work with Mastodon use `/api/v1/instance` to inform their
+        This is mostly provided for Mastodon application compatibility, since many apps that work with Mastodon use `/api/v1/instance` to inform their connection parameters.
-        connection parameters. \n\nHowever, it can also be used by other instances
+
-        for gathering instance information and representing instances in some UI or
+        However, it can also be used by other instances for gathering instance information and representing instances in some UI or other.
-        other."
       operationId: instanceGet
       produces:
       - application/json
@@ -3306,6 +3363,56 @@ paths:
       summary: See public statuses/posts that your instance is aware of.
       tags:
       - timelines
+  /users/{username}/statuses/{status}/replies:
+    get:
+      description: |-
+        Note that the response will be a Collection with a page as `first`, as shown below, if `page` is `false`.
+
+        If `page` is `true`, then the response will be a single `CollectionPage` without the wrapping `Collection`.
+
+        HTTP signature is required on the request.
+      operationId: s2sRepliesGet
+      parameters:
+      - description: Username of the account.
+        in: path
+        name: username
+        required: true
+        type: string
+      - description: ID of the status.
+        in: path
+        name: status
+        required: true
+        type: string
+      - default: false
+        description: Return response as a CollectionPage.
+        in: query
+        name: page
+        type: boolean
+      - default: false
+        description: Return replies only from accounts other than the status owner.
+        in: query
+        name: only_other_accounts
+        type: boolean
+      - description: Minimum ID of the next status, used for paging.
+        in: query
+        name: min_id
+        type: string
+      produces:
+      - application/activity+json
+      responses:
+        "200":
+          description: ""
+          schema:
+            $ref: '#/definitions/swaggerStatusRepliesCollection'
+        "401":
+          description: unauthorized
+        "403":
+          description: forbidden
+        "404":
+          description: not found
+      summary: Get the replies collection for a status.
+      tags:
+      - s2s/federation
 schemes:
 - https
 - http

docs/assets/diagrams/conversation_thread.drawio

@@ -0,0 +1 @@
+<mxfile host="Electron" modified="2021-08-09T09:38:01.312Z" agent="5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) draw.io/13.0.3 Chrome/80.0.3987.163 Electron/8.2.1 Safari/537.36" etag="jpDbRW_Z_HbkNN_9kqIg" version="13.0.3" type="device"><diagram id="xfbHBLe4vMsijS9wwlE7" name="Page-1">7VrBcpswEP0ajs0AMjY+xkncXtqmTaZNTxnFKEYTQIwQsZ2vrwQSBok0rmObmGTG40ELEtq3T29Xsi1wFi8/U5iGX0mAIsu1g6UFzi3XHfku/xaGlTI4pWFOcVCaaoYr/ISk0ZbWHAcoazzICIkYTpvGGUkSNGMNG6SULJqP3ZOo+dYUzpFhuJrByLT+xgELS6vv2Wv7F4TnoXqzY8s7MVQPS0MWwoAsaiZwYYEzSggrr+LlGYoEdgqXst/0mbvVxChK2CYdTp/8m+9T+/pXPP0Bw2sCycPtp0E5yiOMcumwNbAzEiOSIG6/DnEmhhBfUDyH6Er4hmCCk/tcDJ+SjJ1ID9lKwUZJngRIvNmxwGQRYoauUjgTdxecJ9wWsjiStzNGyUMFLwdmImeFKEPLZ911KhA5+RCfMysmJzsAhbsknuv5ZXuxDqOjnglrIRxKG5TMmVdDr8HlFxLf/8B6aGINpo3P61BU/LJ5I4BZWHXcAZq+BqYCqTMwRyaYOnooCU6FAvBWIuisoVKDDi0xu6ld/xEonniydb6UoBaNlWok3I2beqPWSzTX3YqW6ldOEwWG7Gih4K6QnM7Qy4RikM4Re2mRm6Gtxc5rCZ2yURRBhh+b022Lp3zDJcHckYo5LmhSZwA0SpRuyl51/dIH0jgIfG2gEgdjoIJeldvbM843GHdJ0SMmeSZVsG0F8xXHmlyDEZ4n/HrGw40oN4h1iXnCOZU3YhwEovuEogw/wbtiKMGcVHhW+OpNLO9cjJUzkpUpc0frvMq7aqGPzYU+biGLu691Pj5i0fSGby0Fqei+a9kcbCib4y5lczDSZNPZUjY9rQ4COrn2LJuOY1DumxDF/mimHqo2zRwdUjMdt0ei6fhdl5oO6BGcYNA5nC17zlfkoC3zyTa5q4McVC7lzpKQpyUhnRQbJyH7hWy27yTkfdQ9m3MOvKnCZ7Crwmd8YM6Z5z0/URqtelHy6OdBrplV2jiyv5LHPA/qLdjA6xpsc1M+IbyY51opq3rXvlsVUhAThm5NrT22EAAtf1WHWvUqanjIGKiBj7Io1c/mnTY4D1qUuuYu9XjhBKPO4WzZgb67emvT4/nar4FdnM9rJ75b1/iuttM8dI3vtmzTPzj3T73rjHP6dnBrzukb1ENzzjzL6E3Zqf9wB1p+8z1o2emam/jegl2p8u7B5s31n07KhbD+5w64+As=</diagram></mxfile>

docs/federation/behaviors/conversation_threads.md

@@ -0,0 +1,45 @@
+# Conversation Threads
+
+Due to the nature of decentralization and federation, it is practically impossible for any one server on the fediverse to be aware of every post in a given conversation thread.
+
+With that said, it is possible to do 'best effort' dereferencing of threads, whereby remote replies are fetched from one server onto another, to try to more fully flesh out a conversation.
+
+GoToSocial does this by iterating up and down the thread of a conversation, pulling in remote statuses where possible.
+
+## Example
+
+Let's say we have two accounts: `local_account` on `our.server`, and `remote_1` on `remote.1`.
+
+In this scenario, `local_account` follows `remote_1`, so posts from `remote_1` show up in the home timeline of `local_account`.
+
+Now, `remote_1` boosts/reblogs a post from a third account, `remote_2`, residing on server `remote.2`.
+
+`local_account` does not follow `remote_2`, and neither does anybody else on `our.server`, which means that `our.server` has not seen this post by `remote_2` before.
+
+![A diagram of the conversation thread, showing the post from remote_2, and possible ancestor and descendant posts](../../assets/diagrams/conversation_thread.png)
+
+What GoToSocial will do now, is 'dereference' the post by `remote_2` to check if it is part of a thread and, if so, whether any other parts of the thread can be obtained.
+
+GtS begins by checking the `inReplyTo` property of the post, which is set when a post is a reply to another post. [See here](https://www.w3.org/TR/activitystreams-vocabulary/#dfn-inreplyto). If `inReplyTo` is set, GoToSocial derefences the replied-to post. If *this* post also has an `inReplyTo` set, then GoToSocial dereferences that too, and so on.
+
+Once all of these **ancestors** of a status have been retrieved, GtS will begin working down through the **descendants** of posts.
+
+It does this by checking the `replies` property of a derefenced post, and working through replies, and replies of replies. [See here](https://www.w3.org/TR/activitystreams-vocabulary/#dfn-replies).
+
+This process of thread dereferencing will likely involve making multiple HTTP calls to different servers, especially if the thread is long and complicated.
+
+The end result of this dereferencing is that, assuming the reblogged post by `remote_2` was part of a thread, then `local_account` should now be able to see posts in the thread when they open the status on their home timeline. In other words, they will see replies from accounts on other servers (who they may not have come across yet), in addition to any previous and next posts in the thread as posted by `remote_2`.
+
+This gives `local_account` a more complete view on the conversation, as opposed to just seeing the reblogged post in isolation and out of context. It also gives `local_account` the opportunity to discover new accounts to follow, based on replies to `remote_2`.
+
+## Privacy and Security
+
+During the dereferencing process, GoToSocial signs outgoing requests using the key of the actor who received the activity that necessitated dereferencing. To use the above example, this means that all dereferencing requests would be signed by `local_account`. This gives remote servers the ability to refuse these dereferencing requests, assuming that `local_account` is blocked by one or more participants in the conversation.
+
+From GoToSocial's side, domain blocks will be respected during the dereferencing process, to avoid making calls to servers that `our.server` has blocked.
+
+Individual account blocks will also be respected, meaning that `our.server` won't try to dereference posts from accounts blocked by `local_account`.
+
+Finally, GoToSocial expects that remote servers will only list replies that are marked as public (either `to` or `cc`). GtS may *try* to dereference followers-only posts, but it will assume that remote servers will check whether or not `local_account` is allowed to view them, and refuse accordingly.
+
+Of course, when `local_account` opens up the conversation thread in whatever application they are using, GoToSocial will apply the usual post visibility filtering to ensure that they do not see any posts that they shouldn't have access to.

docs/federation/glossary.md

@@ -0,0 +1,27 @@
+# Glossary
+
+Some commonly-used terms in discussions of federation, and their meanings.
+
+### `ActivityPub`
+
+A decentralized social networking protocol based on the ActivityStreams data format. See [here](https://www.w3.org/TR/activitypub/).
+
+GoToSocial uses the ActivityPub protocol to communicate between GtS servers, and with other federated servers like Mastodon, Pixelfed, etc.
+
+### `ActivityStreams`
+
+A model/data format for representing potential and completed activities using JSON. See [here](https://www.w3.org/TR/activitystreams-core/).
+
+GoToSocial uses the ActivityStreams data model to 'speak' ActivityPub with other servers.
+
+### `Actor`
+
+An actor is an ActivityStreams object that is capable of performing some Activity like following, liking, creating a post, reblogging, etc. See [here](https://www.w3.org/TR/activitypub/#actors).
+
+In GoToSocial, each account/user is an actor.
+
+### `Dereference`
+
+To 'dereference' a post or a profile means to make an HTTP call to the server that hosts that post or profile, in order to obtain its ActivityStreams representation.
+
+GoToSocial 'dereferences' posts and profiles on remote servers, in order to convert them to models that GoToSocial can understand and work with.

docs/federation/principles.md

@@ -0,0 +1,9 @@
+# Principles
+
+TODO -- describe the principles GtS uses for federating.
+
+Eg:
+
+* Why federate?
+* Why ActivityPub?
+* Broad overview of how GtS fits into the fediverse.

docs/federation/security.md

@@ -0,0 +1,7 @@
+# Security
+
+TODO: describe the security model we use for federation.
+
+* http signatures
+* behavior for refusing requests
+* how data is protected

internal/ap/extract.go

@@ -16,7 +16,10 @@
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
 
-package typeutils
+// Package ap contains models and utilities for working with activitypub/activitystreams representations.
+//
+// It is built on top of go-fed/activity.
+package ap
 
 import (
 	"crypto/rsa"
@@ -33,7 +36,8 @@ import (
 	"github.com/superseriousbusiness/gotosocial/internal/util"
 )
 
-func extractPreferredUsername(i withPreferredUsername) (string, error) {
+// ExtractPreferredUsername returns a string representation of an interface's preferredUsername property.
+func ExtractPreferredUsername(i WithPreferredUsername) (string, error) {
 	u := i.GetActivityStreamsPreferredUsername()
 	if u == nil || !u.IsXMLSchemaString() {
 		return "", errors.New("preferredUsername was not a string")
@@ -44,7 +48,8 @@ func extractPreferredUsername(i withPreferredUsername) (string, error) {
 	return u.GetXMLSchemaString(), nil
 }
 
-func extractName(i withName) (string, error) {
+// ExtractName returns a string representation of an interface's name property.
+func ExtractName(i WithName) (string, error) {
 	nameProp := i.GetActivityStreamsName()
 	if nameProp == nil {
 		return "", errors.New("activityStreamsName not found")
@@ -60,22 +65,42 @@ func extractName(i withName) (string, error) {
 	return "", errors.New("activityStreamsName not found")
 }
 
-func extractInReplyToURI(i withInReplyTo) (*url.URL, error) {
+// ExtractInReplyToURI extracts the inReplyToURI property (if present) from an interface.
+func ExtractInReplyToURI(i WithInReplyTo) *url.URL {
 	inReplyToProp := i.GetActivityStreamsInReplyTo()
 	if inReplyToProp == nil {
-		return nil, errors.New("in reply to prop was nil")
+		// the property just wasn't set
+		return nil
 	}
 	for iter := inReplyToProp.Begin(); iter != inReplyToProp.End(); iter = iter.Next() {
 		if iter.IsIRI() {
 			if iter.GetIRI() != nil {
-				return iter.GetIRI(), nil
+				return iter.GetIRI()
 			}
 		}
 	}
-	return nil, errors.New("couldn't find iri for in reply to")
+	// couldn't find a URI
+	return nil
 }
 
-func extractTos(i withTo) ([]*url.URL, error) {
+// ExtractURLItems extracts a slice of URLs from a property that has withItems.
+func ExtractURLItems(i WithItems) []*url.URL {
+	urls := []*url.URL{}
+	items := i.GetActivityStreamsItems()
+	if items == nil || items.Len() == 0 {
+		return urls
+	}
+
+	for iter := items.Begin(); iter != items.End(); iter = iter.Next() {
+		if iter.IsIRI() {
+			urls = append(urls, iter.GetIRI())
+		}
+	}
+	return urls
+}
+
+// ExtractTos returns a list of URIs that the activity addresses as To.
+func ExtractTos(i WithTo) ([]*url.URL, error) {
 	to := []*url.URL{}
 	toProp := i.GetActivityStreamsTo()
 	if toProp == nil {
@@ -91,7 +116,8 @@ func extractTos(i withTo) ([]*url.URL, error) {
 	return to, nil
 }
 
-func extractCCs(i withCC) ([]*url.URL, error) {
+// ExtractCCs returns a list of URIs that the activity addresses as CC.
+func ExtractCCs(i WithCC) ([]*url.URL, error) {
 	cc := []*url.URL{}
 	ccProp := i.GetActivityStreamsCc()
 	if ccProp == nil {
@@ -107,7 +133,8 @@ func extractCCs(i withCC) ([]*url.URL, error) {
 	return cc, nil
 }
 
-func extractAttributedTo(i withAttributedTo) (*url.URL, error) {
+// ExtractAttributedTo returns the URL of the actor that the withAttributedTo is attributed to.
+func ExtractAttributedTo(i WithAttributedTo) (*url.URL, error) {
 	attributedToProp := i.GetActivityStreamsAttributedTo()
 	if attributedToProp == nil {
 		return nil, errors.New("attributedToProp was nil")
@@ -122,7 +149,8 @@ func extractAttributedTo(i withAttributedTo) (*url.URL, error) {
 	return nil, errors.New("couldn't find iri for attributed to")
 }
 
-func extractPublished(i withPublished) (time.Time, error) {
+// ExtractPublished extracts the publication time of an activity.
+func ExtractPublished(i WithPublished) (time.Time, error) {
 	publishedProp := i.GetActivityStreamsPublished()
 	if publishedProp == nil {
 		return time.Time{}, errors.New("published prop was nil")
@@ -139,13 +167,13 @@ func extractPublished(i withPublished) (time.Time, error) {
 	return t, nil
 }
 
-// extractIconURL extracts a URL to a supported image file from something like:
+// ExtractIconURL extracts a URL to a supported image file from something like:
 //   "icon": {
 //     "mediaType": "image/jpeg",
 //     "type": "Image",
 //     "url": "http://example.org/path/to/some/file.jpeg"
 //   },
-func extractIconURL(i withIcon) (*url.URL, error) {
+func ExtractIconURL(i WithIcon) (*url.URL, error) {
 	iconProp := i.GetActivityStreamsIcon()
 	if iconProp == nil {
 		return nil, errors.New("icon property was nil")
@@ -166,7 +194,7 @@ func extractIconURL(i withIcon) (*url.URL, error) {
 		}
 
 		// 2. has a URL so we can grab it
-		url, err := extractURL(imageValue)
+		url, err := ExtractURL(imageValue)
 		if err == nil && url != nil {
 			return url, nil
 		}
@@ -175,13 +203,13 @@ func extractIconURL(i withIcon) (*url.URL, error) {
 	return nil, errors.New("could not extract valid image from icon")
 }
 
-// extractImageURL extracts a URL to a supported image file from something like:
+// ExtractImageURL extracts a URL to a supported image file from something like:
 //   "image": {
 //     "mediaType": "image/jpeg",
 //     "type": "Image",
 //     "url": "http://example.org/path/to/some/file.jpeg"
 //   },
-func extractImageURL(i withImage) (*url.URL, error) {
+func ExtractImageURL(i WithImage) (*url.URL, error) {
 	imageProp := i.GetActivityStreamsImage()
 	if imageProp == nil {
 		return nil, errors.New("icon property was nil")
@@ -202,7 +230,7 @@ func extractImageURL(i withImage) (*url.URL, error) {
 		}
 
 		// 2. has a URL so we can grab it
-		url, err := extractURL(imageValue)
+		url, err := ExtractURL(imageValue)
 		if err == nil && url != nil {
 			return url, nil
 		}
@@ -211,7 +239,8 @@ func extractImageURL(i withImage) (*url.URL, error) {
 	return nil, errors.New("could not extract valid image from image property")
 }
 
-func extractSummary(i withSummary) (string, error) {
+// ExtractSummary extracts the summary/content warning of an interface.
+func ExtractSummary(i WithSummary) (string, error) {
 	summaryProp := i.GetActivityStreamsSummary()
 	if summaryProp == nil {
 		return "", errors.New("summary property was nil")
@@ -226,14 +255,16 @@ func extractSummary(i withSummary) (string, error) {
 	return "", errors.New("could not extract summary")
 }
 
-func extractDiscoverable(i withDiscoverable) (bool, error) {
+// ExtractDiscoverable extracts the Discoverable boolean of an interface.
+func ExtractDiscoverable(i WithDiscoverable) (bool, error) {
 	if i.GetTootDiscoverable() == nil {
 		return false, errors.New("discoverable was nil")
 	}
 	return i.GetTootDiscoverable().Get(), nil
 }
 
-func extractURL(i withURL) (*url.URL, error) {
+// ExtractURL extracts the URL property of an interface.
+func ExtractURL(i WithURL) (*url.URL, error) {
 	urlProp := i.GetActivityStreamsUrl()
 	if urlProp == nil {
 		return nil, errors.New("url property was nil")
@@ -248,7 +279,9 @@ func extractURL(i withURL) (*url.URL, error) {
 	return nil, errors.New("could not extract url")
 }
 
-func extractPublicKeyForOwner(i withPublicKey, forOwner *url.URL) (*rsa.PublicKey, *url.URL, error) {
+// ExtractPublicKeyForOwner extracts the public key from an interface, as long as it belongs to the specified owner.
+// It will return the public key itself, the id/URL of the public key, or an error if something goes wrong.
+func ExtractPublicKeyForOwner(i WithPublicKey, forOwner *url.URL) (*rsa.PublicKey, *url.URL, error) {
 	publicKeyProp := i.GetW3IDSecurityV1PublicKey()
 	if publicKeyProp == nil {
 		return nil, nil, errors.New("public key property was nil")
@@ -298,7 +331,8 @@ func extractPublicKeyForOwner(i withPublicKey, forOwner *url.URL) (*rsa.PublicKe
 	return nil, nil, errors.New("couldn't find public key")
 }
 
-func extractContent(i withContent) (string, error) {
+// ExtractContent returns a string representation of the interface's Content property.
+func ExtractContent(i WithContent) (string, error) {
 	contentProperty := i.GetActivityStreamsContent()
 	if contentProperty == nil {
 		return "", errors.New("content property was nil")
@@ -311,7 +345,8 @@ func extractContent(i withContent) (string, error) {
 	return "", errors.New("no content found")
 }
 
-func extractAttachments(i withAttachment) ([]*gtsmodel.MediaAttachment, error) {
+// ExtractAttachments returns a slice of attachments on the interface.
+func ExtractAttachments(i WithAttachment) ([]*gtsmodel.MediaAttachment, error) {
 	attachments := []*gtsmodel.MediaAttachment{}
 	attachmentProp := i.GetActivityStreamsAttachment()
 	if attachmentProp == nil {
@@ -326,7 +361,7 @@ func extractAttachments(i withAttachment) ([]*gtsmodel.MediaAttachment, error) {
 		if !ok {
 			continue
 		}
-		attachment, err := extractAttachment(attachmentable)
+		attachment, err := ExtractAttachment(attachmentable)
 		if err != nil {
 			continue
 		}
@@ -335,12 +370,13 @@ func extractAttachments(i withAttachment) ([]*gtsmodel.MediaAttachment, error) {
 	return attachments, nil
 }
 
-func extractAttachment(i Attachmentable) (*gtsmodel.MediaAttachment, error) {
+// ExtractAttachment returns a gts model of an attachment from an attachmentable interface.
+func ExtractAttachment(i Attachmentable) (*gtsmodel.MediaAttachment, error) {
 	attachment := &gtsmodel.MediaAttachment{
 		File: gtsmodel.File{},
 	}
 
-	attachmentURL, err := extractURL(i)
+	attachmentURL, err := ExtractURL(i)
 	if err != nil {
 		return nil, err
 	}
@@ -356,7 +392,7 @@ func extractAttachment(i Attachmentable) (*gtsmodel.MediaAttachment, error) {
 	attachment.File.ContentType = mediaType.Get()
 	attachment.Type = gtsmodel.FileTypeImage
 
-	name, err := extractName(i)
+	name, err := ExtractName(i)
 	if err == nil {
 		attachment.Description = name
 	}
@@ -376,7 +412,8 @@ func extractAttachment(i Attachmentable) (*gtsmodel.MediaAttachment, error) {
 // 	return i.GetTootBlurhashProperty().Get(), nil
 // }
 
-func extractHashtags(i withTag) ([]*gtsmodel.Tag, error) {
+// ExtractHashtags returns a slice of tags on the interface.
+func ExtractHashtags(i WithTag) ([]*gtsmodel.Tag, error) {
 	tags := []*gtsmodel.Tag{}
 	tagsProp := i.GetActivityStreamsTag()
 	if tagsProp == nil {
@@ -397,7 +434,7 @@ func extractHashtags(i withTag) ([]*gtsmodel.Tag, error) {
 			continue
 		}
 
-		tag, err := extractHashtag(hashtaggable)
+		tag, err := ExtractHashtag(hashtaggable)
 		if err != nil {
 			continue
 		}
@@ -407,7 +444,8 @@ func extractHashtags(i withTag) ([]*gtsmodel.Tag, error) {
 	return tags, nil
 }
 
-func extractHashtag(i Hashtaggable) (*gtsmodel.Tag, error) {
+// ExtractHashtag returns a gtsmodel tag from a hashtaggable.
+func ExtractHashtag(i Hashtaggable) (*gtsmodel.Tag, error) {
 	tag := &gtsmodel.Tag{}
 
 	hrefProp := i.GetActivityStreamsHref()
@@ -416,7 +454,7 @@ func extractHashtag(i Hashtaggable) (*gtsmodel.Tag, error) {
 	}
 	tag.URL = hrefProp.GetIRI().String()
 
-	name, err := extractName(i)
+	name, err := ExtractName(i)
 	if err != nil {
 		return nil, err
 	}
@@ -425,7 +463,8 @@ func extractHashtag(i Hashtaggable) (*gtsmodel.Tag, error) {
 	return tag, nil
 }
 
-func extractEmojis(i withTag) ([]*gtsmodel.Emoji, error) {
+// ExtractEmojis returns a slice of emojis on the interface.
+func ExtractEmojis(i WithTag) ([]*gtsmodel.Emoji, error) {
 	emojis := []*gtsmodel.Emoji{}
 	tagsProp := i.GetActivityStreamsTag()
 	if tagsProp == nil {
@@ -446,7 +485,7 @@ func extractEmojis(i withTag) ([]*gtsmodel.Emoji, error) {
 			continue
 		}
 
-		emoji, err := extractEmoji(emojiable)
+		emoji, err := ExtractEmoji(emojiable)
 		if err != nil {
 			continue
 		}
@@ -456,7 +495,8 @@ func extractEmojis(i withTag) ([]*gtsmodel.Emoji, error) {
 	return emojis, nil
 }
 
-func extractEmoji(i Emojiable) (*gtsmodel.Emoji, error) {
+// ExtractEmoji ...
+func ExtractEmoji(i Emojiable) (*gtsmodel.Emoji, error) {
 	emoji := &gtsmodel.Emoji{}
 
 	idProp := i.GetJSONLDId()
@@ -467,7 +507,7 @@ func extractEmoji(i Emojiable) (*gtsmodel.Emoji, error) {
 	emoji.URI = uri.String()
 	emoji.Domain = uri.Host
 
-	name, err := extractName(i)
+	name, err := ExtractName(i)
 	if err != nil {
 		return nil, err
 	}
@@ -476,7 +516,7 @@ func extractEmoji(i Emojiable) (*gtsmodel.Emoji, error) {
 	if i.GetActivityStreamsIcon() == nil {
 		return nil, errors.New("no icon for emoji")
 	}
-	imageURL, err := extractIconURL(i)
+	imageURL, err := ExtractIconURL(i)
 	if err != nil {
 		return nil, errors.New("no url for emoji image")
 	}
@@ -485,7 +525,8 @@ func extractEmoji(i Emojiable) (*gtsmodel.Emoji, error) {
 	return emoji, nil
 }
 
-func extractMentions(i withTag) ([]*gtsmodel.Mention, error) {
+// ExtractMentions extracts a slice of gtsmodel Mentions from a WithTag interface.
+func ExtractMentions(i WithTag) ([]*gtsmodel.Mention, error) {
 	mentions := []*gtsmodel.Mention{}
 	tagsProp := i.GetActivityStreamsTag()
 	if tagsProp == nil {
@@ -506,7 +547,7 @@ func extractMentions(i withTag) ([]*gtsmodel.Mention, error) {
 			continue
 		}
 
-		mention, err := extractMention(mentionable)
+		mention, err := ExtractMention(mentionable)
 		if err != nil {
 			continue
 		}
@@ -516,10 +557,11 @@ func extractMentions(i withTag) ([]*gtsmodel.Mention, error) {
 	return mentions, nil
 }
 
-func extractMention(i Mentionable) (*gtsmodel.Mention, error) {
+// ExtractMention extracts a gts model mention from a Mentionable.
+func ExtractMention(i Mentionable) (*gtsmodel.Mention, error) {
 	mention := &gtsmodel.Mention{}
 
-	mentionString, err := extractName(i)
+	mentionString, err := ExtractName(i)
 	if err != nil {
 		return nil, err
 	}
@@ -543,7 +585,8 @@ func extractMention(i Mentionable) (*gtsmodel.Mention, error) {
 	return mention, nil
 }
 
-func extractActor(i withActor) (*url.URL, error) {
+// ExtractActor extracts the actor ID/IRI from an interface WithActor.
+func ExtractActor(i WithActor) (*url.URL, error) {
 	actorProp := i.GetActivityStreamsActor()
 	if actorProp == nil {
 		return nil, errors.New("actor property was nil")
@@ -556,7 +599,8 @@ func extractActor(i withActor) (*url.URL, error) {
 	return nil, errors.New("no iri found for actor prop")
 }
 
-func extractObject(i withObject) (*url.URL, error) {
+// ExtractObject extracts a URL object from a WithObject interface.
+func ExtractObject(i WithObject) (*url.URL, error) {
 	objectProp := i.GetActivityStreamsObject()
 	if objectProp == nil {
 		return nil, errors.New("object property was nil")

internal/ap/interfaces.go

@@ -0,0 +1,321 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package ap
+
+import "github.com/go-fed/activity/streams/vocab"
+
+// Accountable represents the minimum activitypub interface for representing an 'account'.
+// This interface is fulfilled by: Person, Application, Organization, Service, and Group
+type Accountable interface {
+	WithJSONLDId
+	WithTypeName
+
+	WithPreferredUsername
+	WithIcon
+	WithName
+	WithImage
+	WithSummary
+	WithDiscoverable
+	WithURL
+	WithPublicKey
+	WithInbox
+	WithOutbox
+	WithFollowing
+	WithFollowers
+	WithFeatured
+}
+
+// Statusable represents the minimum activitypub interface for representing a 'status'.
+// This interface is fulfilled by: Article, Document, Image, Video, Note, Page, Event, Place, Mention, Profile
+type Statusable interface {
+	WithJSONLDId
+	WithTypeName
+
+	WithSummary
+	WithInReplyTo
+	WithPublished
+	WithURL
+	WithAttributedTo
+	WithTo
+	WithCC
+	WithSensitive
+	WithConversation
+	WithContent
+	WithAttachment
+	WithTag
+	WithReplies
+}
+
+// Attachmentable represents the minimum activitypub interface for representing a 'mediaAttachment'.
+// This interface is fulfilled by: Audio, Document, Image, Video
+type Attachmentable interface {
+	WithTypeName
+	WithMediaType
+	WithURL
+	WithName
+}
+
+// Hashtaggable represents the minimum activitypub interface for representing a 'hashtag' tag.
+type Hashtaggable interface {
+	WithTypeName
+	WithHref
+	WithName
+}
+
+// Emojiable represents the minimum interface for an 'emoji' tag.
+type Emojiable interface {
+	WithJSONLDId
+	WithTypeName
+	WithName
+	WithUpdated
+	WithIcon
+}
+
+// Mentionable represents the minimum interface for a 'mention' tag.
+type Mentionable interface {
+	WithName
+	WithHref
+}
+
+// Followable represents the minimum interface for an activitystreams 'follow' activity.
+type Followable interface {
+	WithJSONLDId
+	WithTypeName
+
+	WithActor
+	WithObject
+}
+
+// Likeable represents the minimum interface for an activitystreams 'like' activity.
+type Likeable interface {
+	WithJSONLDId
+	WithTypeName
+
+	WithActor
+	WithObject
+}
+
+// Blockable represents the minimum interface for an activitystreams 'block' activity.
+type Blockable interface {
+	WithJSONLDId
+	WithTypeName
+
+	WithActor
+	WithObject
+}
+
+// Announceable represents the minimum interface for an activitystreams 'announce' activity.
+type Announceable interface {
+	WithJSONLDId
+	WithTypeName
+
+	WithActor
+	WithObject
+	WithPublished
+	WithTo
+	WithCC
+}
+
+// CollectionPageable represents the minimum interface for an activitystreams 'CollectionPage' object.
+type CollectionPageable interface {
+	WithJSONLDId
+	WithTypeName
+
+	WithNext
+	WithPartOf
+	WithItems
+}
+
+// WithJSONLDId represents an activity with JSONLDIdProperty
+type WithJSONLDId interface {
+	GetJSONLDId() vocab.JSONLDIdProperty
+}
+
+// WithTypeName represents an activity with a type name
+type WithTypeName interface {
+	GetTypeName() string
+}
+
+// WithPreferredUsername represents an activity with ActivityStreamsPreferredUsernameProperty
+type WithPreferredUsername interface {
+	GetActivityStreamsPreferredUsername() vocab.ActivityStreamsPreferredUsernameProperty
+}
+
+// WithIcon represents an activity with ActivityStreamsIconProperty
+type WithIcon interface {
+	GetActivityStreamsIcon() vocab.ActivityStreamsIconProperty
+}
+
+// WithName represents an activity with ActivityStreamsNameProperty
+type WithName interface {
+	GetActivityStreamsName() vocab.ActivityStreamsNameProperty
+}
+
+// WithImage represents an activity with ActivityStreamsImageProperty
+type WithImage interface {
+	GetActivityStreamsImage() vocab.ActivityStreamsImageProperty
+}
+
+// WithSummary represents an activity with ActivityStreamsSummaryProperty
+type WithSummary interface {
+	GetActivityStreamsSummary() vocab.ActivityStreamsSummaryProperty
+}
+
+// WithDiscoverable represents an activity with TootDiscoverableProperty
+type WithDiscoverable interface {
+	GetTootDiscoverable() vocab.TootDiscoverableProperty
+}
+
+// WithURL represents an activity with ActivityStreamsUrlProperty
+type WithURL interface {
+	GetActivityStreamsUrl() vocab.ActivityStreamsUrlProperty
+}
+
+// WithPublicKey represents an activity with W3IDSecurityV1PublicKeyProperty
+type WithPublicKey interface {
+	GetW3IDSecurityV1PublicKey() vocab.W3IDSecurityV1PublicKeyProperty
+}
+
+// WithInbox represents an activity with ActivityStreamsInboxProperty
+type WithInbox interface {
+	GetActivityStreamsInbox() vocab.ActivityStreamsInboxProperty
+}
+
+// WithOutbox represents an activity with ActivityStreamsOutboxProperty
+type WithOutbox interface {
+	GetActivityStreamsOutbox() vocab.ActivityStreamsOutboxProperty
+}
+
+// WithFollowing represents an activity with ActivityStreamsFollowingProperty
+type WithFollowing interface {
+	GetActivityStreamsFollowing() vocab.ActivityStreamsFollowingProperty
+}
+
+// WithFollowers represents an activity with ActivityStreamsFollowersProperty
+type WithFollowers interface {
+	GetActivityStreamsFollowers() vocab.ActivityStreamsFollowersProperty
+}
+
+// WithFeatured represents an activity with TootFeaturedProperty
+type WithFeatured interface {
+	GetTootFeatured() vocab.TootFeaturedProperty
+}
+
+// WithAttributedTo represents an activity with ActivityStreamsAttributedToProperty
+type WithAttributedTo interface {
+	GetActivityStreamsAttributedTo() vocab.ActivityStreamsAttributedToProperty
+}
+
+// WithAttachment represents an activity with ActivityStreamsAttachmentProperty
+type WithAttachment interface {
+	GetActivityStreamsAttachment() vocab.ActivityStreamsAttachmentProperty
+}
+
+// WithTo represents an activity with ActivityStreamsToProperty
+type WithTo interface {
+	GetActivityStreamsTo() vocab.ActivityStreamsToProperty
+}
+
+// WithInReplyTo represents an activity with ActivityStreamsInReplyToProperty
+type WithInReplyTo interface {
+	GetActivityStreamsInReplyTo() vocab.ActivityStreamsInReplyToProperty
+}
+
+// WithCC represents an activity with ActivityStreamsCcProperty
+type WithCC interface {
+	GetActivityStreamsCc() vocab.ActivityStreamsCcProperty
+}
+
+// WithSensitive ...
+type WithSensitive interface {
+	// TODO
+}
+
+// WithConversation ...
+type WithConversation interface {
+	// TODO
+}
+
+// WithContent represents an activity with ActivityStreamsContentProperty
+type WithContent interface {
+	GetActivityStreamsContent() vocab.ActivityStreamsContentProperty
+}
+
+// WithPublished represents an activity with ActivityStreamsPublishedProperty
+type WithPublished interface {
+	GetActivityStreamsPublished() vocab.ActivityStreamsPublishedProperty
+}
+
+// WithTag represents an activity with ActivityStreamsTagProperty
+type WithTag interface {
+	GetActivityStreamsTag() vocab.ActivityStreamsTagProperty
+}
+
+// WithReplies represents an activity with ActivityStreamsRepliesProperty
+type WithReplies interface {
+	GetActivityStreamsReplies() vocab.ActivityStreamsRepliesProperty
+}
+
+// WithMediaType represents an activity with ActivityStreamsMediaTypeProperty
+type WithMediaType interface {
+	GetActivityStreamsMediaType() vocab.ActivityStreamsMediaTypeProperty
+}
+
+// type withBlurhash interface {
+// 	GetTootBlurhashProperty() vocab.TootBlurhashProperty
+// }
+
+// type withFocalPoint interface {
+// 	// TODO
+// }
+
+// WithHref represents an activity with ActivityStreamsHrefProperty
+type WithHref interface {
+	GetActivityStreamsHref() vocab.ActivityStreamsHrefProperty
+}
+
+// WithUpdated represents an activity with ActivityStreamsUpdatedProperty
+type WithUpdated interface {
+	GetActivityStreamsUpdated() vocab.ActivityStreamsUpdatedProperty
+}
+
+// WithActor represents an activity with ActivityStreamsActorProperty
+type WithActor interface {
+	GetActivityStreamsActor() vocab.ActivityStreamsActorProperty
+}
+
+// WithObject represents an activity with ActivityStreamsObjectProperty
+type WithObject interface {
+	GetActivityStreamsObject() vocab.ActivityStreamsObjectProperty
+}
+
+// WithNext represents an activity with ActivityStreamsNextProperty
+type WithNext interface {
+	GetActivityStreamsNext() vocab.ActivityStreamsNextProperty
+}
+
+// WithPartOf represents an activity with ActivityStreamsPartOfProperty
+type WithPartOf interface {
+	GetActivityStreamsPartOf() vocab.ActivityStreamsPartOfProperty
+}
+
+// WithItems represents an activity with ActivityStreamsItemsProperty
+type WithItems interface {
+	GetActivityStreamsItems() vocab.ActivityStreamsItemsProperty
+}

internal/api/client/account/accountupdate_test.go

@@ -53,10 +53,10 @@ func (suite *AccountUpdateTestSuite) SetupTest() {
 	suite.db = testrig.NewTestDB()
 	suite.storage = testrig.NewTestStorage()
 	suite.log = testrig.NewTestLog()
-	suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil)), suite.storage)
+	suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage)
 	suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator)
 	suite.accountModule = account.New(suite.config, suite.processor, suite.log).(*account.Module)
-	testrig.StandardDBSetup(suite.db)
+	testrig.StandardDBSetup(suite.db, nil)
 	testrig.StandardStorageSetup(suite.storage, "../../../../testrig/media")
 }
 
@@ -80,6 +80,8 @@ func (suite *AccountUpdateTestSuite) TestAccountUpdateCredentialsPATCHHandler()
 	ctx, _ := gin.CreateTestContext(recorder)
 	ctx.Set(oauth.SessionAuthorizedAccount, suite.testAccounts["local_account_1"])
 	ctx.Set(oauth.SessionAuthorizedToken, oauth.TokenToOauthToken(suite.testTokens["local_account_1"]))
+	ctx.Set(oauth.SessionAuthorizedApplication, suite.testApplications["application_1"])
+	ctx.Set(oauth.SessionAuthorizedUser, suite.testUsers["local_account_1"])
 	ctx.Request = httptest.NewRequest(http.MethodPatch, fmt.Sprintf("http://localhost:8080/%s", account.UpdateCredentialsPath), bytes.NewReader(requestBody.Bytes())) // the endpoint we're hitting
 	ctx.Request.Header.Set("Content-Type", w.FormDataContentType())
 	suite.accountModule.AccountUpdateCredentialsPATCHHandler(ctx)

internal/api/client/fileserver/servefile_test.go

@@ -78,7 +78,7 @@ func (suite *ServeFileTestSuite) SetupSuite() {
 	suite.db = testrig.NewTestDB()
 	suite.log = testrig.NewTestLog()
 	suite.storage = testrig.NewTestStorage()
-	suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil)), suite.storage)
+	suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage)
 	suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator)
 	suite.tc = testrig.NewTestTypeConverter(suite.db)
 	suite.mediaHandler = testrig.NewTestMediaHandler(suite.db, suite.storage)
@@ -95,7 +95,7 @@ func (suite *ServeFileTestSuite) TearDownSuite() {
 }
 
 func (suite *ServeFileTestSuite) SetupTest() {
-	testrig.StandardDBSetup(suite.db)
+	testrig.StandardDBSetup(suite.db, nil)
 	testrig.StandardStorageSetup(suite.storage, "../../../../testrig/media")
 	suite.testTokens = testrig.NewTestTokens()
 	suite.testClients = testrig.NewTestClients()

internal/api/client/media/mediacreate_test.go

@@ -84,7 +84,7 @@ func (suite *MediaCreateTestSuite) SetupSuite() {
 	suite.tc = testrig.NewTestTypeConverter(suite.db)
 	suite.mediaHandler = testrig.NewTestMediaHandler(suite.db, suite.storage)
 	suite.oauthServer = testrig.NewTestOauthServer(suite.db)
-	suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil)), suite.storage)
+	suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage)
 	suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator)
 
 	// setup module being tested
@@ -98,7 +98,7 @@ func (suite *MediaCreateTestSuite) TearDownSuite() {
 }
 
 func (suite *MediaCreateTestSuite) SetupTest() {
-	testrig.StandardDBSetup(suite.db)
+	testrig.StandardDBSetup(suite.db, nil)
 	testrig.StandardStorageSetup(suite.storage, "../../../../testrig/media")
 	suite.testTokens = testrig.NewTestTokens()
 	suite.testClients = testrig.NewTestClients()

internal/api/client/status/statusboost_test.go

@@ -52,10 +52,10 @@ func (suite *StatusBoostTestSuite) SetupTest() {
 	suite.db = testrig.NewTestDB()
 	suite.storage = testrig.NewTestStorage()
 	suite.log = testrig.NewTestLog()
-	suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil)), suite.storage)
+	suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage)
 	suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator)
 	suite.statusModule = status.New(suite.config, suite.processor, suite.log).(*status.Module)
-	testrig.StandardDBSetup(suite.db)
+	testrig.StandardDBSetup(suite.db, nil)
 	testrig.StandardStorageSetup(suite.storage, "../../../../testrig/media")
 }
 

internal/api/client/status/statuscreate_test.go

@@ -58,10 +58,10 @@ func (suite *StatusCreateTestSuite) SetupTest() {
 	suite.storage = testrig.NewTestStorage()
 	suite.log = testrig.NewTestLog()
 	suite.tc = testrig.NewTestTypeConverter(suite.db)
-	suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil)), suite.storage)
+	suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage)
 	suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator)
 	suite.statusModule = status.New(suite.config, suite.processor, suite.log).(*status.Module)
-	testrig.StandardDBSetup(suite.db)
+	testrig.StandardDBSetup(suite.db, nil)
 	testrig.StandardStorageSetup(suite.storage, "../../../../testrig/media")
 }
 

internal/api/client/status/statusfave_test.go

@@ -55,10 +55,10 @@ func (suite *StatusFaveTestSuite) SetupTest() {
 	suite.db = testrig.NewTestDB()
 	suite.storage = testrig.NewTestStorage()
 	suite.log = testrig.NewTestLog()
-	suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil)), suite.storage)
+	suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage)
 	suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator)
 	suite.statusModule = status.New(suite.config, suite.processor, suite.log).(*status.Module)
-	testrig.StandardDBSetup(suite.db)
+	testrig.StandardDBSetup(suite.db, nil)
 	testrig.StandardStorageSetup(suite.storage, "../../../../testrig/media")
 }
 

internal/api/client/status/statusfavedby_test.go

@@ -55,10 +55,10 @@ func (suite *StatusFavedByTestSuite) SetupTest() {
 	suite.db = testrig.NewTestDB()
 	suite.storage = testrig.NewTestStorage()
 	suite.log = testrig.NewTestLog()
-	suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil)), suite.storage)
+	suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage)
 	suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator)
 	suite.statusModule = status.New(suite.config, suite.processor, suite.log).(*status.Module)
-	testrig.StandardDBSetup(suite.db)
+	testrig.StandardDBSetup(suite.db, nil)
 	testrig.StandardStorageSetup(suite.storage, "../../../../testrig/media")
 }
 

internal/api/client/status/statusget_test.go

@@ -45,10 +45,10 @@ func (suite *StatusGetTestSuite) SetupTest() {
 	suite.db = testrig.NewTestDB()
 	suite.storage = testrig.NewTestStorage()
 	suite.log = testrig.NewTestLog()
-	suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil)), suite.storage)
+	suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage)
 	suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator)
 	suite.statusModule = status.New(suite.config, suite.processor, suite.log).(*status.Module)
-	testrig.StandardDBSetup(suite.db)
+	testrig.StandardDBSetup(suite.db, nil)
 	testrig.StandardStorageSetup(suite.storage, "../../../../testrig/media")
 }
 

internal/api/client/status/statusunfave_test.go

@@ -55,10 +55,10 @@ func (suite *StatusUnfaveTestSuite) SetupTest() {
 	suite.db = testrig.NewTestDB()
 	suite.storage = testrig.NewTestStorage()
 	suite.log = testrig.NewTestLog()
-	suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil)), suite.storage)
+	suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage)
 	suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator)
 	suite.statusModule = status.New(suite.config, suite.processor, suite.log).(*status.Module)
-	testrig.StandardDBSetup(suite.db)
+	testrig.StandardDBSetup(suite.db, nil)
 	testrig.StandardStorageSetup(suite.storage, "../../../../testrig/media")
 }
 

internal/api/s2s/user/repliesget.go

@@ -0,0 +1,186 @@
+package user
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"strconv"
+
+	"github.com/gin-gonic/gin"
+	"github.com/sirupsen/logrus"
+	"github.com/superseriousbusiness/gotosocial/internal/util"
+)
+
+// StatusRepliesGETHandler swagger:operation GET /users/{username}/statuses/{status}/replies s2sRepliesGet
+//
+// Get the replies collection for a status.
+//
+// Note that the response will be a Collection with a page as `first`, as shown below, if `page` is `false`.
+//
+// If `page` is `true`, then the response will be a single `CollectionPage` without the wrapping `Collection`.
+//
+// HTTP signature is required on the request.
+//
+// ---
+// tags:
+// - s2s/federation
+//
+// produces:
+// - application/activity+json
+//
+// parameters:
+// - name: username
+//   type: string
+//   description: Username of the account.
+//   in: path
+//   required: true
+// - name: status
+//   type: string
+//   description: ID of the status.
+//   in: path
+//   required: true
+// - name: page
+//   type: boolean
+//   description: Return response as a CollectionPage.
+//   in: query
+//   default: false
+// - name: only_other_accounts
+//   type: boolean
+//   description: Return replies only from accounts other than the status owner.
+//   in: query
+//   default: false
+// - name: min_id
+//   type: string
+//   description: Minimum ID of the next status, used for paging.
+//   in: query
+//
+// responses:
+//   '200':
+//      in: body
+//      schema:
+//        "$ref": "#/definitions/swaggerStatusRepliesCollection"
+//   '400':
+//      description: bad request
+//   '401':
+//      description: unauthorized
+//   '403':
+//      description: forbidden
+//   '404':
+//      description: not found
+func (m *Module) StatusRepliesGETHandler(c *gin.Context) {
+	l := m.log.WithFields(logrus.Fields{
+		"func": "StatusRepliesGETHandler",
+		"url":  c.Request.RequestURI,
+	})
+
+	requestedUsername := c.Param(UsernameKey)
+	if requestedUsername == "" {
+		c.JSON(http.StatusBadRequest, gin.H{"error": "no username specified in request"})
+		return
+	}
+
+	requestedStatusID := c.Param(StatusIDKey)
+	if requestedStatusID == "" {
+		c.JSON(http.StatusBadRequest, gin.H{"error": "no status id specified in request"})
+		return
+	}
+
+	page := false
+	pageString := c.Query(PageKey)
+	if pageString != "" {
+		i, err := strconv.ParseBool(pageString)
+		if err != nil {
+			l.Debugf("error parsing page string: %s", err)
+			c.JSON(http.StatusBadRequest, gin.H{"error": "couldn't parse page query param"})
+			return
+		}
+		page = i
+	}
+
+	onlyOtherAccounts := false
+	onlyOtherAccountsString := c.Query(OnlyOtherAccountsKey)
+	if onlyOtherAccountsString != "" {
+		i, err := strconv.ParseBool(onlyOtherAccountsString)
+		if err != nil {
+			l.Debugf("error parsing only_other_accounts string: %s", err)
+			c.JSON(http.StatusBadRequest, gin.H{"error": "couldn't parse only_other_accounts query param"})
+			return
+		}
+		onlyOtherAccounts = i
+	}
+
+	minID := ""
+	minIDString := c.Query(MinIDKey)
+	if minIDString != "" {
+		minID = minIDString
+	}
+
+	// make sure this actually an AP request
+	format := c.NegotiateFormat(ActivityPubAcceptHeaders...)
+	if format == "" {
+		c.JSON(http.StatusNotAcceptable, gin.H{"error": "could not negotiate format with given Accept header(s)"})
+		return
+	}
+	l.Tracef("negotiated format: %s", format)
+
+	// transfer the signature verifier from the gin context to the request context
+	ctx := c.Request.Context()
+	verifier, signed := c.Get(string(util.APRequestingPublicKeyVerifier))
+	if signed {
+		ctx = context.WithValue(ctx, util.APRequestingPublicKeyVerifier, verifier)
+	}
+
+	replies, err := m.processor.GetFediStatusReplies(ctx, requestedUsername, requestedStatusID, page, onlyOtherAccounts, minID, c.Request.URL)
+	if err != nil {
+		l.Info(err.Error())
+		c.JSON(err.Code(), gin.H{"error": err.Safe()})
+		return
+	}
+
+	b, mErr := json.Marshal(replies)
+	if mErr != nil {
+		err := fmt.Errorf("could not marshal json: %s", mErr)
+		l.Error(err)
+		c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
+		return
+	}
+
+	c.Data(http.StatusOK, format, b)
+}
+
+// SwaggerStatusRepliesCollection represents a response to GET /users/{username}/statuses/{status}/replies.
+// swagger:model swaggerStatusRepliesCollection
+type SwaggerStatusRepliesCollection struct {
+	// ActivityStreams context.
+	// example: https://www.w3.org/ns/activitystreams
+	Context string `json:"@context"`
+	// ActivityStreams ID.
+	// example: https://example.org/users/some_user/statuses/106717595988259568/replies
+	ID string `json:"id"`
+	// ActivityStreams type.
+	// example: Collection
+	Type string `json:"type"`
+	// ActivityStreams first property.
+	First SwaggerStatusRepliesCollectionPage `json:"first"`
+}
+
+// SwaggerStatusRepliesCollectionPage represents one page of a collection.
+// swagger:model swaggerStatusRepliesCollectionPage
+type SwaggerStatusRepliesCollectionPage struct {
+	// ActivityStreams ID.
+	// example: https://example.org/users/some_user/statuses/106717595988259568/replies?page=true
+	ID string `json:"id"`
+	// ActivityStreams type.
+	// example: CollectionPage
+	Type string `json:"type"`
+	// Link to the next page.
+	// example: https://example.org/users/some_user/statuses/106717595988259568/replies?only_other_accounts=true&page=true
+	Next string `json:"next"`
+	// Collection this page belongs to.
+	// example: https://example.org/users/some_user/statuses/106717595988259568/replies
+	PartOf string `json:"partOf"`
+	// Items on this page.
+	// example: ["https://example.org/users/some_other_user/statuses/086417595981111564", "https://another.example.com/users/another_user/statuses/01FCN8XDV3YG7B4R42QA6YQZ9R"]
+	Items []string `json:"items"`
+}

internal/api/s2s/user/repliesget_test.go

@@ -0,0 +1,241 @@
+package user_test
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/gin-gonic/gin"
+	"github.com/go-fed/activity/streams"
+	"github.com/go-fed/activity/streams/vocab"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/api/s2s/user"
+	"github.com/superseriousbusiness/gotosocial/internal/api/security"
+	"github.com/superseriousbusiness/gotosocial/testrig"
+)
+
+type RepliesGetTestSuite struct {
+	UserStandardTestSuite
+}
+
+func (suite *RepliesGetTestSuite) SetupSuite() {
+	suite.testTokens = testrig.NewTestTokens()
+	suite.testClients = testrig.NewTestClients()
+	suite.testApplications = testrig.NewTestApplications()
+	suite.testUsers = testrig.NewTestUsers()
+	suite.testAccounts = testrig.NewTestAccounts()
+	suite.testAttachments = testrig.NewTestAttachments()
+	suite.testStatuses = testrig.NewTestStatuses()
+}
+
+func (suite *RepliesGetTestSuite) SetupTest() {
+	suite.config = testrig.NewTestConfig()
+	suite.db = testrig.NewTestDB()
+	suite.tc = testrig.NewTestTypeConverter(suite.db)
+	suite.storage = testrig.NewTestStorage()
+	suite.log = testrig.NewTestLog()
+	suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage)
+	suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator)
+	suite.userModule = user.New(suite.config, suite.processor, suite.log).(*user.Module)
+	suite.securityModule = security.New(suite.config, suite.db, suite.log).(*security.Module)
+	testrig.StandardDBSetup(suite.db, suite.testAccounts)
+	testrig.StandardStorageSetup(suite.storage, "../../../../testrig/media")
+}
+
+func (suite *RepliesGetTestSuite) TearDownTest() {
+	testrig.StandardDBTeardown(suite.db)
+	testrig.StandardStorageTeardown(suite.storage)
+}
+
+func (suite *RepliesGetTestSuite) TestGetReplies() {
+	// the dereference we're gonna use
+	derefRequests := testrig.NewTestDereferenceRequests(suite.testAccounts)
+	signedRequest := derefRequests["foss_satan_dereference_local_account_1_status_1_replies"]
+	targetAccount := suite.testAccounts["local_account_1"]
+	targetStatus := suite.testStatuses["local_account_1_status_1"]
+
+	tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db)
+	federator := testrig.NewTestFederator(suite.db, tc, suite.storage)
+	processor := testrig.NewTestProcessor(suite.db, suite.storage, federator)
+	userModule := user.New(suite.config, processor, suite.log).(*user.Module)
+
+	// setup request
+	recorder := httptest.NewRecorder()
+	ctx, _ := gin.CreateTestContext(recorder)
+	ctx.Request = httptest.NewRequest(http.MethodGet, targetStatus.URI+"/replies", nil) // the endpoint we're hitting
+	ctx.Request.Header.Set("Signature", signedRequest.SignatureHeader)
+	ctx.Request.Header.Set("Date", signedRequest.DateHeader)
+
+	// we need to pass the context through signature check first to set appropriate values on it
+	suite.securityModule.SignatureCheck(ctx)
+
+	// normally the router would populate these params from the path values,
+	// but because we're calling the function directly, we need to set them manually.
+	ctx.Params = gin.Params{
+		gin.Param{
+			Key:   user.UsernameKey,
+			Value: targetAccount.Username,
+		},
+		gin.Param{
+			Key:   user.StatusIDKey,
+			Value: targetStatus.ID,
+		},
+	}
+
+	// trigger the function being tested
+	userModule.StatusRepliesGETHandler(ctx)
+
+	// check response
+	suite.EqualValues(http.StatusOK, recorder.Code)
+
+	result := recorder.Result()
+	defer result.Body.Close()
+	b, err := ioutil.ReadAll(result.Body)
+	assert.NoError(suite.T(), err)
+	assert.Equal(suite.T(), `{"@context":"https://www.w3.org/ns/activitystreams","first":{"id":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies?page=true","next":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies?only_other_accounts=false\u0026page=true","partOf":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies","type":"CollectionPage"},"id":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies","type":"Collection"}`, string(b))
+
+	// should be a Collection
+	m := make(map[string]interface{})
+	err = json.Unmarshal(b, &m)
+	assert.NoError(suite.T(), err)
+
+	t, err := streams.ToType(context.Background(), m)
+	assert.NoError(suite.T(), err)
+
+	_, ok := t.(vocab.ActivityStreamsCollection)
+	assert.True(suite.T(), ok)
+}
+
+func (suite *RepliesGetTestSuite) TestGetRepliesNext() {
+	// the dereference we're gonna use
+	derefRequests := testrig.NewTestDereferenceRequests(suite.testAccounts)
+	signedRequest := derefRequests["foss_satan_dereference_local_account_1_status_1_replies_next"]
+	targetAccount := suite.testAccounts["local_account_1"]
+	targetStatus := suite.testStatuses["local_account_1_status_1"]
+
+	tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db)
+	federator := testrig.NewTestFederator(suite.db, tc, suite.storage)
+	processor := testrig.NewTestProcessor(suite.db, suite.storage, federator)
+	userModule := user.New(suite.config, processor, suite.log).(*user.Module)
+
+	// setup request
+	recorder := httptest.NewRecorder()
+	ctx, _ := gin.CreateTestContext(recorder)
+	ctx.Request = httptest.NewRequest(http.MethodGet, targetStatus.URI+"/replies?only_other_accounts=false&page=true", nil) // the endpoint we're hitting
+	ctx.Request.Header.Set("Signature", signedRequest.SignatureHeader)
+	ctx.Request.Header.Set("Date", signedRequest.DateHeader)
+
+	// we need to pass the context through signature check first to set appropriate values on it
+	suite.securityModule.SignatureCheck(ctx)
+
+	// normally the router would populate these params from the path values,
+	// but because we're calling the function directly, we need to set them manually.
+	ctx.Params = gin.Params{
+		gin.Param{
+			Key:   user.UsernameKey,
+			Value: targetAccount.Username,
+		},
+		gin.Param{
+			Key:   user.StatusIDKey,
+			Value: targetStatus.ID,
+		},
+	}
+
+	// trigger the function being tested
+	userModule.StatusRepliesGETHandler(ctx)
+
+	// check response
+	suite.EqualValues(http.StatusOK, recorder.Code)
+
+	result := recorder.Result()
+	defer result.Body.Close()
+	b, err := ioutil.ReadAll(result.Body)
+	assert.NoError(suite.T(), err)
+
+	assert.Equal(suite.T(), `{"@context":"https://www.w3.org/ns/activitystreams","id":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies?page=true\u0026only_other_accounts=false","items":"http://localhost:8080/users/1happyturtle/statuses/01FCQSQ667XHJ9AV9T27SJJSX5","next":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies?only_other_accounts=false\u0026page=true\u0026min_id=01FCQSQ667XHJ9AV9T27SJJSX5","partOf":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies","type":"CollectionPage"}`, string(b))
+
+	// should be a Collection
+	m := make(map[string]interface{})
+	err = json.Unmarshal(b, &m)
+	assert.NoError(suite.T(), err)
+
+	t, err := streams.ToType(context.Background(), m)
+	assert.NoError(suite.T(), err)
+
+	page, ok := t.(vocab.ActivityStreamsCollectionPage)
+	assert.True(suite.T(), ok)
+
+	assert.Equal(suite.T(), page.GetActivityStreamsItems().Len(), 1)
+}
+
+func (suite *RepliesGetTestSuite) TestGetRepliesLast() {
+	// the dereference we're gonna use
+	derefRequests := testrig.NewTestDereferenceRequests(suite.testAccounts)
+	signedRequest := derefRequests["foss_satan_dereference_local_account_1_status_1_replies_last"]
+	targetAccount := suite.testAccounts["local_account_1"]
+	targetStatus := suite.testStatuses["local_account_1_status_1"]
+
+	tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db)
+	federator := testrig.NewTestFederator(suite.db, tc, suite.storage)
+	processor := testrig.NewTestProcessor(suite.db, suite.storage, federator)
+	userModule := user.New(suite.config, processor, suite.log).(*user.Module)
+
+	// setup request
+	recorder := httptest.NewRecorder()
+	ctx, _ := gin.CreateTestContext(recorder)
+	ctx.Request = httptest.NewRequest(http.MethodGet, targetStatus.URI+"/replies?only_other_accounts=false&page=true&min_id=01FCQSQ667XHJ9AV9T27SJJSX5", nil) // the endpoint we're hitting
+	ctx.Request.Header.Set("Signature", signedRequest.SignatureHeader)
+	ctx.Request.Header.Set("Date", signedRequest.DateHeader)
+
+	// we need to pass the context through signature check first to set appropriate values on it
+	suite.securityModule.SignatureCheck(ctx)
+
+	// normally the router would populate these params from the path values,
+	// but because we're calling the function directly, we need to set them manually.
+	ctx.Params = gin.Params{
+		gin.Param{
+			Key:   user.UsernameKey,
+			Value: targetAccount.Username,
+		},
+		gin.Param{
+			Key:   user.StatusIDKey,
+			Value: targetStatus.ID,
+		},
+	}
+
+	// trigger the function being tested
+	userModule.StatusRepliesGETHandler(ctx)
+
+	// check response
+	suite.EqualValues(http.StatusOK, recorder.Code)
+
+	result := recorder.Result()
+	defer result.Body.Close()
+	b, err := ioutil.ReadAll(result.Body)
+	assert.NoError(suite.T(), err)
+
+	fmt.Println(string(b))
+	assert.Equal(suite.T(), `{"@context":"https://www.w3.org/ns/activitystreams","id":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies?page=true\u0026only_other_accounts=false\u0026min_id=01FCQSQ667XHJ9AV9T27SJJSX5","items":[],"next":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies?only_other_accounts=false\u0026page=true","partOf":"http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY/replies","type":"CollectionPage"}`, string(b))
+
+	// should be a Collection
+	m := make(map[string]interface{})
+	err = json.Unmarshal(b, &m)
+	assert.NoError(suite.T(), err)
+
+	t, err := streams.ToType(context.Background(), m)
+	assert.NoError(suite.T(), err)
+
+	page, ok := t.(vocab.ActivityStreamsCollectionPage)
+	assert.True(suite.T(), ok)
+
+	assert.Equal(suite.T(), page.GetActivityStreamsItems().Len(), 0)
+}
+
+func TestRepliesGetTestSuite(t *testing.T) {
+	suite.Run(t, new(RepliesGetTestSuite))
+}

internal/api/s2s/user/user.go

@@ -34,6 +34,13 @@ const (
 	UsernameKey = "username"
 	// StatusIDKey is for status IDs
 	StatusIDKey = "status"
+	// OnlyOtherAccountsKey is for filtering status responses.
+	OnlyOtherAccountsKey = "only_other_accounts"
+	// MinIDKey is for filtering status responses.
+	MinIDKey = "min_id"
+	// PageKey is for filtering status responses.
+	PageKey = "page"
+
 	// UsersBasePath is the base path for serving information about Users eg https://example.org/users
 	UsersBasePath = "/" + util.UsersPath
 	// UsersBasePathWithUsername is just the users base path with the Username key in it.
@@ -50,6 +57,8 @@ const (
 	UsersFollowingPath = UsersBasePathWithUsername + "/" + util.FollowingPath
 	// UsersStatusPath is for serving GET requests to a particular status by a user, with the given username key and status ID
 	UsersStatusPath = UsersBasePathWithUsername + "/" + util.StatusesPath + "/:" + StatusIDKey
+	// UsersStatusRepliesPath is for serving the replies collection of a status.
+	UsersStatusRepliesPath = UsersStatusPath + "/replies"
 )
 
 // ActivityPubAcceptHeaders represents the Accept headers mentioned here:
@@ -83,5 +92,6 @@ func (m *Module) Route(s router.Router) error {
 	s.AttachHandler(http.MethodGet, UsersFollowingPath, m.FollowingGETHandler)
 	s.AttachHandler(http.MethodGet, UsersStatusPath, m.StatusGETHandler)
 	s.AttachHandler(http.MethodGet, UsersPublicKeyPath, m.PublicKeyGETHandler)
+	s.AttachHandler(http.MethodGet, UsersStatusRepliesPath, m.StatusRepliesGETHandler)
 	return nil
 }

internal/api/s2s/user/user_test.go

@@ -4,6 +4,7 @@ import (
 	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/suite"
 	"github.com/superseriousbusiness/gotosocial/internal/api/s2s/user"
+	"github.com/superseriousbusiness/gotosocial/internal/api/security"
 	"github.com/superseriousbusiness/gotosocial/internal/blob"
 	"github.com/superseriousbusiness/gotosocial/internal/config"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
@@ -18,13 +19,14 @@ import (
 type UserStandardTestSuite struct {
 	// standard suite interfaces
 	suite.Suite
-	config    *config.Config
+	config         *config.Config
-	db        db.DB
+	db             db.DB
-	log       *logrus.Logger
+	log            *logrus.Logger
-	tc        typeutils.TypeConverter
+	tc             typeutils.TypeConverter
-	federator federation.Federator
+	federator      federation.Federator
-	processor processing.Processor
+	processor      processing.Processor
-	storage   blob.Storage
+	storage        blob.Storage
+	securityModule *security.Module
 
 	// standard suite models
 	testTokens       map[string]*oauth.Token

internal/api/s2s/user/userget_test.go

@@ -1,16 +1,11 @@
 package user_test
 
 import (
-	"bytes"
 	"context"
-	"crypto/x509"
 	"encoding/json"
-	"encoding/pem"
-	"fmt"
 	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
-	"strings"
 	"testing"
 
 	"github.com/gin-gonic/gin"
@@ -19,6 +14,7 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/suite"
 	"github.com/superseriousbusiness/gotosocial/internal/api/s2s/user"
+	"github.com/superseriousbusiness/gotosocial/internal/api/security"
 	"github.com/superseriousbusiness/gotosocial/testrig"
 )
 
@@ -42,10 +38,11 @@ func (suite *UserGetTestSuite) SetupTest() {
 	suite.tc = testrig.NewTestTypeConverter(suite.db)
 	suite.storage = testrig.NewTestStorage()
 	suite.log = testrig.NewTestLog()
-	suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil)), suite.storage)
+	suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db), suite.storage)
 	suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator)
 	suite.userModule = user.New(suite.config, suite.processor, suite.log).(*user.Module)
-	testrig.StandardDBSetup(suite.db)
+	suite.securityModule = security.New(suite.config, suite.db, suite.log).(*security.Module)
+	testrig.StandardDBSetup(suite.db, suite.testAccounts)
 	testrig.StandardStorageSetup(suite.storage, "../../../../testrig/media")
 }
 
@@ -56,48 +53,11 @@ func (suite *UserGetTestSuite) TearDownTest() {
 
 func (suite *UserGetTestSuite) TestGetUser() {
 	// the dereference we're gonna use
-	signedRequest := testrig.NewTestDereferenceRequests(suite.testAccounts)["foss_satan_dereference_zork"]
+	derefRequests := testrig.NewTestDereferenceRequests(suite.testAccounts)
-
+	signedRequest := derefRequests["foss_satan_dereference_zork"]
-	requestingAccount := suite.testAccounts["remote_account_1"]
 	targetAccount := suite.testAccounts["local_account_1"]
 
-	encodedPublicKey, err := x509.MarshalPKIXPublicKey(requestingAccount.PublicKey)
+	tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db)
-	assert.NoError(suite.T(), err)
-	publicKeyBytes := pem.EncodeToMemory(&pem.Block{
-		Type:  "PUBLIC KEY",
-		Bytes: encodedPublicKey,
-	})
-	publicKeyString := strings.ReplaceAll(string(publicKeyBytes), "\n", "\\n")
-
-	// for this test we need the client to return the public key of the requester on the 'remote' instance
-	responseBodyString := fmt.Sprintf(`
-	{
-		"@context": [
-			"https://www.w3.org/ns/activitystreams",
-			"https://w3id.org/security/v1"
-		],
-
-		"id": "%s",
-		"type": "Person",
-		"preferredUsername": "%s",
-		"inbox": "%s",
-
-		"publicKey": {
-			"id": "%s",
-			"owner": "%s",
-			"publicKeyPem": "%s"
-		}
-	}`, requestingAccount.URI, requestingAccount.Username, requestingAccount.InboxURI, requestingAccount.PublicKeyURI, requestingAccount.URI, publicKeyString)
-
-	// create a transport controller whose client will just return the response body string we specified above
-	tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(func(req *http.Request) (*http.Response, error) {
-		r := ioutil.NopCloser(bytes.NewReader([]byte(responseBodyString)))
-		return &http.Response{
-			StatusCode: 200,
-			Body:       r,
-		}, nil
-	}))
-	// get this transport controller embedded right in the user module we're testing
 	federator := testrig.NewTestFederator(suite.db, tc, suite.storage)
 	processor := testrig.NewTestProcessor(suite.db, suite.storage, federator)
 	userModule := user.New(suite.config, processor, suite.log).(*user.Module)
@@ -105,7 +65,12 @@ func (suite *UserGetTestSuite) TestGetUser() {
 	// setup request
 	recorder := httptest.NewRecorder()
 	ctx, _ := gin.CreateTestContext(recorder)
-	ctx.Request = httptest.NewRequest(http.MethodGet, fmt.Sprintf("http://localhost:8080%s", strings.Replace(user.UsersBasePathWithUsername, ":username", targetAccount.Username, 1)), nil) // the endpoint we're hitting
+	ctx.Request = httptest.NewRequest(http.MethodGet, targetAccount.URI, nil) // the endpoint we're hitting
+	ctx.Request.Header.Set("Signature", signedRequest.SignatureHeader)
+	ctx.Request.Header.Set("Date", signedRequest.DateHeader)
+
+	// we need to pass the context through signature check first to set appropriate values on it
+	suite.securityModule.SignatureCheck(ctx)
 
 	// normally the router would populate these params from the path values,
 	// but because we're calling the function directly, we need to set them manually.
@@ -116,11 +81,6 @@ func (suite *UserGetTestSuite) TestGetUser() {
 		},
 	}
 
-	// we need these headers for the request to be validated
-	ctx.Request.Header.Set("Signature", signedRequest.SignatureHeader)
-	ctx.Request.Header.Set("Date", signedRequest.DateHeader)
-	ctx.Request.Header.Set("Digest", signedRequest.DigestHeader)
-
 	// trigger the function being tested
 	userModule.UsersGETHandler(ctx)
 

internal/cliactions/server/server.go

@@ -115,7 +115,7 @@ var Start cliactions.GTSAction = func(ctx context.Context, c *config.Config, log
 	// build backend handlers
 	mediaHandler := media.New(c, dbService, storageBackend, log)
 	oauthServer := oauth.New(dbService, log)
-	transportController := transport.NewController(c, &federation.Clock{}, http.DefaultClient, log)
+	transportController := transport.NewController(c, dbService, &federation.Clock{}, http.DefaultClient, log)
 	federator := federation.NewFederator(dbService, federatingDB, transportController, c, log, typeConverter, mediaHandler)
 	processor := processing.NewProcessor(c, typeConverter, federator, oauthServer, mediaHandler, storageBackend, timelineManager, dbService, log)
 	if err := processor.Start(); err != nil {

internal/cliactions/testrig/testrig.go

@@ -46,7 +46,7 @@ import (
 var Start cliactions.GTSAction = func(ctx context.Context, _ *config.Config, log *logrus.Logger) error {
 	c := testrig.NewTestConfig()
 	dbService := testrig.NewTestDB()
-	testrig.StandardDBSetup(dbService)
+	testrig.StandardDBSetup(dbService, nil)
 	router := testrig.NewTestRouter(dbService)
 	storageBackend := testrig.NewTestStorage()
 	testrig.StandardStorageSetup(storageBackend, "./testrig/media")
@@ -59,7 +59,7 @@ var Start cliactions.GTSAction = func(ctx context.Context, _ *config.Config, log
 			StatusCode: 200,
 			Body:       r,
 		}, nil
-	}))
+	}), dbService)
 	federator := testrig.NewTestFederator(dbService, transportController, storageBackend)
 
 	processor := testrig.NewTestProcessor(dbService, storageBackend, federator)

internal/db/db.go

@@ -218,10 +218,14 @@ type DB interface {
 	GetFaveCountForStatus(status *gtsmodel.Status) (int, error)
 
 	// StatusParents get the parent statuses of a given status.
-	StatusParents(status *gtsmodel.Status) ([]*gtsmodel.Status, error)
+	//
+	// If onlyDirect is true, only the immediate parent will be returned.
+	StatusParents(status *gtsmodel.Status, onlyDirect bool) ([]*gtsmodel.Status, error)
 
 	// StatusChildren gets the child statuses of a given status.
-	StatusChildren(status *gtsmodel.Status) ([]*gtsmodel.Status, error)
+	//
+	// If onlyDirect is true, only the immediate children will be returned.
+	StatusChildren(status *gtsmodel.Status, onlyDirect bool, minID string) ([]*gtsmodel.Status, error)
 
 	// StatusFavedBy checks if a given status has been faved by a given account ID
 	StatusFavedBy(status *gtsmodel.Status, accountID string) (bool, error)

internal/db/pg/statuscontext.go

@@ -25,14 +25,14 @@ import (
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 )
 
-func (ps *postgresService) StatusParents(status *gtsmodel.Status) ([]*gtsmodel.Status, error) {
+func (ps *postgresService) StatusParents(status *gtsmodel.Status, onlyDirect bool) ([]*gtsmodel.Status, error) {
 	parents := []*gtsmodel.Status{}
-	ps.statusParent(status, &parents)
+	ps.statusParent(status, &parents, onlyDirect)
 
 	return parents, nil
 }
 
-func (ps *postgresService) statusParent(status *gtsmodel.Status, foundStatuses *[]*gtsmodel.Status) {
+func (ps *postgresService) statusParent(status *gtsmodel.Status, foundStatuses *[]*gtsmodel.Status, onlyDirect bool) {
 	if status.InReplyToID == "" {
 		return
 	}
@@ -42,13 +42,16 @@ func (ps *postgresService) statusParent(status *gtsmodel.Status, foundStatuses *
 		*foundStatuses = append(*foundStatuses, parentStatus)
 	}
 
-	ps.statusParent(parentStatus, foundStatuses)
+	if onlyDirect {
+		return
+	}
+	ps.statusParent(parentStatus, foundStatuses, false)
 }
 
-func (ps *postgresService) StatusChildren(status *gtsmodel.Status) ([]*gtsmodel.Status, error) {
+func (ps *postgresService) StatusChildren(status *gtsmodel.Status, onlyDirect bool, minID string) ([]*gtsmodel.Status, error) {
 	foundStatuses := &list.List{}
 	foundStatuses.PushFront(status)
-	ps.statusChildren(status, foundStatuses)
+	ps.statusChildren(status, foundStatuses, onlyDirect, minID)
 
 	children := []*gtsmodel.Status{}
 	for e := foundStatuses.Front(); e != nil; e = e.Next() {
@@ -66,11 +69,15 @@ func (ps *postgresService) StatusChildren(status *gtsmodel.Status) ([]*gtsmodel.
 	return children, nil
 }
 
-func (ps *postgresService) statusChildren(status *gtsmodel.Status, foundStatuses *list.List) {
+func (ps *postgresService) statusChildren(status *gtsmodel.Status, foundStatuses *list.List, onlyDirect bool, minID string) {
 	immediateChildren := []*gtsmodel.Status{}
 
-	err := ps.conn.Model(&immediateChildren).Where("in_reply_to_id = ?", status.ID).Select()
+	q := ps.conn.Model(&immediateChildren).Where("in_reply_to_id = ?", status.ID)
-	if err != nil {
+	if minID != "" {
+		q = q.Where("status.id > ?", minID)
+	}
+
+	if err := q.Select(); err != nil {
 		return
 	}
 
@@ -88,6 +95,10 @@ func (ps *postgresService) statusChildren(status *gtsmodel.Status, foundStatuses
 			}
 		}
 
-		ps.statusChildren(child, foundStatuses)
+		// only do one loop if we only want direct children
+		if onlyDirect {
+			return
+		}
+		ps.statusChildren(child, foundStatuses, false, minID)
 	}
 }

internal/federation/authenticate.go

@@ -147,6 +147,7 @@ func (f *federator) AuthenticateFederatedRequest(ctx context.Context, requestedU
 	if strings.EqualFold(requestingHost, f.config.Host) {
 		// LOCAL ACCOUNT REQUEST
 		// the request is coming from INSIDE THE HOUSE so skip the remote dereferencing
+		l.Tracef("proceeding without dereference for local public key %s", requestingPublicKeyID)
 		if err := f.db.GetWhere([]db.Where{{Key: "public_key_uri", Value: requestingPublicKeyID.String()}}, requestingLocalAccount); err != nil {
 			return nil, false, fmt.Errorf("couldn't get local account with public key uri %s from the database: %s", requestingPublicKeyID.String(), err)
 		}
@@ -158,6 +159,7 @@ func (f *federator) AuthenticateFederatedRequest(ctx context.Context, requestedU
 	} else if err := f.db.GetWhere([]db.Where{{Key: "public_key_uri", Value: requestingPublicKeyID.String()}}, requestingRemoteAccount); err == nil {
 		// REMOTE ACCOUNT REQUEST WITH KEY CACHED LOCALLY
 		// this is a remote account and we already have the public key for it so use that
+		l.Tracef("proceeding without dereference for cached public key %s", requestingPublicKeyID)
 		publicKey = requestingRemoteAccount.PublicKey
 		pkOwnerURI, err = url.Parse(requestingRemoteAccount.URI)
 		if err != nil {
@@ -167,7 +169,8 @@ func (f *federator) AuthenticateFederatedRequest(ctx context.Context, requestedU
 		// REMOTE ACCOUNT REQUEST WITHOUT KEY CACHED LOCALLY
 		// the request is remote and we don't have the public key yet,
 		// so we need to authenticate the request properly by dereferencing the remote key
-		transport, err := f.GetTransportForUser(requestedUsername)
+		l.Tracef("proceeding with dereference for uncached public key %s", requestingPublicKeyID)
+		transport, err := f.transportController.NewTransportForUsername(requestedUsername)
 		if err != nil {
 			return nil, false, fmt.Errorf("transport err: %s", err)
 		}
@@ -209,15 +212,28 @@ func (f *federator) AuthenticateFederatedRequest(ctx context.Context, requestedU
 		}
 		pkOwnerURI = pkOwnerProp.GetIRI()
 	}
+
+	// after all that, public key should be defined
 	if publicKey == nil {
 		return nil, false, errors.New("returned public key was empty")
 	}
 
 	// do the actual authentication here!
-	algo := httpsig.RSA_SHA256 // TODO: make this more robust
+	algos := []httpsig.Algorithm{
-	if err := verifier.Verify(publicKey, algo); err != nil {
+		httpsig.RSA_SHA512,
-		return nil, false, nil
+		httpsig.RSA_SHA256,
+		httpsig.ED25519,
 	}
 
-	return pkOwnerURI, true, nil
+	for _, algo := range algos {
+		l.Tracef("trying algo: %s", algo)
+		if err := verifier.Verify(publicKey, algo); err == nil {
+			l.Tracef("authentication for %s PASSED with algorithm %s", pkOwnerURI, algo)
+			return pkOwnerURI, true, nil
+		}
+		l.Tracef("authentication for %s NOT PASSED with algorithm %s: %s", pkOwnerURI, algo, err)
+	}
+
+	l.Infof("authentication not passed for %s", pkOwnerURI)
+	return nil, false, nil
 }

internal/federation/dereference.go

@@ -1,526 +1,32 @@
 package federation
 
 import (
-	"context"
-	"encoding/json"
-	"errors"
-	"fmt"
 	"net/url"
 
-	"github.com/go-fed/activity/streams"
+	"github.com/superseriousbusiness/gotosocial/internal/ap"
-	"github.com/go-fed/activity/streams/vocab"
-	"github.com/sirupsen/logrus"
-	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
-	"github.com/superseriousbusiness/gotosocial/internal/id"
-	"github.com/superseriousbusiness/gotosocial/internal/transport"
-	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
 )
 
-func (f *federator) DereferenceRemoteAccount(username string, remoteAccountID *url.URL) (typeutils.Accountable, error) {
+func (f *federator) GetRemoteAccount(username string, remoteAccountID *url.URL, refresh bool) (*gtsmodel.Account, bool, error) {
-	f.startHandshake(username, remoteAccountID)
+	return f.dereferencer.GetRemoteAccount(username, remoteAccountID, refresh)
-	defer f.stopHandshake(username, remoteAccountID)
-
-	if blocked, err := f.blockedDomain(remoteAccountID.Host); blocked || err != nil {
-		return nil, fmt.Errorf("DereferenceRemoteAccount: domain %s is blocked", remoteAccountID.Host)
-	}
-
-	transport, err := f.GetTransportForUser(username)
-	if err != nil {
-		return nil, fmt.Errorf("transport err: %s", err)
-	}
-
-	b, err := transport.Dereference(context.Background(), remoteAccountID)
-	if err != nil {
-		return nil, fmt.Errorf("error deferencing %s: %s", remoteAccountID.String(), err)
-	}
-
-	m := make(map[string]interface{})
-	if err := json.Unmarshal(b, &m); err != nil {
-		return nil, fmt.Errorf("error unmarshalling bytes into json: %s", err)
-	}
-
-	t, err := streams.ToType(context.Background(), m)
-	if err != nil {
-		return nil, fmt.Errorf("error resolving json into ap vocab type: %s", err)
-	}
-
-	switch t.GetTypeName() {
-	case string(gtsmodel.ActivityStreamsPerson):
-		p, ok := t.(vocab.ActivityStreamsPerson)
-		if !ok {
-			return nil, errors.New("error resolving type as activitystreams person")
-		}
-		return p, nil
-	case string(gtsmodel.ActivityStreamsApplication):
-		p, ok := t.(vocab.ActivityStreamsApplication)
-		if !ok {
-			return nil, errors.New("error resolving type as activitystreams application")
-		}
-		return p, nil
-	case string(gtsmodel.ActivityStreamsService):
-		p, ok := t.(vocab.ActivityStreamsService)
-		if !ok {
-			return nil, errors.New("error resolving type as activitystreams service")
-		}
-		return p, nil
-	}
-
-	return nil, fmt.Errorf("type name %s not supported", t.GetTypeName())
 }
 
-func (f *federator) DereferenceRemoteStatus(username string, remoteStatusID *url.URL) (typeutils.Statusable, error) {
+func (f *federator) GetRemoteStatus(username string, remoteStatusID *url.URL, refresh bool) (*gtsmodel.Status, ap.Statusable, bool, error) {
-	if blocked, err := f.blockedDomain(remoteStatusID.Host); blocked || err != nil {
+	return f.dereferencer.GetRemoteStatus(username, remoteStatusID, refresh)
-		return nil, fmt.Errorf("DereferenceRemoteStatus: domain %s is blocked", remoteStatusID.Host)
-	}
-
-	transport, err := f.GetTransportForUser(username)
-	if err != nil {
-		return nil, fmt.Errorf("transport err: %s", err)
-	}
-
-	b, err := transport.Dereference(context.Background(), remoteStatusID)
-	if err != nil {
-		return nil, fmt.Errorf("error deferencing %s: %s", remoteStatusID.String(), err)
-	}
-
-	m := make(map[string]interface{})
-	if err := json.Unmarshal(b, &m); err != nil {
-		return nil, fmt.Errorf("error unmarshalling bytes into json: %s", err)
-	}
-
-	t, err := streams.ToType(context.Background(), m)
-	if err != nil {
-		return nil, fmt.Errorf("error resolving json into ap vocab type: %s", err)
-	}
-
-	// Article, Document, Image, Video, Note, Page, Event, Place, Mention, Profile
-	switch t.GetTypeName() {
-	case gtsmodel.ActivityStreamsArticle:
-		p, ok := t.(vocab.ActivityStreamsArticle)
-		if !ok {
-			return nil, errors.New("error resolving type as ActivityStreamsArticle")
-		}
-		return p, nil
-	case gtsmodel.ActivityStreamsDocument:
-		p, ok := t.(vocab.ActivityStreamsDocument)
-		if !ok {
-			return nil, errors.New("error resolving type as ActivityStreamsDocument")
-		}
-		return p, nil
-	case gtsmodel.ActivityStreamsImage:
-		p, ok := t.(vocab.ActivityStreamsImage)
-		if !ok {
-			return nil, errors.New("error resolving type as ActivityStreamsImage")
-		}
-		return p, nil
-	case gtsmodel.ActivityStreamsVideo:
-		p, ok := t.(vocab.ActivityStreamsVideo)
-		if !ok {
-			return nil, errors.New("error resolving type as ActivityStreamsVideo")
-		}
-		return p, nil
-	case gtsmodel.ActivityStreamsNote:
-		p, ok := t.(vocab.ActivityStreamsNote)
-		if !ok {
-			return nil, errors.New("error resolving type as ActivityStreamsNote")
-		}
-		return p, nil
-	case gtsmodel.ActivityStreamsPage:
-		p, ok := t.(vocab.ActivityStreamsPage)
-		if !ok {
-			return nil, errors.New("error resolving type as ActivityStreamsPage")
-		}
-		return p, nil
-	case gtsmodel.ActivityStreamsEvent:
-		p, ok := t.(vocab.ActivityStreamsEvent)
-		if !ok {
-			return nil, errors.New("error resolving type as ActivityStreamsEvent")
-		}
-		return p, nil
-	case gtsmodel.ActivityStreamsPlace:
-		p, ok := t.(vocab.ActivityStreamsPlace)
-		if !ok {
-			return nil, errors.New("error resolving type as ActivityStreamsPlace")
-		}
-		return p, nil
-	case gtsmodel.ActivityStreamsProfile:
-		p, ok := t.(vocab.ActivityStreamsProfile)
-		if !ok {
-			return nil, errors.New("error resolving type as ActivityStreamsProfile")
-		}
-		return p, nil
-	}
-
-	return nil, fmt.Errorf("type name %s not supported", t.GetTypeName())
 }
 
-func (f *federator) DereferenceRemoteInstance(username string, remoteInstanceURI *url.URL) (*gtsmodel.Instance, error) {
+func (f *federator) EnrichRemoteStatus(username string, status *gtsmodel.Status) (*gtsmodel.Status, error) {
-	if blocked, err := f.blockedDomain(remoteInstanceURI.Host); blocked || err != nil {
+	return f.dereferencer.EnrichRemoteStatus(username, status)
-		return nil, fmt.Errorf("DereferenceRemoteInstance: domain %s is blocked", remoteInstanceURI.Host)
-	}
-
-	transport, err := f.GetTransportForUser(username)
-	if err != nil {
-		return nil, fmt.Errorf("transport err: %s", err)
-	}
-
-	return transport.DereferenceInstance(context.Background(), remoteInstanceURI)
 }
 
-// dereferenceStatusFields fetches all the information we temporarily pinned to an incoming
+func (f *federator) DereferenceRemoteThread(username string, statusIRI *url.URL) error {
-// federated status, back in the federating db's Create function.
+	return f.dereferencer.DereferenceThread(username, statusIRI)
-//
-// When a status comes in from the federation API, there are certain fields that
-// haven't been dereferenced yet, because we needed to provide a snappy synchronous
-// response to the caller. By the time it reaches this function though, it's being
-// processed asynchronously, so we have all the time in the world to fetch the various
-// bits and bobs that are attached to the status, and properly flesh it out, before we
-// send the status to any timelines and notify people.
-//
-// Things to dereference and fetch here:
-//
-// 1. Media attachments.
-// 2. Hashtags.
-// 3. Emojis.
-// 4. Mentions.
-// 5. Posting account.
-// 6. Replied-to-status.
-//
-// SIDE EFFECTS:
-// This function will deference all of the above, insert them in the database as necessary,
-// and attach them to the status. The status itself will not be added to the database yet,
-// that's up the caller to do.
-func (f *federator) DereferenceStatusFields(status *gtsmodel.Status, requestingUsername string) error {
-	l := f.log.WithFields(logrus.Fields{
-		"func":   "dereferenceStatusFields",
-		"status": fmt.Sprintf("%+v", status),
-	})
-	l.Debug("entering function")
-
-	statusURI, err := url.Parse(status.URI)
-	if err != nil {
-		return fmt.Errorf("DereferenceStatusFields: couldn't parse status URI %s: %s", status.URI, err)
-	}
-	if blocked, err := f.blockedDomain(statusURI.Host); blocked || err != nil {
-		return fmt.Errorf("DereferenceStatusFields: domain %s is blocked", statusURI.Host)
-	}
-
-	t, err := f.GetTransportForUser(requestingUsername)
-	if err != nil {
-		return fmt.Errorf("error creating transport: %s", err)
-	}
-
-	// the status should have an ID by now, but just in case it doesn't let's generate one here
-	// because we'll need it further down
-	if status.ID == "" {
-		newID, err := id.NewULIDFromTime(status.CreatedAt)
-		if err != nil {
-			return err
-		}
-		status.ID = newID
-	}
-
-	// 1. Media attachments.
-	//
-	// At this point we should know:
-	// * the media type of the file we're looking for (a.File.ContentType)
-	// * the blurhash (a.Blurhash)
-	// * the file type (a.Type)
-	// * the remote URL (a.RemoteURL)
-	// This should be enough to pass along to the media processor.
-	attachmentIDs := []string{}
-	for _, a := range status.GTSMediaAttachments {
-		l.Debugf("dereferencing attachment: %+v", a)
-
-		// it might have been processed elsewhere so check first if it's already in the database or not
-		maybeAttachment := &gtsmodel.MediaAttachment{}
-		err := f.db.GetWhere([]db.Where{{Key: "remote_url", Value: a.RemoteURL}}, maybeAttachment)
-		if err == nil {
-			// we already have it in the db, dereferenced, no need to do it again
-			l.Debugf("attachment already exists with id %s", maybeAttachment.ID)
-			attachmentIDs = append(attachmentIDs, maybeAttachment.ID)
-			continue
-		}
-		if _, ok := err.(db.ErrNoEntries); !ok {
-			// we have a real error
-			return fmt.Errorf("error checking db for existence of attachment with remote url %s: %s", a.RemoteURL, err)
-		}
-		// it just doesn't exist yet so carry on
-		l.Debug("attachment doesn't exist yet, calling ProcessRemoteAttachment", a)
-		deferencedAttachment, err := f.mediaHandler.ProcessRemoteAttachment(t, a, status.AccountID)
-		if err != nil {
-			l.Errorf("error dereferencing status attachment: %s", err)
-			continue
-		}
-		l.Debugf("dereferenced attachment: %+v", deferencedAttachment)
-		deferencedAttachment.StatusID = status.ID
-		deferencedAttachment.Description = a.Description
-		if err := f.db.Put(deferencedAttachment); err != nil {
-			return fmt.Errorf("error inserting dereferenced attachment with remote url %s: %s", a.RemoteURL, err)
-		}
-		attachmentIDs = append(attachmentIDs, deferencedAttachment.ID)
-	}
-	status.Attachments = attachmentIDs
-
-	// 2. Hashtags
-
-	// 3. Emojis
-
-	// 4. Mentions
-	// At this point, mentions should have the namestring and mentionedAccountURI set on them.
-	//
-	// We should dereference any accounts mentioned here which we don't have in our db yet, by their URI.
-	mentions := []string{}
-	for _, m := range status.GTSMentions {
-		if m.ID == "" {
-			mID, err := id.NewRandomULID()
-			if err != nil {
-				return err
-			}
-			m.ID = mID
-		}
-
-		uri, err := url.Parse(m.MentionedAccountURI)
-		if err != nil {
-			l.Debugf("error parsing mentioned account uri %s: %s", m.MentionedAccountURI, err)
-			continue
-		}
-
-		m.StatusID = status.ID
-		m.OriginAccountID = status.GTSAuthorAccount.ID
-		m.OriginAccountURI = status.GTSAuthorAccount.URI
-
-		targetAccount := &gtsmodel.Account{}
-		if err := f.db.GetWhere([]db.Where{{Key: "uri", Value: uri.String()}}, targetAccount); err != nil {
-			// proper error
-			if _, ok := err.(db.ErrNoEntries); !ok {
-				return fmt.Errorf("db error checking for account with uri %s", uri.String())
-			}
-
-			// we just don't have it yet, so we should go get it....
-			accountable, err := f.DereferenceRemoteAccount(requestingUsername, uri)
-			if err != nil {
-				// we can't dereference it so just skip it
-				l.Debugf("error dereferencing remote account with uri %s: %s", uri.String(), err)
-				continue
-			}
-
-			targetAccount, err = f.typeConverter.ASRepresentationToAccount(accountable, false)
-			if err != nil {
-				l.Debugf("error converting remote account with uri %s into gts model: %s", uri.String(), err)
-				continue
-			}
-
-			targetAccountID, err := id.NewRandomULID()
-			if err != nil {
-				return err
-			}
-			targetAccount.ID = targetAccountID
-
-			if err := f.db.Put(targetAccount); err != nil {
-				return fmt.Errorf("db error inserting account with uri %s", uri.String())
-			}
-		}
-
-		// by this point, we know the targetAccount exists in our database with an ID :)
-		m.TargetAccountID = targetAccount.ID
-		if err := f.db.Put(m); err != nil {
-			return fmt.Errorf("error creating mention: %s", err)
-		}
-		mentions = append(mentions, m.ID)
-	}
-	status.Mentions = mentions
-
-	return nil
 }
 
-func (f *federator) DereferenceAccountFields(account *gtsmodel.Account, requestingUsername string, refresh bool) error {
+func (f *federator) GetRemoteInstance(username string, remoteInstanceURI *url.URL) (*gtsmodel.Instance, error) {
-	l := f.log.WithFields(logrus.Fields{
+	return f.dereferencer.GetRemoteInstance(username, remoteInstanceURI)
-		"func":               "dereferenceAccountFields",
-		"requestingUsername": requestingUsername,
-	})
-
-	accountURI, err := url.Parse(account.URI)
-	if err != nil {
-		return fmt.Errorf("DereferenceAccountFields: couldn't parse account URI %s: %s", account.URI, err)
-	}
-	if blocked, err := f.blockedDomain(accountURI.Host); blocked || err != nil {
-		return fmt.Errorf("DereferenceAccountFields: domain %s is blocked", accountURI.Host)
-	}
-
-	t, err := f.GetTransportForUser(requestingUsername)
-	if err != nil {
-		return fmt.Errorf("error getting transport for user: %s", err)
-	}
-
-	// fetch the header and avatar
-	if err := f.fetchHeaderAndAviForAccount(account, t, refresh); err != nil {
-		// if this doesn't work, just skip it -- we can do it later
-		l.Debugf("error fetching header/avi for account: %s", err)
-	}
-
-	if err := f.db.UpdateByID(account.ID, account); err != nil {
-		return fmt.Errorf("error updating account in database: %s", err)
-	}
-
-	return nil
 }
 
 func (f *federator) DereferenceAnnounce(announce *gtsmodel.Status, requestingUsername string) error {
-	if announce.GTSBoostedStatus == nil || announce.GTSBoostedStatus.URI == "" {
+	return f.dereferencer.DereferenceAnnounce(announce, requestingUsername)
-		// we can't do anything unfortunately
-		return errors.New("DereferenceAnnounce: no URI to dereference")
-	}
-
-	boostedStatusURI, err := url.Parse(announce.GTSBoostedStatus.URI)
-	if err != nil {
-		return fmt.Errorf("DereferenceAnnounce: couldn't parse boosted status URI %s: %s", announce.GTSBoostedStatus.URI, err)
-	}
-	if blocked, err := f.blockedDomain(boostedStatusURI.Host); blocked || err != nil {
-		return fmt.Errorf("DereferenceAnnounce: domain %s is blocked", boostedStatusURI.Host)
-	}
-
-	// check if we already have the boosted status in the database
-	boostedStatus := &gtsmodel.Status{}
-	err = f.db.GetWhere([]db.Where{{Key: "uri", Value: announce.GTSBoostedStatus.URI}}, boostedStatus)
-	if err == nil {
-		// nice, we already have it so we don't actually need to dereference it from remote
-		announce.Content = boostedStatus.Content
-		announce.ContentWarning = boostedStatus.ContentWarning
-		announce.ActivityStreamsType = boostedStatus.ActivityStreamsType
-		announce.Sensitive = boostedStatus.Sensitive
-		announce.Language = boostedStatus.Language
-		announce.Text = boostedStatus.Text
-		announce.BoostOfID = boostedStatus.ID
-		announce.BoostOfAccountID = boostedStatus.AccountID
-		announce.Visibility = boostedStatus.Visibility
-		announce.VisibilityAdvanced = boostedStatus.VisibilityAdvanced
-		announce.GTSBoostedStatus = boostedStatus
-		return nil
-	}
-
-	// we don't have it so we need to dereference it
-	statusable, err := f.DereferenceRemoteStatus(requestingUsername, boostedStatusURI)
-	if err != nil {
-		return fmt.Errorf("dereferenceAnnounce: error dereferencing remote status with id %s: %s", announce.GTSBoostedStatus.URI, err)
-	}
-
-	// make sure we have the author account in the db
-	attributedToProp := statusable.GetActivityStreamsAttributedTo()
-	for iter := attributedToProp.Begin(); iter != attributedToProp.End(); iter = iter.Next() {
-		accountURI := iter.GetIRI()
-		if accountURI == nil {
-			continue
-		}
-
-		if err := f.db.GetWhere([]db.Where{{Key: "uri", Value: accountURI.String()}}, &gtsmodel.Account{}); err == nil {
-			// we already have it, fine
-			continue
-		}
-
-		// we don't have the boosted status author account yet so dereference it
-		accountable, err := f.DereferenceRemoteAccount(requestingUsername, accountURI)
-		if err != nil {
-			return fmt.Errorf("dereferenceAnnounce: error dereferencing remote account with id %s: %s", accountURI.String(), err)
-		}
-		account, err := f.typeConverter.ASRepresentationToAccount(accountable, false)
-		if err != nil {
-			return fmt.Errorf("dereferenceAnnounce: error converting dereferenced account with id %s into account : %s", accountURI.String(), err)
-		}
-
-		accountID, err := id.NewRandomULID()
-		if err != nil {
-			return err
-		}
-		account.ID = accountID
-
-		if err := f.db.Put(account); err != nil {
-			return fmt.Errorf("dereferenceAnnounce: error putting dereferenced account with id %s into database : %s", accountURI.String(), err)
-		}
-
-		if err := f.DereferenceAccountFields(account, requestingUsername, false); err != nil {
-			return fmt.Errorf("dereferenceAnnounce: error dereferencing fields on account with id %s : %s", accountURI.String(), err)
-		}
-	}
-
-	// now convert the statusable into something we can understand
-	boostedStatus, err = f.typeConverter.ASStatusToStatus(statusable)
-	if err != nil {
-		return fmt.Errorf("dereferenceAnnounce: error converting dereferenced statusable with id %s into status : %s", announce.GTSBoostedStatus.URI, err)
-	}
-
-	boostedStatusID, err := id.NewULIDFromTime(boostedStatus.CreatedAt)
-	if err != nil {
-		return nil
-	}
-	boostedStatus.ID = boostedStatusID
-
-	if err := f.db.Put(boostedStatus); err != nil {
-		return fmt.Errorf("dereferenceAnnounce: error putting dereferenced status with id %s into the db: %s", announce.GTSBoostedStatus.URI, err)
-	}
-
-	// now dereference additional fields straight away (we're already async here so we have time)
-	if err := f.DereferenceStatusFields(boostedStatus, requestingUsername); err != nil {
-		return fmt.Errorf("dereferenceAnnounce: error dereferencing status fields for status with id %s: %s", announce.GTSBoostedStatus.URI, err)
-	}
-
-	// update with the newly dereferenced fields
-	if err := f.db.UpdateByID(boostedStatus.ID, boostedStatus); err != nil {
-		return fmt.Errorf("dereferenceAnnounce: error updating dereferenced status in the db: %s", err)
-	}
-
-	// we have everything we need!
-	announce.Content = boostedStatus.Content
-	announce.ContentWarning = boostedStatus.ContentWarning
-	announce.ActivityStreamsType = boostedStatus.ActivityStreamsType
-	announce.Sensitive = boostedStatus.Sensitive
-	announce.Language = boostedStatus.Language
-	announce.Text = boostedStatus.Text
-	announce.BoostOfID = boostedStatus.ID
-	announce.BoostOfAccountID = boostedStatus.AccountID
-	announce.Visibility = boostedStatus.Visibility
-	announce.VisibilityAdvanced = boostedStatus.VisibilityAdvanced
-	announce.GTSBoostedStatus = boostedStatus
-	return nil
-}
-
-// fetchHeaderAndAviForAccount fetches the header and avatar for a remote account, using a transport
-// on behalf of requestingUsername.
-//
-// targetAccount's AvatarMediaAttachmentID and HeaderMediaAttachmentID will be updated as necessary.
-//
-// SIDE EFFECTS: remote header and avatar will be stored in local storage, and the database will be updated
-// to reflect the creation of these new attachments.
-func (f *federator) fetchHeaderAndAviForAccount(targetAccount *gtsmodel.Account, t transport.Transport, refresh bool) error {
-	accountURI, err := url.Parse(targetAccount.URI)
-	if err != nil {
-		return fmt.Errorf("fetchHeaderAndAviForAccount: couldn't parse account URI %s: %s", targetAccount.URI, err)
-	}
-	if blocked, err := f.blockedDomain(accountURI.Host); blocked || err != nil {
-		return fmt.Errorf("fetchHeaderAndAviForAccount: domain %s is blocked", accountURI.Host)
-	}
-
-	if targetAccount.AvatarRemoteURL != "" && (targetAccount.AvatarMediaAttachmentID == "" || refresh) {
-		a, err := f.mediaHandler.ProcessRemoteHeaderOrAvatar(t, &gtsmodel.MediaAttachment{
-			RemoteURL: targetAccount.AvatarRemoteURL,
-			Avatar:    true,
-		}, targetAccount.ID)
-		if err != nil {
-			return fmt.Errorf("error processing avatar for user: %s", err)
-		}
-		targetAccount.AvatarMediaAttachmentID = a.ID
-	}
-
-	if targetAccount.HeaderRemoteURL != "" && (targetAccount.HeaderMediaAttachmentID == "" || refresh) {
-		a, err := f.mediaHandler.ProcessRemoteHeaderOrAvatar(t, &gtsmodel.MediaAttachment{
-			RemoteURL: targetAccount.HeaderRemoteURL,
-			Header:    true,
-		}, targetAccount.ID)
-		if err != nil {
-			return fmt.Errorf("error processing header for user: %s", err)
-		}
-		targetAccount.HeaderMediaAttachmentID = a.ID
-	}
-	return nil
 }

internal/federation/dereferencing/account.go

@@ -0,0 +1,243 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package dereferencing
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/url"
+
+	"github.com/go-fed/activity/streams"
+	"github.com/go-fed/activity/streams/vocab"
+	"github.com/sirupsen/logrus"
+	"github.com/superseriousbusiness/gotosocial/internal/ap"
+	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+	"github.com/superseriousbusiness/gotosocial/internal/id"
+	"github.com/superseriousbusiness/gotosocial/internal/transport"
+)
+
+// EnrichRemoteAccount takes an account that's already been inserted into the database in a minimal form,
+// and populates it with additional fields, media, etc.
+//
+// EnrichRemoteAccount is mostly useful for calling after an account has been initially created by
+// the federatingDB's Create function, or during the federated authorization flow.
+func (d *deref) EnrichRemoteAccount(username string, account *gtsmodel.Account) (*gtsmodel.Account, error) {
+	if err := d.populateAccountFields(account, username, false); err != nil {
+		return nil, err
+	}
+
+	if err := d.db.UpdateByID(account.ID, account); err != nil {
+		return nil, fmt.Errorf("EnrichRemoteAccount: error updating account: %s", err)
+	}
+
+	return account, nil
+}
+
+// GetRemoteAccount completely dereferences a remote account, converts it to a GtS model account,
+// puts it in the database, and returns it to a caller. The boolean indicates whether the account is new
+// to us or not. If we haven't seen the account before, bool will be true. If we have seen the account before,
+// it will be false.
+//
+// Refresh indicates whether--if the account exists in our db already--it should be refreshed by calling
+// the remote instance again.
+//
+// SIDE EFFECTS: remote account will be stored in the database, or updated if it already exists (and refresh is true).
+func (d *deref) GetRemoteAccount(username string, remoteAccountID *url.URL, refresh bool) (*gtsmodel.Account, bool, error) {
+	new := true
+
+	// check if we already have the account in our db
+	maybeAccount := &gtsmodel.Account{}
+	if err := d.db.GetWhere([]db.Where{{Key: "uri", Value: remoteAccountID.String()}}, maybeAccount); err == nil {
+		// we've seen this account before so it's not new
+		new = false
+
+		// if we're not being asked to refresh, we can just return the maybeAccount as-is and avoid doing any external calls
+		if !refresh {
+			return maybeAccount, new, nil
+		}
+	}
+
+	accountable, err := d.dereferenceAccountable(username, remoteAccountID)
+	if err != nil {
+		return nil, new, fmt.Errorf("FullyDereferenceAccount: error dereferencing accountable: %s", err)
+	}
+
+	gtsAccount, err := d.typeConverter.ASRepresentationToAccount(accountable, false)
+	if err != nil {
+		return nil, new, fmt.Errorf("FullyDereferenceAccount: error converting accountable to account: %s", err)
+	}
+
+	if new {
+		// generate a new id since we haven't seen this account before, and do a put
+		ulid, err := id.NewRandomULID()
+		if err != nil {
+			return nil, new, fmt.Errorf("FullyDereferenceAccount: error generating new id for account: %s", err)
+		}
+		gtsAccount.ID = ulid
+
+		if err := d.populateAccountFields(gtsAccount, username, refresh); err != nil {
+			return nil, new, fmt.Errorf("FullyDereferenceAccount: error populating further account fields: %s", err)
+		}
+
+		if err := d.db.Put(gtsAccount); err != nil {
+			return nil, new, fmt.Errorf("FullyDereferenceAccount: error putting new account: %s", err)
+		}
+	} else {
+		// take the id we already have and do an update
+		gtsAccount.ID = maybeAccount.ID
+
+		if err := d.populateAccountFields(gtsAccount, username, refresh); err != nil {
+			return nil, new, fmt.Errorf("FullyDereferenceAccount: error populating further account fields: %s", err)
+		}
+
+		if err := d.db.UpdateByID(gtsAccount.ID, gtsAccount); err != nil {
+			return nil, new, fmt.Errorf("FullyDereferenceAccount: error updating existing account: %s", err)
+		}
+	}
+
+	return gtsAccount, new, nil
+}
+
+// dereferenceAccountable calls remoteAccountID with a GET request, and tries to parse whatever
+// it finds as something that an account model can be constructed out of.
+//
+// Will work for Person, Application, or Service models.
+func (d *deref) dereferenceAccountable(username string, remoteAccountID *url.URL) (ap.Accountable, error) {
+	d.startHandshake(username, remoteAccountID)
+	defer d.stopHandshake(username, remoteAccountID)
+
+	if blocked, err := d.blockedDomain(remoteAccountID.Host); blocked || err != nil {
+		return nil, fmt.Errorf("DereferenceAccountable: domain %s is blocked", remoteAccountID.Host)
+	}
+
+	transport, err := d.transportController.NewTransportForUsername(username)
+	if err != nil {
+		return nil, fmt.Errorf("DereferenceAccountable: transport err: %s", err)
+	}
+
+	b, err := transport.Dereference(context.Background(), remoteAccountID)
+	if err != nil {
+		return nil, fmt.Errorf("DereferenceAccountable: error deferencing %s: %s", remoteAccountID.String(), err)
+	}
+
+	m := make(map[string]interface{})
+	if err := json.Unmarshal(b, &m); err != nil {
+		return nil, fmt.Errorf("DereferenceAccountable: error unmarshalling bytes into json: %s", err)
+	}
+
+	t, err := streams.ToType(context.Background(), m)
+	if err != nil {
+		return nil, fmt.Errorf("DereferenceAccountable: error resolving json into ap vocab type: %s", err)
+	}
+
+	switch t.GetTypeName() {
+	case string(gtsmodel.ActivityStreamsPerson):
+		p, ok := t.(vocab.ActivityStreamsPerson)
+		if !ok {
+			return nil, errors.New("DereferenceAccountable: error resolving type as activitystreams person")
+		}
+		return p, nil
+	case string(gtsmodel.ActivityStreamsApplication):
+		p, ok := t.(vocab.ActivityStreamsApplication)
+		if !ok {
+			return nil, errors.New("DereferenceAccountable: error resolving type as activitystreams application")
+		}
+		return p, nil
+	case string(gtsmodel.ActivityStreamsService):
+		p, ok := t.(vocab.ActivityStreamsService)
+		if !ok {
+			return nil, errors.New("DereferenceAccountable: error resolving type as activitystreams service")
+		}
+		return p, nil
+	}
+
+	return nil, fmt.Errorf("DereferenceAccountable: type name %s not supported", t.GetTypeName())
+}
+
+// populateAccountFields populates any fields on the given account that weren't populated by the initial
+// dereferencing. This includes things like header and avatar etc.
+func (d *deref) populateAccountFields(account *gtsmodel.Account, requestingUsername string, refresh bool) error {
+	l := d.log.WithFields(logrus.Fields{
+		"func":               "PopulateAccountFields",
+		"requestingUsername": requestingUsername,
+	})
+
+	accountURI, err := url.Parse(account.URI)
+	if err != nil {
+		return fmt.Errorf("PopulateAccountFields: couldn't parse account URI %s: %s", account.URI, err)
+	}
+	if blocked, err := d.blockedDomain(accountURI.Host); blocked || err != nil {
+		return fmt.Errorf("PopulateAccountFields: domain %s is blocked", accountURI.Host)
+	}
+
+	t, err := d.transportController.NewTransportForUsername(requestingUsername)
+	if err != nil {
+		return fmt.Errorf("PopulateAccountFields: error getting transport for user: %s", err)
+	}
+
+	// fetch the header and avatar
+	if err := d.fetchHeaderAndAviForAccount(account, t, refresh); err != nil {
+		// if this doesn't work, just skip it -- we can do it later
+		l.Debugf("error fetching header/avi for account: %s", err)
+	}
+
+	return nil
+}
+
+// fetchHeaderAndAviForAccount fetches the header and avatar for a remote account, using a transport
+// on behalf of requestingUsername.
+//
+// targetAccount's AvatarMediaAttachmentID and HeaderMediaAttachmentID will be updated as necessary.
+//
+// SIDE EFFECTS: remote header and avatar will be stored in local storage.
+func (d *deref) fetchHeaderAndAviForAccount(targetAccount *gtsmodel.Account, t transport.Transport, refresh bool) error {
+	accountURI, err := url.Parse(targetAccount.URI)
+	if err != nil {
+		return fmt.Errorf("fetchHeaderAndAviForAccount: couldn't parse account URI %s: %s", targetAccount.URI, err)
+	}
+	if blocked, err := d.blockedDomain(accountURI.Host); blocked || err != nil {
+		return fmt.Errorf("fetchHeaderAndAviForAccount: domain %s is blocked", accountURI.Host)
+	}
+
+	if targetAccount.AvatarRemoteURL != "" && (targetAccount.AvatarMediaAttachmentID == "" || refresh) {
+		a, err := d.mediaHandler.ProcessRemoteHeaderOrAvatar(t, &gtsmodel.MediaAttachment{
+			RemoteURL: targetAccount.AvatarRemoteURL,
+			Avatar:    true,
+		}, targetAccount.ID)
+		if err != nil {
+			return fmt.Errorf("error processing avatar for user: %s", err)
+		}
+		targetAccount.AvatarMediaAttachmentID = a.ID
+	}
+
+	if targetAccount.HeaderRemoteURL != "" && (targetAccount.HeaderMediaAttachmentID == "" || refresh) {
+		a, err := d.mediaHandler.ProcessRemoteHeaderOrAvatar(t, &gtsmodel.MediaAttachment{
+			RemoteURL: targetAccount.HeaderRemoteURL,
+			Header:    true,
+		}, targetAccount.ID)
+		if err != nil {
+			return fmt.Errorf("error processing header for user: %s", err)
+		}
+		targetAccount.HeaderMediaAttachmentID = a.ID
+	}
+	return nil
+}

internal/federation/dereferencing/announce.go

@@ -0,0 +1,65 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package dereferencing
+
+import (
+	"errors"
+	"fmt"
+	"net/url"
+
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+)
+
+func (d *deref) DereferenceAnnounce(announce *gtsmodel.Status, requestingUsername string) error {
+	if announce.GTSBoostedStatus == nil || announce.GTSBoostedStatus.URI == "" {
+		// we can't do anything unfortunately
+		return errors.New("DereferenceAnnounce: no URI to dereference")
+	}
+
+	boostedStatusURI, err := url.Parse(announce.GTSBoostedStatus.URI)
+	if err != nil {
+		return fmt.Errorf("DereferenceAnnounce: couldn't parse boosted status URI %s: %s", announce.GTSBoostedStatus.URI, err)
+	}
+	if blocked, err := d.blockedDomain(boostedStatusURI.Host); blocked || err != nil {
+		return fmt.Errorf("DereferenceAnnounce: domain %s is blocked", boostedStatusURI.Host)
+	}
+
+	// dereference statuses in the thread of the boosted status
+	if err := d.DereferenceThread(requestingUsername, boostedStatusURI); err != nil {
+		return fmt.Errorf("DereferenceAnnounce: error dereferencing thread of boosted status: %s", err)
+	}
+
+	boostedStatus, _, _, err := d.GetRemoteStatus(requestingUsername, boostedStatusURI, false)
+	if err != nil {
+		return fmt.Errorf("DereferenceAnnounce: error dereferencing remote status with id %s: %s", announce.GTSBoostedStatus.URI, err)
+	}
+
+	announce.Content = boostedStatus.Content
+	announce.ContentWarning = boostedStatus.ContentWarning
+	announce.ActivityStreamsType = boostedStatus.ActivityStreamsType
+	announce.Sensitive = boostedStatus.Sensitive
+	announce.Language = boostedStatus.Language
+	announce.Text = boostedStatus.Text
+	announce.BoostOfID = boostedStatus.ID
+	announce.BoostOfAccountID = boostedStatus.AccountID
+	announce.Visibility = boostedStatus.Visibility
+	announce.VisibilityAdvanced = boostedStatus.VisibilityAdvanced
+	announce.GTSBoostedStatus = boostedStatus
+	return nil
+}

internal/federation/dereferencing/blocked.go

@@ -0,0 +1,41 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package dereferencing
+
+import (
+	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+)
+
+func (d *deref) blockedDomain(host string) (bool, error) {
+	b := &gtsmodel.DomainBlock{}
+	err := d.db.GetWhere([]db.Where{{Key: "domain", Value: host, CaseInsensitive: true}}, b)
+	if err == nil {
+		// block exists
+		return true, nil
+	}
+
+	if _, ok := err.(db.ErrNoEntries); ok {
+		// there are no entries so there's no block
+		return false, nil
+	}
+
+	// there's an actual error
+	return false, err
+}

internal/federation/dereferencing/collectionpage.go

@@ -0,0 +1,70 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package dereferencing
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/url"
+
+	"github.com/go-fed/activity/streams"
+	"github.com/go-fed/activity/streams/vocab"
+	"github.com/superseriousbusiness/gotosocial/internal/ap"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+)
+
+// DereferenceCollectionPage returns the activitystreams CollectionPage at the specified IRI, or an error if something goes wrong.
+func (d *deref) DereferenceCollectionPage(username string, pageIRI *url.URL) (ap.CollectionPageable, error) {
+	if blocked, err := d.blockedDomain(pageIRI.Host); blocked || err != nil {
+		return nil, fmt.Errorf("DereferenceCollectionPage: domain %s is blocked", pageIRI.Host)
+	}
+
+	transport, err := d.transportController.NewTransportForUsername(username)
+	if err != nil {
+		return nil, fmt.Errorf("DereferenceCollectionPage: error creating transport: %s", err)
+	}
+
+	b, err := transport.Dereference(context.Background(), pageIRI)
+	if err != nil {
+		return nil, fmt.Errorf("DereferenceCollectionPage: error deferencing %s: %s", pageIRI.String(), err)
+	}
+
+	m := make(map[string]interface{})
+	if err := json.Unmarshal(b, &m); err != nil {
+		return nil, fmt.Errorf("DereferenceCollectionPage: error unmarshalling bytes into json: %s", err)
+	}
+
+	t, err := streams.ToType(context.Background(), m)
+	if err != nil {
+		return nil, fmt.Errorf("DereferenceCollectionPage: error resolving json into ap vocab type: %s", err)
+	}
+
+	if t.GetTypeName() != gtsmodel.ActivityStreamsCollectionPage {
+		return nil, fmt.Errorf("DereferenceCollectionPage: type name %s not supported", t.GetTypeName())
+	}
+
+	p, ok := t.(vocab.ActivityStreamsCollectionPage)
+	if !ok {
+		return nil, errors.New("DereferenceCollectionPage: error resolving type as activitystreams collection page")
+	}
+
+	return p, nil
+}

internal/federation/dereferencing/dereferencer.go

@@ -0,0 +1,73 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package dereferencing
+
+import (
+	"net/url"
+	"sync"
+
+	"github.com/sirupsen/logrus"
+	"github.com/superseriousbusiness/gotosocial/internal/ap"
+	"github.com/superseriousbusiness/gotosocial/internal/config"
+	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+	"github.com/superseriousbusiness/gotosocial/internal/media"
+	"github.com/superseriousbusiness/gotosocial/internal/transport"
+	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
+)
+
+// Dereferencer wraps logic and functionality for doing dereferencing of remote accounts, statuses, etc, from federated instances.
+type Dereferencer interface {
+	GetRemoteAccount(username string, remoteAccountID *url.URL, refresh bool) (*gtsmodel.Account, bool, error)
+	EnrichRemoteAccount(username string, account *gtsmodel.Account) (*gtsmodel.Account, error)
+
+	GetRemoteStatus(username string, remoteStatusID *url.URL, refresh bool) (*gtsmodel.Status, ap.Statusable, bool, error)
+	EnrichRemoteStatus(username string, status *gtsmodel.Status) (*gtsmodel.Status, error)
+
+	GetRemoteInstance(username string, remoteInstanceURI *url.URL) (*gtsmodel.Instance, error)
+
+	DereferenceAnnounce(announce *gtsmodel.Status, requestingUsername string) error
+	DereferenceThread(username string, statusIRI *url.URL) error
+
+	Handshaking(username string, remoteAccountID *url.URL) bool
+}
+
+type deref struct {
+	log                 *logrus.Logger
+	db                  db.DB
+	typeConverter       typeutils.TypeConverter
+	transportController transport.Controller
+	mediaHandler        media.Handler
+	config              *config.Config
+	handshakes          map[string][]*url.URL
+	handshakeSync       *sync.Mutex // mutex to lock/unlock when checking or updating the handshakes map
+}
+
+// NewDereferencer returns a Dereferencer initialized with the given parameters.
+func NewDereferencer(config *config.Config, db db.DB, typeConverter typeutils.TypeConverter, transportController transport.Controller, mediaHandler media.Handler, log *logrus.Logger) Dereferencer {
+	return &deref{
+		log:                 log,
+		db:                  db,
+		typeConverter:       typeConverter,
+		transportController: transportController,
+		mediaHandler:        mediaHandler,
+		config:              config,
+		handshakeSync:       &sync.Mutex{},
+	}
+}

internal/federation/dereferencing/handshake.go

@@ -0,0 +1,98 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package dereferencing
+
+import "net/url"
+
+func (d *deref) Handshaking(username string, remoteAccountID *url.URL) bool {
+	d.handshakeSync.Lock()
+	defer d.handshakeSync.Unlock()
+
+	if d.handshakes == nil {
+		// handshakes isn't even initialized yet so we can't be handshaking with anyone
+		return false
+	}
+
+	remoteIDs, ok := d.handshakes[username]
+	if !ok {
+		// user isn't handshaking with anyone, bail
+		return false
+	}
+
+	for _, id := range remoteIDs {
+		if id.String() == remoteAccountID.String() {
+			// we are currently handshaking with the remote account, yep
+			return true
+		}
+	}
+
+	// didn't find it which means we're not handshaking
+	return false
+}
+
+func (d *deref) startHandshake(username string, remoteAccountID *url.URL) {
+	d.handshakeSync.Lock()
+	defer d.handshakeSync.Unlock()
+
+	// lazily initialize handshakes
+	if d.handshakes == nil {
+		d.handshakes = make(map[string][]*url.URL)
+	}
+
+	remoteIDs, ok := d.handshakes[username]
+	if !ok {
+		// there was nothing in there yet, so just add this entry and return
+		d.handshakes[username] = []*url.URL{remoteAccountID}
+		return
+	}
+
+	// add the remote ID to the slice
+	remoteIDs = append(remoteIDs, remoteAccountID)
+	d.handshakes[username] = remoteIDs
+}
+
+func (d *deref) stopHandshake(username string, remoteAccountID *url.URL) {
+	d.handshakeSync.Lock()
+	defer d.handshakeSync.Unlock()
+
+	if d.handshakes == nil {
+		return
+	}
+
+	remoteIDs, ok := d.handshakes[username]
+	if !ok {
+		// there was nothing in there yet anyway so just bail
+		return
+	}
+
+	newRemoteIDs := []*url.URL{}
+	for _, id := range remoteIDs {
+		if id.String() != remoteAccountID.String() {
+			newRemoteIDs = append(newRemoteIDs, id)
+		}
+	}
+
+	if len(newRemoteIDs) == 0 {
+		// there are no handshakes so just remove this user entry from the map and save a few bytes
+		delete(d.handshakes, username)
+	} else {
+		// there are still other handshakes ongoing
+		d.handshakes[username] = newRemoteIDs
+	}
+}

internal/federation/dereferencing/instance.go

@@ -0,0 +1,40 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package dereferencing
+
+import (
+	"context"
+	"fmt"
+	"net/url"
+
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+)
+
+func (d *deref) GetRemoteInstance(username string, remoteInstanceURI *url.URL) (*gtsmodel.Instance, error) {
+	if blocked, err := d.blockedDomain(remoteInstanceURI.Host); blocked || err != nil {
+		return nil, fmt.Errorf("GetRemoteInstance: domain %s is blocked", remoteInstanceURI.Host)
+	}
+
+	transport, err := d.transportController.NewTransportForUsername(username)
+	if err != nil {
+		return nil, fmt.Errorf("transport err: %s", err)
+	}
+
+	return transport.DereferenceInstance(context.Background(), remoteInstanceURI)
+}

internal/federation/dereferencing/status.go

@@ -0,0 +1,369 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package dereferencing
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/url"
+
+	"github.com/go-fed/activity/streams"
+	"github.com/go-fed/activity/streams/vocab"
+	"github.com/sirupsen/logrus"
+	"github.com/superseriousbusiness/gotosocial/internal/ap"
+	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+	"github.com/superseriousbusiness/gotosocial/internal/id"
+)
+
+// EnrichRemoteStatus takes a status that's already been inserted into the database in a minimal form,
+// and populates it with additional fields, media, etc.
+//
+// EnrichRemoteStatus is mostly useful for calling after a status has been initially created by
+// the federatingDB's Create function, but additional dereferencing is needed on it.
+func (d *deref) EnrichRemoteStatus(username string, status *gtsmodel.Status) (*gtsmodel.Status, error) {
+	if err := d.populateStatusFields(status, username); err != nil {
+		return nil, err
+	}
+
+	if err := d.db.UpdateByID(status.ID, status); err != nil {
+		return nil, fmt.Errorf("EnrichRemoteStatus: error updating status: %s", err)
+	}
+
+	return status, nil
+}
+
+// GetRemoteStatus completely dereferences a remote status, converts it to a GtS model status,
+// puts it in the database, and returns it to a caller. The boolean indicates whether the status is new
+// to us or not. If we haven't seen the status before, bool will be true. If we have seen the status before,
+// it will be false.
+//
+// If refresh is true, then even if we have the status in our database already, it will be dereferenced from its
+// remote representation, as will its owner.
+//
+// If a dereference was performed, then the function also returns the ap.Statusable representation for further processing.
+//
+// SIDE EFFECTS: remote status will be stored in the database, and the remote status owner will also be stored.
+func (d *deref) GetRemoteStatus(username string, remoteStatusID *url.URL, refresh bool) (*gtsmodel.Status, ap.Statusable, bool, error) {
+	new := true
+
+	// check if we already have the status in our db
+	maybeStatus := &gtsmodel.Status{}
+	if err := d.db.GetWhere([]db.Where{{Key: "uri", Value: remoteStatusID.String()}}, maybeStatus); err == nil {
+		// we've seen this status before so it's not new
+		new = false
+
+		// if we're not being asked to refresh, we can just return the maybeStatus as-is and avoid doing any external calls
+		if !refresh {
+			return maybeStatus, nil, new, nil
+		}
+	}
+
+	statusable, err := d.dereferenceStatusable(username, remoteStatusID)
+	if err != nil {
+		return nil, statusable, new, fmt.Errorf("GetRemoteStatus: error dereferencing statusable: %s", err)
+	}
+
+	accountURI, err := ap.ExtractAttributedTo(statusable)
+	if err != nil {
+		return nil, statusable, new, fmt.Errorf("GetRemoteStatus: error extracting attributedTo: %s", err)
+	}
+
+	// do this so we know we have the remote account of the status in the db
+	_, _, err = d.GetRemoteAccount(username, accountURI, false)
+	if err != nil {
+		return nil, statusable, new, fmt.Errorf("GetRemoteStatus: couldn't derive status author: %s", err)
+	}
+
+	gtsStatus, err := d.typeConverter.ASStatusToStatus(statusable)
+	if err != nil {
+		return nil, statusable, new, fmt.Errorf("GetRemoteStatus: error converting statusable to status: %s", err)
+	}
+
+	if new {
+		ulid, err := id.NewULIDFromTime(gtsStatus.CreatedAt)
+		if err != nil {
+			return nil, statusable, new, fmt.Errorf("GetRemoteStatus: error generating new id for status: %s", err)
+		}
+		gtsStatus.ID = ulid
+
+		if err := d.populateStatusFields(gtsStatus, username); err != nil {
+			return nil, statusable, new, fmt.Errorf("GetRemoteStatus: error populating status fields: %s", err)
+		}
+
+		if err := d.db.Put(gtsStatus); err != nil {
+			return nil, statusable, new, fmt.Errorf("GetRemoteStatus: error putting new status: %s", err)
+		}
+	} else {
+		gtsStatus.ID = maybeStatus.ID
+
+		if err := d.populateStatusFields(gtsStatus, username); err != nil {
+			return nil, statusable, new, fmt.Errorf("GetRemoteStatus: error populating status fields: %s", err)
+		}
+
+		if err := d.db.UpdateByID(gtsStatus.ID, gtsStatus); err != nil {
+			return nil, statusable, new, fmt.Errorf("GetRemoteStatus: error updating status: %s", err)
+		}
+	}
+
+	return gtsStatus, statusable, new, nil
+}
+
+func (d *deref) dereferenceStatusable(username string, remoteStatusID *url.URL) (ap.Statusable, error) {
+	if blocked, err := d.blockedDomain(remoteStatusID.Host); blocked || err != nil {
+		return nil, fmt.Errorf("DereferenceStatusable: domain %s is blocked", remoteStatusID.Host)
+	}
+
+	transport, err := d.transportController.NewTransportForUsername(username)
+	if err != nil {
+		return nil, fmt.Errorf("DereferenceStatusable: transport err: %s", err)
+	}
+
+	b, err := transport.Dereference(context.Background(), remoteStatusID)
+	if err != nil {
+		return nil, fmt.Errorf("DereferenceStatusable: error deferencing %s: %s", remoteStatusID.String(), err)
+	}
+
+	m := make(map[string]interface{})
+	if err := json.Unmarshal(b, &m); err != nil {
+		return nil, fmt.Errorf("DereferenceStatusable: error unmarshalling bytes into json: %s", err)
+	}
+
+	t, err := streams.ToType(context.Background(), m)
+	if err != nil {
+		return nil, fmt.Errorf("DereferenceStatusable: error resolving json into ap vocab type: %s", err)
+	}
+
+	// Article, Document, Image, Video, Note, Page, Event, Place, Mention, Profile
+	switch t.GetTypeName() {
+	case gtsmodel.ActivityStreamsArticle:
+		p, ok := t.(vocab.ActivityStreamsArticle)
+		if !ok {
+			return nil, errors.New("DereferenceStatusable: error resolving type as ActivityStreamsArticle")
+		}
+		return p, nil
+	case gtsmodel.ActivityStreamsDocument:
+		p, ok := t.(vocab.ActivityStreamsDocument)
+		if !ok {
+			return nil, errors.New("DereferenceStatusable: error resolving type as ActivityStreamsDocument")
+		}
+		return p, nil
+	case gtsmodel.ActivityStreamsImage:
+		p, ok := t.(vocab.ActivityStreamsImage)
+		if !ok {
+			return nil, errors.New("DereferenceStatusable: error resolving type as ActivityStreamsImage")
+		}
+		return p, nil
+	case gtsmodel.ActivityStreamsVideo:
+		p, ok := t.(vocab.ActivityStreamsVideo)
+		if !ok {
+			return nil, errors.New("DereferenceStatusable: error resolving type as ActivityStreamsVideo")
+		}
+		return p, nil
+	case gtsmodel.ActivityStreamsNote:
+		p, ok := t.(vocab.ActivityStreamsNote)
+		if !ok {
+			return nil, errors.New("DereferenceStatusable: error resolving type as ActivityStreamsNote")
+		}
+		return p, nil
+	case gtsmodel.ActivityStreamsPage:
+		p, ok := t.(vocab.ActivityStreamsPage)
+		if !ok {
+			return nil, errors.New("DereferenceStatusable: error resolving type as ActivityStreamsPage")
+		}
+		return p, nil
+	case gtsmodel.ActivityStreamsEvent:
+		p, ok := t.(vocab.ActivityStreamsEvent)
+		if !ok {
+			return nil, errors.New("DereferenceStatusable: error resolving type as ActivityStreamsEvent")
+		}
+		return p, nil
+	case gtsmodel.ActivityStreamsPlace:
+		p, ok := t.(vocab.ActivityStreamsPlace)
+		if !ok {
+			return nil, errors.New("DereferenceStatusable: error resolving type as ActivityStreamsPlace")
+		}
+		return p, nil
+	case gtsmodel.ActivityStreamsProfile:
+		p, ok := t.(vocab.ActivityStreamsProfile)
+		if !ok {
+			return nil, errors.New("DereferenceStatusable: error resolving type as ActivityStreamsProfile")
+		}
+		return p, nil
+	}
+
+	return nil, fmt.Errorf("DereferenceStatusable: type name %s not supported", t.GetTypeName())
+}
+
+// populateStatusFields fetches all the information we temporarily pinned to an incoming
+// federated status, back in the federating db's Create function.
+//
+// When a status comes in from the federation API, there are certain fields that
+// haven't been dereferenced yet, because we needed to provide a snappy synchronous
+// response to the caller. By the time it reaches this function though, it's being
+// processed asynchronously, so we have all the time in the world to fetch the various
+// bits and bobs that are attached to the status, and properly flesh it out, before we
+// send the status to any timelines and notify people.
+//
+// Things to dereference and fetch here:
+//
+// 1. Media attachments.
+// 2. Hashtags.
+// 3. Emojis.
+// 4. Mentions.
+// 5. Posting account.
+// 6. Replied-to-status.
+//
+// SIDE EFFECTS:
+// This function will deference all of the above, insert them in the database as necessary,
+// and attach them to the status. The status itself will not be added to the database yet,
+// that's up the caller to do.
+func (d *deref) populateStatusFields(status *gtsmodel.Status, requestingUsername string) error {
+	l := d.log.WithFields(logrus.Fields{
+		"func":   "dereferenceStatusFields",
+		"status": fmt.Sprintf("%+v", status),
+	})
+	l.Debug("entering function")
+
+	// make sure we have a status URI and that the domain in question isn't blocked
+	statusURI, err := url.Parse(status.URI)
+	if err != nil {
+		return fmt.Errorf("DereferenceStatusFields: couldn't parse status URI %s: %s", status.URI, err)
+	}
+	if blocked, err := d.blockedDomain(statusURI.Host); blocked || err != nil {
+		return fmt.Errorf("DereferenceStatusFields: domain %s is blocked", statusURI.Host)
+	}
+
+	// we can continue -- create a new transport here because we'll probably need it
+	t, err := d.transportController.NewTransportForUsername(requestingUsername)
+	if err != nil {
+		return fmt.Errorf("error creating transport: %s", err)
+	}
+
+	// in case the status doesn't have an id yet (ie., it hasn't entered the database yet), then create one
+	if status.ID == "" {
+		newID, err := id.NewULIDFromTime(status.CreatedAt)
+		if err != nil {
+			return err
+		}
+		status.ID = newID
+	}
+
+	// 1. Media attachments.
+	//
+	// At this point we should know:
+	// * the media type of the file we're looking for (a.File.ContentType)
+	// * the blurhash (a.Blurhash)
+	// * the file type (a.Type)
+	// * the remote URL (a.RemoteURL)
+	// This should be enough to pass along to the media processor.
+	attachmentIDs := []string{}
+	for _, a := range status.GTSMediaAttachments {
+		l.Tracef("dereferencing attachment: %+v", a)
+
+		// it might have been processed elsewhere so check first if it's already in the database or not
+		maybeAttachment := &gtsmodel.MediaAttachment{}
+		err := d.db.GetWhere([]db.Where{{Key: "remote_url", Value: a.RemoteURL}}, maybeAttachment)
+		if err == nil {
+			// we already have it in the db, dereferenced, no need to do it again
+			l.Tracef("attachment already exists with id %s", maybeAttachment.ID)
+			attachmentIDs = append(attachmentIDs, maybeAttachment.ID)
+			continue
+		}
+		if _, ok := err.(db.ErrNoEntries); !ok {
+			// we have a real error
+			return fmt.Errorf("error checking db for existence of attachment with remote url %s: %s", a.RemoteURL, err)
+		}
+		// it just doesn't exist yet so carry on
+		l.Debug("attachment doesn't exist yet, calling ProcessRemoteAttachment", a)
+		deferencedAttachment, err := d.mediaHandler.ProcessRemoteAttachment(t, a, status.AccountID)
+		if err != nil {
+			l.Errorf("error dereferencing status attachment: %s", err)
+			continue
+		}
+		l.Debugf("dereferenced attachment: %+v", deferencedAttachment)
+		deferencedAttachment.StatusID = status.ID
+		deferencedAttachment.Description = a.Description
+		if err := d.db.Put(deferencedAttachment); err != nil {
+			return fmt.Errorf("error inserting dereferenced attachment with remote url %s: %s", a.RemoteURL, err)
+		}
+		attachmentIDs = append(attachmentIDs, deferencedAttachment.ID)
+	}
+	status.Attachments = attachmentIDs
+
+	// 2. Hashtags
+
+	// 3. Emojis
+
+	// 4. Mentions
+	// At this point, mentions should have the namestring and mentionedAccountURI set on them.
+	//
+	// We should dereference any accounts mentioned here which we don't have in our db yet, by their URI.
+	mentions := []string{}
+	for _, m := range status.GTSMentions {
+
+		if m.ID != "" {
+			continue
+			// we've already populated this mention, since it has an ID
+		}
+
+		mID, err := id.NewRandomULID()
+		if err != nil {
+			return err
+		}
+		m.ID = mID
+
+		uri, err := url.Parse(m.MentionedAccountURI)
+		if err != nil {
+			l.Debugf("error parsing mentioned account uri %s: %s", m.MentionedAccountURI, err)
+			continue
+		}
+
+		m.StatusID = status.ID
+		m.OriginAccountID = status.GTSAuthorAccount.ID
+		m.OriginAccountURI = status.GTSAuthorAccount.URI
+
+		targetAccount, _, err := d.GetRemoteAccount(requestingUsername, uri, false)
+		if err != nil {
+			continue
+		}
+
+		// by this point, we know the targetAccount exists in our database with an ID :)
+		m.TargetAccountID = targetAccount.ID
+		if err := d.db.Put(m); err != nil {
+			return fmt.Errorf("error creating mention: %s", err)
+		}
+		mentions = append(mentions, m.ID)
+	}
+	status.Mentions = mentions
+
+	// status has replyToURI but we don't have an ID yet for the status it replies to
+	if status.InReplyToURI != "" && status.InReplyToID == "" {
+		replyToStatus := &gtsmodel.Status{}
+		if err := d.db.GetWhere([]db.Where{{Key: "uri", Value: status.InReplyToURI}}, replyToStatus); err == nil {
+			// we have the status
+			status.InReplyToID = replyToStatus.ID
+			status.InReplyToAccountID = replyToStatus.AccountID
+		}
+	}
+
+	return nil
+}

internal/federation/dereferencing/thread.go

@@ -0,0 +1,250 @@
+/*
+   GoToSocial
+   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+   This program is free software: you can redistribute it and/or modify
+   it under the terms of the GNU Affero General Public License as published by
+   the Free Software Foundation, either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Affero General Public License for more details.
+
+   You should have received a copy of the GNU Affero General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package dereferencing
+
+import (
+	"fmt"
+	"net/url"
+
+	"github.com/sirupsen/logrus"
+	"github.com/superseriousbusiness/gotosocial/internal/ap"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+	"github.com/superseriousbusiness/gotosocial/internal/util"
+)
+
+// DereferenceThread takes a statusable (something that has withReplies and withInReplyTo),
+// and dereferences statusables in the conversation.
+//
+// This process involves working up and down the chain of replies, and parsing through the collections of IDs
+// presented by remote instances as part of their replies collections, and will likely involve making several calls to
+// multiple different hosts.
+func (d *deref) DereferenceThread(username string, statusIRI *url.URL) error {
+	l := d.log.WithFields(logrus.Fields{
+		"func":      "DereferenceThread",
+		"username":  username,
+		"statusIRI": statusIRI.String(),
+	})
+	l.Debug("entering DereferenceThread")
+
+	// if it's our status we already have everything stashed so we can bail early
+	if statusIRI.Host == d.config.Host {
+		l.Debug("iri belongs to us, bailing")
+		return nil
+	}
+
+	// first make sure we have this status in our db
+	_, statusable, _, err := d.GetRemoteStatus(username, statusIRI, true)
+	if err != nil {
+		return fmt.Errorf("DereferenceThread: error getting status with id %s: %s", statusIRI.String(), err)
+	}
+
+	// first iterate up through ancestors, dereferencing if necessary as we go
+	if err := d.iterateAncestors(username, *statusIRI); err != nil {
+		return fmt.Errorf("error iterating ancestors of status %s: %s", statusIRI.String(), err)
+	}
+
+	// now iterate down through descendants, again dereferencing as we go
+	if err := d.iterateDescendants(username, *statusIRI, statusable); err != nil {
+		return fmt.Errorf("error iterating descendants of status %s: %s", statusIRI.String(), err)
+	}
+
+	return nil
+}
+
+// iterateAncestors has the goal of reaching the oldest ancestor of a given status, and stashing all statuses along the way.
+func (d *deref) iterateAncestors(username string, statusIRI url.URL) error {
+	l := d.log.WithFields(logrus.Fields{
+		"func":      "iterateAncestors",
+		"username":  username,
+		"statusIRI": statusIRI.String(),
+	})
+	l.Debug("entering iterateAncestors")
+
+	// if it's our status we don't need to dereference anything so we can immediately move up the chain
+	if statusIRI.Host == d.config.Host {
+		l.Debug("iri belongs to us, moving up to next ancestor")
+
+		// since this is our status, we know we can extract the id from the status path
+		_, id, err := util.ParseStatusesPath(&statusIRI)
+		if err != nil {
+			return err
+		}
+
+		status := &gtsmodel.Status{}
+		if err := d.db.GetByID(id, status); err != nil {
+			return err
+		}
+
+		if status.InReplyToURI == "" {
+			// status doesn't reply to anything
+			return nil
+		}
+		nextIRI, err := url.Parse(status.URI)
+		if err != nil {
+			return err
+		}
+		return d.iterateAncestors(username, *nextIRI)
+	}
+
+	// If we reach here, we're looking at a remote status -- make sure we have it in our db by calling GetRemoteStatus
+	// We call it with refresh to true because we want the statusable representation to parse inReplyTo from.
+	status, statusable, _, err := d.GetRemoteStatus(username, &statusIRI, true)
+	if err != nil {
+		l.Debugf("error getting remote status: %s", err)
+		return nil
+	}
+
+	inReplyTo := ap.ExtractInReplyToURI(statusable)
+	if inReplyTo == nil || inReplyTo.String() == "" {
+		// status doesn't reply to anything
+		return nil
+	}
+
+	// get the ancestor status into our database if we don't have it yet
+	if _, _, _, err := d.GetRemoteStatus(username, inReplyTo, false); err != nil {
+		l.Debugf("error getting remote status: %s", err)
+		return nil
+	}
+
+	// now enrich the current status, since we should have the ancestor in the db
+	if _, err := d.EnrichRemoteStatus(username, status); err != nil {
+		l.Debugf("error enriching remote status: %s", err)
+		return nil
+	}
+
+	// now move up to the next ancestor
+	return d.iterateAncestors(username, *inReplyTo)
+}
+
+func (d *deref) iterateDescendants(username string, statusIRI url.URL, statusable ap.Statusable) error {
+	l := d.log.WithFields(logrus.Fields{
+		"func":      "iterateDescendants",
+		"username":  username,
+		"statusIRI": statusIRI.String(),
+	})
+	l.Debug("entering iterateDescendants")
+
+	// if it's our status we already have descendants stashed so we can bail early
+	if statusIRI.Host == d.config.Host {
+		l.Debug("iri belongs to us, bailing")
+		return nil
+	}
+
+	replies := statusable.GetActivityStreamsReplies()
+	if replies == nil || !replies.IsActivityStreamsCollection() {
+		l.Debug("no replies, bailing")
+		return nil
+	}
+
+	repliesCollection := replies.GetActivityStreamsCollection()
+	if repliesCollection == nil {
+		l.Debug("replies collection is nil, bailing")
+		return nil
+	}
+
+	first := repliesCollection.GetActivityStreamsFirst()
+	if first == nil {
+		l.Debug("replies collection has no first, bailing")
+		return nil
+	}
+
+	firstPage := first.GetActivityStreamsCollectionPage()
+	if firstPage == nil {
+		l.Debug("first has no collection page, bailing")
+		return nil
+	}
+
+	firstPageNext := firstPage.GetActivityStreamsNext()
+	if firstPageNext == nil || !firstPageNext.IsIRI() {
+		l.Debug("next is not an iri, bailing")
+		return nil
+	}
+
+	var foundReplies int
+	currentPageIRI := firstPageNext.GetIRI()
+
+pageLoop:
+	for {
+		l.Debugf("dereferencing page %s", currentPageIRI)
+		nextPage, err := d.DereferenceCollectionPage(username, currentPageIRI)
+		if err != nil {
+			return nil
+		}
+
+		// next items could be either a list of URLs or a list of statuses
+
+		nextItems := nextPage.GetActivityStreamsItems()
+		if nextItems.Len() == 0 {
+			// no items on this page, which means we're done
+			break pageLoop
+		}
+
+		// have a look through items and see what we can find
+		for iter := nextItems.Begin(); iter != nextItems.End(); iter = iter.Next() {
+			// We're looking for a url to feed to GetRemoteStatus.
+			// Items can be either an IRI, or a Note.
+			// If a note, we grab the ID from it and call it, rather than parsing the note.
+
+			var itemURI *url.URL
+			if iter.IsIRI() {
+				// iri, easy
+				itemURI = iter.GetIRI()
+			} else if iter.IsActivityStreamsNote() {
+				// note, get the id from it to use as iri
+				n := iter.GetActivityStreamsNote()
+				id := n.GetJSONLDId()
+				if id != nil && id.IsIRI() {
+					itemURI = id.GetIRI()
+				}
+			} else {
+				// if it's not an iri or a note, we don't know how to process it
+				continue
+			}
+
+			if itemURI.Host == d.config.Host {
+				// skip if the reply is from us -- we already have it then
+				continue
+			}
+
+			// we can confidently say now that we found something
+			foundReplies = foundReplies + 1
+
+			// get the remote statusable and put it in the db
+			_, statusable, new, err := d.GetRemoteStatus(username, itemURI, false)
+			if new && err == nil && statusable != nil {
+				// now iterate descendants of *that* status
+				if err := d.iterateDescendants(username, *itemURI, statusable); err != nil {
+					continue
+				}
+			}
+		}
+
+		next := nextPage.GetActivityStreamsNext()
+		if next != nil && next.IsIRI() {
+			l.Debug("setting next page")
+			currentPageIRI = next.GetIRI()
+		} else {
+			l.Debug("no next page, bailing")
+			break pageLoop
+		}
+	}
+
+	l.Debugf("foundReplies %d", foundReplies)
+	return nil
+}

internal/federation/federatingdb/update.go

@@ -9,8 +9,8 @@ import (
 	"github.com/go-fed/activity/streams"
 	"github.com/go-fed/activity/streams/vocab"
 	"github.com/sirupsen/logrus"
+	"github.com/superseriousbusiness/gotosocial/internal/ap"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
-	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
 	"github.com/superseriousbusiness/gotosocial/internal/util"
 )
 
@@ -78,7 +78,7 @@ func (f *federatingDB) Update(ctx context.Context, asType vocab.Type) error {
 		typeName == gtsmodel.ActivityStreamsPerson ||
 		typeName == gtsmodel.ActivityStreamsService {
 		// it's an UPDATE to some kind of account
-		var accountable typeutils.Accountable
+		var accountable ap.Accountable
 
 		switch asType.GetTypeName() {
 		case gtsmodel.ActivityStreamsApplication:

internal/federation/federatingprotocol.go

@@ -31,7 +31,6 @@ import (
 	"github.com/sirupsen/logrus"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
-	"github.com/superseriousbusiness/gotosocial/internal/id"
 	"github.com/superseriousbusiness/gotosocial/internal/util"
 )
 
@@ -139,7 +138,7 @@ func (f *federator) AuthenticatePostInbox(ctx context.Context, w http.ResponseWr
 		}
 
 		// we don't have an entry for this instance yet so dereference it
-		i, err = f.DereferenceRemoteInstance(username, &url.URL{
+		i, err = f.GetRemoteInstance(username, &url.URL{
 			Scheme: publicKeyOwnerURI.Scheme,
 			Host:   publicKeyOwnerURI.Host,
 		})
@@ -153,51 +152,9 @@ func (f *federator) AuthenticatePostInbox(ctx context.Context, w http.ResponseWr
 		}
 	}
 
-	requestingAccount := &gtsmodel.Account{}
+	requestingAccount, _, err := f.GetRemoteAccount(username, publicKeyOwnerURI, false)
-	if err := f.db.GetWhere([]db.Where{{Key: "uri", Value: publicKeyOwnerURI.String()}}, requestingAccount); err != nil {
+	if err != nil {
-		// there's been a proper error so return it
+		return nil, false, fmt.Errorf("couldn't get remote account: %s", err)
-		if _, ok := err.(db.ErrNoEntries); !ok {
-			return ctx, false, fmt.Errorf("error getting requesting account with public key id %s: %s", publicKeyOwnerURI.String(), err)
-		}
-
-		// we don't know this account (yet) so let's dereference it right now
-		person, err := f.DereferenceRemoteAccount(requestedAccount.Username, publicKeyOwnerURI)
-		if err != nil {
-			return ctx, false, fmt.Errorf("error dereferencing account with public key id %s: %s", publicKeyOwnerURI.String(), err)
-		}
-
-		a, err := f.typeConverter.ASRepresentationToAccount(person, false)
-		if err != nil {
-			return ctx, false, fmt.Errorf("error converting person with public key id %s to account: %s", publicKeyOwnerURI.String(), err)
-		}
-
-		aID, err := id.NewRandomULID()
-		if err != nil {
-			return ctx, false, err
-		}
-		a.ID = aID
-
-		if err := f.db.Put(a); err != nil {
-			l.Errorf("error inserting dereferenced remote account: %s", err)
-		}
-
-		requestingAccount = a
-
-		// send the newly dereferenced account into the processor channel for further async processing
-		fromFederatorChanI := ctx.Value(util.APFromFederatorChanKey)
-		if fromFederatorChanI == nil {
-			l.Error("from federator channel wasn't set on context")
-		}
-		fromFederatorChan, ok := fromFederatorChanI.(chan gtsmodel.FromFederator)
-		if !ok {
-			l.Error("from federator channel was set on context but couldn't be parsed")
-		}
-
-		fromFederatorChan <- gtsmodel.FromFederator{
-			APObjectType:   gtsmodel.ActivityStreamsProfile,
-			APActivityType: gtsmodel.ActivityStreamsCreate,
-			GTSModel:       requestingAccount,
-		}
 	}
 
 	withRequester := context.WithValue(ctx, util.APRequestingAccount, requestingAccount)

internal/federation/federator.go

@@ -21,12 +21,13 @@ package federation
 import (
 	"context"
 	"net/url"
-	"sync"
 
 	"github.com/go-fed/activity/pub"
 	"github.com/sirupsen/logrus"
+	"github.com/superseriousbusiness/gotosocial/internal/ap"
 	"github.com/superseriousbusiness/gotosocial/internal/config"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing"
 	"github.com/superseriousbusiness/gotosocial/internal/federation/federatingdb"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/media"
@@ -40,6 +41,7 @@ type Federator interface {
 	FederatingActor() pub.FederatingActor
 	// FederatingDB returns the underlying FederatingDB interface.
 	FederatingDB() federatingdb.DB
+
 	// AuthenticateFederatedRequest can be used to check the authenticity of incoming http-signed requests for federating resources.
 	// The given username will be used to create a transport for making outgoing requests. See the implementation for more detailed comments.
 	//
@@ -49,29 +51,21 @@ type Federator interface {
 	//
 	// If something goes wrong during authentication, nil, false, and an error will be returned.
 	AuthenticateFederatedRequest(ctx context.Context, username string) (*url.URL, bool, error)
+
 	// FingerRemoteAccount performs a webfinger lookup for a remote account, using the .well-known path. It will return the ActivityPub URI for that
 	// account, or an error if it doesn't exist or can't be retrieved.
 	FingerRemoteAccount(requestingUsername string, targetUsername string, targetDomain string) (*url.URL, error)
-	// DereferenceRemoteAccount can be used to get the representation of a remote account, based on the account ID (which is a URI).
+
-	// The given username will be used to create a transport for making outgoing requests. See the implementation for more detailed comments.
+	DereferenceRemoteThread(username string, statusURI *url.URL) error
-	DereferenceRemoteAccount(username string, remoteAccountID *url.URL) (typeutils.Accountable, error)
-	// DereferenceRemoteStatus can be used to get the representation of a remote status, based on its ID (which is a URI).
-	// The given username will be used to create a transport for making outgoing requests. See the implementation for more detailed comments.
-	DereferenceRemoteStatus(username string, remoteStatusID *url.URL) (typeutils.Statusable, error)
-	// DereferenceRemoteInstance takes the URL of a remote instance, and a username (optional) to spin up a transport with. It then
-	// does its damnedest to get some kind of information back about the instance, trying /api/v1/instance, then /.well-known/nodeinfo
-	DereferenceRemoteInstance(username string, remoteInstanceURI *url.URL) (*gtsmodel.Instance, error)
-	// DereferenceStatusFields does further dereferencing on a status.
-	DereferenceStatusFields(status *gtsmodel.Status, requestingUsername string) error
-	// DereferenceAccountFields does further dereferencing on an account.
-	DereferenceAccountFields(account *gtsmodel.Account, requestingUsername string, refresh bool) error
-	// DereferenceAnnounce does further dereferencing on an announce.
 	DereferenceAnnounce(announce *gtsmodel.Status, requestingUsername string) error
-	// GetTransportForUser returns a new transport initialized with the key credentials belonging to the given username.
+
-	// This can be used for making signed http requests.
+	GetRemoteAccount(username string, remoteAccountID *url.URL, refresh bool) (*gtsmodel.Account, bool, error)
-	//
+
-	// If username is an empty string, our instance user's credentials will be used instead.
+	GetRemoteStatus(username string, remoteStatusID *url.URL, refresh bool) (*gtsmodel.Status, ap.Statusable, bool, error)
-	GetTransportForUser(username string) (transport.Transport, error)
+	EnrichRemoteStatus(username string, status *gtsmodel.Status) (*gtsmodel.Status, error)
+
+	GetRemoteInstance(username string, remoteInstanceURI *url.URL) (*gtsmodel.Instance, error)
+
 	// Handshaking returns true if the given username is currently in the process of dereferencing the remoteAccountID.
 	Handshaking(username string, remoteAccountID *url.URL) bool
 	pub.CommonBehavior
@@ -85,16 +79,17 @@ type federator struct {
 	clock               pub.Clock
 	typeConverter       typeutils.TypeConverter
 	transportController transport.Controller
+	dereferencer        dereferencing.Dereferencer
 	mediaHandler        media.Handler
 	actor               pub.FederatingActor
 	log                 *logrus.Logger
-	handshakes          map[string][]*url.URL
-	handshakeSync       *sync.Mutex // mutex to lock/unlock when checking or updating the handshakes map
 }
 
 // NewFederator returns a new federator
 func NewFederator(db db.DB, federatingDB federatingdb.DB, transportController transport.Controller, config *config.Config, log *logrus.Logger, typeConverter typeutils.TypeConverter, mediaHandler media.Handler) Federator {
 
+	dereferencer := dereferencing.NewDereferencer(config, db, typeConverter, transportController, mediaHandler, log)
+
 	clock := &Clock{}
 	f := &federator{
 		config:              config,
@@ -103,9 +98,9 @@ func NewFederator(db db.DB, federatingDB federatingdb.DB, transportController tr
 		clock:               &Clock{},
 		typeConverter:       typeConverter,
 		transportController: transportController,
+		dereferencer:        dereferencer,
 		mediaHandler:        mediaHandler,
 		log:                 log,
-		handshakeSync:       &sync.Mutex{},
 	}
 	actor := newFederatingActor(f, f, federatingDB, clock)
 	f.actor = actor

internal/federation/federator_test.go

@@ -69,7 +69,7 @@ func (suite *ProtocolTestSuite) SetupSuite() {
 }
 
 func (suite *ProtocolTestSuite) SetupTest() {
-	testrig.StandardDBSetup(suite.db)
+	testrig.StandardDBSetup(suite.db, suite.accounts)
 
 }
 
@@ -87,7 +87,7 @@ func (suite *ProtocolTestSuite) TestPostInboxRequestBodyHook() {
 	// setup transport controller with a no-op client so we don't make external calls
 	tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(func(req *http.Request) (*http.Response, error) {
 		return nil, nil
-	}))
+	}), suite.db)
 	// setup module being tested
 	federator := federation.NewFederator(suite.db, testrig.NewTestFederatingDB(suite.db), tc, suite.config, suite.log, suite.typeConverter, testrig.NewTestMediaHandler(suite.db, suite.storage))
 
@@ -152,7 +152,7 @@ func (suite *ProtocolTestSuite) TestAuthenticatePostInbox() {
 			StatusCode: 200,
 			Body:       r,
 		}, nil
-	}))
+	}), suite.db)
 
 	// now setup module being tested, with the mock transport controller
 	federator := federation.NewFederator(suite.db, testrig.NewTestFederatingDB(suite.db), tc, suite.config, suite.log, suite.typeConverter, testrig.NewTestMediaHandler(suite.db, suite.storage))

internal/federation/finger.go

@@ -34,7 +34,7 @@ func (f *federator) FingerRemoteAccount(requestingUsername string, targetUsernam
 		return nil, fmt.Errorf("FingerRemoteAccount: domain %s is blocked", targetDomain)
 	}
 
-	t, err := f.GetTransportForUser(requestingUsername)
+	t, err := f.transportController.NewTransportForUsername(requestingUsername)
 	if err != nil {
 		return nil, fmt.Errorf("FingerRemoteAccount: error getting transport for username %s while dereferencing @%s@%s: %s", requestingUsername, targetUsername, targetDomain, err)
 	}

internal/federation/handshake.go

@@ -3,78 +3,5 @@ package federation
 import "net/url"
 
 func (f *federator) Handshaking(username string, remoteAccountID *url.URL) bool {
-	f.handshakeSync.Lock()
+	return f.dereferencer.Handshaking(username, remoteAccountID)
-	defer f.handshakeSync.Unlock()
-
-	if f.handshakes == nil {
-		// handshakes isn't even initialized yet so we can't be handshaking with anyone
-		return false
-	}
-
-	remoteIDs, ok := f.handshakes[username]
-	if !ok {
-		// user isn't handshaking with anyone, bail
-		return false
-	}
-
-	for _, id := range remoteIDs {
-		if id.String() == remoteAccountID.String() {
-			// we are currently handshaking with the remote account, yep
-			return true
-		}
-	}
-
-	// didn't find it which means we're not handshaking
-	return false
-}
-
-func (f *federator) startHandshake(username string, remoteAccountID *url.URL) {
-	f.handshakeSync.Lock()
-	defer f.handshakeSync.Unlock()
-
-	// lazily initialize handshakes
-	if f.handshakes == nil {
-		f.handshakes = make(map[string][]*url.URL)
-	}
-
-	remoteIDs, ok := f.handshakes[username]
-	if !ok {
-		// there was nothing in there yet, so just add this entry and return
-		f.handshakes[username] = []*url.URL{remoteAccountID}
-		return
-	}
-
-	// add the remote ID to the slice
-	remoteIDs = append(remoteIDs, remoteAccountID)
-	f.handshakes[username] = remoteIDs
-}
-
-func (f *federator) stopHandshake(username string, remoteAccountID *url.URL) {
-	f.handshakeSync.Lock()
-	defer f.handshakeSync.Unlock()
-
-	if f.handshakes == nil {
-		return
-	}
-
-	remoteIDs, ok := f.handshakes[username]
-	if !ok {
-		// there was nothing in there yet anyway so just bail
-		return
-	}
-
-	newRemoteIDs := []*url.URL{}
-	for _, id := range remoteIDs {
-		if id.String() != remoteAccountID.String() {
-			newRemoteIDs = append(newRemoteIDs, id)
-		}
-	}
-
-	if len(newRemoteIDs) == 0 {
-		// there are no handshakes so just remove this user entry from the map and save a few bytes
-		delete(f.handshakes, username)
-	} else {
-		// there are still other handshakes ongoing
-		f.handshakes[username] = newRemoteIDs
-	}
 }

internal/federation/transport.go

@@ -6,8 +6,6 @@ import (
 	"net/url"
 
 	"github.com/go-fed/activity/pub"
-	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
-	"github.com/superseriousbusiness/gotosocial/internal/transport"
 	"github.com/superseriousbusiness/gotosocial/internal/util"
 )
 
@@ -35,7 +33,6 @@ import (
 // returned Transport so that any private credentials are able to be
 // garbage collected.
 func (f *federator) NewTransport(ctx context.Context, actorBoxIRI *url.URL, gofedAgent string) (pub.Transport, error) {
-
 	var username string
 	var err error
 
@@ -53,32 +50,5 @@ func (f *federator) NewTransport(ctx context.Context, actorBoxIRI *url.URL, gofe
 		return nil, fmt.Errorf("id %s was neither an inbox path nor an outbox path", actorBoxIRI.String())
 	}
 
-	account := &gtsmodel.Account{}
+	return f.transportController.NewTransportForUsername(username)
-	if err := f.db.GetLocalAccountByUsername(username, account); err != nil {
-		return nil, fmt.Errorf("error getting account with username %s from the db: %s", username, err)
-	}
-
-	return f.transportController.NewTransport(account.PublicKeyURI, account.PrivateKey)
-}
-
-func (f *federator) GetTransportForUser(username string) (transport.Transport, error) {
-	// We need an account to use to create a transport for dereferecing something.
-	// If a username has been given, we can fetch the account with that username and use it.
-	// Otherwise, we can take the instance account and use those credentials to make the request.
-	ourAccount := &gtsmodel.Account{}
-	var u string
-	if username == "" {
-		u = f.config.Host
-	} else {
-		u = username
-	}
-	if err := f.db.GetLocalAccountByUsername(u, ourAccount); err != nil {
-		return nil, fmt.Errorf("error getting account %s from db: %s", username, err)
-	}
-
-	transport, err := f.transportController.NewTransport(ourAccount.PublicKeyURI, ourAccount.PrivateKey)
-	if err != nil {
-		return nil, fmt.Errorf("error creating transport for user %s: %s", username, err)
-	}
-	return transport, nil
 }

internal/gtsmodel/activitystreams.go

@@ -43,6 +43,10 @@ const (
 	ActivityStreamsTombstone = "Tombstone"
 	// ActivityStreamsVideo https://www.w3.org/TR/activitystreams-vocabulary/#dfn-video
 	ActivityStreamsVideo = "Video"
+	//ActivityStreamsCollection https://www.w3.org/TR/activitystreams-vocabulary/#dfn-collection
+	ActivityStreamsCollection = "Collection"
+	// ActivityStreamsCollectionPage https://www.w3.org/TR/activitystreams-vocabulary/#dfn-collectionpage
+	ActivityStreamsCollectionPage = "CollectionPage"
 )
 
 const (

internal/gtsmodel/status.go

@@ -18,7 +18,9 @@
 
 package gtsmodel
 
-import "time"
+import (
+	"time"
+)
 
 // Status represents a user-created 'post' or 'status' in the database, either remote or local
 type Status struct {

internal/processing/account/get.go

@@ -36,15 +36,6 @@ func (p *processor) Get(requestingAccount *gtsmodel.Account, targetAccountID str
 		return nil, fmt.Errorf("db error: %s", err)
 	}
 
-	// lazily dereference things on the account if it hasn't been done yet
-	var requestingUsername string
-	if requestingAccount != nil {
-		requestingUsername = requestingAccount.Username
-	}
-	if err := p.federator.DereferenceAccountFields(targetAccount, requestingUsername, false); err != nil {
-		p.log.WithField("func", "AccountGet").Debugf("dereferencing account: %s", err)
-	}
-
 	var blocked bool
 	var err error
 	if requestingAccount != nil {

internal/processing/account/getfollowers.go

@@ -63,12 +63,6 @@ func (p *processor) FollowersGet(requestingAccount *gtsmodel.Account, targetAcco
 			return nil, gtserror.NewErrorInternalError(err)
 		}
 
-		// derefence account fields in case we haven't done it already
-		if err := p.federator.DereferenceAccountFields(a, requestingAccount.Username, false); err != nil {
-			// don't bail if we can't fetch them, we'll try another time
-			p.log.WithField("func", "AccountFollowersGet").Debugf("error dereferencing account fields: %s", err)
-		}
-
 		account, err := p.tc.AccountToMastoPublic(a)
 		if err != nil {
 			return nil, gtserror.NewErrorInternalError(err)

internal/processing/account/getfollowing.go

@@ -63,12 +63,6 @@ func (p *processor) FollowingGet(requestingAccount *gtsmodel.Account, targetAcco
 			return nil, gtserror.NewErrorInternalError(err)
 		}
 
-		// derefence account fields in case we haven't done it already
-		if err := p.federator.DereferenceAccountFields(a, requestingAccount.Username, false); err != nil {
-			// don't bail if we can't fetch them, we'll try another time
-			p.log.WithField("func", "AccountFollowingGet").Debugf("error dereferencing account fields: %s", err)
-		}
-
 		account, err := p.tc.AccountToMastoPublic(a)
 		if err != nil {
 			return nil, gtserror.NewErrorInternalError(err)

internal/processing/federation.go

@@ -31,65 +31,9 @@ import (
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
-	"github.com/superseriousbusiness/gotosocial/internal/id"
 	"github.com/superseriousbusiness/gotosocial/internal/util"
 )
 
-// dereferenceFediRequest authenticates the HTTP signature of an incoming federation request, using the given
-// username to perform the validation. It will *also* dereference the originator of the request and return it as a gtsmodel account
-// for further processing. NOTE that this function will have the side effect of putting the dereferenced account into the database,
-// and passing it into the processor through a channel for further asynchronous processing.
-func (p *processor) dereferenceFediRequest(username string, requestingAccountURI *url.URL) (*gtsmodel.Account, error) {
-	// OK now we can do the dereferencing part
-	// we might already have an entry for this account so check that first
-	requestingAccount := &gtsmodel.Account{}
-
-	err := p.db.GetWhere([]db.Where{{Key: "uri", Value: requestingAccountURI.String()}}, requestingAccount)
-	if err == nil {
-		// we do have it yay, return it
-		return requestingAccount, nil
-	}
-
-	if _, ok := err.(db.ErrNoEntries); !ok {
-		// something has actually gone wrong so bail
-		return nil, fmt.Errorf("database error getting account with uri %s: %s", requestingAccountURI.String(), err)
-	}
-
-	// we just don't have an entry for this account yet
-	// what we do now should depend on our chosen federation method
-	// for now though, we'll just dereference it
-	// TODO: slow-fed
-	requestingPerson, err := p.federator.DereferenceRemoteAccount(username, requestingAccountURI)
-	if err != nil {
-		return nil, fmt.Errorf("couldn't dereference %s: %s", requestingAccountURI.String(), err)
-	}
-
-	// convert it to our internal account representation
-	requestingAccount, err = p.tc.ASRepresentationToAccount(requestingPerson, false)
-	if err != nil {
-		return nil, fmt.Errorf("couldn't convert dereferenced uri %s to gtsmodel account: %s", requestingAccountURI.String(), err)
-	}
-
-	requestingAccountID, err := id.NewRandomULID()
-	if err != nil {
-		return nil, err
-	}
-	requestingAccount.ID = requestingAccountID
-
-	if err := p.db.Put(requestingAccount); err != nil {
-		return nil, fmt.Errorf("database error inserting account with uri %s: %s", requestingAccountURI.String(), err)
-	}
-
-	// put it in our channel to queue it for async processing
-	p.fromFederator <- gtsmodel.FromFederator{
-		APObjectType:   gtsmodel.ActivityStreamsProfile,
-		APActivityType: gtsmodel.ActivityStreamsCreate,
-		GTSModel:       requestingAccount,
-	}
-
-	return requestingAccount, nil
-}
-
 func (p *processor) GetFediUser(ctx context.Context, requestedUsername string, requestURL *url.URL) (interface{}, gtserror.WithCode) {
 	// get the account the request is referring to
 	requestedAccount := &gtsmodel.Account{}
@@ -112,9 +56,9 @@ func (p *processor) GetFediUser(ctx context.Context, requestedUsername string, r
 			return nil, gtserror.NewErrorNotAuthorized(errors.New("not authorized"), "not authorized")
 		}
 
-		// if we're already handshaking/dereferencing a remote account, we can skip the dereferencing part
+		// if we're not already handshaking/dereferencing a remote account, dereference it now
 		if !p.federator.Handshaking(requestedUsername, requestingAccountURI) {
-			requestingAccount, err := p.dereferenceFediRequest(requestedUsername, requestingAccountURI)
+			requestingAccount, _, err := p.federator.GetRemoteAccount(requestedUsername, requestingAccountURI, false)
 			if err != nil {
 				return nil, gtserror.NewErrorNotAuthorized(err)
 			}
@@ -158,7 +102,7 @@ func (p *processor) GetFediFollowers(ctx context.Context, requestedUsername stri
 		return nil, gtserror.NewErrorNotAuthorized(errors.New("not authorized"), "not authorized")
 	}
 
-	requestingAccount, err := p.dereferenceFediRequest(requestedUsername, requestingAccountURI)
+	requestingAccount, _, err := p.federator.GetRemoteAccount(requestedUsername, requestingAccountURI, false)
 	if err != nil {
 		return nil, gtserror.NewErrorNotAuthorized(err)
 	}
@@ -203,7 +147,7 @@ func (p *processor) GetFediFollowing(ctx context.Context, requestedUsername stri
 		return nil, gtserror.NewErrorNotAuthorized(errors.New("not authorized"), "not authorized")
 	}
 
-	requestingAccount, err := p.dereferenceFediRequest(requestedUsername, requestingAccountURI)
+	requestingAccount, _, err := p.federator.GetRemoteAccount(requestedUsername, requestingAccountURI, false)
 	if err != nil {
 		return nil, gtserror.NewErrorNotAuthorized(err)
 	}
@@ -248,7 +192,7 @@ func (p *processor) GetFediStatus(ctx context.Context, requestedUsername string,
 		return nil, gtserror.NewErrorNotAuthorized(errors.New("not authorized"), "not authorized")
 	}
 
-	requestingAccount, err := p.dereferenceFediRequest(requestedUsername, requestingAccountURI)
+	requestingAccount, _, err := p.federator.GetRemoteAccount(requestedUsername, requestingAccountURI, false)
 	if err != nil {
 		return nil, gtserror.NewErrorNotAuthorized(err)
 	}
@@ -295,6 +239,139 @@ func (p *processor) GetFediStatus(ctx context.Context, requestedUsername string,
 	return data, nil
 }
 
+func (p *processor) GetFediStatusReplies(ctx context.Context, requestedUsername string, requestedStatusID string, page bool, onlyOtherAccounts bool, minID string, requestURL *url.URL) (interface{}, gtserror.WithCode) {
+	// get the account the request is referring to
+	requestedAccount := &gtsmodel.Account{}
+	if err := p.db.GetLocalAccountByUsername(requestedUsername, requestedAccount); err != nil {
+		return nil, gtserror.NewErrorNotFound(fmt.Errorf("database error getting account with username %s: %s", requestedUsername, err))
+	}
+
+	// authenticate the request
+	requestingAccountURI, authenticated, err := p.federator.AuthenticateFederatedRequest(ctx, requestedUsername)
+	if err != nil || !authenticated {
+		return nil, gtserror.NewErrorNotAuthorized(errors.New("not authorized"), "not authorized")
+	}
+
+	requestingAccount, _, err := p.federator.GetRemoteAccount(requestedUsername, requestingAccountURI, false)
+	if err != nil {
+		return nil, gtserror.NewErrorNotAuthorized(err)
+	}
+
+	// authorize the request:
+	// 1. check if a block exists between the requester and the requestee
+	blocked, err := p.db.Blocked(requestedAccount.ID, requestingAccount.ID)
+	if err != nil {
+		return nil, gtserror.NewErrorInternalError(err)
+	}
+
+	if blocked {
+		return nil, gtserror.NewErrorNotAuthorized(fmt.Errorf("block exists between accounts %s and %s", requestedAccount.ID, requestingAccount.ID))
+	}
+
+	// get the status out of the database here
+	s := &gtsmodel.Status{}
+	if err := p.db.GetWhere([]db.Where{
+		{Key: "id", Value: requestedStatusID},
+		{Key: "account_id", Value: requestedAccount.ID},
+	}, s); err != nil {
+		return nil, gtserror.NewErrorNotFound(fmt.Errorf("database error getting status with id %s and account id %s: %s", requestedStatusID, requestedAccount.ID, err))
+	}
+
+	visible, err := p.filter.StatusVisible(s, requestingAccount)
+	if err != nil {
+		return nil, gtserror.NewErrorInternalError(err)
+	}
+	if !visible {
+		return nil, gtserror.NewErrorNotFound(fmt.Errorf("status with id %s not visible to user with id %s", s.ID, requestingAccount.ID))
+	}
+
+	var data map[string]interface{}
+
+	// now there are three scenarios:
+	// 1. we're asked for the whole collection and not a page -- we can just return the collection, with no items, but a link to 'first' page.
+	// 2. we're asked for a page but only_other_accounts has not been set in the query -- so we should just return the first page of the collection, with no items.
+	// 3. we're asked for a page, and only_other_accounts has been set, and min_id has optionally been set -- so we need to return some actual items!
+
+	if !page {
+		// scenario 1
+
+		// get the collection
+		collection, err := p.tc.StatusToASRepliesCollection(s, onlyOtherAccounts)
+		if err != nil {
+			return nil, gtserror.NewErrorInternalError(err)
+		}
+
+		data, err = streams.Serialize(collection)
+		if err != nil {
+			return nil, gtserror.NewErrorInternalError(err)
+		}
+	} else if page && requestURL.Query().Get("only_other_accounts") == "" {
+		// scenario 2
+
+		// get the collection
+		collection, err := p.tc.StatusToASRepliesCollection(s, onlyOtherAccounts)
+		if err != nil {
+			return nil, gtserror.NewErrorInternalError(err)
+		}
+		// but only return the first page
+		data, err = streams.Serialize(collection.GetActivityStreamsFirst().GetActivityStreamsCollectionPage())
+		if err != nil {
+			return nil, gtserror.NewErrorInternalError(err)
+		}
+	} else {
+		// scenario 3
+		// get immediate children
+		replies, err := p.db.StatusChildren(s, true, minID)
+		if err != nil {
+			return nil, gtserror.NewErrorInternalError(err)
+		}
+
+		// filter children and extract URIs
+		replyURIs := map[string]*url.URL{}
+		for _, r := range replies {
+			// only show public or unlocked statuses as replies
+			if r.Visibility != gtsmodel.VisibilityPublic && r.Visibility != gtsmodel.VisibilityUnlocked {
+				continue
+			}
+
+			// respect onlyOtherAccounts parameter
+			if onlyOtherAccounts && r.AccountID == requestedAccount.ID {
+				continue
+			}
+
+			// only show replies that the status owner can see
+			visibleToStatusOwner, err := p.filter.StatusVisible(r, requestedAccount)
+			if err != nil || !visibleToStatusOwner {
+				continue
+			}
+
+			// only show replies that the requester can see
+			visibleToRequester, err := p.filter.StatusVisible(r, requestingAccount)
+			if err != nil || !visibleToRequester {
+				continue
+			}
+
+			rURI, err := url.Parse(r.URI)
+			if err != nil {
+				continue
+			}
+
+			replyURIs[r.ID] = rURI
+		}
+
+		repliesPage, err := p.tc.StatusURIsToASRepliesPage(s, onlyOtherAccounts, minID, replyURIs)
+		if err != nil {
+			return nil, gtserror.NewErrorInternalError(err)
+		}
+		data, err = streams.Serialize(repliesPage)
+		if err != nil {
+			return nil, gtserror.NewErrorInternalError(err)
+		}
+	}
+
+	return data, nil
+}
+
 func (p *processor) GetWebfingerAccount(ctx context.Context, requestedUsername string, requestURL *url.URL) (*apimodel.WellKnownResponse, gtserror.WithCode) {
 	// get the account the request is referring to
 	requestedAccount := &gtsmodel.Account{}

internal/processing/fromfederator.go

@@ -21,6 +21,7 @@ package processing
 import (
 	"errors"
 	"fmt"
+	"net/url"
 
 	"github.com/sirupsen/logrus"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
@@ -47,36 +48,21 @@ func (p *processor) processFromFederator(federatorMsg gtsmodel.FromFederator) er
 				return errors.New("note was not parseable as *gtsmodel.Status")
 			}
 
-			l.Trace("will now derefence incoming status")
+			status, err := p.federator.EnrichRemoteStatus(federatorMsg.ReceivingAccount.Username, incomingStatus)
-			if err := p.federator.DereferenceStatusFields(incomingStatus, federatorMsg.ReceivingAccount.Username); err != nil {
+			if err != nil {
-				return fmt.Errorf("error dereferencing status from federator: %s", err)
-			}
-			if err := p.db.UpdateByID(incomingStatus.ID, incomingStatus); err != nil {
-				return fmt.Errorf("error updating dereferenced status in the db: %s", err)
-			}
-
-			if err := p.timelineStatus(incomingStatus); err != nil {
 				return err
 			}
 
-			if err := p.notifyStatus(incomingStatus); err != nil {
+			if err := p.timelineStatus(status); err != nil {
 				return err
 			}
 
+			if err := p.notifyStatus(status); err != nil {
+				return err
+			}
 		case gtsmodel.ActivityStreamsProfile:
 			// CREATE AN ACCOUNT
-			incomingAccount, ok := federatorMsg.GTSModel.(*gtsmodel.Account)
+			// nothing to do here
-			if !ok {
-				return errors.New("profile was not parseable as *gtsmodel.Account")
-			}
-
-			l.Trace("will now derefence incoming account")
-			if err := p.federator.DereferenceAccountFields(incomingAccount, "", false); err != nil {
-				return fmt.Errorf("error dereferencing account from federator: %s", err)
-			}
-			if err := p.db.UpdateByID(incomingAccount.ID, incomingAccount); err != nil {
-				return fmt.Errorf("error updating dereferenced account in the db: %s", err)
-			}
 		case gtsmodel.ActivityStreamsLike:
 			// CREATE A FAVE
 			incomingFave, ok := federatorMsg.GTSModel.(*gtsmodel.StatusFave)
@@ -154,12 +140,13 @@ func (p *processor) processFromFederator(federatorMsg gtsmodel.FromFederator) er
 				return errors.New("profile was not parseable as *gtsmodel.Account")
 			}
 
-			l.Trace("will now derefence incoming account")
+			incomingAccountURI, err := url.Parse(incomingAccount.URI)
-			if err := p.federator.DereferenceAccountFields(incomingAccount, federatorMsg.ReceivingAccount.Username, true); err != nil {
+			if err != nil {
-				return fmt.Errorf("error dereferencing account from federator: %s", err)
+				return err
 			}
-			if err := p.db.UpdateByID(incomingAccount.ID, incomingAccount); err != nil {
+
-				return fmt.Errorf("error updating dereferenced account in the db: %s", err)
+			if _, _, err := p.federator.GetRemoteAccount(federatorMsg.ReceivingAccount.Username, incomingAccountURI, true); err != nil {
+				return fmt.Errorf("error dereferencing account from federator: %s", err)
 			}
 		}
 	case gtsmodel.ActivityStreamsDelete:

internal/processing/processor.go

@@ -191,6 +191,10 @@ type Processor interface {
 	// authentication before returning a JSON serializable interface to the caller.
 	GetFediStatus(ctx context.Context, requestedUsername string, requestedStatusID string, requestURL *url.URL) (interface{}, gtserror.WithCode)
 
+	// GetFediStatus handles the getting of a fedi/activitypub representation of replies to a status, performing appropriate
+	// authentication before returning a JSON serializable interface to the caller.
+	GetFediStatusReplies(ctx context.Context, requestedUsername string, requestedStatusID string, page bool, onlyOtherAccounts bool, minID string, requestURL *url.URL) (interface{}, gtserror.WithCode)
+
 	// GetWebfingerAccount handles the GET for a webfinger resource. Most commonly, it will be used for returning account lookups.
 	GetWebfingerAccount(ctx context.Context, requestedUsername string, requestURL *url.URL) (*apimodel.WellKnownResponse, gtserror.WithCode)
 

internal/processing/search.go

@@ -19,7 +19,6 @@
 package processing
 
 import (
-	"errors"
 	"fmt"
 	"net/url"
 	"strings"
@@ -29,7 +28,6 @@ import (
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
-	"github.com/superseriousbusiness/gotosocial/internal/id"
 	"github.com/superseriousbusiness/gotosocial/internal/oauth"
 	"github.com/superseriousbusiness/gotosocial/internal/util"
 )
@@ -122,6 +120,11 @@ func (p *processor) SearchGet(authed *oauth.Auth, searchQuery *apimodel.SearchQu
 }
 
 func (p *processor) searchStatusByURI(authed *oauth.Auth, uri *url.URL, resolve bool) (*gtsmodel.Status, error) {
+	l := p.log.WithFields(logrus.Fields{
+		"func":    "searchStatusByURI",
+		"uri":     uri.String(),
+		"resolve": resolve,
+	})
 
 	maybeStatus := &gtsmodel.Status{}
 	if err := p.db.GetWhere([]db.Where{{Key: "uri", Value: uri.String(), CaseInsensitive: true}}, maybeStatus); err == nil {
@@ -134,57 +137,12 @@ func (p *processor) searchStatusByURI(authed *oauth.Auth, uri *url.URL, resolve
 
 	// we don't have it locally so dereference it if we're allowed to
 	if resolve {
-		statusable, err := p.federator.DereferenceRemoteStatus(authed.Account.Username, uri)
+		status, _, _, err := p.federator.GetRemoteStatus(authed.Account.Username, uri, true)
 		if err == nil {
-			// it IS a status!
+			if err := p.federator.DereferenceRemoteThread(authed.Account.Username, uri); err != nil {
-
+				// try to deref the thread while we're here
-			// extract the status owner's IRI from the statusable
+				l.Debugf("searchStatusByURI: error dereferencing remote thread: %s", err)
-			var statusOwnerURI *url.URL
-			statusAttributedTo := statusable.GetActivityStreamsAttributedTo()
-			for i := statusAttributedTo.Begin(); i != statusAttributedTo.End(); i = i.Next() {
-				if i.IsIRI() {
-					statusOwnerURI = i.GetIRI()
-					break
-				}
 			}
-			if statusOwnerURI == nil {
-				return nil, errors.New("couldn't extract ownerAccountURI from statusable")
-			}
-
-			// make sure the status owner exists in the db by searching for it
-			_, err := p.searchAccountByURI(authed, statusOwnerURI, resolve)
-			if err != nil {
-				return nil, err
-			}
-
-			// we have the status owner, we have the dereferenced status, so now we should finish dereferencing the status properly
-
-			// first turn it into a gtsmodel.Status
-			status, err := p.tc.ASStatusToStatus(statusable)
-			if err != nil {
-				return nil, gtserror.NewErrorInternalError(err)
-			}
-
-			statusID, err := id.NewULIDFromTime(status.CreatedAt)
-			if err != nil {
-				return nil, err
-			}
-			status.ID = statusID
-
-			if err := p.db.Put(status); err != nil {
-				return nil, fmt.Errorf("error putting status in the db: %s", err)
-			}
-
-			// properly dereference everything in the status (media attachments etc)
-			if err := p.federator.DereferenceStatusFields(status, authed.Account.Username); err != nil {
-				return nil, fmt.Errorf("error dereferencing status fields: %s", err)
-			}
-
-			// update with the nicely dereferenced status
-			if err := p.db.UpdateByID(status.ID, status); err != nil {
-				return nil, fmt.Errorf("error updating status in the db: %s", err)
-			}
-
 			return status, nil
 		}
 	}
@@ -202,31 +160,10 @@ func (p *processor) searchAccountByURI(authed *oauth.Auth, uri *url.URL, resolve
 	}
 	if resolve {
 		// we don't have it locally so try and dereference it
-		accountable, err := p.federator.DereferenceRemoteAccount(authed.Account.Username, uri)
+		account, _, err := p.federator.GetRemoteAccount(authed.Account.Username, uri, true)
 		if err != nil {
 			return nil, fmt.Errorf("searchAccountByURI: error dereferencing account with uri %s: %s", uri.String(), err)
 		}
-
-		// it IS an account!
-		account, err := p.tc.ASRepresentationToAccount(accountable, false)
-		if err != nil {
-			return nil, fmt.Errorf("searchAccountByURI: error dereferencing account with uri %s: %s", uri.String(), err)
-		}
-
-		accountID, err := id.NewRandomULID()
-		if err != nil {
-			return nil, err
-		}
-		account.ID = accountID
-
-		if err := p.db.Put(account); err != nil {
-			return nil, fmt.Errorf("searchAccountByURI: error inserting account with uri %s: %s", uri.String(), err)
-		}
-
-		if err := p.federator.DereferenceAccountFields(account, authed.Account.Username, false); err != nil {
-			return nil, fmt.Errorf("searchAccountByURI: error further dereferencing account with uri %s: %s", uri.String(), err)
-		}
-
 		return account, nil
 	}
 	return nil, nil
@@ -275,35 +212,12 @@ func (p *processor) searchAccountByMention(authed *oauth.Auth, mention string, r
 			return nil, fmt.Errorf("searchAccountByMention: error fingering remote account with username %s and domain %s: %s", username, domain, err)
 		}
 
-		// dereference the account based on the URI we retrieved from the webfinger lookup
+		// we don't have it locally so try and dereference it
-		accountable, err := p.federator.DereferenceRemoteAccount(authed.Account.Username, acctURI)
+		account, _, err := p.federator.GetRemoteAccount(authed.Account.Username, acctURI, true)
 		if err != nil {
-			// something went wrong doing the dereferencing so we can't process the request
+			return nil, fmt.Errorf("searchAccountByMention: error dereferencing account with uri %s: %s", acctURI.String(), err)
-			return nil, fmt.Errorf("searchAccountByMention: error dereferencing remote account with uri %s: %s", acctURI.String(), err)
-		}
-
-		// convert the dereferenced account to the gts model of that account
-		foundAccount, err := p.tc.ASRepresentationToAccount(accountable, false)
-		if err != nil {
-			// something went wrong doing the conversion to a gtsmodel.Account so we can't process the request
-			return nil, fmt.Errorf("searchAccountByMention: error converting account with uri %s: %s", acctURI.String(), err)
-		}
-
-		foundAccountID, err := id.NewULID()
-		if err != nil {
-			return nil, err
-		}
-		foundAccount.ID = foundAccountID
-
-		// put this new account in our database
-		if err := p.db.Put(foundAccount); err != nil {
-			return nil, fmt.Errorf("searchAccountByMention: error inserting account with uri %s: %s", acctURI.String(), err)
-		}
-
-		// properly dereference all the fields on the account immediately
-		if err := p.federator.DereferenceAccountFields(foundAccount, authed.Account.Username, true); err != nil {
-			return nil, fmt.Errorf("searchAccountByMention: error dereferencing fields on account with uri %s: %s", acctURI.String(), err)
 		}
+		return account, nil
 	}
 
 	return nil, nil

internal/processing/status/context.go

@@ -33,7 +33,7 @@ func (p *processor) Context(account *gtsmodel.Account, targetStatusID string) (*
 		return nil, gtserror.NewErrorForbidden(fmt.Errorf("account with id %s does not have permission to view status %s", account.ID, targetStatusID))
 	}
 
-	parents, err := p.db.StatusParents(targetStatus)
+	parents, err := p.db.StatusParents(targetStatus, false)
 	if err != nil {
 		return nil, gtserror.NewErrorInternalError(err)
 	}
@@ -51,7 +51,7 @@ func (p *processor) Context(account *gtsmodel.Account, targetStatusID string) (*
 		return context.Ancestors[i].ID < context.Ancestors[j].ID
 	})
 
-	children, err := p.db.StatusChildren(targetStatus)
+	children, err := p.db.StatusChildren(targetStatus, false, "")
 	if err != nil {
 		return nil, gtserror.NewErrorInternalError(err)
 	}

internal/text/link_test.go

@@ -86,7 +86,7 @@ func (suite *LinkTestSuite) SetupTest() {
 	suite.log = testrig.NewTestLog()
 	suite.formatter = text.NewFormatter(suite.config, suite.db, suite.log)
 
-	testrig.StandardDBSetup(suite.db)
+	testrig.StandardDBSetup(suite.db, nil)
 }
 
 func (suite *LinkTestSuite) TearDownTest() {

internal/text/plain_test.go

@@ -57,7 +57,7 @@ func (suite *PlainTestSuite) SetupTest() {
 	suite.log = testrig.NewTestLog()
 	suite.formatter = text.NewFormatter(suite.config, suite.db, suite.log)
 
-	testrig.StandardDBSetup(suite.db)
+	testrig.StandardDBSetup(suite.db, nil)
 }
 
 func (suite *PlainTestSuite) TearDownTest() {

internal/transport/controller.go

@@ -27,15 +27,19 @@ import (
 	"github.com/go-fed/httpsig"
 	"github.com/sirupsen/logrus"
 	"github.com/superseriousbusiness/gotosocial/internal/config"
+	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 )
 
 // Controller generates transports for use in making federation requests to other servers.
 type Controller interface {
 	NewTransport(pubKeyID string, privkey crypto.PrivateKey) (Transport, error)
+	NewTransportForUsername(username string) (Transport, error)
 }
 
 type controller struct {
 	config   *config.Config
+	db       db.DB
 	clock    pub.Clock
 	client   pub.HttpClient
 	appAgent string
@@ -43,9 +47,10 @@ type controller struct {
 }
 
 // NewController returns an implementation of the Controller interface for creating new transports
-func NewController(config *config.Config, clock pub.Clock, client pub.HttpClient, log *logrus.Logger) Controller {
+func NewController(config *config.Config, db db.DB, clock pub.Clock, client pub.HttpClient, log *logrus.Logger) Controller {
 	return &controller{
 		config:   config,
+		db:       db,
 		clock:    clock,
 		client:   client,
 		appAgent: fmt.Sprintf("%s %s", config.ApplicationName, config.Host),
@@ -55,10 +60,10 @@ func NewController(config *config.Config, clock pub.Clock, client pub.HttpClient
 
 // NewTransport returns a new http signature transport with the given public key id (a URL), and the given private key.
 func (c *controller) NewTransport(pubKeyID string, privkey crypto.PrivateKey) (Transport, error) {
-	prefs := []httpsig.Algorithm{httpsig.RSA_SHA256, httpsig.RSA_SHA512}
+	prefs := []httpsig.Algorithm{httpsig.RSA_SHA512}
 	digestAlgo := httpsig.DigestSha256
-	getHeaders := []string{"(request-target)", "host", "date"}
+	getHeaders := []string{httpsig.RequestTarget, "host", "date"}
-	postHeaders := []string{"(request-target)", "host", "date", "digest"}
+	postHeaders := []string{httpsig.RequestTarget, "host", "date", "digest"}
 
 	getSigner, _, err := httpsig.NewSigner(prefs, digestAlgo, getHeaders, httpsig.Signature, 120)
 	if err != nil {
@@ -85,3 +90,25 @@ func (c *controller) NewTransport(pubKeyID string, privkey crypto.PrivateKey) (T
 		log:          c.log,
 	}, nil
 }
+
+func (c *controller) NewTransportForUsername(username string) (Transport, error) {
+	// We need an account to use to create a transport for dereferecing something.
+	// If a username has been given, we can fetch the account with that username and use it.
+	// Otherwise, we can take the instance account and use those credentials to make the request.
+	ourAccount := &gtsmodel.Account{}
+	var u string
+	if username == "" {
+		u = c.config.Host
+	} else {
+		u = username
+	}
+	if err := c.db.GetLocalAccountByUsername(u, ourAccount); err != nil {
+		return nil, fmt.Errorf("error getting account %s from db: %s", username, err)
+	}
+
+	transport, err := c.NewTransport(ourAccount.PublicKeyURI, ourAccount.PrivateKey)
+	if err != nil {
+		return nil, fmt.Errorf("error creating transport for user %s: %s", username, err)
+	}
+	return transport, nil
+}

internal/typeutils/asinterfaces.go

@@ -1,265 +0,0 @@
-/*
-   GoToSocial
-   Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
-
-   This program is free software: you can redistribute it and/or modify
-   it under the terms of the GNU Affero General Public License as published by
-   the Free Software Foundation, either version 3 of the License, or
-   (at your option) any later version.
-
-   This program is distributed in the hope that it will be useful,
-   but WITHOUT ANY WARRANTY; without even the implied warranty of
-   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-   GNU Affero General Public License for more details.
-
-   You should have received a copy of the GNU Affero General Public License
-   along with this program.  If not, see <http://www.gnu.org/licenses/>.
-*/
-
-package typeutils
-
-import "github.com/go-fed/activity/streams/vocab"
-
-// Accountable represents the minimum activitypub interface for representing an 'account'.
-// This interface is fulfilled by: Person, Application, Organization, Service, and Group
-type Accountable interface {
-	withJSONLDId
-	withTypeName
-
-	withPreferredUsername
-	withIcon
-	withName
-	withImage
-	withSummary
-	withDiscoverable
-	withURL
-	withPublicKey
-	withInbox
-	withOutbox
-	withFollowing
-	withFollowers
-	withFeatured
-}
-
-// Statusable represents the minimum activitypub interface for representing a 'status'.
-// This interface is fulfilled by: Article, Document, Image, Video, Note, Page, Event, Place, Mention, Profile
-type Statusable interface {
-	withJSONLDId
-	withTypeName
-
-	withSummary
-	withInReplyTo
-	withPublished
-	withURL
-	withAttributedTo
-	withTo
-	withCC
-	withSensitive
-	withConversation
-	withContent
-	withAttachment
-	withTag
-	withReplies
-}
-
-// Attachmentable represents the minimum activitypub interface for representing a 'mediaAttachment'.
-// This interface is fulfilled by: Audio, Document, Image, Video
-type Attachmentable interface {
-	withTypeName
-	withMediaType
-	withURL
-	withName
-}
-
-// Hashtaggable represents the minimum activitypub interface for representing a 'hashtag' tag.
-type Hashtaggable interface {
-	withTypeName
-	withHref
-	withName
-}
-
-// Emojiable represents the minimum interface for an 'emoji' tag.
-type Emojiable interface {
-	withJSONLDId
-	withTypeName
-	withName
-	withUpdated
-	withIcon
-}
-
-// Mentionable represents the minimum interface for a 'mention' tag.
-type Mentionable interface {
-	withName
-	withHref
-}
-
-// Followable represents the minimum interface for an activitystreams 'follow' activity.
-type Followable interface {
-	withJSONLDId
-	withTypeName
-
-	withActor
-	withObject
-}
-
-// Likeable represents the minimum interface for an activitystreams 'like' activity.
-type Likeable interface {
-	withJSONLDId
-	withTypeName
-
-	withActor
-	withObject
-}
-
-// Blockable represents the minimum interface for an activitystreams 'block' activity.
-type Blockable interface {
-	withJSONLDId
-	withTypeName
-
-	withActor
-	withObject
-}
-
-// Announceable represents the minimum interface for an activitystreams 'announce' activity.
-type Announceable interface {
-	withJSONLDId
-	withTypeName
-
-	withActor
-	withObject
-	withPublished
-	withTo
-	withCC
-}
-
-type withJSONLDId interface {
-	GetJSONLDId() vocab.JSONLDIdProperty
-}
-
-type withTypeName interface {
-	GetTypeName() string
-}
-
-type withPreferredUsername interface {
-	GetActivityStreamsPreferredUsername() vocab.ActivityStreamsPreferredUsernameProperty
-}
-
-type withIcon interface {
-	GetActivityStreamsIcon() vocab.ActivityStreamsIconProperty
-}
-
-type withName interface {
-	GetActivityStreamsName() vocab.ActivityStreamsNameProperty
-}
-
-type withImage interface {
-	GetActivityStreamsImage() vocab.ActivityStreamsImageProperty
-}
-
-type withSummary interface {
-	GetActivityStreamsSummary() vocab.ActivityStreamsSummaryProperty
-}
-
-type withDiscoverable interface {
-	GetTootDiscoverable() vocab.TootDiscoverableProperty
-}
-
-type withURL interface {
-	GetActivityStreamsUrl() vocab.ActivityStreamsUrlProperty
-}
-
-type withPublicKey interface {
-	GetW3IDSecurityV1PublicKey() vocab.W3IDSecurityV1PublicKeyProperty
-}
-
-type withInbox interface {
-	GetActivityStreamsInbox() vocab.ActivityStreamsInboxProperty
-}
-
-type withOutbox interface {
-	GetActivityStreamsOutbox() vocab.ActivityStreamsOutboxProperty
-}
-
-type withFollowing interface {
-	GetActivityStreamsFollowing() vocab.ActivityStreamsFollowingProperty
-}
-
-type withFollowers interface {
-	GetActivityStreamsFollowers() vocab.ActivityStreamsFollowersProperty
-}
-
-type withFeatured interface {
-	GetTootFeatured() vocab.TootFeaturedProperty
-}
-
-type withAttributedTo interface {
-	GetActivityStreamsAttributedTo() vocab.ActivityStreamsAttributedToProperty
-}
-
-type withAttachment interface {
-	GetActivityStreamsAttachment() vocab.ActivityStreamsAttachmentProperty
-}
-
-type withTo interface {
-	GetActivityStreamsTo() vocab.ActivityStreamsToProperty
-}
-
-type withInReplyTo interface {
-	GetActivityStreamsInReplyTo() vocab.ActivityStreamsInReplyToProperty
-}
-
-type withCC interface {
-	GetActivityStreamsCc() vocab.ActivityStreamsCcProperty
-}
-
-type withSensitive interface {
-	// TODO
-}
-
-type withConversation interface {
-	// TODO
-}
-
-type withContent interface {
-	GetActivityStreamsContent() vocab.ActivityStreamsContentProperty
-}
-
-type withPublished interface {
-	GetActivityStreamsPublished() vocab.ActivityStreamsPublishedProperty
-}
-
-type withTag interface {
-	GetActivityStreamsTag() vocab.ActivityStreamsTagProperty
-}
-
-type withReplies interface {
-	GetActivityStreamsReplies() vocab.ActivityStreamsRepliesProperty
-}
-
-type withMediaType interface {
-	GetActivityStreamsMediaType() vocab.ActivityStreamsMediaTypeProperty
-}
-
-// type withBlurhash interface {
-// 	GetTootBlurhashProperty() vocab.TootBlurhashProperty
-// }
-
-// type withFocalPoint interface {
-// 	// TODO
-// }
-
-type withHref interface {
-	GetActivityStreamsHref() vocab.ActivityStreamsHrefProperty
-}
-
-type withUpdated interface {
-	GetActivityStreamsUpdated() vocab.ActivityStreamsUpdatedProperty
-}
-
-type withActor interface {
-	GetActivityStreamsActor() vocab.ActivityStreamsActorProperty
-}
-
-type withObject interface {
-	GetActivityStreamsObject() vocab.ActivityStreamsObjectProperty
-}

internal/typeutils/astointernal.go

@@ -24,11 +24,12 @@ import (
 	"net/url"
 	"strings"
 
+	"github.com/superseriousbusiness/gotosocial/internal/ap"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 )
 
-func (c *converter) ASRepresentationToAccount(accountable Accountable, update bool) (*gtsmodel.Account, error) {
+func (c *converter) ASRepresentationToAccount(accountable ap.Accountable, update bool) (*gtsmodel.Account, error) {
 	// first check if we actually already know this account
 	uriProp := accountable.GetJSONLDId()
 	if uriProp == nil || !uriProp.IsIRI() {
@@ -55,7 +56,7 @@ func (c *converter) ASRepresentationToAccount(accountable Accountable, update bo
 
 	// Username aka preferredUsername
 	// We need this one so bail if it's not set.
-	username, err := extractPreferredUsername(accountable)
+	username, err := ap.ExtractPreferredUsername(accountable)
 	if err != nil {
 		return nil, fmt.Errorf("couldn't extract username: %s", err)
 	}
@@ -66,27 +67,27 @@ func (c *converter) ASRepresentationToAccount(accountable Accountable, update bo
 
 	// avatar aka icon
 	// if this one isn't extractable in a format we recognise we'll just skip it
-	if avatarURL, err := extractIconURL(accountable); err == nil {
+	if avatarURL, err := ap.ExtractIconURL(accountable); err == nil {
 		acct.AvatarRemoteURL = avatarURL.String()
 	}
 
 	// header aka image
 	// if this one isn't extractable in a format we recognise we'll just skip it
-	if headerURL, err := extractImageURL(accountable); err == nil {
+	if headerURL, err := ap.ExtractImageURL(accountable); err == nil {
 		acct.HeaderRemoteURL = headerURL.String()
 	}
 
 	// display name aka name
 	// we default to the username, but take the more nuanced name property if it exists
 	acct.DisplayName = username
-	if displayName, err := extractName(accountable); err == nil {
+	if displayName, err := ap.ExtractName(accountable); err == nil {
 		acct.DisplayName = displayName
 	}
 
 	// TODO: fields aka attachment array
 
 	// note aka summary
-	note, err := extractSummary(accountable)
+	note, err := ap.ExtractSummary(accountable)
 	if err == nil && note != "" {
 		acct.Note = note
 	}
@@ -110,13 +111,13 @@ func (c *converter) ASRepresentationToAccount(accountable Accountable, update bo
 	// discoverable
 	// default to false -- take custom value if it's set though
 	acct.Discoverable = false
-	discoverable, err := extractDiscoverable(accountable)
+	discoverable, err := ap.ExtractDiscoverable(accountable)
 	if err == nil {
 		acct.Discoverable = discoverable
 	}
 
 	// url property
-	url, err := extractURL(accountable)
+	url, err := ap.ExtractURL(accountable)
 	if err == nil {
 		// take the URL if we can find it
 		acct.URL = url.String()
@@ -155,7 +156,7 @@ func (c *converter) ASRepresentationToAccount(accountable Accountable, update bo
 	// TODO: alsoKnownAs
 
 	// publicKey
-	pkey, pkeyURL, err := extractPublicKeyForOwner(accountable, uri)
+	pkey, pkeyURL, err := ap.ExtractPublicKeyForOwner(accountable, uri)
 	if err != nil {
 		return nil, fmt.Errorf("couldn't get public key for person %s: %s", uri.String(), err)
 	}
@@ -165,7 +166,7 @@ func (c *converter) ASRepresentationToAccount(accountable Accountable, update bo
 	return acct, nil
 }
 
-func (c *converter) ASStatusToStatus(statusable Statusable) (*gtsmodel.Status, error) {
+func (c *converter) ASStatusToStatus(statusable ap.Statusable) (*gtsmodel.Status, error) {
 	status := &gtsmodel.Status{}
 
 	// uri at which this status is reachable
@@ -176,49 +177,49 @@ func (c *converter) ASStatusToStatus(statusable Statusable) (*gtsmodel.Status, e
 	status.URI = uriProp.GetIRI().String()
 
 	// web url for viewing this status
-	if statusURL, err := extractURL(statusable); err == nil {
+	if statusURL, err := ap.ExtractURL(statusable); err == nil {
 		status.URL = statusURL.String()
 	}
 
 	// the html-formatted content of this status
-	if content, err := extractContent(statusable); err == nil {
+	if content, err := ap.ExtractContent(statusable); err == nil {
 		status.Content = content
 	}
 
 	// attachments to dereference and fetch later on (we don't do that here)
-	if attachments, err := extractAttachments(statusable); err == nil {
+	if attachments, err := ap.ExtractAttachments(statusable); err == nil {
 		status.GTSMediaAttachments = attachments
 	}
 
 	// hashtags to dereference later on
-	if hashtags, err := extractHashtags(statusable); err == nil {
+	if hashtags, err := ap.ExtractHashtags(statusable); err == nil {
 		status.GTSTags = hashtags
 	}
 
 	// emojis to dereference and fetch later on
-	if emojis, err := extractEmojis(statusable); err == nil {
+	if emojis, err := ap.ExtractEmojis(statusable); err == nil {
 		status.GTSEmojis = emojis
 	}
 
 	// mentions to dereference later on
-	if mentions, err := extractMentions(statusable); err == nil {
+	if mentions, err := ap.ExtractMentions(statusable); err == nil {
 		status.GTSMentions = mentions
 	}
 
 	// cw string for this status
-	if cw, err := extractSummary(statusable); err == nil {
+	if cw, err := ap.ExtractSummary(statusable); err == nil {
 		status.ContentWarning = cw
 	}
 
 	// when was this status created?
-	published, err := extractPublished(statusable)
+	published, err := ap.ExtractPublished(statusable)
 	if err == nil {
 		status.CreatedAt = published
 	}
 
 	// which account posted this status?
 	// if we don't know the account yet we can dereference it later
-	attributedTo, err := extractAttributedTo(statusable)
+	attributedTo, err := ap.ExtractAttributedTo(statusable)
 	if err != nil {
 		return nil, errors.New("attributedTo was empty")
 	}
@@ -233,8 +234,8 @@ func (c *converter) ASStatusToStatus(statusable Statusable) (*gtsmodel.Status, e
 	status.GTSAuthorAccount = statusOwner
 
 	// check if there's a post that this is a reply to
-	inReplyToURI, err := extractInReplyToURI(statusable)
+	inReplyToURI := ap.ExtractInReplyToURI(statusable)
-	if err == nil {
+	if inReplyToURI != nil {
 		// something is set so we can at least set this field on the
 		// status and dereference using this later if we need to
 		status.InReplyToURI = inReplyToURI.String()
@@ -259,12 +260,12 @@ func (c *converter) ASStatusToStatus(statusable Statusable) (*gtsmodel.Status, e
 	// visibility entry for this status
 	var visibility gtsmodel.Visibility
 
-	to, err := extractTos(statusable)
+	to, err := ap.ExtractTos(statusable)
 	if err != nil {
 		return nil, fmt.Errorf("error extracting TO values: %s", err)
 	}
 
-	cc, err := extractCCs(statusable)
+	cc, err := ap.ExtractCCs(statusable)
 	if err != nil {
 		return nil, fmt.Errorf("error extracting CC values: %s", err)
 	}
@@ -315,7 +316,7 @@ func (c *converter) ASStatusToStatus(statusable Statusable) (*gtsmodel.Status, e
 	return status, nil
 }
 
-func (c *converter) ASFollowToFollowRequest(followable Followable) (*gtsmodel.FollowRequest, error) {
+func (c *converter) ASFollowToFollowRequest(followable ap.Followable) (*gtsmodel.FollowRequest, error) {
 
 	idProp := followable.GetJSONLDId()
 	if idProp == nil || !idProp.IsIRI() {
@@ -323,7 +324,7 @@ func (c *converter) ASFollowToFollowRequest(followable Followable) (*gtsmodel.Fo
 	}
 	uri := idProp.GetIRI().String()
 
-	origin, err := extractActor(followable)
+	origin, err := ap.ExtractActor(followable)
 	if err != nil {
 		return nil, errors.New("error extracting actor property from follow")
 	}
@@ -332,7 +333,7 @@ func (c *converter) ASFollowToFollowRequest(followable Followable) (*gtsmodel.Fo
 		return nil, fmt.Errorf("error extracting account with uri %s from the database: %s", origin.String(), err)
 	}
 
-	target, err := extractObject(followable)
+	target, err := ap.ExtractObject(followable)
 	if err != nil {
 		return nil, errors.New("error extracting object property from follow")
 	}
@@ -350,14 +351,14 @@ func (c *converter) ASFollowToFollowRequest(followable Followable) (*gtsmodel.Fo
 	return followRequest, nil
 }
 
-func (c *converter) ASFollowToFollow(followable Followable) (*gtsmodel.Follow, error) {
+func (c *converter) ASFollowToFollow(followable ap.Followable) (*gtsmodel.Follow, error) {
 	idProp := followable.GetJSONLDId()
 	if idProp == nil || !idProp.IsIRI() {
 		return nil, errors.New("no id property set on follow, or was not an iri")
 	}
 	uri := idProp.GetIRI().String()
 
-	origin, err := extractActor(followable)
+	origin, err := ap.ExtractActor(followable)
 	if err != nil {
 		return nil, errors.New("error extracting actor property from follow")
 	}
@@ -366,7 +367,7 @@ func (c *converter) ASFollowToFollow(followable Followable) (*gtsmodel.Follow, e
 		return nil, fmt.Errorf("error extracting account with uri %s from the database: %s", origin.String(), err)
 	}
 
-	target, err := extractObject(followable)
+	target, err := ap.ExtractObject(followable)
 	if err != nil {
 		return nil, errors.New("error extracting object property from follow")
 	}
@@ -384,14 +385,14 @@ func (c *converter) ASFollowToFollow(followable Followable) (*gtsmodel.Follow, e
 	return follow, nil
 }
 
-func (c *converter) ASLikeToFave(likeable Likeable) (*gtsmodel.StatusFave, error) {
+func (c *converter) ASLikeToFave(likeable ap.Likeable) (*gtsmodel.StatusFave, error) {
 	idProp := likeable.GetJSONLDId()
 	if idProp == nil || !idProp.IsIRI() {
 		return nil, errors.New("no id property set on like, or was not an iri")
 	}
 	uri := idProp.GetIRI().String()
 
-	origin, err := extractActor(likeable)
+	origin, err := ap.ExtractActor(likeable)
 	if err != nil {
 		return nil, errors.New("error extracting actor property from like")
 	}
@@ -400,7 +401,7 @@ func (c *converter) ASLikeToFave(likeable Likeable) (*gtsmodel.StatusFave, error
 		return nil, fmt.Errorf("error extracting account with uri %s from the database: %s", origin.String(), err)
 	}
 
-	target, err := extractObject(likeable)
+	target, err := ap.ExtractObject(likeable)
 	if err != nil {
 		return nil, errors.New("error extracting object property from like")
 	}
@@ -426,14 +427,14 @@ func (c *converter) ASLikeToFave(likeable Likeable) (*gtsmodel.StatusFave, error
 	}, nil
 }
 
-func (c *converter) ASBlockToBlock(blockable Blockable) (*gtsmodel.Block, error) {
+func (c *converter) ASBlockToBlock(blockable ap.Blockable) (*gtsmodel.Block, error) {
 	idProp := blockable.GetJSONLDId()
 	if idProp == nil || !idProp.IsIRI() {
 		return nil, errors.New("ASBlockToBlock: no id property set on block, or was not an iri")
 	}
 	uri := idProp.GetIRI().String()
 
-	origin, err := extractActor(blockable)
+	origin, err := ap.ExtractActor(blockable)
 	if err != nil {
 		return nil, errors.New("ASBlockToBlock: error extracting actor property from block")
 	}
@@ -442,7 +443,7 @@ func (c *converter) ASBlockToBlock(blockable Blockable) (*gtsmodel.Block, error)
 		return nil, fmt.Errorf("ASBlockToBlock: error extracting account with uri %s from the database: %s", origin.String(), err)
 	}
 
-	target, err := extractObject(blockable)
+	target, err := ap.ExtractObject(blockable)
 	if err != nil {
 		return nil, errors.New("ASBlockToBlock: error extracting object property from block")
 	}
@@ -461,7 +462,7 @@ func (c *converter) ASBlockToBlock(blockable Blockable) (*gtsmodel.Block, error)
 	}, nil
 }
 
-func (c *converter) ASAnnounceToStatus(announceable Announceable) (*gtsmodel.Status, bool, error) {
+func (c *converter) ASAnnounceToStatus(announceable ap.Announceable) (*gtsmodel.Status, bool, error) {
 	status := &gtsmodel.Status{}
 	isNew := true
 
@@ -480,7 +481,7 @@ func (c *converter) ASAnnounceToStatus(announceable Announceable) (*gtsmodel.Sta
 	status.URI = uri
 
 	// get the URI of the announced/boosted status
-	boostedStatusURI, err := extractObject(announceable)
+	boostedStatusURI, err := ap.ExtractObject(announceable)
 	if err != nil {
 		return nil, isNew, fmt.Errorf("ASAnnounceToStatus: error getting object from announce: %s", err)
 	}
@@ -491,7 +492,7 @@ func (c *converter) ASAnnounceToStatus(announceable Announceable) (*gtsmodel.Sta
 	}
 
 	// get the published time for the announce
-	published, err := extractPublished(announceable)
+	published, err := ap.ExtractPublished(announceable)
 	if err != nil {
 		return nil, isNew, fmt.Errorf("ASAnnounceToStatus: error extracting published time: %s", err)
 	}
@@ -499,7 +500,7 @@ func (c *converter) ASAnnounceToStatus(announceable Announceable) (*gtsmodel.Sta
 	status.UpdatedAt = published
 
 	// get the actor's IRI (ie., the person who boosted the status)
-	actor, err := extractActor(announceable)
+	actor, err := ap.ExtractActor(announceable)
 	if err != nil {
 		return nil, isNew, fmt.Errorf("ASAnnounceToStatus: error extracting actor: %s", err)
 	}
@@ -522,12 +523,12 @@ func (c *converter) ASAnnounceToStatus(announceable Announceable) (*gtsmodel.Sta
 	// parse the visibility from the To and CC entries
 	var visibility gtsmodel.Visibility
 
-	to, err := extractTos(announceable)
+	to, err := ap.ExtractTos(announceable)
 	if err != nil {
 		return nil, isNew, fmt.Errorf("error extracting TO values: %s", err)
 	}
 
-	cc, err := extractCCs(announceable)
+	cc, err := ap.ExtractCCs(announceable)
 	if err != nil {
 		return nil, isNew, fmt.Errorf("error extracting CC values: %s", err)
 	}

internal/typeutils/astointernal_test.go

@@ -28,6 +28,7 @@ import (
 	"github.com/go-fed/activity/streams/vocab"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/ap"
 	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
 	"github.com/superseriousbusiness/gotosocial/testrig"
 )
@@ -342,7 +343,7 @@ func (suite *ASToInternalTestSuite) SetupSuite() {
 }
 
 func (suite *ASToInternalTestSuite) SetupTest() {
-	testrig.StandardDBSetup(suite.db)
+	testrig.StandardDBSetup(suite.db, nil)
 }
 
 func (suite *ASToInternalTestSuite) TestParsePerson() {
@@ -364,7 +365,7 @@ func (suite *ASToInternalTestSuite) TestParseGargron() {
 	t, err := streams.ToType(context.Background(), m)
 	assert.NoError(suite.T(), err)
 
-	rep, ok := t.(typeutils.Accountable)
+	rep, ok := t.(ap.Accountable)
 	assert.True(suite.T(), ok)
 
 	acct, err := suite.typeconverter.ASRepresentationToAccount(rep, false)
@@ -391,7 +392,7 @@ func (suite *ASToInternalTestSuite) TestParseStatus() {
 	first := obj.Begin()
 	assert.NotNil(suite.T(), first)
 
-	rep, ok := first.GetType().(typeutils.Statusable)
+	rep, ok := first.GetType().(ap.Statusable)
 	assert.True(suite.T(), ok)
 
 	status, err := suite.typeconverter.ASStatusToStatus(rep)
@@ -418,7 +419,7 @@ func (suite *ASToInternalTestSuite) TestParseStatusWithMention() {
 	first := obj.Begin()
 	assert.NotNil(suite.T(), first)
 
-	rep, ok := first.GetType().(typeutils.Statusable)
+	rep, ok := first.GetType().(ap.Statusable)
 	assert.True(suite.T(), ok)
 
 	status, err := suite.typeconverter.ASStatusToStatus(rep)

internal/typeutils/converter.go

@@ -19,7 +19,10 @@
 package typeutils
 
 import (
+	"net/url"
+
 	"github.com/go-fed/activity/streams/vocab"
+	"github.com/superseriousbusiness/gotosocial/internal/ap"
 	"github.com/superseriousbusiness/gotosocial/internal/api/model"
 	"github.com/superseriousbusiness/gotosocial/internal/config"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
@@ -99,17 +102,17 @@ type TypeConverter interface {
 	// If update is false, and the account is already known in the database, then the existing account entry will be returned.
 	// If update is true, then even if the account is already known, all fields in the accountable will be parsed and a new *gtsmodel.Account
 	// will be generated. This is useful when one needs to force refresh of an account, eg., during an Update of a Profile.
-	ASRepresentationToAccount(accountable Accountable, update bool) (*gtsmodel.Account, error)
+	ASRepresentationToAccount(accountable ap.Accountable, update bool) (*gtsmodel.Account, error)
 	// ASStatus converts a remote activitystreams 'status' representation into a gts model status.
-	ASStatusToStatus(statusable Statusable) (*gtsmodel.Status, error)
+	ASStatusToStatus(statusable ap.Statusable) (*gtsmodel.Status, error)
 	// ASFollowToFollowRequest converts a remote activitystreams `follow` representation into gts model follow request.
-	ASFollowToFollowRequest(followable Followable) (*gtsmodel.FollowRequest, error)
+	ASFollowToFollowRequest(followable ap.Followable) (*gtsmodel.FollowRequest, error)
 	// ASFollowToFollowRequest converts a remote activitystreams `follow` representation into gts model follow.
-	ASFollowToFollow(followable Followable) (*gtsmodel.Follow, error)
+	ASFollowToFollow(followable ap.Followable) (*gtsmodel.Follow, error)
 	// ASLikeToFave converts a remote activitystreams 'like' representation into a gts model status fave.
-	ASLikeToFave(likeable Likeable) (*gtsmodel.StatusFave, error)
+	ASLikeToFave(likeable ap.Likeable) (*gtsmodel.StatusFave, error)
 	// ASBlockToBlock converts a remote activity streams 'block' representation into a gts model block.
-	ASBlockToBlock(blockable Blockable) (*gtsmodel.Block, error)
+	ASBlockToBlock(blockable ap.Blockable) (*gtsmodel.Block, error)
 	// ASAnnounceToStatus converts an activitystreams 'announce' into a status.
 	//
 	// The returned bool indicates whether this status is new (true) or not new (false).
@@ -122,7 +125,7 @@ type TypeConverter interface {
 	// This is useful when multiple users on an instance might receive the same boost, and we only want to process the boost once.
 	//
 	// NOTE -- this is different from one status being boosted multiple times! In this case, new boosts should indeed be created.
-	ASAnnounceToStatus(announceable Announceable) (status *gtsmodel.Status, new bool, err error)
+	ASAnnounceToStatus(announceable ap.Announceable) (status *gtsmodel.Status, new bool, err error)
 
 	/*
 		INTERNAL (gts) MODEL TO ACTIVITYSTREAMS MODEL
@@ -150,7 +153,10 @@ type TypeConverter interface {
 	BoostToAS(boostWrapperStatus *gtsmodel.Status, boostingAccount *gtsmodel.Account, boostedAccount *gtsmodel.Account) (vocab.ActivityStreamsAnnounce, error)
 	// BlockToAS converts a gts model block into an activityStreams BLOCK, suitable for federation.
 	BlockToAS(block *gtsmodel.Block) (vocab.ActivityStreamsBlock, error)
-
+	// StatusToASRepliesCollection converts a gts model status into an activityStreams REPLIES collection.
+	StatusToASRepliesCollection(status *gtsmodel.Status, onlyOtherAccounts bool) (vocab.ActivityStreamsCollection, error)
+	// StatusURIsToASRepliesPage returns a collection page with appropriate next/part of pagination.
+	StatusURIsToASRepliesPage(status *gtsmodel.Status, onlyOtherAccounts bool, minID string, replies map[string]*url.URL) (vocab.ActivityStreamsCollectionPage, error)
 	/*
 		INTERNAL (gts) MODEL TO INTERNAL MODEL
 	*/

internal/typeutils/converter_test.go

@@ -21,6 +21,7 @@ package typeutils_test
 import (
 	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/ap"
 	"github.com/superseriousbusiness/gotosocial/internal/config"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
@@ -34,7 +35,7 @@ type ConverterStandardTestSuite struct {
 	db       db.DB
 	log      *logrus.Logger
 	accounts map[string]*gtsmodel.Account
-	people   map[string]typeutils.Accountable
+	people   map[string]ap.Accountable
 
 	typeconverter typeutils.TypeConverter
 }

internal/typeutils/internaltoas.go

@@ -27,6 +27,7 @@ import (
 	"github.com/go-fed/activity/streams"
 	"github.com/go-fed/activity/streams/vocab"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
+	"github.com/superseriousbusiness/gotosocial/internal/gtserror"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 )
 
@@ -505,7 +506,14 @@ func (c *converter) StatusToAS(s *gtsmodel.Status) (vocab.ActivityStreamsNote, e
 	status.SetActivityStreamsAttachment(attachmentProp)
 
 	// replies
-	// TODO
+	repliesCollection, err := c.StatusToASRepliesCollection(s, false)
+	if err != nil {
+		return nil, fmt.Errorf("error creating repliesCollection: %s", err)
+	}
+
+	repliesProp := streams.NewActivityStreamsRepliesProperty()
+	repliesProp.SetActivityStreamsCollection(repliesCollection)
+	status.SetActivityStreamsReplies(repliesProp)
 
 	return status, nil
 }
@@ -850,3 +858,138 @@ func (c *converter) BlockToAS(b *gtsmodel.Block) (vocab.ActivityStreamsBlock, er
 
 	return block, nil
 }
+
+/*
+	the goal is to end up with something like this:
+
+	{
+		"@context": "https://www.w3.org/ns/activitystreams",
+		"id": "https://example.org/users/whatever/statuses/01FCNEXAGAKPEX1J7VJRPJP490/replies",
+		"type": "Collection",
+		"first": {
+		"id": "https://example.org/users/whatever/statuses/01FCNEXAGAKPEX1J7VJRPJP490/replies?page=true",
+		"type": "CollectionPage",
+		"next": "https://example.org/users/whatever/statuses/01FCNEXAGAKPEX1J7VJRPJP490/replies?only_other_accounts=true&page=true",
+		"partOf": "https://example.org/users/whatever/statuses/01FCNEXAGAKPEX1J7VJRPJP490/replies",
+		"items": []
+		}
+	}
+*/
+func (c *converter) StatusToASRepliesCollection(status *gtsmodel.Status, onlyOtherAccounts bool) (vocab.ActivityStreamsCollection, error) {
+	collectionID := fmt.Sprintf("%s/replies", status.URI)
+	collectionIDURI, err := url.Parse(collectionID)
+	if err != nil {
+		return nil, err
+	}
+
+	collection := streams.NewActivityStreamsCollection()
+
+	// collection.id
+	collectionIDProp := streams.NewJSONLDIdProperty()
+	collectionIDProp.SetIRI(collectionIDURI)
+	collection.SetJSONLDId(collectionIDProp)
+
+	// first
+	first := streams.NewActivityStreamsFirstProperty()
+	firstPage := streams.NewActivityStreamsCollectionPage()
+
+	// first.id
+	firstPageIDProp := streams.NewJSONLDIdProperty()
+	firstPageID, err := url.Parse(fmt.Sprintf("%s?page=true", collectionID))
+	if err != nil {
+		return nil, gtserror.NewErrorInternalError(err)
+	}
+	firstPageIDProp.SetIRI(firstPageID)
+	firstPage.SetJSONLDId(firstPageIDProp)
+
+	// first.next
+	nextProp := streams.NewActivityStreamsNextProperty()
+	nextPropID, err := url.Parse(fmt.Sprintf("%s?only_other_accounts=%t&page=true", collectionID, onlyOtherAccounts))
+	if err != nil {
+		return nil, gtserror.NewErrorInternalError(err)
+	}
+	nextProp.SetIRI(nextPropID)
+	firstPage.SetActivityStreamsNext(nextProp)
+
+	// first.partOf
+	partOfProp := streams.NewActivityStreamsPartOfProperty()
+	partOfProp.SetIRI(collectionIDURI)
+	firstPage.SetActivityStreamsPartOf(partOfProp)
+
+	first.SetActivityStreamsCollectionPage(firstPage)
+
+	// collection.first
+	collection.SetActivityStreamsFirst(first)
+
+	return collection, nil
+}
+
+/*
+	the goal is to end up with something like this:
+	{
+		"@context": "https://www.w3.org/ns/activitystreams",
+		"id": "https://example.org/users/whatever/statuses/01FCNEXAGAKPEX1J7VJRPJP490/replies?only_other_accounts=true&page=true",
+		"type": "CollectionPage",
+		"next": "https://example.org/users/whatever/statuses/01FCNEXAGAKPEX1J7VJRPJP490/replies?min_id=106720870266901180&only_other_accounts=true&page=true",
+		"partOf": "https://example.org/users/whatever/statuses/01FCNEXAGAKPEX1J7VJRPJP490/replies",
+		"items": [
+			"https://example.com/users/someone/statuses/106720752853216226",
+			"https://somewhere.online/users/eeeeeeeeeep/statuses/106720870163727231"
+		]
+	}
+*/
+func (c *converter) StatusURIsToASRepliesPage(status *gtsmodel.Status, onlyOtherAccounts bool, minID string, replies map[string]*url.URL) (vocab.ActivityStreamsCollectionPage, error) {
+	collectionID := fmt.Sprintf("%s/replies", status.URI)
+
+	page := streams.NewActivityStreamsCollectionPage()
+
+	// .id
+	pageIDProp := streams.NewJSONLDIdProperty()
+	pageIDString := fmt.Sprintf("%s?page=true&only_other_accounts=%t", collectionID, onlyOtherAccounts)
+	if minID != "" {
+		pageIDString = fmt.Sprintf("%s&min_id=%s", pageIDString, minID)
+	}
+
+	pageID, err := url.Parse(pageIDString)
+	if err != nil {
+		return nil, gtserror.NewErrorInternalError(err)
+	}
+	pageIDProp.SetIRI(pageID)
+	page.SetJSONLDId(pageIDProp)
+
+	// .partOf
+	collectionIDURI, err := url.Parse(collectionID)
+	if err != nil {
+		return nil, err
+	}
+	partOfProp := streams.NewActivityStreamsPartOfProperty()
+	partOfProp.SetIRI(collectionIDURI)
+	page.SetActivityStreamsPartOf(partOfProp)
+
+	// .items
+	items := streams.NewActivityStreamsItemsProperty()
+	var highestID string
+	for k, v := range replies {
+		items.AppendIRI(v)
+		if k > highestID {
+			highestID = k
+		}
+	}
+	page.SetActivityStreamsItems(items)
+
+	// .next
+	nextProp := streams.NewActivityStreamsNextProperty()
+	nextPropIDString := fmt.Sprintf("%s?only_other_accounts=%t&page=true", collectionID, onlyOtherAccounts)
+	if highestID != "" {
+		nextPropIDString = fmt.Sprintf("%s&min_id=%s", nextPropIDString, highestID)
+	}
+
+	nextPropID, err := url.Parse(nextPropIDString)
+	if err != nil {
+		return nil, gtserror.NewErrorInternalError(err)
+	}
+	nextProp.SetIRI(nextPropID)
+	page.SetActivityStreamsNext(nextProp)
+
+	return page, nil
+}

internal/typeutils/internaltoas_test.go

@@ -47,7 +47,7 @@ func (suite *InternalToASTestSuite) SetupSuite() {
 }
 
 func (suite *InternalToASTestSuite) SetupTest() {
-	testrig.StandardDBSetup(suite.db)
+	testrig.StandardDBSetup(suite.db, nil)
 }
 
 // TearDownTest drops tables to make sure there's no data in the db

testrig/db.go

@@ -65,7 +65,14 @@ func NewTestDB() db.DB {
 }
 
 // StandardDBSetup populates a given db with all the necessary tables/models for perfoming tests.
-func StandardDBSetup(db db.DB) {
+//
+// The accounts parameter is provided in case the db should be populated with a certain set of accounts.
+// If accounts is nil, then the standard test accounts will be used.
+//
+// When testing http signatures, you should pass into this function the same accounts map that you generated
+// signatures with, otherwise this function will randomly generate new keys for accounts and signature
+// verification will fail.
+func StandardDBSetup(db db.DB, accounts map[string]*gtsmodel.Account) {
 	for _, m := range testModels {
 		if err := db.CreateTable(m); err != nil {
 			panic(err)
@@ -96,9 +103,17 @@ func StandardDBSetup(db db.DB) {
 		}
 	}
 
-	for _, v := range NewTestAccounts() {
+	if accounts == nil {
-		if err := db.Put(v); err != nil {
+		for _, v := range NewTestAccounts() {
-			panic(err)
+			if err := db.Put(v); err != nil {
+				panic(err)
+			}
+		}
+	} else {
+		for _, v := range accounts {
+			if err := db.Put(v); err != nil {
+				panic(err)
+			}
 		}
 	}
 

testrig/testmodels.go

@@ -36,9 +36,9 @@ import (
 	"github.com/go-fed/activity/pub"
 	"github.com/go-fed/activity/streams"
 	"github.com/go-fed/activity/streams/vocab"
+	"github.com/superseriousbusiness/gotosocial/internal/ap"
 	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 	"github.com/superseriousbusiness/gotosocial/internal/oauth"
-	"github.com/superseriousbusiness/gotosocial/internal/typeutils"
 )
 
 // NewTestTokens returns a map of tokens keyed according to which account the token belongs to.
@@ -443,9 +443,9 @@ func NewTestAccounts() map[string]*gtsmodel.Account {
 			FeaturedCollectionURI: "http://fossbros-anonymous.io/users/foss_satan/collections/featured",
 			ActorType:             gtsmodel.ActivityStreamsPerson,
 			AlsoKnownAs:           "",
-			PrivateKey:            nil,
+			PrivateKey:            &rsa.PrivateKey{},
 			PublicKey:             &rsa.PublicKey{},
-			PublicKeyURI:          "http://fossbros-anonymous.io/users/foss_satan#main-key",
+			PublicKeyURI:          "http://fossbros-anonymous.io/users/foss_satan/main-key",
 			SensitizedAt:          time.Time{},
 			SilencedAt:            time.Time{},
 			SuspendedAt:           time.Time{},
@@ -1033,6 +1033,32 @@ func NewTestStatuses() map[string]*gtsmodel.Status {
 			},
 			ActivityStreamsType: gtsmodel.ActivityStreamsNote,
 		},
+		"local_account_2_status_5": {
+			ID:                       "01FCQSQ667XHJ9AV9T27SJJSX5",
+			URI:                      "http://localhost:8080/users/1happyturtle/statuses/01FCQSQ667XHJ9AV9T27SJJSX5",
+			URL:                      "http://localhost:8080/@1happyturtle/statuses/01FCQSQ667XHJ9AV9T27SJJSX5",
+			Content:                  "🐢 hi zork! 🐢",
+			CreatedAt:                time.Now().Add(-1 * time.Minute),
+			UpdatedAt:                time.Now().Add(-1 * time.Minute),
+			Local:                    true,
+			AccountID:                "01F8MH5NBDF2MV7CTC4Q5128HF",
+			InReplyToID:              "01F8MHAMCHF6Y650WCRSCP4WMY",
+			InReplyToAccountID:       "01F8MH1H7YV1Z7D2C8K2730QBF",
+			InReplyToURI:             "http://localhost:8080/users/the_mighty_zork/statuses/01F8MHAMCHF6Y650WCRSCP4WMY",
+			BoostOfID:                "",
+			ContentWarning:           "",
+			Visibility:               gtsmodel.VisibilityPublic,
+			Sensitive:                false,
+			Language:                 "en",
+			CreatedWithApplicationID: "01F8MGYG9E893WRHW0TAEXR8GJ",
+			VisibilityAdvanced: &gtsmodel.VisibilityAdvanced{
+				Federated: true,
+				Boostable: true,
+				Replyable: true,
+				Likeable:  true,
+			},
+			ActivityStreamsType: gtsmodel.ActivityStreamsNote,
+		},
 	}
 }
 
@@ -1155,14 +1181,14 @@ func NewTestActivities(accounts map[string]*gtsmodel.Account) map[string]Activit
 }
 
 // NewTestFediPeople returns a bunch of activity pub Person representations for testing converters and so on.
-func NewTestFediPeople() map[string]typeutils.Accountable {
+func NewTestFediPeople() map[string]ap.Accountable {
 	newPerson1Priv, err := rsa.GenerateKey(rand.Reader, 2048)
 	if err != nil {
 		panic(err)
 	}
 	newPerson1Pub := &newPerson1Priv.PublicKey
 
-	return map[string]typeutils.Accountable{
+	return map[string]ap.Accountable{
 		"new_person_1": newPerson(
 			URLMustParse("https://unknown-instance.com/users/brand_new_person"),
 			URLMustParse("https://unknown-instance.com/users/brand_new_person/following"),
@@ -1187,13 +1213,47 @@ func NewTestFediPeople() map[string]typeutils.Accountable {
 
 // NewTestDereferenceRequests returns a map of incoming dereference requests, with their signatures.
 func NewTestDereferenceRequests(accounts map[string]*gtsmodel.Account) map[string]ActivityWithSignature {
-	sig, digest, date := getSignatureForDereference(accounts["remote_account_1"].PublicKeyURI, accounts["remote_account_1"].PrivateKey, URLMustParse(accounts["local_account_1"].URI))
+	var sig, digest, date string
+	var target *url.URL
+	statuses := NewTestStatuses()
+
+	target = URLMustParse(accounts["local_account_1"].URI)
+	sig, digest, date = getSignatureForDereference(accounts["remote_account_1"].PublicKeyURI, accounts["remote_account_1"].PrivateKey, target)
+	fossSatanDereferenceZork := ActivityWithSignature{
+		SignatureHeader: sig,
+		DigestHeader:    digest,
+		DateHeader:      date,
+	}
+
+	target = URLMustParse(statuses["local_account_1_status_1"].URI + "/replies")
+	sig, digest, date = getSignatureForDereference(accounts["remote_account_1"].PublicKeyURI, accounts["remote_account_1"].PrivateKey, target)
+	fossSatanDereferenceLocalAccount1Status1Replies := ActivityWithSignature{
+		SignatureHeader: sig,
+		DigestHeader:    digest,
+		DateHeader:      date,
+	}
+
+	target = URLMustParse(statuses["local_account_1_status_1"].URI + "/replies?only_other_accounts=false&page=true")
+	sig, digest, date = getSignatureForDereference(accounts["remote_account_1"].PublicKeyURI, accounts["remote_account_1"].PrivateKey, target)
+	fossSatanDereferenceLocalAccount1Status1RepliesNext := ActivityWithSignature{
+		SignatureHeader: sig,
+		DigestHeader:    digest,
+		DateHeader:      date,
+	}
+
+	target = URLMustParse(statuses["local_account_1_status_1"].URI + "/replies?only_other_accounts=false&page=true&min_id=01FCQSQ667XHJ9AV9T27SJJSX5")
+	sig, digest, date = getSignatureForDereference(accounts["remote_account_1"].PublicKeyURI, accounts["remote_account_1"].PrivateKey, target)
+	fossSatanDereferenceLocalAccount1Status1RepliesLast := ActivityWithSignature{
+		SignatureHeader: sig,
+		DigestHeader:    digest,
+		DateHeader:      date,
+	}
+
 	return map[string]ActivityWithSignature{
-		"foss_satan_dereference_zork": {
+		"foss_satan_dereference_zork":                                  fossSatanDereferenceZork,
-			SignatureHeader: sig,
+		"foss_satan_dereference_local_account_1_status_1_replies":      fossSatanDereferenceLocalAccount1Status1Replies,
-			DigestHeader:    digest,
+		"foss_satan_dereference_local_account_1_status_1_replies_next": fossSatanDereferenceLocalAccount1Status1RepliesNext,
-			DateHeader:      date,
+		"foss_satan_dereference_local_account_1_status_1_replies_last": fossSatanDereferenceLocalAccount1Status1RepliesLast,
-		},
 	}
 }
 
@@ -1215,7 +1275,7 @@ func getSignatureForActivity(activity pub.Activity, pubKeyID string, privkey cry
 	}
 
 	// use the client to create a new transport
-	c := NewTestTransportController(client)
+	c := NewTestTransportController(client, NewTestDB())
 	tp, err := c.NewTransport(pubKeyID, privkey)
 	if err != nil {
 		panic(err)
@@ -1247,7 +1307,6 @@ func getSignatureForDereference(pubKeyID string, privkey crypto.PrivateKey, dest
 	client := &mockHTTPClient{
 		do: func(req *http.Request) (*http.Response, error) {
 			signatureHeader = req.Header.Get("Signature")
-			digestHeader = req.Header.Get("Digest")
 			dateHeader = req.Header.Get("Date")
 			r := ioutil.NopCloser(bytes.NewReader([]byte{})) // we only need this so the 'close' func doesn't nil out
 			return &http.Response{
@@ -1258,7 +1317,7 @@ func getSignatureForDereference(pubKeyID string, privkey crypto.PrivateKey, dest
 	}
 
 	// use the client to create a new transport
-	c := NewTestTransportController(client)
+	c := NewTestTransportController(client, NewTestDB())
 	tp, err := c.NewTransport(pubKeyID, privkey)
 	if err != nil {
 		panic(err)
@@ -1290,7 +1349,7 @@ func newPerson(
 	avatarURL *url.URL,
 	avatarContentType string,
 	headerURL *url.URL,
-	headerContentType string) typeutils.Accountable {
+	headerContentType string) ap.Accountable {
 	person := streams.NewActivityStreamsPerson()
 
 	// id should be the activitypub URI of this user

testrig/transportcontroller.go

@@ -24,6 +24,7 @@ import (
 	"net/http"
 
 	"github.com/go-fed/activity/pub"
+	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/federation"
 	"github.com/superseriousbusiness/gotosocial/internal/transport"
 )
@@ -37,8 +38,8 @@ import (
 // Unlike the other test interfaces provided in this package, you'll probably want to call this function
 // PER TEST rather than per suite, so that the do function can be set on a test by test (or even more granular)
 // basis.
-func NewTestTransportController(client pub.HttpClient) transport.Controller {
+func NewTestTransportController(client pub.HttpClient, db db.DB) transport.Controller {
-	return transport.NewController(NewTestConfig(), &federation.Clock{}, client, NewTestLog())
+	return transport.NewController(NewTestConfig(), db, &federation.Clock{}, client, NewTestLog())
 }
 
 // NewMockHTTPClient returns a client that conforms to the pub.HttpClient interface,