diff --git a/tests/users/models/test_identity.py b/tests/users/models/test_identity.py
index 7e68795..582c824 100644
--- a/tests/users/models/test_identity.py
+++ b/tests/users/models/test_identity.py
@@ -168,6 +168,24 @@ def test_fetch_actor(httpx_mock, config_system):
             "url": "https://example.com/test-actor/view/",
         },
     )
+    httpx_mock.add_response(
+        url="https://example.com/test-actor/collections/featured/",
+        json={
+            "type": "Collection",
+            "totalItems": 1,
+            "orderedItems": [
+                {
+                    "id": "https://example.com/test-actor/posts/123456789",
+                    "type": "Note",
+                    "attributedTo": "https://example.com/test-actor/",
+                    "content": "<p>Test post</p>",
+                    "published": "2022-11-02T00:00:00Z",
+                    "to": "as:Public",
+                    "url": "https://example.com/test-actor/posts/123456789",
+                }
+            ],
+        },
+    )
     identity.fetch_actor()
 
     # Verify the data arrived
diff --git a/users/models/identity.py b/users/models/identity.py
index bebaa31..b714122 100644
--- a/users/models/identity.py
+++ b/users/models/identity.py
@@ -30,6 +30,7 @@ from core.uris import (
     RelativeAbsoluteUrl,
     StaticAbsoluteUrl,
 )
+from stator.exceptions import TryAgainLater
 from stator.models import State, StateField, StateGraph, StatorModel
 from users.models.domain import Domain
 from users.models.system_actor import SystemActor
@@ -740,7 +741,11 @@ class Identity(StatorModel):
                 response.raise_for_status()
             except (httpx.HTTPError, ssl.SSLCertVerificationError) as ex:
                 response = getattr(ex, "response", None)
-                if (
+                if isinstance(ex, httpx.TimeoutException) or (
+                    response and response.status_code in [408, 504]
+                ):
+                    raise TryAgainLater() from ex
+                elif (
                     response
                     and response.status_code < 500
                     and response.status_code not in [400, 401, 403, 404, 406, 410]
@@ -793,7 +798,11 @@ class Identity(StatorModel):
                 response.raise_for_status()
             except (httpx.HTTPError, ssl.SSLCertVerificationError) as ex:
                 response = getattr(ex, "response", None)
-                if (
+                if isinstance(ex, httpx.TimeoutException) or (
+                    response and response.status_code in [408, 504]
+                ):
+                    raise TryAgainLater() from ex
+                elif (
                     response
                     and response.status_code < 500
                     and response.status_code not in [401, 403, 404, 406, 410]
@@ -846,6 +855,8 @@ class Identity(StatorModel):
                 method="get",
                 uri=self.actor_uri,
             )
+        except httpx.TimeoutException:
+            raise TryAgainLater()
         except (httpx.RequestError, ssl.SSLCertVerificationError):
             return False
         content_type = response.headers.get("content-type")
@@ -854,10 +865,11 @@ class Identity(StatorModel):
             return False
         status_code = response.status_code
         if status_code >= 400:
+            if status_code in [408, 504]:
+                raise TryAgainLater()
             if status_code == 410 and self.pk:
                 # Their account got deleted, so let's do the same.
                 Identity.objects.filter(pk=self.pk).delete()
-
             if status_code < 500 and status_code not in [401, 403, 404, 406, 410]:
                 capture_message(
                     f"Client error fetching actor at {self.actor_uri}: {status_code}",
@@ -923,18 +935,19 @@ class Identity(StatorModel):
                 )
         # Now go do webfinger with that info to see if we can get a canonical domain
         actor_url_parts = urlparse(self.actor_uri)
+        self.domain = Domain.get_remote_domain(actor_url_parts.hostname)
         if self.username:
-            webfinger_actor, webfinger_handle = self.fetch_webfinger(
-                f"{self.username}@{actor_url_parts.hostname}"
-            )
-            if webfinger_handle:
-                webfinger_username, webfinger_domain = webfinger_handle.split("@")
-                self.username = webfinger_username
-                self.domain = Domain.get_remote_domain(webfinger_domain)
-            else:
-                self.domain = Domain.get_remote_domain(actor_url_parts.hostname)
-        else:
-            self.domain = Domain.get_remote_domain(actor_url_parts.hostname)
+            try:
+                webfinger_actor, webfinger_handle = self.fetch_webfinger(
+                    f"{self.username}@{actor_url_parts.hostname}"
+                )
+                if webfinger_handle:
+                    webfinger_username, webfinger_domain = webfinger_handle.split("@")
+                    self.username = webfinger_username
+                    self.domain = Domain.get_remote_domain(webfinger_domain)
+            except TryAgainLater:
+                # continue with original domain when webfinger times out
+                pass
         # Emojis (we need the domain so we do them here)
         for tag in get_list(document, "tag"):
             if tag["type"].lower() in ["toot:emoji", "emoji"]:
diff --git a/users/views/activitypub.py b/users/views/activitypub.py
index 461840e..a43301c 100644
--- a/users/views/activitypub.py
+++ b/users/views/activitypub.py
@@ -20,6 +20,7 @@ from core.signatures import (
     VerificationFormatError,
 )
 from core.views import StaticContentView
+from stator.exceptions import TryAgainLater
 from takahe import __version__
 from users.models import Identity, InboxMessage, SystemActor
 from users.shortcuts import by_handle_or_404
@@ -150,7 +151,13 @@ class Inbox(View):
 
         if not identity.public_key:
             # See if we can fetch it right now
-            identity.fetch_actor()
+            try:
+                identity.fetch_actor()
+            except TryAgainLater:
+                exceptions.capture_message(
+                    f"Inbox error: timed out fetching actor {document['actor']}"
+                )
+                return HttpResponse(status=504)
 
         if not identity.public_key:
             exceptions.capture_message(