diff --git a/fedireads/incoming.py b/fedireads/incoming.py
index 6ca51d0e9..00fea094c 100644
--- a/fedireads/incoming.py
+++ b/fedireads/incoming.py
@@ -54,7 +54,7 @@ def shared_inbox(request):
         key = get_public_key(key_actor)
 
         signature.verify(key, request)
-    except ValueError:
+    except (ValueError, requests.exceptions.HTTPError):
         return HttpResponse(status=401)
 
     handlers = {
@@ -91,20 +91,8 @@ def shared_inbox(request):
 
 def get_public_key(key_actor):
     ''' try a stored key or load it from remote '''
-    try:
-        user = models.User.objects.get(remote_id=key_actor)
-        public_key = user.public_key
-    except models.User.DoesNotExist:
-        response = requests.get(
-            key_actor,
-            headers={'Accept': 'application/activity+json'}
-        )
-        if not response.ok:
-            raise ValueError('Could not load public key')
-        user_data = response.json()
-        public_key = user_data['publicKey']['publicKeyPem']
-
-    return public_key
+    user = get_or_create_remote_user(key_actor)
+    return user.public_key
 
 @app.task
 def handle_follow(activity):
diff --git a/fedireads/remote_user.py b/fedireads/remote_user.py
index 81c8c34c1..72a0391c3 100644
--- a/fedireads/remote_user.py
+++ b/fedireads/remote_user.py
@@ -36,7 +36,8 @@ def get_or_create_remote_user(actor):
         user.save()
 
     avatar = get_avatar(data)
-    user.avatar.save(*avatar)
+    if avatar:
+        user.avatar.save(*avatar)
 
     if user.fedireads_user:
         get_remote_reviews(user)
diff --git a/fedireads/tests/test_signing.py b/fedireads/tests/test_signing.py
index b93d263e9..62bdc761f 100644
--- a/fedireads/tests/test_signing.py
+++ b/fedireads/tests/test_signing.py
@@ -1,6 +1,7 @@
 import time
 from collections import namedtuple
 from urllib.parse import urlsplit
+import pathlib
 
 import json
 import responses
@@ -73,13 +74,26 @@ class Signature(TestCase):
 
     @responses.activate
     def test_remote_signer(self):
+        datafile = pathlib.Path(__file__).parent.joinpath('data/ap_user.json')
+        data = json.loads(datafile.read_bytes())
+        data['id'] = self.fake_remote.remote_id
+        data['publicKey']['publicKeyPem'] = self.fake_remote.public_key
+        del data['icon'] # Avoid having to return an avatar.
         responses.add(
             responses.GET,
             self.fake_remote.remote_id,
-            json={'publicKey': {
-                'publicKeyPem': self.fake_remote.public_key
-            }},
+            json=data,
             status=200)
+        responses.add(
+            responses.GET,
+            'https://localhost/.well-known/nodeinfo',
+            status=404)
+        responses.add(
+            responses.GET,
+            'https://example.com/user/mouse/outbox?page=true',
+            json={'orderedItems': []},
+            status=200
+        )
 
         response = self.send_test_request(sender=self.fake_remote)
         self.assertEqual(response.status_code, 200)