Merge branch 'main' into book-series-v1

bookwyrm/activitypub/base_activity.py

@@ -2,11 +2,16 @@
 from dataclasses import dataclass, fields, MISSING
 from json import JSONEncoder
 import logging
+import requests
 
 from django.apps import apps
 from django.db import IntegrityError, transaction
+from django.utils.http import http_date
 
+from bookwyrm import models
 from bookwyrm.connectors import ConnectorException, get_data
+from bookwyrm.signatures import make_signature
+from bookwyrm.settings import DOMAIN, INSTANCE_ACTOR_USERNAME
 from bookwyrm.tasks import app, MEDIUM
 
 logger = logging.getLogger(__name__)
@@ -246,10 +251,10 @@ def set_related_field(
 
 def get_model_from_type(activity_type):
     """given the activity, what type of model"""
-    models = apps.get_models()
+    activity_models = apps.get_models()
     model = [
         m
-        for m in models
+        for m in activity_models
         if hasattr(m, "activity_serializer")
         and hasattr(m.activity_serializer, "type")
         and m.activity_serializer.type == activity_type
@@ -275,10 +280,16 @@ def resolve_remote_id(
     # load the data and create the object
     try:
         data = get_data(remote_id)
-    except ConnectorException:
+    except ConnectionError:
+        logger.info("Could not connect to host for remote_id: %s", remote_id)
+        return None
+    except requests.HTTPError as e:
+        if (e.response is not None) and e.response.status_code == 401:
+            # This most likely means it's a mastodon with secure fetch enabled.
+            data = get_activitypub_data(remote_id)
+        else:
             logger.info("Could not connect to host for remote_id: %s", remote_id)
             return None
-
     # determine the model implicitly, if not provided
     # or if it's a model with subclasses like Status, check again
     if not model or hasattr(model.objects, "select_subclasses"):
@@ -297,6 +308,51 @@ def resolve_remote_id(
     return item.to_model(model=model, instance=result, save=save)
 
 
+def get_representative():
+    """Get or create an actor representing the instance
+    to sign requests to 'secure mastodon' servers"""
+    username = f"{INSTANCE_ACTOR_USERNAME}@{DOMAIN}"
+    email = "bookwyrm@localhost"
+    try:
+        user = models.User.objects.get(username=username)
+    except models.User.DoesNotExist:
+        user = models.User.objects.create_user(
+            username=username,
+            email=email,
+            local=True,
+            localname=INSTANCE_ACTOR_USERNAME,
+        )
+    return user
+
+
+def get_activitypub_data(url):
+    """wrapper for request.get"""
+    now = http_date()
+    sender = get_representative()
+    if not sender.key_pair.private_key:
+        # this shouldn't happen. it would be bad if it happened.
+        raise ValueError("No private key found for sender")
+    try:
+        resp = requests.get(
+            url,
+            headers={
+                "Accept": "application/json; charset=utf-8",
+                "Date": now,
+                "Signature": make_signature("get", sender, url, now),
+            },
+        )
+    except requests.RequestException:
+        raise ConnectorException()
+    if not resp.ok:
+        resp.raise_for_status()
+    try:
+        data = resp.json()
+    except ValueError:
+        raise ConnectorException()
+
+    return data
+
+
 @dataclass(init=False)
 class Link(ActivityObject):
     """for tagging a book in a status"""

bookwyrm/connectors/abstract_connector.py

@@ -244,6 +244,10 @@ def get_data(url, params=None, timeout=settings.QUERY_TIMEOUT):
         raise ConnectorException(err)
 
     if not resp.ok:
+        if resp.status_code == 401:
+            # this is probably an AUTHORIZED_FETCH issue
+            resp.raise_for_status()
+        else:
             raise ConnectorException()
     try:
         data = resp.json()

bookwyrm/models/activitypub_mixin.py

@@ -543,7 +543,7 @@ async def sign_and_send(
     headers = {
         "Date": now,
         "Digest": digest,
-        "Signature": make_signature(sender, destination, now, digest),
+        "Signature": make_signature("post", sender, destination, now, digest),
         "Content-Type": "application/activity+json; charset=utf-8",
         "User-Agent": USER_AGENT,
     }

bookwyrm/settings.py

@@ -373,3 +373,9 @@ TWO_FACTOR_LOGIN_MAX_SECONDS = 60
 HTTP_X_FORWARDED_PROTO = env.bool("SECURE_PROXY_SSL_HEADER", False)
 if HTTP_X_FORWARDED_PROTO:
     SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https")
+
+# Instance Actor for signing GET requests to "secure mode"
+# Mastodon servers.
+# Do not change this setting unless you already have an existing
+# user with the same username - in which case you should change it!
+INSTANCE_ACTOR_USERNAME = "bookwyrm.instance.actor"

bookwyrm/signatures.py

@@ -22,22 +22,26 @@ def create_key_pair():
     return private_key, public_key
 
 
-def make_signature(sender, destination, date, digest):
+def make_signature(method, sender, destination, date, digest=None):
     """uses a private key to sign an outgoing message"""
     inbox_parts = urlparse(destination)
     signature_headers = [
-        f"(request-target): post {inbox_parts.path}",
+        f"(request-target): {method} {inbox_parts.path}",
         f"host: {inbox_parts.netloc}",
         f"date: {date}",
-        f"digest: {digest}",
     ]
+    headers = "(request-target) host date"
+    if digest is not None:
+        signature_headers.append(f"digest: {digest}")
+        headers = "(request-target) host date digest"
+
     message_to_sign = "\n".join(signature_headers)
     signer = pkcs1_15.new(RSA.import_key(sender.key_pair.private_key))
     signed_message = signer.sign(SHA256.new(message_to_sign.encode("utf8")))
     signature = {
         "keyId": f"{sender.remote_id}#main-key",
         "algorithm": "rsa-sha256",
-        "headers": "(request-target) host date digest",
+        "headers": headers,
         "signature": b64encode(signed_message).decode("utf8"),
     }
     return ",".join(f'{k}="{v}"' for (k, v) in signature.items())

bookwyrm/templates/settings/celery.html

@@ -13,24 +13,30 @@
 <section class="block content">
     <h2>{% trans "Queues" %}</h2>
     <div class="columns has-text-centered">
-        <div class="column is-4">
+        <div class="column is-3">
             <div class="notification">
                 <p class="header">{% trans "Low priority" %}</p>
                 <p class="title is-5">{{ queues.low_priority|intcomma }}</p>
             </div>
         </div>
-        <div class="column is-4">
+        <div class="column is-3">
             <div class="notification">
                 <p class="header">{% trans "Medium priority" %}</p>
                 <p class="title is-5">{{ queues.medium_priority|intcomma }}</p>
             </div>
         </div>
-        <div class="column is-4">
+        <div class="column is-3">
             <div class="notification">
                 <p class="header">{% trans "High priority" %}</p>
                 <p class="title is-5">{{ queues.high_priority|intcomma }}</p>
             </div>
         </div>
+        <div class="column is-3">
+            <div class="notification">
+                <p class="header">{% trans "Imports" %}</p>
+                <p class="title is-5">{{ queues.imports|intcomma }}</p>
+            </div>
+        </div>
     </div>
 </section>
 {% else %}

bookwyrm/tests/activitypub/test_base_activity.py

@@ -14,6 +14,7 @@ from bookwyrm.activitypub.base_activity import (
     ActivityObject,
     resolve_remote_id,
     set_related_field,
+    get_representative,
 )
 from bookwyrm.activitypub import ActivitySerializerError
 from bookwyrm import models
@@ -52,6 +53,11 @@ class BaseActivity(TestCase):
         image.save(output, format=image.format)
         self.image_data = output.getvalue()
 
+    def test_get_representative_not_existing(self, *_):
+        """test that an instance representative actor is created if it does not exist"""
+        representative = get_representative()
+        self.assertIsInstance(representative, models.User)
+
     def test_init(self, *_):
         """simple successfuly init"""
         instance = ActivityObject(id="a", type="b")

bookwyrm/tests/test_signing.py

@@ -86,7 +86,9 @@ class Signature(TestCase):
         now = date or http_date()
         data = json.dumps(get_follow_activity(sender, self.rat))
         digest = digest or make_digest(data)
-        signature = make_signature(signer or sender, self.rat.inbox, now, digest)
+        signature = make_signature(
+            "post", signer or sender, self.rat.inbox, now, digest
+        )
         with patch("bookwyrm.views.inbox.activity_task.apply_async"):
             with patch("bookwyrm.models.user.set_remote_server.delay"):
                 return self.send(signature, now, send_data or data, digest)

bookwyrm/views/admin/celery_status.py

@@ -36,6 +36,7 @@ class CeleryStatus(View):
                 "low_priority": r.llen("low_priority"),
                 "medium_priority": r.llen("medium_priority"),
                 "high_priority": r.llen("high_priority"),
+                "imports": r.llen("imports"),
             }
         # pylint: disable=broad-except
         except Exception as err:

contrib/systemd/bookwyrm-worker.service

@@ -6,7 +6,7 @@ After=network.target postgresql.service redis.service
 User=bookwyrm
 Group=bookwyrm
 WorkingDirectory=/opt/bookwyrm/
-ExecStart=/opt/bookwyrm/venv/bin/celery -A celerywyrm worker -l info -Q high_priority,medium_priority,low_priority
+ExecStart=/opt/bookwyrm/venv/bin/celery -A celerywyrm worker -l info -Q high_priority,medium_priority,low_priority,import
 StandardOutput=journal
 StandardError=inherit
 

requirements.txt

@@ -4,24 +4,24 @@ celery==5.2.7
 colorthief==0.2.1
 Django==3.2.16
 django-celery-beat==2.4.0
-django-compressor==4.1
+django-compressor==4.3.1
 django-imagekit==4.1.0
-django-model-utils==4.2.0
+django-model-utils==4.3.1
 django-sass-processor==1.2.2
 environs==9.5.0
 flower==1.2.0
 libsass==0.22.0
-Markdown==3.3.3
+Markdown==3.4.1
-Pillow>=9.3.0
+Pillow==9.4.0
 psycopg2==2.9.5
 pycryptodome==3.16.0
 python-dateutil==2.8.2
 redis==3.4.1
-requests==2.28.1
+requests==2.28.2
 responses==0.22.0
 pytz>=2022.7
-boto3==1.26.32
+boto3==1.26.57
-django-storages==1.11.1
+django-storages==1.13.2
 django-redis==5.2.0
 opentelemetry-api==1.11.1
 opentelemetry-exporter-otlp-proto-grpc==1.11.1
@@ -29,7 +29,7 @@ opentelemetry-instrumentation-celery==0.30b1
 opentelemetry-instrumentation-django==0.30b1
 opentelemetry-sdk==1.11.1
 protobuf==3.20.*
-pyotp==2.6.0
+pyotp==2.8.0
 qrcode==7.3.1
 
 # Dev