Merge branch 'main' into check-version-number

.env.example

@@ -137,3 +137,10 @@ TWO_FACTOR_LOGIN_MAX_SECONDS=60
 # and AWS_S3_CUSTOM_DOMAIN (if used) are added by default.
 # Value should be a comma-separated list of host names.
 CSP_ADDITIONAL_HOSTS=
+
+# The last number here means "megabytes"
+# Increase if users are having trouble uploading BookWyrm export files.
+DATA_UPLOAD_MAX_MEMORY_SIZE = (1024**2 * 100)
+
+# Time before being logged out (in seconds)
+# SESSION_COOKIE_AGE=2592000   # current default: 30 days

bookwyrm/book_search.py

@@ -43,6 +43,7 @@ def search(
     min_confidence: float = 0,
     filters: Optional[list[Any]] = None,
     return_first: bool = False,
+    books: Optional[QuerySet[models.Edition]] = None,
 ) -> Union[Optional[models.Edition], QuerySet[models.Edition]]:
     """search your local database"""
     filters = filters or []
@@ -54,13 +55,15 @@ def search(
     # first, try searching unique identifiers
     # unique identifiers never have spaces, title/author usually do
     if not " " in query:
-        results = search_identifiers(query, *filters, return_first=return_first)
+        results = search_identifiers(
+            query, *filters, return_first=return_first, books=books
+        )
 
     # if there were no identifier results...
     if not results:
         # then try searching title/author
         results = search_title_author(
-            query, min_confidence, *filters, return_first=return_first
+            query, min_confidence, *filters, return_first=return_first, books=books
         )
     return results
 
@@ -98,9 +101,17 @@ def format_search_result(search_result):
 
 
 def search_identifiers(
-    query, *filters, return_first=False
+    query,
+    *filters,
+    return_first=False,
+    books=None,
 ) -> Union[Optional[models.Edition], QuerySet[models.Edition]]:
-    """tries remote_id, isbn; defined as dedupe fields on the model"""
+    """search Editions by deduplication fields
+
+    Best for cases when we can assume someone is searching for an exact match on
+    commonly unique data identifiers like isbn or specific library ids.
+    """
+    books = books or models.Edition.objects
     if connectors.maybe_isbn(query):
         # Oh did you think the 'S' in ISBN stood for 'standard'?
         normalized_isbn = query.strip().upper().rjust(10, "0")
@@ -111,7 +122,7 @@ def search_identifiers(
         for f in models.Edition._meta.get_fields()
         if hasattr(f, "deduplication_field") and f.deduplication_field
     ]
-    results = models.Edition.objects.filter(
+    results = books.filter(
         *filters, reduce(operator.or_, (Q(**f) for f in or_filters))
     ).distinct()
 
@@ -121,12 +132,17 @@ def search_identifiers(
 
 
 def search_title_author(
-    query, min_confidence, *filters, return_first=False
+    query,
+    min_confidence,
+    *filters,
+    return_first=False,
+    books=None,
 ) -> QuerySet[models.Edition]:
     """searches for title and author"""
+    books = books or models.Edition.objects
     query = SearchQuery(query, config="simple") | SearchQuery(query, config="english")
     results = (
-        models.Edition.objects.filter(*filters, search_vector=query)
+        books.filter(*filters, search_vector=query)
         .annotate(rank=SearchRank(F("search_vector"), query))
         .filter(rank__gt=min_confidence)
         .order_by("-rank")

bookwyrm/management/commands/erase_deleted_user_data.py

@@ -0,0 +1,43 @@
+""" Erase any data stored about deleted users """
+import sys
+from django.core.management.base import BaseCommand, CommandError
+from bookwyrm import models
+from bookwyrm.models.user import erase_user_data
+
+# pylint: disable=missing-function-docstring
+class Command(BaseCommand):
+    """command-line options"""
+
+    help = "Remove Two Factor Authorisation from user"
+
+    def add_arguments(self, parser):  # pylint: disable=no-self-use
+        parser.add_argument(
+            "--dryrun",
+            action="store_true",
+            help="Preview users to be cleared without altering the database",
+        )
+
+    def handle(self, *args, **options):  # pylint: disable=unused-argument
+
+        # Check for anything fishy
+        bad_state = models.User.objects.filter(is_deleted=True, is_active=True)
+        if bad_state.exists():
+            raise CommandError(
+                f"{bad_state.count()} user(s) marked as both active and deleted"
+            )
+
+        deleted_users = models.User.objects.filter(is_deleted=True)
+        self.stdout.write(f"Found {deleted_users.count()} deleted users")
+        if options["dryrun"]:
+            self.stdout.write("\n".join(u.username for u in deleted_users[:5]))
+            if deleted_users.count() > 5:
+                self.stdout.write("... and more")
+            sys.exit()
+
+        self.stdout.write("Erasing user data:")
+        for user_id in deleted_users.values_list("id", flat=True):
+            erase_user_data.delay(user_id)
+            self.stdout.write(".", ending="")
+
+        self.stdout.write("")
+        self.stdout.write("Tasks created successfully")

bookwyrm/migrations/0184_auto_20231106_0421.py

@@ -22,17 +22,6 @@ def update_deleted_users(apps, schema_editor):
     ).update(is_deleted=True)
 
 
-def erase_deleted_user_data(apps, schema_editor):
-    """Retroactively clear user data"""
-    for user in User.objects.filter(is_deleted=True):
-        user.erase_user_data()
-        user.save(
-            broadcast=False,
-            update_fields=["email", "avatar", "preview_image", "summary", "name"],
-        )
-        user.erase_user_statuses(broadcast=False)
-
-
 class Migration(migrations.Migration):
 
     dependencies = [
@@ -43,7 +32,4 @@ class Migration(migrations.Migration):
         migrations.RunPython(
             update_deleted_users, reverse_code=migrations.RunPython.noop
         ),
-        migrations.RunPython(
-            erase_deleted_user_data, reverse_code=migrations.RunPython.noop
-        ),
     ]

bookwyrm/migrations/0192_sitesettings_user_exports_enabled.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.23 on 2024-01-16 10:28
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0191_merge_20240102_0326"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="sitesettings",
+            name="user_exports_enabled",
+            field=models.BooleanField(default=False),
+        ),
+    ]

bookwyrm/models/activitypub_mixin.py

@@ -152,8 +152,9 @@ class ActivitypubMixin:
         # find anyone who's tagged in a status, for example
         mentions = self.recipients if hasattr(self, "recipients") else []
 
-        # we always send activities to explicitly mentioned users' inboxes
-        recipients = [u.inbox for u in mentions or [] if not u.local]
+        # we always send activities to explicitly mentioned users (using shared inboxes
+        # where available to avoid duplicate submissions to a given instance)
+        recipients = {u.shared_inbox or u.inbox for u in mentions if not u.local}
 
         # unless it's a dm, all the followers should receive the activity
         if privacy != "direct":
@@ -173,18 +174,18 @@ class ActivitypubMixin:
             if user:
                 queryset = queryset.filter(following=user)
 
-            # ideally, we will send to shared inboxes for efficiency
-            shared_inboxes = (
-                queryset.filter(shared_inbox__isnull=False)
-                .values_list("shared_inbox", flat=True)
-                .distinct()
+            # as above, we prefer shared inboxes if available
+            recipients.update(
+                queryset.filter(shared_inbox__isnull=False).values_list(
+                    "shared_inbox", flat=True
+                )
             )
-            # but not everyone has a shared inbox
-            inboxes = queryset.filter(shared_inbox__isnull=True).values_list(
-                "inbox", flat=True
+            recipients.update(
+                queryset.filter(shared_inbox__isnull=True).values_list(
+                    "inbox", flat=True
+                )
             )
-            recipients += list(shared_inboxes) + list(inboxes)
-        return list(set(recipients))
+        return list(recipients)
 
     def to_activity_dataclass(self):
         """convert from a model to an activity"""

bookwyrm/models/site.py

@@ -99,6 +99,7 @@ class SiteSettings(SiteModel):
     imports_enabled = models.BooleanField(default=True)
     import_size_limit = models.IntegerField(default=0)
     import_limit_reset = models.IntegerField(default=0)
+    user_exports_enabled = models.BooleanField(default=False)
     user_import_time_limit = models.IntegerField(default=48)
 
     field_tracker = FieldTracker(fields=["name", "instance_tagline", "logo"])

bookwyrm/models/status.py

@@ -12,6 +12,8 @@ from django.db.models import Q
 from django.dispatch import receiver
 from django.template.loader import get_template
 from django.utils import timezone
+from django.utils.translation import gettext_lazy as _
+from django.utils.translation import ngettext_lazy
 from model_utils import FieldTracker
 from model_utils.managers import InheritanceManager
 
@@ -107,14 +109,14 @@ class Status(OrderedCollectionPageMixin, BookWyrmModel):
     @property
     def recipients(self):
         """tagged users who definitely need to get this status in broadcast"""
-        mentions = [u for u in self.mention_users.all() if not u.local]
+        mentions = {u for u in self.mention_users.all() if not u.local}
         if (
             hasattr(self, "reply_parent")
             and self.reply_parent
             and not self.reply_parent.user.local
         ):
-            mentions.append(self.reply_parent.user)
-        return list(set(mentions))
+            mentions.add(self.reply_parent.user)
+        return list(mentions)
 
     @classmethod
     def ignore_activity(
@@ -178,6 +180,24 @@ class Status(OrderedCollectionPageMixin, BookWyrmModel):
         """you can't boost dms"""
         return self.privacy in ["unlisted", "public"]
 
+    @property
+    def page_title(self):
+        """title of the page when only this status is shown"""
+        return _("%(display_name)s's status") % {"display_name": self.user.display_name}
+
+    @property
+    def page_description(self):
+        """description of the page in meta tags when only this status is shown"""
+        return None
+
+    @property
+    def page_image(self):
+        """image to use as preview in meta tags when only this status is shown"""
+        if self.mention_books.exists():
+            book = self.mention_books.first()
+            return book.preview_image or book.cover
+        return self.user.preview_image
+
     def to_replies(self, **kwargs):
         """helper function for loading AP serialized replies to a status"""
         return self.to_ordered_collection(
@@ -301,6 +321,10 @@ class BookStatus(Status):
 
         abstract = True
 
+    @property
+    def page_image(self):
+        return self.book.preview_image or self.book.cover or super().page_image
+
 
 class Comment(BookStatus):
     """like a review but without a rating and transient"""
@@ -332,6 +356,13 @@ class Comment(BookStatus):
 
     activity_serializer = activitypub.Comment
 
+    @property
+    def page_title(self):
+        return _("%(display_name)s's comment on %(book_title)s") % {
+            "display_name": self.user.display_name,
+            "book_title": self.book.title,
+        }
+
 
 class Quotation(BookStatus):
     """like a review but without a rating and transient"""
@@ -374,6 +405,13 @@ class Quotation(BookStatus):
 
     activity_serializer = activitypub.Quotation
 
+    @property
+    def page_title(self):
+        return _("%(display_name)s's quote from %(book_title)s") % {
+            "display_name": self.user.display_name,
+            "book_title": self.book.title,
+        }
+
 
 class Review(BookStatus):
     """a book review"""
@@ -403,6 +441,13 @@ class Review(BookStatus):
         """indicate the book in question for mastodon (or w/e) users"""
         return self.content
 
+    @property
+    def page_title(self):
+        return _("%(display_name)s's review of %(book_title)s") % {
+            "display_name": self.user.display_name,
+            "book_title": self.book.title,
+        }
+
     activity_serializer = activitypub.Review
     pure_type = "Article"
 
@@ -426,6 +471,18 @@ class ReviewRating(Review):
         template = get_template("snippets/generated_status/rating.html")
         return template.render({"book": self.book, "rating": self.rating}).strip()
 
+    @property
+    def page_description(self):
+        return ngettext_lazy(
+            "%(display_name)s rated %(book_title)s: %(display_rating).1f star",
+            "%(display_name)s rated %(book_title)s: %(display_rating).1f stars",
+            "display_rating",
+        ) % {
+            "display_name": self.user.display_name,
+            "book_title": self.book.title,
+            "display_rating": self.rating,
+        }
+
     activity_serializer = activitypub.Rating
     pure_type = "Note"
 

bookwyrm/models/user.py

@@ -523,6 +523,20 @@ class KeyPair(ActivitypubMixin, BookWyrmModel):
         return super().save(*args, **kwargs)
 
 
+@app.task(queue=MISC)
+def erase_user_data(user_id):
+    """Erase any custom data about this user asynchronously
+    This is for deleted historical user data that pre-dates data
+    being cleared automatically"""
+    user = User.objects.get(id=user_id)
+    user.erase_user_data()
+    user.save(
+        broadcast=False,
+        update_fields=["email", "avatar", "preview_image", "summary", "name"],
+    )
+    user.erase_user_statuses(broadcast=False)
+
+
 @app.task(queue=MISC)
 def set_remote_server(user_id, allow_external_connections=False):
     """figure out the user's remote server in the background"""

bookwyrm/settings.py

@@ -30,6 +30,9 @@ RELEASE_API = env(
 
 PAGE_LENGTH = env.int("PAGE_LENGTH", 15)
 DEFAULT_LANGUAGE = env("DEFAULT_LANGUAGE", "English")
+# TODO: extend maximum age to 1 year once termination of active sessions
+# is implemented (see bookwyrm-social#2278, bookwyrm-social#3082).
+SESSION_COOKIE_AGE = env.int("SESSION_COOKIE_AGE", 3600 * 24 * 30)  # 1 month
 
 JS_CACHE = "8a89cad7"
 
@@ -347,8 +350,7 @@ USE_L10N = True
 USE_TZ = True
 
 
-agent = requests.utils.default_user_agent()
-USER_AGENT = f"{agent} (BookWyrm/{VERSION}; +https://{DOMAIN}/)"
+USER_AGENT = f"BookWyrm (BookWyrm/{VERSION}; +https://{DOMAIN}/)"
 
 # Imagekit generated thumbnails
 ENABLE_THUMBNAIL_GENERATION = env.bool("ENABLE_THUMBNAIL_GENERATION", False)
@@ -442,3 +444,5 @@ if HTTP_X_FORWARDED_PROTO:
 # Do not change this setting unless you already have an existing
 # user with the same username - in which case you should change it!
 INSTANCE_ACTOR_USERNAME = "bookwyrm.instance.actor"
+
+DATA_UPLOAD_MAX_MEMORY_SIZE = env.int("DATA_UPLOAD_MAX_MEMORY_SIZE", (1024**2 * 100))

bookwyrm/static/js/autocomplete.js

@@ -111,6 +111,10 @@ const tries = {
             },
         },
         f: {
+            b: {
+                2: "FB2",
+                3: "FB3",
+            },
             l: {
                 a: {
                     c: "FLAC",

bookwyrm/templates/about/about.html

@@ -31,10 +31,10 @@
         </p>
     </div>
 
-    <div class="columns">
+    <div class="columns is-multiline">
         {% if superlatives.top_rated %}
         {% with book=superlatives.top_rated.default_edition rating=superlatives.top_rated.rating %}
-        <div class="column is-one-third is-flex">
+        <div class="column is-half-tablet is-one-third-desktop is-flex-grow-1 is-flex">
             <div class="media notification is-clipped">
                 <div class="media-left">
                     <a href="{{ book.local_path }}">
@@ -53,7 +53,7 @@
 
         {% if superlatives.wanted %}
         {% with book=superlatives.wanted.default_edition %}
-        <div class="column is-one-third is-flex">
+        <div class="column is-half-tablet is-one-third-desktop is-flex-grow-1 is-flex">
             <div class="media notification is-clipped">
                 <div class="media-left">
                     <a href="{{ book.local_path }}">
@@ -72,7 +72,7 @@
 
         {% if superlatives.controversial %}
         {% with book=superlatives.controversial.default_edition %}
-        <div class="column is-one-third is-flex">
+        <div class="column is-half-tablet is-one-third-desktop is-flex-grow-1 is-flex">
             <div class="media notification is-clipped">
                 <div class="media-left">
                     <a href="{{ book.local_path }}">

bookwyrm/templates/book/book.html

@@ -9,7 +9,8 @@
 {% block title %}{{ book|book_title }}{% endblock %}
 
 {% block opengraph %}
-    {% include 'snippets/opengraph.html' with title=book.title description=book|book_description image=book.preview_image %}
+    {% firstof book.preview_image book.cover as book_image %}
+    {% include 'snippets/opengraph.html' with title=book.title description=book|book_description image=book_image %}
 {% endblock %}
 
 {% block content %}

bookwyrm/templates/confirm_email/confirm_email.html

@@ -6,8 +6,8 @@
 {% block content %}
 <h1 class="title">{% trans "Confirm your email address" %}</h1>
 
-<div class="columns">
-    <div class="column">
+<div class="columns is-multiline">
+    <div class="column is-full is-half-desktop">
         <div class="block content">
             <section class="block">
                 <p>{% trans "A confirmation code has been sent to the email address you used to register your account." %}</p>

bookwyrm/templates/feed/feed.html

@@ -41,7 +41,7 @@
     </section>
     {% endif %}
 
-    {% if annual_summary_year and tab.key == 'home' %}
+    {% if annual_summary_year and tab.key == 'home' and has_summary_read_throughs %}
     <section class="block is-hidden" data-hide="hide_annual_summary_{{ annual_summary_year }}">
         {% include 'feed/summary_card.html' with year=annual_summary_year %}
         <hr>

bookwyrm/templates/feed/status.html

@@ -2,13 +2,11 @@
 {% load feed_page_tags %}
 {% load i18n %}
 
+{% block title %}{{ title }}{% endblock %}
+
+
 {% block opengraph %}
-    {% firstof status.book status.mention_books.first as book %}
-    {% if book %}
-        {% include 'snippets/opengraph.html' with image=preview %}
-    {% else %}
-        {% include 'snippets/opengraph.html' %}
-    {% endif %}
+    {% include 'snippets/opengraph.html' with image=page_image %}
 {% endblock %}
 
 

bookwyrm/templates/landing/invite.html

@@ -6,8 +6,8 @@
 {% block content %}
 
 <h1 class="title">{% trans "Create an Account" %}</h1>
-<div class="columns">
-    <div class="column">
+<div class="columns is-multiline">
+    <div class="column is-full is-half-desktop">
         <div class="block">
             {% if valid %}
             <div>

bookwyrm/templates/landing/login.html

@@ -6,7 +6,7 @@
 {% block content %}
 <h1 class="title">{% trans "Log in" %}</h1>
 <div class="columns is-multiline">
-    <div class="column is-half">
+    <div class="column {% if site.allow_registration %}is-half{% else %}is-full is-half-desktop{% endif %}">
         {% if login_form.non_field_errors %}
         <p class="notification is-danger">{{ login_form.non_field_errors }}</p>
         {% endif %}
@@ -20,13 +20,15 @@
             <div class="field">
                 <label class="label" for="id_localname_confirm">{% trans "Username:" %}</label>
                 <div class="control">
-                    <input type="text" name="localname" maxlength="255" class="input" required="" id="id_localname_confirm" value="{{ login_form.localname.value|default:'' }}">
+                    <input type="text" name="localname" maxlength="255" class="input" required=""
+                        id="id_localname_confirm" value="{{ login_form.localname.value|default:'' }}">
                 </div>
             </div>
             <div class="field">
                 <label class="label" for="id_password_confirm">{% trans "Password:" %}</label>
                 <div class="control">
-                    <input type="password" name="password" maxlength="128" class="input" required="" id="id_password_confirm" aria-describedby="desc_password">
+                    <input type="password" name="password" maxlength="128" class="input" required=""
+                        id="id_password_confirm" aria-describedby="desc_password">
                 </div>
 
                 {% include 'snippets/form_errors.html' with errors_list=login_form.password.errors id="desc_password" %}
@@ -58,10 +60,10 @@
             {% include 'snippets/about.html' %}
 
             <p class="block">
-            <a href="{% url 'about' %}">{% trans "More about this site" %}</a>
+                <a href="{% url 'about' %}">{% trans "More about this site" %}</a>
             </p>
         </div>
     </div>
 </div>
 
-{% endblock %}
+{% endblock %}

bookwyrm/templates/landing/password_reset.html

@@ -4,8 +4,8 @@
 {% block title %}{% trans "Reset Password" %}{% endblock %}
 
 {% block content %}
-<div class="columns">
-    <div class="column">
+<div class="columns is-multiline">
+    <div class="column is-full is-half-desktop">
         <div class="block">
             <h1 class="title">{% trans "Reset Password" %}</h1>
 

bookwyrm/templates/landing/reactivate.html

@@ -6,7 +6,7 @@
 {% block content %}
 <h1 class="title">{% trans "Reactivate Account" %}</h1>
 <div class="columns is-multiline">
-    <div class="column is-half">
+    <div class="column {% if site.allow_registration %}is-half{% else %}is-full is-half-desktop{% endif %}">
         {% if login_form.non_field_errors %}
         <p class="notification is-danger">{{ login_form.non_field_errors }}</p>
         {% endif %}
@@ -16,13 +16,15 @@
             <div class="field">
                 <label class="label" for="id_localname_confirm">{% trans "Username:" %}</label>
                 <div class="control">
-                    <input type="text" name="localname" maxlength="255" class="input" required="" id="id_localname_confirm" value="{{ login_form.localname.value|default:'' }}">
+                    <input type="text" name="localname" maxlength="255" class="input" required=""
+                        id="id_localname_confirm" value="{{ login_form.localname.value|default:'' }}">
                 </div>
             </div>
             <div class="field">
                 <label class="label" for="id_password_confirm">{% trans "Password:" %}</label>
                 <div class="control">
-                    <input type="password" name="password" maxlength="128" class="input" required="" id="id_password_confirm" aria-describedby="desc_password">
+                    <input type="password" name="password" maxlength="128" class="input" required=""
+                        id="id_password_confirm" aria-describedby="desc_password">
                 </div>
 
                 {% include 'snippets/form_errors.html' with errors_list=login_form.password.errors id="desc_password" %}
@@ -51,10 +53,10 @@
             {% include 'snippets/about.html' %}
 
             <p class="block">
-            <a href="{% url 'about' %}">{% trans "More about this site" %}</a>
+                <a href="{% url 'about' %}">{% trans "More about this site" %}</a>
             </p>
         </div>
     </div>
 </div>
 
-{% endblock %}
+{% endblock %}

bookwyrm/templates/moved.html

@@ -23,7 +23,7 @@
 
           <div class="notification is-warning">
             <p>
-                {% id_to_username request.user.moved_to  as username %}
+                {% id_to_username request.user.moved_to as username %}
                 {% blocktrans trimmed with moved_to=user.moved_to %}
                     <strong>You have moved your account</strong> to <a href="{{ moved_to }}">{{ username }}</a>
                 {% endblocktrans %}

bookwyrm/templates/notifications/item.html

@@ -10,9 +10,7 @@
 {% elif notification.notification_type == 'FOLLOW' %}
     {% include 'notifications/items/follow.html' %}
 {% elif notification.notification_type == 'FOLLOW_REQUEST' %}
-    {% if notification.related_users.0.is_active %}
-        {% include 'notifications/items/follow_request.html' %}
-    {% endif %}
+    {% include 'notifications/items/follow_request.html' %}
 {% elif notification.notification_type == 'IMPORT' %}
     {% include 'notifications/items/import.html' %}
 {% elif notification.notification_type == 'USER_IMPORT' %}

bookwyrm/templates/notifications/items/move_user.html

@@ -14,7 +14,7 @@
 
 {% block description %}
     {% if related_user_moved_to %}
-        {% id_to_username request.user.moved_to  as username %}
+        {% id_to_username related_user_moved_to as username %}
         {% blocktrans trimmed %}
         {{ related_user }} has moved to <a href="{{ related_user_moved_to }}">{{ username }}</a>
         {% endblocktrans %}

bookwyrm/templates/preferences/export-user.html

@@ -46,7 +46,11 @@
             {% trans "If you wish to migrate any statuses (comments, reviews, or quotes) you must either set the account you are moving to as an <strong>alias</strong> of this one, or <strong>move</strong> this account to the new account, before you import your user data." %}
             {% endspaceless %}
         </p>
-    {% if next_available %}
+    {% if not site.user_exports_enabled %}
+    <p class="notification is-danger">
+        {% trans "New user exports are currently disabled." %}
+    </p>
+    {% elif next_available %}
     <p class="notification is-warning">
         {% blocktrans trimmed %}
         You will be able to create a new export file at {{ next_available }}

bookwyrm/templates/settings/imports/imports.html

@@ -90,6 +90,33 @@
             </div>
         </form>
     </details>
+
+    {% if site.user_exports_enabled %}
+    <details class="details-panel box">
+        <summary>
+            <span role="heading" aria-level="2" class="title is-6">
+                {% trans "Disable starting new user exports" %}
+            </span>
+            <span class="details-close icon icon-x" aria-hidden="true"></span>
+        </summary>
+        <form
+            name="disable-user-exports"
+            id="disable-user-exports"
+            method="POST"
+            action="{% url 'settings-user-exports-disable' %}"
+        >
+            <div class="notification">
+                {% trans "This is only intended to be used when things have gone very wrong with exports and you need to pause the feature while addressing issues." %}
+                {% trans "While exports are disabled, users will not be allowed to start new user exports, but existing exports will not be affected." %}
+            </div>
+            {% csrf_token %}
+            <div class="control">
+                <button type="submit" class="button is-danger">
+                    {% trans "Disable user exports" %}
+                </button>
+            </div>
+        </form>
+    </details>
     <details class="details-panel box">
         <summary>
             <span role="heading" aria-level="2" class="title is-6">
@@ -108,7 +135,7 @@
                 {% trans "Set the value to 0 to not enforce any limit." %}
             </div>
             <div class="align.to-t">
-                <label for="limit">{% trans "Restrict user imports and exports to once every " %}</label>
+                <label for="limit">{% trans "Limit how often users can import and export user data" %}</label>
                 <input name="limit" class="input is-w-xs is-h-em" type="text" placeholder="0" value="{{ user_import_time_limit }}">
                 <label>{% trans "hours" %}</label>
                 {% csrf_token %}
@@ -120,6 +147,28 @@
             </div>
         </form>
     </details>
+    {% else %}
+    <form
+        name="enable-user-imports"
+        id="enable-user-imports"
+        method="POST"
+        action="{% url 'settings-user-exports-enable' %}"
+        class="box"
+    >
+        <div class="notification is-danger is-light">
+            <p class="my-2">{% trans "Users are currently unable to start new user exports. This is the default setting." %}</p>
+            {% if use_s3 %}
+            <p>{% trans "It is not currently possible to provide user exports when using s3 storage. The BookWyrm development team are working on a fix for this." %}</p>
+            {% endif %}
+        </div>
+        {% csrf_token %}
+        <div class="control">
+            <button type="submit" class="button is-success" {% if use_s3 %}disabled{% endif %}>
+                {% trans "Enable user exports" %}
+            </button>
+        </div>
+    </form>
+    {% endif %}
 </div>
 <div class="block">
     <h4 class="title is-4">{% trans "Book Imports" %}</h4>

bookwyrm/templates/shelf/shelf.html

@@ -101,7 +101,6 @@
                     {% plural %}
                         {{ formatted_count }} books
                     {% endblocktrans %}
-
                     {% if books.has_other_pages %}
                         {% blocktrans trimmed with start=books.start_index end=books.end_index %}
                         (showing {{ start }}-{{ end }})
@@ -111,6 +110,8 @@
                 {% endif %}
                 {% endwith %}
             </h2>
+            {% include 'shelf/shelves_filters.html' with user=user query=query %}
+
         </div>
         {% if is_self and shelf.id %}
         <div class="column is-narrow">
@@ -209,7 +210,17 @@
             </tbody>
         </table>
         {% else %}
-        <p><em>{% trans "This shelf is empty." %}</em></p>
+        <p>
+            <em>
+                {% if shelves_filter_query %}
+                    {% blocktrans trimmed %}
+                        We couldn't find any books that matched {{ shelves_filter_query }}
+                    {% endblocktrans %}
+                {% else %}
+                    {% trans "This shelf is empty." %}
+                {% endif %}
+            </em>
+        </p>
         {% endif %}
 
     </div>

bookwyrm/templates/shelf/shelves_filter_field.html

@@ -0,0 +1,9 @@
+{% extends 'snippets/filters_panel/filter_field.html' %}
+{% load i18n %}
+
+{% block filter %}
+  <div class="control">
+    <label class="label" for="filter_query">{% trans 'Filter by keyword' %}</label>
+    <input aria-label="Filter by keyword" id="my-books-filter" class="input" type="text" name="filter" placeholder="{% trans 'Enter text here' %}" value="{{ shelves_filter_query|default:'' }}" spellcheck="false" />
+  </div>
+{% endblock %}

bookwyrm/templates/shelf/shelves_filters.html

@@ -0,0 +1,5 @@
+{% extends 'snippets/filters_panel/filters_panel.html' %}
+
+{% block filter_fields %}
+  {% include 'shelf/shelves_filter_field.html' %}
+{% endblock %}

bookwyrm/templates/snippets/opengraph.html

@@ -1,24 +1,25 @@
 {% load static %}
 
-{% if preview_images_enabled is True %}
+{% firstof image site.preview_image as page_image %}
+{% if page_image %}
     <meta name="twitter:card" content="summary_large_image">
-    {% if image %}
-        <meta name="twitter:image" content="{{ media_full_url }}{{ image }}">
-        <meta name="og:image" content="{{ media_full_url }}{{ image }}">
-    {% else %}
-        <meta name="twitter:image" content="{{ media_full_url }}{{ site.preview_image }}">
-        <meta name="og:image" content="{{ media_full_url }}{{ site.preview_image }}">
-    {% endif %}
+    <meta name="twitter:image" content="{{ media_full_url }}{{ page_image }}">
+    <meta name="og:image" content="{{ media_full_url }}{{ page_image }}">
+{% elif site.logo %}
+    <meta name="twitter:card" content="summary">
+    <meta name="twitter:image" content="{{ media_full_url }}{{ site.logo }}">
+    <meta name="twitter:image:alt" content="{{ site.name }} Logo">
+    <meta name="og:image" content="{{ media_full_url }}{{ site.logo }}">
 {% else %}
     <meta name="twitter:card" content="summary">
-    <meta name="twitter:image" content="{% if site.logo %}{{ media_full_url }}{{ site.logo }}{% else %}{% static "images/logo.png" %}{% endif %}">
-    <meta name="og:image" content="{% if site.logo %}{{ media_full_url }}{{ site.logo }}{% else %}{% static "images/logo.png" %}{% endif %}">
+    <meta name="twitter:image" content="{% static "images/logo.png" %}">
+    <meta name="twitter:image:alt" content="BookWyrm Logo">
+    <meta name="og:image" content="{% static "images/logo.png" %}">
 {% endif %}
 
-<meta name="twitter:image:alt" content="BookWyrm Logo">
-
 <meta name="twitter:title" content="{% if title %}{{ title }} - {% endif %}{{ site.name }}">
 <meta name="og:title" content="{% if title %}{{ title }} - {% endif %}{{ site.name }}">
 
-<meta name="twitter:description" content="{% if description %}{{ description }}{% else %}{{ site.instance_tagline }}{% endif %}">
-<meta name="og:description" content="{% if description %}{{ description }}{% else %}{{ site.instance_tagline }}{% endif %}">
+{% firstof description site.instance_tagline as description %}
+<meta name="twitter:description" content="{{ description }}">
+<meta name="og:description" content="{{ description }}">

bookwyrm/templatetags/utilities.py

@@ -125,7 +125,8 @@ def id_to_username(user_id):
         name = parts[-1]
         value = f"{name}@{domain}"
 
-    return value
+        return value
+    return "a new user account"
 
 
 @register.filter(name="get_file_size")

bookwyrm/tests/models/test_activitypub_mixin.py

@@ -227,14 +227,18 @@ class ActivitypubMixins(TestCase):
                 shared_inbox="http://example.com/inbox",
                 outbox="https://example.com/users/nutria/outbox",
             )
-        MockSelf = namedtuple("Self", ("privacy", "user"))
-        mock_self = MockSelf("public", self.local_user)
+        MockSelf = namedtuple("Self", ("privacy", "user", "recipients"))
         self.local_user.followers.add(self.remote_user)
         self.local_user.followers.add(another_remote_user)
 
+        mock_self = MockSelf("public", self.local_user, [])
         recipients = ActivitypubMixin.get_recipients(mock_self)
-        self.assertEqual(len(recipients), 1)
-        self.assertEqual(recipients[0], "http://example.com/inbox")
+        self.assertCountEqual(recipients, ["http://example.com/inbox"])
+
+        # should also work with recipient that is a follower
+        mock_self.recipients.append(another_remote_user)
+        recipients = ActivitypubMixin.get_recipients(mock_self)
+        self.assertCountEqual(recipients, ["http://example.com/inbox"])
 
     def test_get_recipients_software(self, *_):
         """should differentiate between bookwyrm and other remote users"""

bookwyrm/tests/views/preferences/test_export.py

@@ -18,7 +18,9 @@ class ExportViews(TestCase):
     """viewing and creating statuses"""
 
     @classmethod
-    def setUpTestData(self):  # pylint: disable=bad-classmethod-argument
+    def setUpTestData(
+        self,
+    ):  # pylint: disable=bad-classmethod-argument, disable=invalid-name
         """we need basic test data and mocks"""
         with patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"), patch(
             "bookwyrm.activitystreams.populate_stream_task.delay"
@@ -40,6 +42,7 @@ class ExportViews(TestCase):
             bnf_id="beep",
         )
 
+    # pylint: disable=invalid-name
     def setUp(self):
         """individual test setup"""
         self.factory = RequestFactory()
@@ -53,11 +56,12 @@ class ExportViews(TestCase):
 
     def test_export_file(self, *_):
         """simple export"""
-        models.ShelfBook.objects.create(
+        shelfbook = models.ShelfBook.objects.create(
             shelf=self.local_user.shelf_set.first(),
             user=self.local_user,
             book=self.book,
         )
+        book_date = str.encode(f"{shelfbook.shelved_date.date()}")
         request = self.factory.post("")
         request.user = self.local_user
         export = views.Export.as_view()(request)
@@ -66,7 +70,7 @@ class ExportViews(TestCase):
         # pylint: disable=line-too-long
         self.assertEqual(
             export.content,
-            b"title,author_text,remote_id,openlibrary_key,inventaire_id,librarything_key,goodreads_key,bnf_id,viaf,wikidata,asin,aasin,isfdb,isbn_10,isbn_13,oclc_number,rating,review_name,review_cw,review_content\r\nTest Book,,"
-            + self.book.remote_id.encode("utf-8")
-            + b",,,,,beep,,,,,,123456789X,9781234567890,,,,,\r\n",
+            b"title,author_text,remote_id,openlibrary_key,inventaire_id,librarything_key,goodreads_key,bnf_id,viaf,wikidata,asin,aasin,isfdb,isbn_10,isbn_13,oclc_number,start_date,finish_date,stopped_date,rating,review_name,review_cw,review_content,review_published,shelf,shelf_name,shelf_date\r\n"
+            + b"Test Book,,%b,,,,,beep,,,,,,123456789X,9781234567890,,,,,,,,,,to-read,To Read,%b\r\n"
+            % (self.book.remote_id.encode("utf-8"), book_date),
         )

bookwyrm/urls.py

@@ -338,6 +338,16 @@ urlpatterns = [
         views.disable_imports,
         name="settings-imports-disable",
     ),
+    re_path(
+        r"^settings/user-exports/enable/?$",
+        views.enable_user_exports,
+        name="settings-user-exports-enable",
+    ),
+    re_path(
+        r"^settings/user-exports/disable/?$",
+        views.disable_user_exports,
+        name="settings-user-exports-disable",
+    ),
     re_path(
         r"^settings/imports/enable/?$",
         views.enable_imports,

bookwyrm/views/__init__.py

@@ -19,6 +19,8 @@ from .admin.imports import (
     set_import_size_limit,
     set_user_import_completed,
     set_user_import_limit,
+    enable_user_exports,
+    disable_user_exports,
 )
 from .admin.ip_blocklist import IPBlocklist
 from .admin.invite import ManageInvites, Invite, InviteRequest

bookwyrm/views/admin/imports.py

@@ -9,7 +9,7 @@ from django.views.decorators.http import require_POST
 
 from bookwyrm import models
 from bookwyrm.views.helpers import redirect_to_referer
-from bookwyrm.settings import PAGE_LENGTH
+from bookwyrm.settings import PAGE_LENGTH, USE_S3
 
 
 # pylint: disable=no-self-use
@@ -59,6 +59,7 @@ class ImportList(View):
             "import_size_limit": site_settings.import_size_limit,
             "import_limit_reset": site_settings.import_limit_reset,
             "user_import_time_limit": site_settings.user_import_time_limit,
+            "use_s3": USE_S3,
         }
         return TemplateResponse(request, "settings/imports/imports.html", data)
 
@@ -126,3 +127,25 @@ def set_user_import_limit(request):
     site.user_import_time_limit = int(request.POST.get("limit"))
     site.save(update_fields=["user_import_time_limit"])
     return redirect("settings-imports")
+
+
+@require_POST
+@permission_required("bookwyrm.edit_instance_settings", raise_exception=True)
+# pylint: disable=unused-argument
+def enable_user_exports(request):
+    """Allow users to export account data"""
+    site = models.SiteSettings.objects.get()
+    site.user_exports_enabled = True
+    site.save(update_fields=["user_exports_enabled"])
+    return redirect("settings-imports")
+
+
+@require_POST
+@permission_required("bookwyrm.edit_instance_settings", raise_exception=True)
+# pylint: disable=unused-argument
+def disable_user_exports(request):
+    """Don't allow users to export account data"""
+    site = models.SiteSettings.objects.get()
+    site.user_exports_enabled = False
+    site.save(update_fields=["user_exports_enabled"])
+    return redirect("settings-imports")

bookwyrm/views/feed.py

@@ -1,4 +1,5 @@
 """ non-interactive pages """
+from datetime import date
 from django.contrib.auth.decorators import login_required
 from django.core.paginator import Paginator
 from django.db.models import Q
@@ -52,6 +53,19 @@ class Feed(View):
 
         suggestions = suggested_users.get_suggestions(request.user)
 
+        cutoff = (
+            date(get_annual_summary_year(), 12, 31)
+            if get_annual_summary_year()
+            else None
+        )
+        readthroughs = (
+            models.ReadThrough.objects.filter(
+                user=request.user, finish_date__lte=cutoff
+            )
+            if get_annual_summary_year()
+            else []
+        )
+
         data = {
             **feed_page_data(request.user),
             **{
@@ -66,6 +80,7 @@ class Feed(View):
                 "path": f"/{tab['key']}",
                 "annual_summary_year": get_annual_summary_year(),
                 "has_tour": True,
+                "has_summary_read_throughs": len(readthroughs),
             },
         }
         return TemplateResponse(request, "feed/feed.html", data)
@@ -185,19 +200,15 @@ class Status(View):
             params=[status.id, visible_thread, visible_thread],
         )
 
-        preview = None
-        if hasattr(status, "book"):
-            preview = status.book.preview_image
-        elif status.mention_books.exists():
-            preview = status.mention_books.first().preview_image
-
         data = {
             **feed_page_data(request.user),
             **{
                 "status": status,
                 "children": children,
                 "ancestors": ancestors,
-                "preview": preview,
+                "title": status.page_title,
+                "description": status.page_description,
+                "page_image": status.page_image,
             },
         }
         return TemplateResponse(request, "feed/status.html", data)

bookwyrm/views/preferences/export.py

@@ -17,7 +17,8 @@ from bookwyrm import models
 from bookwyrm.models.bookwyrm_export_job import BookwyrmExportJob
 from bookwyrm.settings import PAGE_LENGTH
 
-# pylint: disable=no-self-use
+
+# pylint: disable=no-self-use,too-many-locals
 @method_decorator(login_required, name="dispatch")
 class Export(View):
     """Let users export data"""
@@ -54,7 +55,19 @@ class Export(View):
         fields = (
             ["title", "author_text"]
             + deduplication_fields
-            + ["rating", "review_name", "review_cw", "review_content"]
+            + [
+                "start_date",
+                "finish_date",
+                "stopped_date",
+                "rating",
+                "review_name",
+                "review_cw",
+                "review_content",
+                "review_published",
+                "shelf",
+                "shelf_name",
+                "shelf_date",
+            ]
         )
         writer.writerow(fields)
 
@@ -70,6 +83,24 @@ class Export(View):
 
             book.rating = review_rating.rating if review_rating else None
 
+            readthrough = (
+                models.ReadThrough.objects.filter(user=request.user, book=book)
+                .order_by("-start_date", "-finish_date")
+                .first()
+            )
+            if readthrough:
+                book.start_date = (
+                    readthrough.start_date.date() if readthrough.start_date else None
+                )
+                book.finish_date = (
+                    readthrough.finish_date.date() if readthrough.finish_date else None
+                )
+                book.stopped_date = (
+                    readthrough.stopped_date.date()
+                    if readthrough.stopped_date
+                    else None
+                )
+
             review = (
                 models.Review.objects.filter(
                     user=request.user, book=book, content__isnull=False
@@ -78,9 +109,27 @@ class Export(View):
                 .first()
             )
             if review:
+                book.review_published = (
+                    review.published_date.date() if review.published_date else None
+                )
                 book.review_name = review.name
                 book.review_cw = review.content_warning
-                book.review_content = review.raw_content
+                book.review_content = (
+                    review.raw_content if review.raw_content else review.content
+                )  # GoodReads imported reviews do not have raw_content, but content.
+
+            shelfbook = (
+                models.ShelfBook.objects.filter(user=request.user, book=book)
+                .order_by("-shelved_date", "-created_date", "-updated_date")
+                .last()
+            )
+            if shelfbook:
+                book.shelf = shelfbook.shelf.identifier
+                book.shelf_name = shelfbook.shelf.name
+                book.shelf_date = (
+                    shelfbook.shelved_date.date() if shelfbook.shelved_date else None
+                )
+
             writer.writerow([getattr(book, field, "") or "" for field in fields])
 
         return HttpResponse(

bookwyrm/views/search.py

@@ -51,7 +51,7 @@ class Search(View):
 def api_book_search(request):
     """Return books via API response"""
     query = request.GET.get("q")
-    query = isbn_check(query)
+    query = isbn_check_and_format(query)
     min_confidence = request.GET.get("min_confidence", 0)
     # only return local book results via json so we don't cascade
     book_results = search(query, min_confidence=min_confidence)
@@ -64,7 +64,7 @@ def book_search(request):
     """the real business is elsewhere"""
     query = request.GET.get("q")
     # check if query is isbn
-    query = isbn_check(query)
+    query = isbn_check_and_format(query)
     min_confidence = request.GET.get("min_confidence", 0)
     search_remote = request.GET.get("remote", False) and request.user.is_authenticated
 
@@ -159,7 +159,7 @@ def list_search(request):
     return TemplateResponse(request, "search/list.html", data)
 
 
-def isbn_check(query):
+def isbn_check_and_format(query):
     """isbn10 or isbn13 check, if so remove separators"""
     if query:
         su_num = re.sub(r"(?<=\d)\D(?=\d|[xX])", "", query)

bookwyrm/views/shelf/shelf.py

@@ -15,12 +15,14 @@ from bookwyrm import forms, models
 from bookwyrm.activitypub import ActivitypubResponse
 from bookwyrm.settings import PAGE_LENGTH
 from bookwyrm.views.helpers import is_api_request, get_user_from_username
+from bookwyrm.book_search import search
 
 
 # pylint: disable=no-self-use
 class Shelf(View):
     """shelf page"""
 
+    # pylint: disable=R0914
     def get(self, request, username, shelf_identifier=None):
         """display a shelf"""
         user = get_user_from_username(request.user, username)
@@ -32,6 +34,8 @@ class Shelf(View):
         else:
             shelves = models.Shelf.privacy_filter(request.user).filter(user=user).all()
 
+        shelves_filter_query = request.GET.get("filter")
+
         # get the shelf and make sure the logged in user should be able to see it
         if shelf_identifier:
             shelf = get_object_or_404(user.shelf_set, identifier=shelf_identifier)
@@ -42,6 +46,7 @@ class Shelf(View):
             FakeShelf = namedtuple(
                 "Shelf", ("identifier", "name", "user", "books", "privacy")
             )
+
             books = (
                 models.Edition.viewer_aware_objects(request.user)
                 .filter(
@@ -50,6 +55,7 @@ class Shelf(View):
                 )
                 .distinct()
             )
+
             shelf = FakeShelf("all", _("All books"), user, books, "public")
 
         if is_api_request(request) and shelf_identifier:
@@ -86,6 +92,9 @@ class Shelf(View):
 
         books = sort_books(books, request.GET.get("sort"))
 
+        if shelves_filter_query:
+            books = search(shelves_filter_query, books=books)
+
         paginated = Paginator(
             books,
             PAGE_LENGTH,
@@ -103,6 +112,8 @@ class Shelf(View):
             "page_range": paginated.get_elided_page_range(
                 page.number, on_each_side=2, on_ends=1
             ),
+            "shelves_filter_query": shelves_filter_query,
+            "size": "small",
         }
 
         return TemplateResponse(request, "shelf/shelf.html", data)

bw-dev

@@ -246,6 +246,9 @@ case "$CMD" in
     remove_remote_user_preview_images)
         runweb python manage.py remove_remote_user_preview_images
         ;;
+    erase_deleted_user_data)
+        runweb python manage.py erase_deleted_user_data "$@"
+        ;;
     copy_media_to_s3)
         awscommand "bookwyrm_media_volume:/images"\
             "s3 cp /images s3://${AWS_STORAGE_BUCKET_NAME}/images\
@@ -297,7 +300,7 @@ case "$CMD" in
         echo "Unrecognised command. Try:"
         echo "    setup"
         echo "    up [container]"
-        echo "    down"        
+        echo "    down"
         echo "    service_ports_web"
         echo "    initdb"
         echo "    resetdb"

dev-tools/Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.9
+FROM python:3.9-bookworm
 WORKDIR /app/dev-tools
 
 ENV PATH="/app/dev-tools/node_modules/.bin:$PATH"

nginx/development

@@ -64,13 +64,18 @@ server {
     # directly serve images and static files from the
     # bookwyrm filesystem using sendfile.
     # make the logs quieter by not reporting these requests
-    location ~ ^/(images|static)/ {
+    location ~ \.(bmp|ico|jpg|jpeg|png|svg|tif|tiff|ttf|webp|css|js)$ {
         root /app;
         try_files $uri =404;
         add_header X-Cache-Status STATIC;
         access_log off;
     }
 
+    # block access to any non-image files from images or static
+    location ~ ^/images/ {
+        return 403;
+    }
+
     # monitor the celery queues with flower, no caching enabled
     location /flower/ {
        proxy_pass http://flower:8888;

nginx/production

@@ -96,12 +96,17 @@ server {
 #     # directly serve images and static files from the
 #     # bookwyrm filesystem using sendfile.
 #     # make the logs quieter by not reporting these requests
-#     location ~ ^/(images|static)/ {
+#     location ~ \.(bmp|ico|jpg|jpeg|png|svg|tif|tiff|ttf|webp|css|js)$ {
 #         root /app;
 #         try_files $uri =404;
 #         add_header X-Cache-Status STATIC;
 #         access_log off;
 #     }
+
+#     # block access to any non-image files from images or static
+#     location ~ ^/images/ {
+#          return 403;
+#     }
 #
 #     # monitor the celery queues with flower, no caching enabled
 #     location /flower/ {

redis.conf

@@ -2,6 +2,9 @@ bind 127.0.0.1 ::1
 protected-mode yes
 port 6379
 
+auto-aof-rewrite-percentage 50
+auto-aof-rewrite-min-size 128mb
+
 rename-command FLUSHDB ""
 rename-command FLUSHALL ""
 rename-command DEBUG ""

requirements.txt

@@ -1,4 +1,4 @@
-aiohttp==3.9.0
+aiohttp==3.9.2
 bleach==5.0.1
 celery==5.2.7
 colorthief==0.2.1
@@ -16,7 +16,7 @@ libsass==0.22.0
 Markdown==3.4.1
 Pillow==10.0.1
 psycopg2==2.9.5
-pycryptodome==3.16.0
+pycryptodome==3.19.1
 python-dateutil==2.8.2
 redis==4.5.4
 requests==2.31.0