Merge branch 'main' into images-django-imagekit

.env.dev.example

@@ -3,6 +3,7 @@ SECRET_KEY="7(2w1sedok=aznpq)ta1mc4i%4h=xx@hxwx*o57ctsuml0x%fr"
 
 # SECURITY WARNING: don't run with debug turned on in production!
 DEBUG=true
+USE_HTTPS=false
 
 DOMAIN=your.domain.here
 #EMAIL=your@email.here
@@ -45,6 +46,22 @@ EMAIL_USE_SSL=false
 # Thumbnails Generation
 ENABLE_THUMBNAIL_GENERATION=false
 
+# S3 configuration
+USE_S3=false
+AWS_ACCESS_KEY_ID=
+AWS_SECRET_ACCESS_KEY=
+
+# Commented are example values if you use a non-AWS, S3-compatible service
+# AWS S3 should work with only AWS_STORAGE_BUCKET_NAME and AWS_S3_REGION_NAME
+# non-AWS S3-compatible services will need AWS_STORAGE_BUCKET_NAME,
+# along with both AWS_S3_CUSTOM_DOMAIN and AWS_S3_ENDPOINT_URL
+
+# AWS_STORAGE_BUCKET_NAME=        # "example-bucket-name"
+# AWS_S3_CUSTOM_DOMAIN=None       # "example-bucket-name.s3.fr-par.scw.cloud"
+# AWS_S3_REGION_NAME=None         # "fr-par"
+# AWS_S3_ENDPOINT_URL=None        # "https://s3.fr-par.scw.cloud"
+
+
 # Preview image generation can be computing and storage intensive
 # ENABLE_PREVIEW_IMAGES=True
 

.env.prod.example

@@ -3,6 +3,7 @@ SECRET_KEY="7(2w1sedok=aznpq)ta1mc4i%4h=xx@hxwx*o57ctsuml0x%fr"
 
 # SECURITY WARNING: don't run with debug turned on in production!
 DEBUG=false
+USE_HTTPS=true
 
 DOMAIN=your.domain.here
 EMAIL=your@email.here
@@ -45,6 +46,22 @@ EMAIL_USE_SSL=false
 # Thumbnails Generation
 ENABLE_THUMBNAIL_GENERATION=false
 
+# S3 configuration
+USE_S3=false
+AWS_ACCESS_KEY_ID=
+AWS_SECRET_ACCESS_KEY=
+
+# Commented are example values if you use a non-AWS, S3-compatible service
+# AWS S3 should work with only AWS_STORAGE_BUCKET_NAME and AWS_S3_REGION_NAME
+# non-AWS S3-compatible services will need AWS_STORAGE_BUCKET_NAME,
+# along with both AWS_S3_CUSTOM_DOMAIN and AWS_S3_ENDPOINT_URL
+
+# AWS_STORAGE_BUCKET_NAME=        # "example-bucket-name"
+# AWS_S3_CUSTOM_DOMAIN=None       # "example-bucket-name.s3.fr-par.scw.cloud"
+# AWS_S3_REGION_NAME=None         # "fr-par"
+# AWS_S3_ENDPOINT_URL=None        # "https://s3.fr-par.scw.cloud"
+
+
 # Preview image generation can be computing and storage intensive
 # ENABLE_PREVIEW_IMAGES=True
 

.github/ISSUE_TEMPLATE/bug_report.md

@@ -2,7 +2,7 @@
 name: Bug report
 about: Create a report to help us improve
 title: ''
-labels: ''
+labels: 'bug'
 assignees: ''
 
 ---
@@ -23,6 +23,14 @@ A clear and concise description of what you expected to happen.
 **Screenshots**
 If applicable, add screenshots to help explain your problem.
 
+**Instance**
+On which BookWyrm instance did you encounter this problem.
+
+**Additional context**
+Add any other context about the problem here.
+
+---
+
 **Desktop (please complete the following information):**
     - OS: [e.g. iOS]
     - Browser [e.g. chrome, safari]
@@ -33,6 +41,3 @@ If applicable, add screenshots to help explain your problem.
     - OS: [e.g. iOS8.1]
     - Browser [e.g. stock browser, safari]
     - Version [e.g. 22]
-
-**Additional context**
-Add any other context about the problem here.

.github/workflows/django-tests.yml

@@ -20,7 +20,7 @@ jobs:
 
     services:
       postgres:
-        image: postgres:10
+        image: postgres:12
         env:
           POSTGRES_USER: postgres
           POSTGRES_PASSWORD: hunter2
@@ -66,4 +66,4 @@ jobs:
         EMAIL_USE_TLS: true
         ENABLE_PREVIEW_IMAGES: true
       run: |
-        python manage.py test
+        pytest

bookwyrm/connectors/abstract_connector.py

@@ -2,11 +2,10 @@
 from abc import ABC, abstractmethod
 from dataclasses import asdict, dataclass
 import logging
-from urllib3.exceptions import RequestError
 
 from django.db import transaction
 import requests
-from requests.exceptions import SSLError
+from requests.exceptions import RequestException
 
 from bookwyrm import activitypub, models, settings
 from .connector_manager import load_more_data, ConnectorException
@@ -237,7 +236,7 @@ def get_data(url, params=None, timeout=10):
             },
             timeout=timeout,
         )
-    except (RequestError, SSLError, ConnectionError) as err:
+    except RequestException as err:
         logger.exception(err)
         raise ConnectorException()
 
@@ -262,7 +261,7 @@ def get_image(url, timeout=10):
             },
             timeout=timeout,
         )
-    except (RequestError, SSLError) as err:
+    except RequestException as err:
         logger.exception(err)
         return None
     if not resp.ok:

bookwyrm/connectors/self_connector.py

@@ -2,7 +2,7 @@
 from functools import reduce
 import operator
 
-from django.contrib.postgres.search import SearchRank, SearchVector
+from django.contrib.postgres.search import SearchRank, SearchQuery
 from django.db.models import OuterRef, Subquery, F, Q
 
 from bookwyrm import models
@@ -13,7 +13,7 @@ class Connector(AbstractConnector):
     """instantiate a connector"""
 
     # pylint: disable=arguments-differ
-    def search(self, query, min_confidence=0.1, raw=False, filters=None):
+    def search(self, query, min_confidence=0, raw=False, filters=None):
         """search your local database"""
         filters = filters or []
         if not query:
@@ -141,16 +141,11 @@ def search_identifiers(query, *filters):
 
 def search_title_author(query, min_confidence, *filters):
     """searches for title and author"""
-    vector = (
-        SearchVector("title", weight="A")
-        + SearchVector("subtitle", weight="B")
-        + SearchVector("authors__name", weight="C")
-        + SearchVector("series", weight="D")
-    )
-
+    query = SearchQuery(query, config="simple") | SearchQuery(query, config="english")
     results = (
-        models.Edition.objects.annotate(rank=SearchRank(vector, query))
-        .filter(*filters, rank__gt=min_confidence)
+        models.Edition.objects.filter(*filters, search_vector=query)
+        .annotate(rank=SearchRank(F("search_vector"), query))
+        .filter(rank__gt=min_confidence)
         .order_by("-rank")
     )
 

bookwyrm/context_processors.py

@@ -12,10 +12,7 @@ def site_settings(request):  # pylint: disable=unused-argument
         "site": models.SiteSettings.objects.get(),
         "active_announcements": models.Announcement.active_announcements(),
         "enable_thumbnail_generation": settings.ENABLE_THUMBNAIL_GENERATION,
-        "static_url": settings.STATIC_URL,
-        "media_url": settings.MEDIA_URL,
-        "static_path": settings.STATIC_PATH,
-        "media_path": settings.MEDIA_PATH,
+        "media_full_url": settings.MEDIA_FULL_URL,
         "preview_images_enabled": settings.ENABLE_PREVIEW_IMAGES,
         "request_protocol": request_protocol,
     }

bookwyrm/forms.py

@@ -183,6 +183,7 @@ class EditionForm(CustomForm):
             "parent_work",
             "shelves",
             "connector",
+            "search_vector",
         ]
 
 
@@ -194,6 +195,7 @@ class AuthorForm(CustomForm):
             "origin_id",
             "created_date",
             "updated_date",
+            "search_vector",
         ]
 
 

bookwyrm/importers/importer.py

@@ -2,6 +2,8 @@
 import csv
 import logging
 
+from django.utils import timezone
+
 from bookwyrm import models
 from bookwyrm.models import ImportJob, ImportItem
 from bookwyrm.tasks import app
@@ -100,7 +102,10 @@ def handle_imported_book(source, user, item, include_reviews, privacy):
     # shelve the book if it hasn't been shelved already
     if item.shelf and not existing_shelf:
         desired_shelf = models.Shelf.objects.get(identifier=item.shelf, user=user)
-        models.ShelfBook.objects.create(book=item.book, shelf=desired_shelf, user=user)
+        shelved_date = item.date_added or timezone.now()
+        models.ShelfBook.objects.create(
+            book=item.book, shelf=desired_shelf, user=user, shelved_date=shelved_date
+        )
 
     for read in item.reads:
         # check for an existing readthrough with the same dates

bookwyrm/migrations/0077_auto_20210623_2155.py

@@ -0,0 +1,126 @@
+# Generated by Django 3.2.4 on 2021-06-23 21:55
+
+import django.contrib.postgres.indexes
+import django.contrib.postgres.search
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0076_preview_images"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="author",
+            name="search_vector",
+            field=django.contrib.postgres.search.SearchVectorField(null=True),
+        ),
+        migrations.AddField(
+            model_name="book",
+            name="search_vector",
+            field=django.contrib.postgres.search.SearchVectorField(null=True),
+        ),
+        migrations.AddIndex(
+            model_name="author",
+            index=django.contrib.postgres.indexes.GinIndex(
+                fields=["search_vector"], name="bookwyrm_au_search__b050a8_gin"
+            ),
+        ),
+        migrations.AddIndex(
+            model_name="book",
+            index=django.contrib.postgres.indexes.GinIndex(
+                fields=["search_vector"], name="bookwyrm_bo_search__51beb3_gin"
+            ),
+        ),
+        migrations.RunSQL(
+            sql="""
+                CREATE FUNCTION book_trigger() RETURNS trigger AS $$
+                begin
+                    new.search_vector :=
+                        coalesce(
+                            NULLIF(setweight(to_tsvector('english', coalesce(new.title, '')), 'A'), ''),
+                            setweight(to_tsvector('simple', coalesce(new.title, '')), 'A')
+                        ) ||
+                        setweight(to_tsvector('english', coalesce(new.subtitle, '')), 'B') ||
+                        (SELECT setweight(to_tsvector('simple', coalesce(array_to_string(array_agg(bookwyrm_author.name), ' '), '')), 'C')
+                            FROM bookwyrm_book
+                            LEFT OUTER JOIN bookwyrm_book_authors
+                            ON bookwyrm_book.id = bookwyrm_book_authors.book_id
+                            LEFT OUTER JOIN bookwyrm_author
+                            ON bookwyrm_book_authors.author_id = bookwyrm_author.id
+                            WHERE bookwyrm_book.id = new.id
+                        ) ||
+                        setweight(to_tsvector('english', coalesce(new.series, '')), 'D');
+                    return new;
+                end
+                $$ LANGUAGE plpgsql;
+
+                CREATE TRIGGER search_vector_trigger
+                BEFORE INSERT OR UPDATE OF title, subtitle, series, search_vector
+                ON bookwyrm_book
+                FOR EACH ROW EXECUTE FUNCTION book_trigger();
+
+                UPDATE bookwyrm_book SET search_vector = NULL;
+            """,
+            reverse_sql="""
+                DROP TRIGGER IF EXISTS search_vector_trigger
+                ON bookwyrm_book;
+                DROP FUNCTION IF EXISTS book_trigger;
+            """,
+        ),
+        # when an author is edited
+        migrations.RunSQL(
+            sql="""
+                CREATE FUNCTION author_trigger() RETURNS trigger AS $$
+                begin
+                    WITH book AS (
+                        SELECT bookwyrm_book.id as row_id
+                        FROM bookwyrm_author
+                        LEFT OUTER JOIN bookwyrm_book_authors
+                        ON bookwyrm_book_authors.id = new.id
+                        LEFT OUTER JOIN bookwyrm_book
+                        ON bookwyrm_book.id = bookwyrm_book_authors.book_id
+                    )
+                    UPDATE bookwyrm_book SET search_vector = ''
+                    FROM book
+                    WHERE id = book.row_id;
+                    return new;
+                end
+                $$ LANGUAGE plpgsql;
+
+                CREATE TRIGGER author_search_vector_trigger
+                AFTER UPDATE OF name
+                ON bookwyrm_author
+                FOR EACH ROW EXECUTE FUNCTION author_trigger();
+            """,
+            reverse_sql="""
+                DROP TRIGGER IF EXISTS author_search_vector_trigger
+                ON bookwyrm_author;
+                DROP FUNCTION IF EXISTS author_trigger;
+            """,
+        ),
+        # when an author is added to or removed from a book
+        migrations.RunSQL(
+            sql="""
+                CREATE FUNCTION book_authors_trigger() RETURNS trigger AS $$
+                begin
+                    UPDATE bookwyrm_book SET search_vector = ''
+                    WHERE id = coalesce(new.book_id, old.book_id);
+                    return new;
+                end
+                $$ LANGUAGE plpgsql;
+
+                CREATE TRIGGER book_authors_search_vector_trigger
+                AFTER INSERT OR DELETE
+                ON bookwyrm_book_authors
+                FOR EACH ROW EXECUTE FUNCTION book_authors_trigger();
+            """,
+            reverse_sql="""
+                DROP TRIGGER IF EXISTS book_authors_search_vector_trigger
+                ON bookwyrm_book_authors;
+                DROP FUNCTION IF EXISTS book_authors_trigger;
+            """,
+        ),
+    ]

bookwyrm/migrations/0078_add_shelved_date.py

@@ -0,0 +1,34 @@
+# Generated by Django 3.2.4 on 2021-07-03 08:25
+
+from django.db import migrations, models
+import django.utils.timezone
+
+
+def copy_created_date(app_registry, schema_editor):
+    db_alias = schema_editor.connection.alias
+    ShelfBook = app_registry.get_model("bookwyrm", "ShelfBook")
+    ShelfBook.objects.all().update(shelved_date=models.F("created_date"))
+
+
+def do_nothing(app_registry, schema_editor):
+    pass
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0077_auto_20210623_2155"),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name="shelfbook",
+            options={"ordering": ("-shelved_date",)},
+        ),
+        migrations.AddField(
+            model_name="shelfbook",
+            name="shelved_date",
+            field=models.DateTimeField(default=django.utils.timezone.now),
+        ),
+        migrations.RunPython(copy_created_date, reverse_code=do_nothing),
+    ]

bookwyrm/models/activitypub_mixin.py

@@ -30,6 +30,7 @@ logger = logging.getLogger(__name__)
 PropertyField = namedtuple("PropertyField", ("set_activity_from_field"))
 
 
+# pylint: disable=invalid-name
 def set_activity_from_property_field(activity, obj, field):
     """assign a model property value to the activity json"""
     activity[field[1]] = getattr(obj, field[0])
@@ -318,7 +319,9 @@ class OrderedCollectionPageMixin(ObjectMixin):
 
         remote_id = remote_id or self.remote_id
         if page:
-            return to_ordered_collection_page(queryset, remote_id, **kwargs)
+            if isinstance(page, list) and len(page) > 0:
+                page = page[0]
+            return to_ordered_collection_page(queryset, remote_id, page=page, **kwargs)
 
         if collection_only or not hasattr(self, "activity_serializer"):
             serializer = activitypub.OrderedCollection

bookwyrm/models/author.py

@@ -1,4 +1,5 @@
 """ database schema for info about authors """
+from django.contrib.postgres.indexes import GinIndex
 from django.db import models
 
 from bookwyrm import activitypub
@@ -37,3 +38,8 @@ class Author(BookDataModel):
         return "https://%s/author/%s" % (DOMAIN, self.id)
 
     activity_serializer = activitypub.Author
+
+    class Meta:
+        """sets up postgres GIN index field"""
+
+        indexes = (GinIndex(fields=["search_vector"]),)

bookwyrm/models/book.py

@@ -1,6 +1,8 @@
 """ database schema for books and shelves """
 import re
 
+from django.contrib.postgres.search import SearchVectorField
+from django.contrib.postgres.indexes import GinIndex
 from django.db import models
 from django.dispatch import receiver
 from model_utils import FieldTracker
@@ -40,6 +42,7 @@ class BookDataModel(ObjectMixin, BookWyrmModel):
     bnf_id = fields.CharField(  # Bibliothèque nationale de France
         max_length=255, blank=True, null=True, deduplication_field=True
     )
+    search_vector = SearchVectorField(null=True)
 
     last_edited_by = fields.ForeignKey(
         "User",
@@ -182,6 +185,11 @@ class Book(BookDataModel):
             self.title,
         )
 
+    class Meta:
+        """sets up postgres GIN index field"""
+
+        indexes = (GinIndex(fields=["search_vector"]),)
+
 
 class Work(OrderedCollectionPageMixin, Book):
     """a work (an abstract concept of a book that manifests in an edition)"""

bookwyrm/models/fields.py

@@ -408,7 +408,8 @@ class ImageField(ActivitypubFieldMixin, models.ImageField):
             return None
 
         image_content = ContentFile(response.content)
-        image_name = str(uuid4()) + "." + imghdr.what(None, image_content.read())
+        extension = imghdr.what(None, image_content.read()) or ""
+        image_name = "{:s}.{:s}".format(str(uuid4()), extension)
         return [image_name, image_content]
 
     def formfield(self, **kwargs):

bookwyrm/models/shelf.py

@@ -1,6 +1,7 @@
 """ puttin' books on shelves """
 import re
 from django.db import models
+from django.utils import timezone
 
 from bookwyrm import activitypub
 from .activitypub_mixin import CollectionItemMixin, OrderedCollectionMixin
@@ -69,6 +70,7 @@ class ShelfBook(CollectionItemMixin, BookWyrmModel):
         "Edition", on_delete=models.PROTECT, activitypub_field="book"
     )
     shelf = models.ForeignKey("Shelf", on_delete=models.PROTECT)
+    shelved_date = models.DateTimeField(default=timezone.now)
     user = fields.ForeignKey(
         "User", on_delete=models.PROTECT, activitypub_field="actor"
     )
@@ -86,4 +88,4 @@ class ShelfBook(CollectionItemMixin, BookWyrmModel):
         you can't put a book on shelf twice"""
 
         unique_together = ("book", "shelf")
-        ordering = ("-created_date",)
+        ordering = ("-shelved_date", "-created_date", "-updated_date")

bookwyrm/preview_images.py

@@ -11,6 +11,7 @@ from PIL import Image, ImageDraw, ImageFont, ImageOps, ImageColor
 
 from django.core.files.base import ContentFile
 from django.core.files.uploadedfile import InMemoryUploadedFile
+from django.core.files.storage import default_storage
 from django.db.models import Avg
 
 from bookwyrm import models, settings
@@ -319,9 +320,9 @@ def save_and_cleanup(image, instance=None):
 
     try:
         try:
-            old_path = instance.preview_image.path
+            old_path = instance.preview_image.name
         except ValueError:
-            old_path = ""
+            old_path = None
 
         # Save
         image.save(image_buffer, format="jpeg", quality=75)
@@ -342,8 +343,8 @@ def save_and_cleanup(image, instance=None):
             instance.save()
 
         # Clean up old file after saving
-        if os.path.exists(old_path):
-            os.remove(old_path)
+        if old_path and default_storage.exists(old_path):
+            default_storage.delete(old_path)
 
     finally:
         image_buffer.close()

bookwyrm/settings.py

@@ -58,6 +58,7 @@ SECRET_KEY = env("SECRET_KEY")
 
 # SECURITY WARNING: don't run with debug turned on in production!
 DEBUG = env.bool("DEBUG", True)
+USE_HTTPS = env.bool("USE_HTTPS", False)
 
 ALLOWED_HOSTS = env.list("ALLOWED_HOSTS", ["*"])
 
@@ -75,6 +76,7 @@ INSTALLED_APPS = [
     "bookwyrm",
     "celery",
     "imagekit",
+    "storages",
 ]
 
 MIDDLEWARE = [
@@ -167,6 +169,7 @@ LANGUAGES = [
     ("es", _("Spanish")),
     ("fr-fr", _("French")),
     ("zh-hans", _("Simplified Chinese")),
+    ("zh-hant", _("Traditional Chinese")),
 ]
 
 
@@ -179,17 +182,6 @@ USE_L10N = True
 USE_TZ = True
 
 
-# Static files (CSS, JavaScript, Images)
-# https://docs.djangoproject.com/en/3.2/howto/static-files/
-
-PROJECT_DIR = os.path.dirname(os.path.abspath(__file__))
-STATIC_URL = "/static/"
-STATIC_PATH = "%s/%s" % (DOMAIN, env("STATIC_ROOT", "static"))
-STATIC_ROOT = os.path.join(BASE_DIR, env("STATIC_ROOT", "static"))
-MEDIA_URL = "/images/"
-MEDIA_PATH = "%s/%s" % (DOMAIN, env("MEDIA_ROOT", "images"))
-MEDIA_ROOT = os.path.join(BASE_DIR, env("MEDIA_ROOT", "images"))
-
 USER_AGENT = "%s (BookWyrm/%s; +https://%s/)" % (
     requests.utils.default_user_agent(),
     VERSION,
@@ -199,3 +191,45 @@ USER_AGENT = "%s (BookWyrm/%s; +https://%s/)" % (
 # Imagekit generated thumbnails
 ENABLE_THUMBNAIL_GENERATION = env.bool("ENABLE_THUMBNAIL_GENERATION", False)
 IMAGEKIT_CACHEFILE_DIR = "thumbnails"
+
+# Static files (CSS, JavaScript, Images)
+# https://docs.djangoproject.com/en/3.2/howto/static-files/
+
+PROJECT_DIR = os.path.dirname(os.path.abspath(__file__))
+
+# Storage
+
+PROTOCOL = "http"
+if USE_HTTPS:
+    PROTOCOL = "https"
+
+USE_S3 = env.bool("USE_S3", False)
+
+if USE_S3:
+    # AWS settings
+    AWS_ACCESS_KEY_ID = env("AWS_ACCESS_KEY_ID")
+    AWS_SECRET_ACCESS_KEY = env("AWS_SECRET_ACCESS_KEY")
+    AWS_STORAGE_BUCKET_NAME = env("AWS_STORAGE_BUCKET_NAME")
+    AWS_S3_CUSTOM_DOMAIN = env("AWS_S3_CUSTOM_DOMAIN")
+    AWS_S3_REGION_NAME = env("AWS_S3_REGION_NAME", "")
+    AWS_S3_ENDPOINT_URL = env("AWS_S3_ENDPOINT_URL")
+    AWS_DEFAULT_ACL = "public-read"
+    AWS_S3_OBJECT_PARAMETERS = {"CacheControl": "max-age=86400"}
+    # S3 Static settings
+    STATIC_LOCATION = "static"
+    STATIC_URL = "https://%s/%s/" % (AWS_S3_CUSTOM_DOMAIN, STATIC_LOCATION)
+    STATICFILES_STORAGE = "bookwyrm.storage_backends.StaticStorage"
+    # S3 Media settings
+    MEDIA_LOCATION = "images"
+    MEDIA_URL = "https://%s/%s/" % (AWS_S3_CUSTOM_DOMAIN, MEDIA_LOCATION)
+    MEDIA_FULL_URL = MEDIA_URL
+    DEFAULT_FILE_STORAGE = "bookwyrm.storage_backends.ImagesStorage"
+    # I don't know if it's used, but the site crashes without it
+    STATIC_ROOT = os.path.join(BASE_DIR, env("STATIC_ROOT", "static"))
+    MEDIA_ROOT = os.path.join(BASE_DIR, env("MEDIA_ROOT", "images"))
+else:
+    STATIC_URL = "/static/"
+    STATIC_ROOT = os.path.join(BASE_DIR, env("STATIC_ROOT", "static"))
+    MEDIA_URL = "/images/"
+    MEDIA_FULL_URL = "%s://%s%s" % (PROTOCOL, DOMAIN, MEDIA_URL)
+    MEDIA_ROOT = os.path.join(BASE_DIR, env("MEDIA_ROOT", "images"))

bookwyrm/static/css/bookwyrm.css

@@ -72,6 +72,14 @@ body {
     flex-grow: 1;
 }
 
+.preserve-whitespace p {
+    white-space: pre-wrap !important;
+}
+
+.display-inline p {
+    display: inline !important;
+}
+
 /** Shelving
  ******************************************************************************/
 

bookwyrm/storage_backends.py

@@ -0,0 +1,17 @@
+"""Handles backends for storages"""
+from storages.backends.s3boto3 import S3Boto3Storage
+
+
+class StaticStorage(S3Boto3Storage):  # pylint: disable=abstract-method
+    """Storage class for Static contents"""
+
+    location = "static"
+    default_acl = "public-read"
+
+
+class ImagesStorage(S3Boto3Storage):  # pylint: disable=abstract-method
+    """Storage class for Image files"""
+
+    location = "images"
+    default_acl = "public-read"
+    file_overwrite = False

bookwyrm/tasks.py

@@ -7,6 +7,5 @@ from bookwyrm import settings
 # set the default Django settings module for the 'celery' program.
 os.environ.setdefault("DJANGO_SETTINGS_MODULE", "celerywyrm.settings")
 app = Celery(
-    "tasks",
-    broker=settings.CELERY_BROKER,
+    "tasks", broker=settings.CELERY_BROKER, backend=settings.CELERY_RESULT_BACKEND
 )

bookwyrm/templates/book/book.html

@@ -1,5 +1,10 @@
 {% extends 'layout.html' %}
-{% load i18n %}{% load bookwyrm_tags %}{% load humanize %}{% load utilities %}{% load layout %}
+{% load i18n %}
+{% load bookwyrm_tags %}
+{% load humanize %}
+{% load utilities %}
+{% load static %}
+{% load layout %}
 
 {% block title %}{{ book|book_title }}{% endblock %}
 
@@ -47,7 +52,7 @@
 
         {% if user_authenticated and can_edit_book %}
         <div class="column is-narrow">
-            <a href="{{ book.id }}/edit">
+            <a href="{% url 'edit-book' book.id %}">
                 <span class="icon icon-pencil" title="{% trans "Edit Book" %}" aria-hidden=True></span>
                 <span class="is-hidden-mobile">{% trans "Edit Book" %}</span>
             </a>
@@ -209,24 +214,24 @@
                     <ul>
                         {% url 'book' book.id as tab_url %}
                         <li {% if tab_url == request.path %}class="is-active"{% endif %}>
-                            <a href="{{ tab_url }}">{% trans "Reviews" %} ({{ review_count }})</a>
+                            <a href="{{ tab_url }}#reviews">{% trans "Reviews" %} ({{ review_count }})</a>
                         </li>
                         {% if user_statuses.review_count %}
                         {% url 'book-user-statuses' book.id 'review' as tab_url %}
                         <li {% if tab_url == request.path %}class="is-active"{% endif %}>
-                            <a href="{{ tab_url }}">{% trans "Your reviews" %} ({{ user_statuses.review_count }})</a>
+                            <a href="{{ tab_url }}#reviews">{% trans "Your reviews" %} ({{ user_statuses.review_count }})</a>
                         </li>
                         {% endif %}
                         {% if user_statuses.comment_count %}
                         {% url 'book-user-statuses' book.id 'comment' as tab_url %}
                         <li {% if tab_url == request.path %}class="is-active"{% endif %}>
-                            <a href="{{ tab_url }}">{% trans "Your comments" %} ({{ user_statuses.comment_count }})</a>
+                            <a href="{{ tab_url }}#reviews">{% trans "Your comments" %} ({{ user_statuses.comment_count }})</a>
                         </li>
                         {% endif %}
                         {% if user_statuses.quotation_count %}
                         {% url 'book-user-statuses' book.id 'quote' as tab_url %}
                         <li {% if tab_url == request.path %}class="is-active"{% endif %}>
-                            <a href="{{ tab_url }}">{% trans "Your quotes" %} ({{ user_statuses.quotation_count }})</a>
+                            <a href="{{ tab_url }}#reviews">{% trans "Your quotes" %} ({{ user_statuses.quotation_count }})</a>
                         </li>
                         {% endif %}
                     </ul>
@@ -321,5 +326,5 @@
 {% endblock %}
 
 {% block scripts %}
-<script src="/static/js/vendor/tabs.js"></script>
+<script src="{% static "js/vendor/tabs.js" %}"></script>
 {% endblock %}

bookwyrm/templates/directory/user_card.html

@@ -18,7 +18,7 @@
             </div>
         </div>
 
-        <div>
+        <div class="display-inline">
         {% if user.summary %}
             {{ user.summary|to_markdown|safe|truncatechars_html:40 }}
         {% else %}&nbsp;{% endif %}

bookwyrm/templates/feed/direct_messages.html

@@ -1,4 +1,4 @@
-{% extends 'feed/feed_layout.html' %}
+{% extends 'feed/layout.html' %}
 {% load i18n %}
 {% block panel %}
 

bookwyrm/templates/feed/feed.html

@@ -1,4 +1,4 @@
-{% extends 'feed/feed_layout.html' %}
+{% extends 'feed/layout.html' %}
 {% load i18n %}
 
 {% block panel %}

bookwyrm/templates/feed/feed_layout.html

@@ -1,107 +0,0 @@
-{% extends 'layout.html' %}
-{% load i18n %}
-
-{% block title %}{% trans "Updates" %}{% endblock %}
-
-{% block content %}
-<div class="columns">
-    {% if user.is_authenticated %}
-    <div class="column is-one-third">
-        <h2 class="title is-5">{% trans "Your books" %}</h2>
-        {% if not suggested_books %}
-        <p>{% trans "There are no books here right now! Try searching for a book to get started" %}</p>
-        {% else %}
-        {% with active_book=request.GET.book %}
-        <div class="tab-group">
-            <div class="tabs is-small">
-                <ul role="tablist">
-                {% for shelf in suggested_books %}
-                {% if shelf.books %}
-                {% with shelf_counter=forloop.counter %}
-                    <li>
-                        <p>
-                            {% if shelf.identifier == 'to-read' %}{% trans "To Read" %}
-                            {% elif shelf.identifier == 'reading' %}{% trans "Currently Reading" %}
-                            {% elif shelf.identifier == 'read' %}{% trans "Read" %}
-                            {% else %}{{ shelf.name }}{% endif %}
-                        </p>
-                        <div class="tabs is-small is-toggle">
-                            <ul>
-                                {% for book in shelf.books %}
-                                <li class="{% if active_book == book.id|stringformat:'d' %}is-active{% elif not active_book and shelf_counter == 1 and forloop.first %}is-active{% endif %}">
-                                    <a
-                                        href="{{ request.path }}?book={{ book.id }}"
-                                        id="tab-book-{{ book.id }}"
-                                        role="tab"
-                                        aria-label="{{ book.title }}"
-                                        aria-selected="{% if active_book == book.id|stringformat:'d' %}true{% elif shelf_counter == 1 and forloop.first %}true{% else %}false{% endif %}"
-                                        aria-controls="book-{{ book.id }}">
-                                        {% include 'snippets/book_cover.html' with book=book cover_class='is-h-m' %}
-                                    </a>
-                                </li>
-                                {% endfor %}
-                            </ul>
-                        </div>
-                    </li>
-                {% endwith %}
-                {% endif %}
-                {% endfor %}
-                </ul>
-            </div>
-            {% for shelf in suggested_books %}
-            {% with shelf_counter=forloop.counter %}
-            {% for book in shelf.books %}
-            <div
-                class="suggested-tabs card"
-                role="tabpanel"
-                id="book-{{ book.id }}"
-                {% if active_book and active_book == book.id|stringformat:'d' %}{% elif not active_book and shelf_counter == 1 and forloop.first %}{% else %} hidden{% endif %}
-                aria-labelledby="tab-book-{{ book.id }}">
-
-                <div class="card-header">
-                    <div class="card-header-title">
-                        <div>
-                            <p class="mb-2">{% include 'snippets/book_titleby.html' with book=book %}</p>
-                            {% include 'snippets/shelve_button/shelve_button.html' with book=book %}
-                        </div>
-                    </div>
-                    <div class="card-header-icon is-hidden-tablet">
-                        {% trans "Close" as button_text %}
-                        {% include 'snippets/toggle/toggle_button.html' with label=button_text controls_text="book" controls_uid=book.id class="delete" nonbutton=True pressed=True %}
-                    </div>
-                </div>
-                <div class="card-content">
-                {% include 'snippets/create_status.html' with book=book %}
-                </div>
-            </div>
-            {% endfor %}
-            {% endwith %}
-            {% endfor %}
-        </div>
-        {% endwith %}
-        {% endif %}
-
-        {% if goal %}
-        <section class="section">
-            <div class="block">
-                <h3 class="title is-4">{% blocktrans with yar=goal.year %}{{ year }} Reading Goal{% endblocktrans %}</h3>
-                {% include 'snippets/goal_progress.html' with goal=goal %}
-            </div>
-        </section>
-        {% endif %}
-    </div>
-    {% endif %}
-
-    <div class="column is-two-thirds" id="feed">
-        {% block panel %}{% endblock %}
-
-        {% if activities %}
-        {% include 'snippets/pagination.html' with page=activities path=path anchor="#feed" %}
-        {% endif %}
-    </div>
-</div>
-{% endblock %}
-
-{% block scripts %}
-<script src="/static/js/vendor/tabs.js"></script>
-{% endblock %}

bookwyrm/templates/feed/layout.html

@@ -0,0 +1,110 @@
+{% extends 'layout.html' %}
+{% load i18n %}
+{% load static %}
+
+{% block title %}{% trans "Updates" %}{% endblock %}
+
+{% block content %}
+<div class="columns">
+    {% if user.is_authenticated %}
+    <div class="column is-one-third">
+        <section class="block">
+            <h2 class="title is-4">{% trans "Your books" %}</h2>
+            {% if not suggested_books %}
+            <p>{% trans "There are no books here right now! Try searching for a book to get started" %}</p>
+            {% else %}
+            {% with active_book=request.GET.book %}
+            <div class="tab-group">
+                <div class="tabs is-small">
+                    <ul role="tablist">
+                    {% for shelf in suggested_books %}
+                    {% if shelf.books %}
+                    {% with shelf_counter=forloop.counter %}
+                        <li>
+                            <p>
+                                {% if shelf.identifier == 'to-read' %}{% trans "To Read" %}
+                                {% elif shelf.identifier == 'reading' %}{% trans "Currently Reading" %}
+                                {% elif shelf.identifier == 'read' %}{% trans "Read" %}
+                                {% else %}{{ shelf.name }}{% endif %}
+                            </p>
+                            <div class="tabs is-small is-toggle">
+                                <ul>
+                                    {% for book in shelf.books %}
+                                    <li class="{% if active_book == book.id|stringformat:'d' %}is-active{% elif not active_book and shelf_counter == 1 and forloop.first %}is-active{% endif %}">
+                                        <a
+                                            href="{{ request.path }}?book={{ book.id }}"
+                                            id="tab-book-{{ book.id }}"
+                                            role="tab"
+                                            aria-label="{{ book.title }}"
+                                            aria-selected="{% if active_book == book.id|stringformat:'d' %}true{% elif shelf_counter == 1 and forloop.first %}true{% else %}false{% endif %}"
+                                            aria-controls="book-{{ book.id }}">
+                                            {% include 'snippets/book_cover.html' with book=book cover_class='is-h-m' %}
+                                        </a>
+                                    </li>
+                                    {% endfor %}
+                                </ul>
+                            </div>
+                        </li>
+                    {% endwith %}
+                    {% endif %}
+                    {% endfor %}
+                    </ul>
+                </div>
+                {% for shelf in suggested_books %}
+                {% with shelf_counter=forloop.counter %}
+                {% for book in shelf.books %}
+                <div
+                    class="suggested-tabs card"
+                    role="tabpanel"
+                    id="book-{{ book.id }}"
+                    {% if active_book and active_book == book.id|stringformat:'d' %}{% elif not active_book and shelf_counter == 1 and forloop.first %}{% else %} hidden{% endif %}
+                    aria-labelledby="tab-book-{{ book.id }}">
+
+                    <div class="card-header">
+                        <div class="card-header-title">
+                            <div>
+                                <p class="mb-2">{% include 'snippets/book_titleby.html' with book=book %}</p>
+                                {% include 'snippets/shelve_button/shelve_button.html' with book=book %}
+                            </div>
+                        </div>
+                        <div class="card-header-icon is-hidden-tablet">
+                            {% trans "Close" as button_text %}
+                            {% include 'snippets/toggle/toggle_button.html' with label=button_text controls_text="book" controls_uid=book.id class="delete" nonbutton=True pressed=True %}
+                        </div>
+                    </div>
+                    <div class="card-content">
+                    {% include 'snippets/create_status.html' with book=book %}
+                    </div>
+                </div>
+                {% endfor %}
+                {% endwith %}
+                {% endfor %}
+            </div>
+            {% endwith %}
+            {% endif %}
+        </section>
+
+        {% if goal %}
+        <section class="block">
+            <div class="block">
+                <h3 class="title is-4">{% blocktrans with yar=goal.year %}{{ year }} Reading Goal{% endblocktrans %}</h3>
+                {% include 'snippets/goal_progress.html' with goal=goal %}
+            </div>
+        </section>
+        {% endif %}
+    </div>
+    {% endif %}
+
+    <div class="column is-two-thirds" id="feed">
+        {% block panel %}{% endblock %}
+
+        {% if activities %}
+        {% include 'snippets/pagination.html' with page=activities path=path anchor="#feed" %}
+        {% endif %}
+    </div>
+</div>
+{% endblock %}
+
+{% block scripts %}
+<script src="{% static "js/vendor/tabs.js" %}"></script>
+{% endblock %}

bookwyrm/templates/feed/status.html

@@ -1,4 +1,4 @@
-{% extends 'feed/feed_layout.html' %}
+{% extends 'feed/layout.html' %}
 {% load i18n %}
 
 {% block panel %}

bookwyrm/templates/get_started/layout.html

@@ -1,5 +1,6 @@
 {% extends 'layout.html' %}
 {% load i18n %}
+{% load static %}
 
 {% block title %}{% trans "Welcome" %}{% endblock %}
 
@@ -9,7 +10,7 @@
     <div class="modal-background"></div>
     <div class="modal-card is-fullwidth">
         <header class="modal-card-head">
-            <img class="image logo mr-2" src="{% if site.logo_small %}/images/{{ site.logo_small }}{% else %}/static/images/logo-small.png{% endif %}" aria-hidden="true">
+            <img class="image logo mr-2" src="{% if site.logo_small %}{% get_media_prefix %}{{ site.logo_small }}{% else %}{% static "images/logo-small.png" %}{% endif %}" aria-hidden="true">
             <h1 class="modal-card-title" id="get-started-header">
                 {% blocktrans %}Welcome to {{ site_name }}!{% endblocktrans %}
                 <span class="subtitle is-block">

bookwyrm/templates/import_status.html

@@ -1,6 +1,7 @@
 {% extends 'layout.html' %}
 {% load i18n %}
 {% load humanize %}
+{% load static %}
 
 {% block title %}{% trans "Import Status" %}{% endblock %}
 
@@ -156,5 +157,5 @@
 {% endspaceless %}{% endblock %}
 
 {% block scripts %}
-<script src="/static/js/check_all.js"></script>
+<script src="{% static "js/check_all.js" %}"></script>
 {% endblock %}

bookwyrm/templates/layout.html

@@ -1,14 +1,16 @@
-{% load layout %}{% load i18n %}
+{% load layout %}
+{% load i18n %}
+{% load static %}
 <!DOCTYPE html>
 <html lang="{% get_lang %}">
 <head>
     <title>{% block title %}BookWyrm{% endblock %} | {{ site.name }}</title>
     <meta name="viewport" content="width=device-width, initial-scale=1">
-    <link rel="stylesheet" href="/static/css/vendor/bulma.min.css">
-    <link rel="stylesheet" href="/static/css/vendor/icons.css">
-    <link rel="stylesheet" href="/static/css/bookwyrm.css">
+    <link rel="stylesheet" href="{% static "css/vendor/bulma.min.css" %}">
+    <link rel="stylesheet" href="{% static "css/vendor/icons.css" %}">
+    <link rel="stylesheet" href="{% static "css/bookwyrm.css" %}">
 
-    <link rel="shortcut icon" type="image/x-icon" href="{% if site.favicon %}{{ media_url }}{{ site.favicon }}{% else %}/static/images/favicon.ico{% endif %}">
+    <link rel="shortcut icon" type="image/x-icon" href="{% if site.favicon %}{% get_media_prefix %}{{ site.favicon }}{% else %}{% static "images/favicon.ico" %}{% endif %}">
 
     {% if preview_images_enabled is True %}
     <meta name="twitter:card" content="summary_large_image">
@@ -30,7 +32,7 @@
     <div class="container">
         <div class="navbar-brand">
             <a class="navbar-item" href="/">
-                <img class="image logo" src="{% if site.logo_small %}{{ media_url }}{{ site.logo_small }}{% else %}/static/images/logo-small.png{% endif %}" alt="Home page">
+                <img class="image logo" src="{% if site.logo_small %}{% get_media_prefix %}{{ site.logo_small }}{% else %}{% static "images/logo-small.png" %}{% endif %}" alt="Home page">
             </a>
             <form class="navbar-item column" action="/search/">
                 <div class="field has-addons">
@@ -242,8 +244,8 @@
 <script>
     var csrf_token = '{{ csrf_token }}';
 </script>
-<script src="/static/js/bookwyrm.js"></script>
-<script src="/static/js/localstorage.js"></script>
+<script src="{% static "js/bookwyrm.js" %}"></script>
+<script src="{% static "js/localstorage.js" %}"></script>
 {% block scripts %}{% endblock %}
 </body>
 </html>

bookwyrm/templates/snippets/about.html

@@ -1,7 +1,9 @@
+{% load static %}
+
 <div class="columns">
     <div class="column is-narrow is-hidden-mobile">
-        <figure class="block">
-            <img src="{% if site.logo %}/images/{{ site.logo }}{% else %}/static/images/logo.png{% endif %}" alt="BookWyrm logo">
+        <figure class="block is-w-xl">
+            <img src="{% if site.logo %}/images/{{ site.logo }}{% else %}{% static "images/logo.png" %}{% endif %}" alt="BookWyrm logo">
         </figure>
     </div>
     <div class="content">

bookwyrm/templates/snippets/avatar.html

@@ -1,2 +1,4 @@
-<img class="avatar image {% if large %}is-96x96{% elif medium %}is-48x48{% else %}is-32x32{% endif %}" src="{% if user.avatar %}/images/{{ user.avatar }}{% else %}/static/images/default_avi.jpg{% endif %}" {% if ariaHide %}aria-hidden="true"{% endif %} alt="{{ user.alt_text }}">
+{% load static %}
+
+<img class="avatar image {% if large %}is-96x96{% elif medium %}is-48x48{% else %}is-32x32{% endif %}" src="{% if user.avatar %}{% get_media_prefix %}{{ user.avatar }}{% else %}{% static "images/default_avi.jpg" %}{% endif %}" {% if ariaHide %}aria-hidden="true"{% endif %} alt="{{ user.alt_text }}">
 

bookwyrm/templates/snippets/book_cover.html

@@ -109,10 +109,14 @@
 
             <img
                 class="book-cover"
-                src="{% get_media_prefix %}{{ book.cover }}"
                 itemprop="thumbnailUrl"
-                alt="{{ book.alt_text|default:'' }}"
+                {% if book.cover %}
+                src="{% if img_path is None %}{% get_media_prefix %}{% else %}{{ img_path }}{% endif %}{{ book.cover }}"
+                {% else %}
+                src="{% static "images/no_cover.jpg" %}"
+                alt="{% trans "No cover" %}"
             >
+
         {% endif %}
     </picture>
 {% endif %}

bookwyrm/templates/snippets/opengraph_images.html

@@ -1,12 +1,14 @@
+{% load static %}
+
 {% if preview_images_enabled is True %}
     {% if image %}
-        <meta name="twitter:image" content="{{ request.scheme }}://{{ media_path }}{{ image }}">
-        <meta name="og:image" content="{{ request.scheme }}://{{ media_path }}{{ image }}">
+        <meta name="twitter:image" content="{{ media_full_url }}{{ image }}">
+        <meta name="og:image" content="{{ media_full_url }}{{ image }}">
     {% else %}
-        <meta name="twitter:image" content="{{ request.scheme }}://{{ media_path }}{{ site.preview_image }}">
-        <meta name="og:image" content="{{ request.scheme }}://{{ media_path }}{{ site.preview_image }}">
+        <meta name="twitter:image" content="{{ media_full_url }}{{ site.preview_image }}">
+        <meta name="og:image" content="{{ media_full_url }}{{ site.preview_image }}">
     {% endif %}
 {% else %}
-    <meta name="twitter:image" content="{{ request.scheme }}://{% if site.logo %}{{ media_path }}{{ site.logo }}{% else %}{{ static_path }}/images/logo.png{% endif %}">
-    <meta name="og:image" content="{{ request.scheme }}://{% if site.logo %}{{ media_path }}{{ site.logo }}{% else %}{{ static_path }}/images/logo.png{% endif %}">
+    <meta name="twitter:image" content="{% if site.logo %}{{ media_full_url }}{{ site.logo }}{% else %}{% static "images/logo.png" %}{% endif %}">
+    <meta name="og:image" content="{% if site.logo %}{{ media_full_url }}{{ site.logo }}{% else %}{% static "images/logo.png" %}{% endif %}">
 {% endif %}

bookwyrm/templates/snippets/status/content_status.html

@@ -1,6 +1,7 @@
 {% load bookwyrm_tags %}
 {% load markdown %}
 {% load i18n %}
+{% load static %}
 
 {% with status_type=status.status_type %}
 <div
@@ -111,12 +112,12 @@
                                 <div class="column is-narrow">
                                     <figure class="image is-128x128">
                                         <a
-                                            href="/images/{{ attachment.image }}"
+                                            href="{% get_media_prefix %}{{ attachment.image }}"
                                             target="_blank"
                                             aria-label="{% trans 'Open image in new window' %}"
                                         >
                                             <img
-                                                src="/images/{{ attachment.image }}"
+                                                src="{% get_media_prefix %}{{ attachment.image }}"
 
                                                 {% if attachment.caption %}
                                                     alt="{{ attachment.caption }}"

bookwyrm/templates/snippets/status/status_header.html

@@ -2,6 +2,7 @@
 {% load status_display %}
 {% load i18n %}
 {% load humanize %}
+{% load static %}
 
 <div class="media">
     <figure class="media-left" aria-hidden="true">
@@ -18,7 +19,7 @@
                 itemtype="https://schema.org/Person"
             >
                 {% if status.user.avatar %}
-                    <meta itemprop="image" content="/images/{{ status.user.avatar }}">
+                    <meta itemprop="image" content="{% get_media_prefix %}{{ status.user.avatar }}">
                 {% endif %}
 
                 <a

bookwyrm/templates/user/layout.html

@@ -28,8 +28,10 @@
         </div>
 
         {% if user.summary %}
-        <div class="column box has-background-white-bis content">
+        {% spaceless %}
+        <div class="column box has-background-white-bis content preserve-whitespace">
             {{ user.summary|to_markdown|safe }}
+        {% endspaceless %}
         </div>
         {% endif %}
     </div>

bookwyrm/templates/user/shelf/shelf.html

@@ -103,7 +103,7 @@
                         {% include 'snippets/authors.html' %}
                     </td>
                     <td data-title="{% trans "Shelved" %}">
-                        {{ book.created_date|naturalday }}
+                        {{ book.shelved_date|naturalday }}
                     </td>
                     {% latest_read_through book user as read_through %}
                     <td data-title="{% trans "Started" %}">

bookwyrm/tests/connectors/test_self_connector.py

@@ -43,68 +43,69 @@ class SelfConnector(TestCase):
         self.assertEqual(result.year, 1980)
         self.assertEqual(result.connector, self.connector)
 
-    def test_search_rank(self):
+    @patch("bookwyrm.preview_images.generate_edition_preview_image_task.delay")
+    def test_search_rank(self, _):
         """prioritize certain results"""
         author = models.Author.objects.create(name="Anonymous")
-        with patch("bookwyrm.preview_images.generate_edition_preview_image_task.delay"):
-            edition = models.Edition.objects.create(
-                title="Edition of Example Work",
-                published_date=datetime.datetime(1980, 5, 10, tzinfo=timezone.utc),
-                parent_work=models.Work.objects.create(title=""),
-            )
-            # author text is rank C
-            edition.authors.add(author)
+        edition = models.Edition.objects.create(
+            title="Edition of Example Work",
+            published_date=datetime.datetime(1980, 5, 10, tzinfo=timezone.utc),
+            parent_work=models.Work.objects.create(title=""),
+        )
+        # author text is rank B
+        edition.authors.add(author)
 
-            # series is rank D
-            models.Edition.objects.create(
-                title="Another Edition",
-                series="Anonymous",
-                parent_work=models.Work.objects.create(title=""),
-            )
-            # subtitle is rank B
-            models.Edition.objects.create(
-                title="More Editions",
-                subtitle="The Anonymous Edition",
-                parent_work=models.Work.objects.create(title=""),
-            )
-            # title is rank A
-            models.Edition.objects.create(title="Anonymous")
-            # doesn't rank in this search
-            edition = models.Edition.objects.create(
-                title="An Edition", parent_work=models.Work.objects.create(title="")
-            )
+        # series is rank D
+        models.Edition.objects.create(
+            title="Another Edition",
+            series="Anonymous",
+            parent_work=models.Work.objects.create(title=""),
+        )
+        # subtitle is rank B
+        models.Edition.objects.create(
+            title="More Editions",
+            subtitle="The Anonymous Edition",
+            parent_work=models.Work.objects.create(title=""),
+        )
+        # title is rank A
+        models.Edition.objects.create(title="Anonymous")
+        # doesn't rank in this search
+        models.Edition.objects.create(
+            title="An Edition", parent_work=models.Work.objects.create(title="")
+        )
 
-            results = self.connector.search("Anonymous")
-        self.assertEqual(len(results), 3)
+        results = self.connector.search("Anonymous")
+        self.assertEqual(len(results), 4)
         self.assertEqual(results[0].title, "Anonymous")
         self.assertEqual(results[1].title, "More Editions")
         self.assertEqual(results[2].title, "Edition of Example Work")
+        self.assertEqual(results[3].title, "Another Edition")
 
-    def test_search_multiple_editions(self):
+    @patch("bookwyrm.preview_images.generate_edition_preview_image_task.delay")
+    def test_search_multiple_editions(self, _):
         """it should get rid of duplicate editions for the same work"""
-        with patch("bookwyrm.preview_images.generate_edition_preview_image_task.delay"):
-            work = models.Work.objects.create(title="Work Title")
-            edition_1 = models.Edition.objects.create(
-                title="Edition 1 Title", parent_work=work
-            )
-            edition_2 = models.Edition.objects.create(
-                title="Edition 2 Title",
-                parent_work=work,
-                edition_rank=20,  # that's default babey
-            )
-            edition_3 = models.Edition.objects.create(title="Fish", parent_work=work)
+        work = models.Work.objects.create(title="Work Title")
+        edition_1 = models.Edition.objects.create(
+            title="Edition 1 Title", parent_work=work
+        )
+        edition_2 = models.Edition.objects.create(
+            title="Edition 2 Title",
+            parent_work=work,
+            isbn_13="123456789",  # this is now the defualt edition
+        )
+        edition_3 = models.Edition.objects.create(title="Fish", parent_work=work)
 
-            # pick the best edition
-            results = self.connector.search("Edition 1 Title")
-            self.assertEqual(len(results), 1)
-            self.assertEqual(results[0].key, edition_1.remote_id)
+        # pick the best edition
+        results = self.connector.search("Edition 1 Title")
+        self.assertEqual(len(results), 1)
+        self.assertEqual(results[0].key, edition_1.remote_id)
 
-            # pick the default edition when no match is best
-            results = self.connector.search("Edition Title")
-            self.assertEqual(len(results), 1)
-            self.assertEqual(results[0].key, edition_2.remote_id)
+        # pick the default edition when no match is best
+        results = self.connector.search("Edition Title")
+        self.assertEqual(len(results), 1)
+        self.assertEqual(results[0].key, edition_2.remote_id)
 
-            # only matches one edition, so no deduplication takes place
-            results = self.connector.search("Fish")
-            self.assertEqual(len(results), 1)
-            self.assertEqual(results[0].key, edition_3.remote_id)
+        # only matches one edition, so no deduplication takes place
+        results = self.connector.search("Fish")
+        self.assertEqual(len(results), 1)
+        self.assertEqual(results[0].key, edition_3.remote_id)

bookwyrm/tests/importers/test_goodreads_import.py

@@ -3,6 +3,8 @@ from collections import namedtuple
 import csv
 import pathlib
 from unittest.mock import patch
+import datetime
+import pytz
 
 from django.test import TestCase
 import responses
@@ -13,6 +15,10 @@ from bookwyrm.importers.importer import import_data, handle_imported_book
 from bookwyrm.settings import DOMAIN
 
 
+def make_date(*args):
+    return datetime.datetime(*args, tzinfo=pytz.UTC)
+
+
 class GoodreadsImport(TestCase):
     """importing from goodreads csv"""
 
@@ -130,22 +136,25 @@ class GoodreadsImport(TestCase):
 
         shelf.refresh_from_db()
         self.assertEqual(shelf.books.first(), self.book)
+        self.assertEqual(
+            shelf.shelfbook_set.first().shelved_date, make_date(2020, 10, 21)
+        )
 
         readthrough = models.ReadThrough.objects.get(user=self.user)
         self.assertEqual(readthrough.book, self.book)
-        # I can't remember how to create dates and I don't want to look it up.
-        self.assertEqual(readthrough.start_date.year, 2020)
-        self.assertEqual(readthrough.start_date.month, 10)
-        self.assertEqual(readthrough.start_date.day, 21)
-        self.assertEqual(readthrough.finish_date.year, 2020)
-        self.assertEqual(readthrough.finish_date.month, 10)
-        self.assertEqual(readthrough.finish_date.day, 25)
+        self.assertEqual(readthrough.start_date, make_date(2020, 10, 21))
+        self.assertEqual(readthrough.finish_date, make_date(2020, 10, 25))
 
     def test_handle_imported_book_already_shelved(self):
         """goodreads import added a book, this adds related connections"""
         with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"):
             shelf = self.user.shelf_set.filter(identifier="to-read").first()
-            models.ShelfBook.objects.create(shelf=shelf, user=self.user, book=self.book)
+            models.ShelfBook.objects.create(
+                shelf=shelf,
+                user=self.user,
+                book=self.book,
+                shelved_date=make_date(2020, 2, 2),
+            )
 
         import_job = models.ImportJob.objects.create(user=self.user)
         datafile = pathlib.Path(__file__).parent.joinpath("../data/goodreads.csv")
@@ -164,15 +173,15 @@ class GoodreadsImport(TestCase):
 
         shelf.refresh_from_db()
         self.assertEqual(shelf.books.first(), self.book)
+        self.assertEqual(
+            shelf.shelfbook_set.first().shelved_date, make_date(2020, 2, 2)
+        )
         self.assertIsNone(self.user.shelf_set.get(identifier="read").books.first())
+
         readthrough = models.ReadThrough.objects.get(user=self.user)
         self.assertEqual(readthrough.book, self.book)
-        self.assertEqual(readthrough.start_date.year, 2020)
-        self.assertEqual(readthrough.start_date.month, 10)
-        self.assertEqual(readthrough.start_date.day, 21)
-        self.assertEqual(readthrough.finish_date.year, 2020)
-        self.assertEqual(readthrough.finish_date.month, 10)
-        self.assertEqual(readthrough.finish_date.day, 25)
+        self.assertEqual(readthrough.start_date, make_date(2020, 10, 21))
+        self.assertEqual(readthrough.finish_date, make_date(2020, 10, 25))
 
     def test_handle_import_twice(self):
         """re-importing books"""
@@ -198,16 +207,14 @@ class GoodreadsImport(TestCase):
 
         shelf.refresh_from_db()
         self.assertEqual(shelf.books.first(), self.book)
+        self.assertEqual(
+            shelf.shelfbook_set.first().shelved_date, make_date(2020, 10, 21)
+        )
 
         readthrough = models.ReadThrough.objects.get(user=self.user)
         self.assertEqual(readthrough.book, self.book)
-        # I can't remember how to create dates and I don't want to look it up.
-        self.assertEqual(readthrough.start_date.year, 2020)
-        self.assertEqual(readthrough.start_date.month, 10)
-        self.assertEqual(readthrough.start_date.day, 21)
-        self.assertEqual(readthrough.finish_date.year, 2020)
-        self.assertEqual(readthrough.finish_date.month, 10)
-        self.assertEqual(readthrough.finish_date.day, 25)
+        self.assertEqual(readthrough.start_date, make_date(2020, 10, 21))
+        self.assertEqual(readthrough.finish_date, make_date(2020, 10, 25))
 
     @patch("bookwyrm.activitystreams.ActivityStream.add_status")
     def test_handle_imported_book_review(self, _):
@@ -229,9 +236,7 @@ class GoodreadsImport(TestCase):
         review = models.Review.objects.get(book=self.book, user=self.user)
         self.assertEqual(review.content, "mixed feelings")
         self.assertEqual(review.rating, 2)
-        self.assertEqual(review.published_date.year, 2019)
-        self.assertEqual(review.published_date.month, 7)
-        self.assertEqual(review.published_date.day, 8)
+        self.assertEqual(review.published_date, make_date(2019, 7, 8))
         self.assertEqual(review.privacy, "unlisted")
 
     @patch("bookwyrm.activitystreams.ActivityStream.add_status")
@@ -256,9 +261,7 @@ class GoodreadsImport(TestCase):
         review = models.ReviewRating.objects.get(book=self.book, user=self.user)
         self.assertIsInstance(review, models.ReviewRating)
         self.assertEqual(review.rating, 2)
-        self.assertEqual(review.published_date.year, 2019)
-        self.assertEqual(review.published_date.month, 7)
-        self.assertEqual(review.published_date.day, 8)
+        self.assertEqual(review.published_date, make_date(2019, 7, 8))
         self.assertEqual(review.privacy, "unlisted")
 
     def test_handle_imported_book_reviews_disabled(self):

bookwyrm/tests/importers/test_librarything_import.py

@@ -2,6 +2,8 @@
 import csv
 import pathlib
 from unittest.mock import patch
+import datetime
+import pytz
 
 from django.test import TestCase
 import responses
@@ -12,6 +14,10 @@ from bookwyrm.importers.importer import import_data, handle_imported_book
 from bookwyrm.settings import DOMAIN
 
 
+def make_date(*args):
+    return datetime.datetime(*args, tzinfo=pytz.UTC)
+
+
 class LibrarythingImport(TestCase):
     """importing from librarything tsv"""
 
@@ -125,13 +131,8 @@ class LibrarythingImport(TestCase):
 
         readthrough = models.ReadThrough.objects.get(user=self.user)
         self.assertEqual(readthrough.book, self.book)
-        # I can't remember how to create dates and I don't want to look it up.
-        self.assertEqual(readthrough.start_date.year, 2007)
-        self.assertEqual(readthrough.start_date.month, 4)
-        self.assertEqual(readthrough.start_date.day, 16)
-        self.assertEqual(readthrough.finish_date.year, 2007)
-        self.assertEqual(readthrough.finish_date.month, 5)
-        self.assertEqual(readthrough.finish_date.day, 8)
+        self.assertEqual(readthrough.start_date, make_date(2007, 4, 16))
+        self.assertEqual(readthrough.finish_date, make_date(2007, 5, 8))
 
     def test_handle_imported_book_already_shelved(self):
         """librarything import added a book, this adds related connections"""
@@ -160,14 +161,11 @@ class LibrarythingImport(TestCase):
         shelf.refresh_from_db()
         self.assertEqual(shelf.books.first(), self.book)
         self.assertIsNone(self.user.shelf_set.get(identifier="read").books.first())
+
         readthrough = models.ReadThrough.objects.get(user=self.user)
         self.assertEqual(readthrough.book, self.book)
-        self.assertEqual(readthrough.start_date.year, 2007)
-        self.assertEqual(readthrough.start_date.month, 4)
-        self.assertEqual(readthrough.start_date.day, 16)
-        self.assertEqual(readthrough.finish_date.year, 2007)
-        self.assertEqual(readthrough.finish_date.month, 5)
-        self.assertEqual(readthrough.finish_date.day, 8)
+        self.assertEqual(readthrough.start_date, make_date(2007, 4, 16))
+        self.assertEqual(readthrough.finish_date, make_date(2007, 5, 8))
 
     def test_handle_import_twice(self):
         """re-importing books"""
@@ -198,13 +196,8 @@ class LibrarythingImport(TestCase):
 
         readthrough = models.ReadThrough.objects.get(user=self.user)
         self.assertEqual(readthrough.book, self.book)
-        # I can't remember how to create dates and I don't want to look it up.
-        self.assertEqual(readthrough.start_date.year, 2007)
-        self.assertEqual(readthrough.start_date.month, 4)
-        self.assertEqual(readthrough.start_date.day, 16)
-        self.assertEqual(readthrough.finish_date.year, 2007)
-        self.assertEqual(readthrough.finish_date.month, 5)
-        self.assertEqual(readthrough.finish_date.day, 8)
+        self.assertEqual(readthrough.start_date, make_date(2007, 4, 16))
+        self.assertEqual(readthrough.finish_date, make_date(2007, 5, 8))
 
     @patch("bookwyrm.activitystreams.ActivityStream.add_status")
     def test_handle_imported_book_review(self, _):
@@ -226,9 +219,7 @@ class LibrarythingImport(TestCase):
         review = models.Review.objects.get(book=self.book, user=self.user)
         self.assertEqual(review.content, "chef d'oeuvre")
         self.assertEqual(review.rating, 5)
-        self.assertEqual(review.published_date.year, 2007)
-        self.assertEqual(review.published_date.month, 5)
-        self.assertEqual(review.published_date.day, 8)
+        self.assertEqual(review.published_date, make_date(2007, 5, 8))
         self.assertEqual(review.privacy, "unlisted")
 
     def test_handle_imported_book_reviews_disabled(self):

bookwyrm/tests/models/test_activitypub_mixin.py

@@ -9,11 +9,19 @@ from django.test import TestCase
 from bookwyrm.activitypub.base_activity import ActivityObject
 from bookwyrm import models
 from bookwyrm.models import base_model
-from bookwyrm.models.activitypub_mixin import ActivitypubMixin
-from bookwyrm.models.activitypub_mixin import ActivityMixin, ObjectMixin
+from bookwyrm.models.activitypub_mixin import (
+    ActivitypubMixin,
+    ActivityMixin,
+    ObjectMixin,
+    OrderedCollectionMixin,
+    to_ordered_collection_page,
+)
+from bookwyrm.settings import PAGE_LENGTH
 
 
+# pylint: disable=invalid-name
 @patch("bookwyrm.activitystreams.ActivityStream.add_status")
+@patch("bookwyrm.preview_images.generate_user_preview_image_task.delay")
 class ActivitypubMixins(TestCase):
     """functionality shared across models"""
 
@@ -45,8 +53,7 @@ class ActivitypubMixins(TestCase):
             "published": "2020-12-04T17:52:22.623807+00:00",
         }
 
-    # ActivitypubMixin
-    def test_to_activity(self, _):
+    def test_to_activity(self, *_):
         """model to ActivityPub json"""
 
         @dataclass(init=False)
@@ -67,7 +74,7 @@ class ActivitypubMixins(TestCase):
         self.assertEqual(activity["id"], "https://www.example.com/test")
         self.assertEqual(activity["type"], "Test")
 
-    def test_find_existing_by_remote_id(self, _):
+    def test_find_existing_by_remote_id(self, *_):
         """attempt to match a remote id to an object in the db"""
         # uses a different remote id scheme
         # this isn't really part of this test directly but it's helpful to state
@@ -101,7 +108,7 @@ class ActivitypubMixins(TestCase):
         # test subclass match
         result = models.Status.find_existing_by_remote_id("https://comment.net")
 
-    def test_find_existing(self, _):
+    def test_find_existing(self, *_):
         """match a blob of data to a model"""
         with patch("bookwyrm.preview_images.generate_edition_preview_image_task.delay"):
             book = models.Edition.objects.create(
@@ -112,7 +119,7 @@ class ActivitypubMixins(TestCase):
         result = models.Edition.find_existing({"openlibraryKey": "OL1234"})
         self.assertEqual(result, book)
 
-    def test_get_recipients_public_object(self, _):
+    def test_get_recipients_public_object(self, *_):
         """determines the recipients for an object's broadcast"""
         MockSelf = namedtuple("Self", ("privacy"))
         mock_self = MockSelf("public")
@@ -120,7 +127,7 @@ class ActivitypubMixins(TestCase):
         self.assertEqual(len(recipients), 1)
         self.assertEqual(recipients[0], self.remote_user.inbox)
 
-    def test_get_recipients_public_user_object_no_followers(self, _):
+    def test_get_recipients_public_user_object_no_followers(self, *_):
         """determines the recipients for a user's object broadcast"""
         MockSelf = namedtuple("Self", ("privacy", "user"))
         mock_self = MockSelf("public", self.local_user)
@@ -128,7 +135,7 @@ class ActivitypubMixins(TestCase):
         recipients = ActivitypubMixin.get_recipients(mock_self)
         self.assertEqual(len(recipients), 0)
 
-    def test_get_recipients_public_user_object(self, _):
+    def test_get_recipients_public_user_object(self, *_):
         """determines the recipients for a user's object broadcast"""
         MockSelf = namedtuple("Self", ("privacy", "user"))
         mock_self = MockSelf("public", self.local_user)
@@ -138,22 +145,21 @@ class ActivitypubMixins(TestCase):
         self.assertEqual(len(recipients), 1)
         self.assertEqual(recipients[0], self.remote_user.inbox)
 
-    def test_get_recipients_public_user_object_with_mention(self, _):
+    def test_get_recipients_public_user_object_with_mention(self, *_):
         """determines the recipients for a user's object broadcast"""
         MockSelf = namedtuple("Self", ("privacy", "user"))
         mock_self = MockSelf("public", self.local_user)
         self.local_user.followers.add(self.remote_user)
-        with patch("bookwyrm.preview_images.generate_user_preview_image_task.delay"):
-            with patch("bookwyrm.models.user.set_remote_server.delay"):
-                another_remote_user = models.User.objects.create_user(
-                    "nutria",
-                    "nutria@nutria.com",
-                    "nutriaword",
-                    local=False,
-                    remote_id="https://example.com/users/nutria",
-                    inbox="https://example.com/users/nutria/inbox",
-                    outbox="https://example.com/users/nutria/outbox",
-                )
+        with patch("bookwyrm.models.user.set_remote_server.delay"):
+            another_remote_user = models.User.objects.create_user(
+                "nutria",
+                "nutria@nutria.com",
+                "nutriaword",
+                local=False,
+                remote_id="https://example.com/users/nutria",
+                inbox="https://example.com/users/nutria/inbox",
+                outbox="https://example.com/users/nutria/outbox",
+            )
         MockSelf = namedtuple("Self", ("privacy", "user", "recipients"))
         mock_self = MockSelf("public", self.local_user, [another_remote_user])
 
@@ -162,22 +168,21 @@ class ActivitypubMixins(TestCase):
         self.assertTrue(another_remote_user.inbox in recipients)
         self.assertTrue(self.remote_user.inbox in recipients)
 
-    def test_get_recipients_direct(self, _):
+    def test_get_recipients_direct(self, *_):
         """determines the recipients for a user's object broadcast"""
         MockSelf = namedtuple("Self", ("privacy", "user"))
         mock_self = MockSelf("public", self.local_user)
         self.local_user.followers.add(self.remote_user)
-        with patch("bookwyrm.preview_images.generate_user_preview_image_task.delay"):
-            with patch("bookwyrm.models.user.set_remote_server.delay"):
-                another_remote_user = models.User.objects.create_user(
-                    "nutria",
-                    "nutria@nutria.com",
-                    "nutriaword",
-                    local=False,
-                    remote_id="https://example.com/users/nutria",
-                    inbox="https://example.com/users/nutria/inbox",
-                    outbox="https://example.com/users/nutria/outbox",
-                )
+        with patch("bookwyrm.models.user.set_remote_server.delay"):
+            another_remote_user = models.User.objects.create_user(
+                "nutria",
+                "nutria@nutria.com",
+                "nutriaword",
+                local=False,
+                remote_id="https://example.com/users/nutria",
+                inbox="https://example.com/users/nutria/inbox",
+                outbox="https://example.com/users/nutria/outbox",
+            )
         MockSelf = namedtuple("Self", ("privacy", "user", "recipients"))
         mock_self = MockSelf("direct", self.local_user, [another_remote_user])
 
@@ -185,22 +190,21 @@ class ActivitypubMixins(TestCase):
         self.assertEqual(len(recipients), 1)
         self.assertEqual(recipients[0], another_remote_user.inbox)
 
-    def test_get_recipients_combine_inboxes(self, _):
+    def test_get_recipients_combine_inboxes(self, *_):
         """should combine users with the same shared_inbox"""
         self.remote_user.shared_inbox = "http://example.com/inbox"
         self.remote_user.save(broadcast=False)
-        with patch("bookwyrm.preview_images.generate_user_preview_image_task.delay"):
-            with patch("bookwyrm.models.user.set_remote_server.delay"):
-                another_remote_user = models.User.objects.create_user(
-                    "nutria",
-                    "nutria@nutria.com",
-                    "nutriaword",
-                    local=False,
-                    remote_id="https://example.com/users/nutria",
-                    inbox="https://example.com/users/nutria/inbox",
-                    shared_inbox="http://example.com/inbox",
-                    outbox="https://example.com/users/nutria/outbox",
-                )
+        with patch("bookwyrm.models.user.set_remote_server.delay"):
+            another_remote_user = models.User.objects.create_user(
+                "nutria",
+                "nutria@nutria.com",
+                "nutriaword",
+                local=False,
+                remote_id="https://example.com/users/nutria",
+                inbox="https://example.com/users/nutria/inbox",
+                shared_inbox="http://example.com/inbox",
+                outbox="https://example.com/users/nutria/outbox",
+            )
         MockSelf = namedtuple("Self", ("privacy", "user"))
         mock_self = MockSelf("public", self.local_user)
         self.local_user.followers.add(self.remote_user)
@@ -210,20 +214,19 @@ class ActivitypubMixins(TestCase):
         self.assertEqual(len(recipients), 1)
         self.assertEqual(recipients[0], "http://example.com/inbox")
 
-    def test_get_recipients_software(self, _):
+    def test_get_recipients_software(self, *_):
         """should differentiate between bookwyrm and other remote users"""
-        with patch("bookwyrm.preview_images.generate_user_preview_image_task.delay"):
-            with patch("bookwyrm.models.user.set_remote_server.delay"):
-                another_remote_user = models.User.objects.create_user(
-                    "nutria",
-                    "nutria@nutria.com",
-                    "nutriaword",
-                    local=False,
-                    remote_id="https://example.com/users/nutria",
-                    inbox="https://example.com/users/nutria/inbox",
-                    outbox="https://example.com/users/nutria/outbox",
-                    bookwyrm_user=False,
-                )
+        with patch("bookwyrm.models.user.set_remote_server.delay"):
+            another_remote_user = models.User.objects.create_user(
+                "nutria",
+                "nutria@nutria.com",
+                "nutriaword",
+                local=False,
+                remote_id="https://example.com/users/nutria",
+                inbox="https://example.com/users/nutria/inbox",
+                outbox="https://example.com/users/nutria/outbox",
+                bookwyrm_user=False,
+            )
         MockSelf = namedtuple("Self", ("privacy", "user"))
         mock_self = MockSelf("public", self.local_user)
         self.local_user.followers.add(self.remote_user)
@@ -241,7 +244,7 @@ class ActivitypubMixins(TestCase):
         self.assertEqual(recipients[0], another_remote_user.inbox)
 
     # ObjectMixin
-    def test_object_save_create(self, _):
+    def test_object_save_create(self, *_):
         """should save uneventufully when broadcast is disabled"""
 
         class Success(Exception):
@@ -272,7 +275,7 @@ class ActivitypubMixins(TestCase):
         ObjectModel(user=self.local_user).save(broadcast=False)
         ObjectModel(user=None).save()
 
-    def test_object_save_update(self, _):
+    def test_object_save_update(self, *_):
         """should save uneventufully when broadcast is disabled"""
 
         class Success(Exception):
@@ -298,7 +301,7 @@ class ActivitypubMixins(TestCase):
         with self.assertRaises(Success):
             UpdateObjectModel(id=1, last_edited_by=self.local_user).save()
 
-    def test_object_save_delete(self, _):
+    def test_object_save_delete(self, *_):
         """should create delete activities when objects are deleted by flag"""
 
         class ActivitySuccess(Exception):
@@ -320,7 +323,7 @@ class ActivitypubMixins(TestCase):
         with self.assertRaises(ActivitySuccess):
             DeletableObjectModel(id=1, user=self.local_user, deleted=True).save()
 
-    def test_to_delete_activity(self, _):
+    def test_to_delete_activity(self, *_):
         """wrapper for Delete activity"""
         MockSelf = namedtuple("Self", ("remote_id", "to_activity"))
         mock_self = MockSelf(
@@ -335,7 +338,7 @@ class ActivitypubMixins(TestCase):
             activity["cc"], ["https://www.w3.org/ns/activitystreams#Public"]
         )
 
-    def test_to_update_activity(self, _):
+    def test_to_update_activity(self, *_):
         """ditto above but for Update"""
         MockSelf = namedtuple("Self", ("remote_id", "to_activity"))
         mock_self = MockSelf(
@@ -352,8 +355,7 @@ class ActivitypubMixins(TestCase):
         )
         self.assertIsInstance(activity["object"], dict)
 
-    # Activity mixin
-    def test_to_undo_activity(self, _):
+    def test_to_undo_activity(self, *_):
         """and again, for Undo"""
         MockSelf = namedtuple("Self", ("remote_id", "to_activity", "user"))
         mock_self = MockSelf(
@@ -366,3 +368,59 @@ class ActivitypubMixins(TestCase):
         self.assertEqual(activity["actor"], self.local_user.remote_id)
         self.assertEqual(activity["type"], "Undo")
         self.assertIsInstance(activity["object"], dict)
+
+    @patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay")
+    def test_to_ordered_collection_page(self, *_):
+        """make sure the paged results of an ordered collection work"""
+        self.assertEqual(PAGE_LENGTH, 15)
+        for number in range(0, 2 * PAGE_LENGTH):
+            models.Status.objects.create(
+                user=self.local_user,
+                content="test status {:d}".format(number),
+            )
+        page_1 = to_ordered_collection_page(
+            models.Status.objects.all(), "http://fish.com/", page=1
+        )
+        self.assertEqual(page_1.partOf, "http://fish.com/")
+        self.assertEqual(page_1.id, "http://fish.com/?page=1")
+        self.assertEqual(page_1.next, "http://fish.com/?page=2")
+        self.assertEqual(page_1.orderedItems[0]["content"], "test status 29")
+        self.assertEqual(page_1.orderedItems[1]["content"], "test status 28")
+
+        page_2 = to_ordered_collection_page(
+            models.Status.objects.all(), "http://fish.com/", page=2
+        )
+        self.assertEqual(page_2.partOf, "http://fish.com/")
+        self.assertEqual(page_2.id, "http://fish.com/?page=2")
+        self.assertEqual(page_2.orderedItems[0]["content"], "test status 14")
+        self.assertEqual(page_2.orderedItems[-1]["content"], "test status 0")
+
+    @patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay")
+    def test_to_ordered_collection(self, *_):
+        """convert a queryset into an ordered collection object"""
+        self.assertEqual(PAGE_LENGTH, 15)
+
+        for number in range(0, 2 * PAGE_LENGTH):
+            models.Status.objects.create(
+                user=self.local_user,
+                content="test status {:d}".format(number),
+            )
+
+        MockSelf = namedtuple("Self", ("remote_id"))
+        mock_self = MockSelf("")
+
+        collection = OrderedCollectionMixin.to_ordered_collection(
+            mock_self, models.Status.objects.all(), remote_id="http://fish.com/"
+        )
+
+        self.assertEqual(collection.totalItems, 30)
+        self.assertEqual(collection.first, "http://fish.com/?page=1")
+        self.assertEqual(collection.last, "http://fish.com/?page=2")
+
+        page_2 = OrderedCollectionMixin.to_ordered_collection(
+            mock_self, models.Status.objects.all(), remote_id="http://fish.com/", page=2
+        )
+        self.assertEqual(page_2.partOf, "http://fish.com/")
+        self.assertEqual(page_2.id, "http://fish.com/?page=2")
+        self.assertEqual(page_2.orderedItems[0]["content"], "test status 14")
+        self.assertEqual(page_2.orderedItems[-1]["content"], "test status 0")

bookwyrm/tests/models/test_book_model.py

@@ -1,8 +1,9 @@
 """ testing models """
+from unittest.mock import patch
+
 from dateutil.parser import parse
 from django.test import TestCase
 from django.utils import timezone
-from unittest.mock import patch
 
 from bookwyrm import models, settings
 from bookwyrm.models.book import isbn_10_to_13, isbn_13_to_10

bookwyrm/tests/test_postgres.py

@@ -0,0 +1,77 @@
+""" django configuration of postgres  """
+from unittest.mock import patch
+from django.test import TestCase
+
+from bookwyrm import models
+
+
+@patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay")
+@patch("bookwyrm.preview_images.generate_edition_preview_image_task.delay")
+class PostgresTriggers(TestCase):
+    """special migrations, fancy stuff ya know"""
+
+    def test_search_vector_on_create(self, *_):
+        """make sure that search_vector is being set correctly on create"""
+        book = models.Edition.objects.create(title="The Long Goodbye")
+        book.refresh_from_db()
+        self.assertEqual(book.search_vector, "'goodby':3A 'long':2A")
+
+    def test_search_vector_on_update(self, *_):
+        """make sure that search_vector is being set correctly on edit"""
+        book = models.Edition.objects.create(title="The Long Goodbye")
+        book.title = "The Even Longer Goodbye"
+        book.save(broadcast=False)
+        book.refresh_from_db()
+        self.assertEqual(book.search_vector, "'even':2A 'goodby':4A 'longer':3A")
+
+    def test_search_vector_fields(self, *_):
+        """use multiple fields to create search vector"""
+        author = models.Author.objects.create(name="The Rays")
+        book = models.Edition.objects.create(
+            title="The Long Goodbye",
+            subtitle="wow cool",
+            series="series name",
+            languages=["irrelevent"],
+        )
+        book.authors.add(author)
+        book.refresh_from_db()
+        self.assertEqual(
+            book.search_vector,
+            "'cool':5B 'goodby':3A 'long':2A 'name':9 'rays':7C 'seri':8 'the':6C 'wow':4B",
+        )
+
+    def test_seach_vector_on_author_update(self, *_):
+        """update search when an author name changes"""
+        author = models.Author.objects.create(name="The Rays")
+        book = models.Edition.objects.create(
+            title="The Long Goodbye",
+        )
+        book.authors.add(author)
+        author.name = "Jeremy"
+        author.save(broadcast=False)
+        book.refresh_from_db()
+
+        self.assertEqual(book.search_vector, "'goodby':3A 'jeremy':4C 'long':2A")
+
+    def test_seach_vector_on_author_delete(self, *_):
+        """update search when an author name changes"""
+        author = models.Author.objects.create(name="Jeremy")
+        book = models.Edition.objects.create(
+            title="The Long Goodbye",
+        )
+
+        book.authors.add(author)
+        book.refresh_from_db()
+        self.assertEqual(book.search_vector, "'goodby':3A 'jeremy':4C 'long':2A")
+
+        book.authors.remove(author)
+        book.refresh_from_db()
+        self.assertEqual(book.search_vector, "'goodby':3A 'long':2A")
+
+    def test_search_vector_stop_word_fallback(self, *_):
+        """use a fallback when removing stop words leads to an empty vector"""
+        book = models.Edition.objects.create(
+            title="there there",
+        )
+        book.refresh_from_db()
+        self.assertEqual(book.search_vector, "'there':1A,2A")

bookwyrm/tests/views/test_status.py

@@ -8,6 +8,7 @@ from bookwyrm import forms, models, views
 from bookwyrm.settings import DOMAIN
 
 
+# pylint: disable=invalid-name
 @patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay")
 class StatusViews(TestCase):
     """viewing and creating statuses"""
@@ -318,6 +319,15 @@ class StatusViews(TestCase):
             '<p><em>hi</em> and <a href="http://fish.com">fish.com</a> ' "is rad</p>",
         )
 
+    def test_to_markdown_detect_url(self, _):
+        """this is mostly handled in other places, but nonetheless"""
+        text = "http://fish.com/@hello#okay"
+        result = views.status.to_markdown(text)
+        self.assertEqual(
+            result,
+            '<p><a href="http://fish.com/@hello#okay">fish.com/@hello#okay</a></p>',
+        )
+
     def test_to_markdown_link(self, _):
         """this is mostly handled in other places, but nonetheless"""
         text = "[hi](http://fish.com) is <marquee>rad</marquee>"

bookwyrm/urls.py

@@ -192,8 +192,8 @@ urlpatterns = [
     re_path(r"^import/?$", views.Import.as_view(), name="import"),
     re_path(r"^import/(\d+)/?$", views.ImportStatus.as_view(), name="import-status"),
     # users
-    re_path(r"%s/?$" % USER_PATH, views.User.as_view(), name="user-feed"),
     re_path(r"%s\.json$" % USER_PATH, views.User.as_view()),
+    re_path(r"%s/?$" % USER_PATH, views.User.as_view(), name="user-feed"),
     re_path(r"%s/rss" % USER_PATH, views.rss_feed.RssFeed(), name="user-rss"),
     re_path(
         r"%s/followers(.json)?/?$" % USER_PATH,
@@ -295,7 +295,7 @@ urlpatterns = [
         views.Book.as_view(),
         name="book-user-statuses",
     ),
-    re_path(r"%s/edit/?$" % BOOK_PATH, views.EditBook.as_view()),
+    re_path(r"%s/edit/?$" % BOOK_PATH, views.EditBook.as_view(), name="edit-book"),
     re_path(r"%s/confirm/?$" % BOOK_PATH, views.ConfirmEditBook.as_view()),
     re_path(r"^create-book/?$", views.EditBook.as_view(), name="create-book"),
     re_path(r"^create-book/confirm?$", views.ConfirmEditBook.as_view()),

bookwyrm/views/books.py

@@ -59,7 +59,7 @@ class Book(View):
                 queryset = book.comment_set
             else:
                 queryset = book.quotation_set
-            queryset = queryset.filter(user=request.user)
+            queryset = queryset.filter(user=request.user, deleted=False)
         else:
             queryset = reviews.exclude(Q(content__isnull=True) | Q(content=""))
         queryset = queryset.select_related("user")
@@ -102,10 +102,11 @@ class Book(View):
                 book__parent_work=book.parent_work,
             ).select_related("shelf", "book")
 
+            filters = {"user": request.user, "deleted": False}
             data["user_statuses"] = {
-                "review_count": book.review_set.filter(user=request.user).count(),
-                "comment_count": book.comment_set.filter(user=request.user).count(),
-                "quotation_count": book.quotation_set.filter(user=request.user).count(),
+                "review_count": book.review_set.filter(**filters).count(),
+                "comment_count": book.comment_set.filter(**filters).count(),
+                "quotation_count": book.quotation_set.filter(**filters).count(),
             }
 
         return TemplateResponse(request, "book/book.html", data)

bookwyrm/views/import_data.py

@@ -84,7 +84,9 @@ class ImportStatus(View):
 
         try:
             task = app.AsyncResult(job.task_id)
-        except ValueError:
+            # triggers attribute error if the task won't load
+            task.status  # pylint: disable=pointless-statement
+        except (ValueError, AttributeError):
             task = None
 
         items = job.items.order_by("index").all()

bookwyrm/views/search.py

@@ -23,7 +23,7 @@ class Search(View):
     def get(self, request):
         """that search bar up top"""
         query = request.GET.get("q")
-        min_confidence = request.GET.get("min_confidence", 0.1)
+        min_confidence = request.GET.get("min_confidence", 0)
         search_type = request.GET.get("type")
         search_remote = (
             request.GET.get("remote", False) and request.user.is_authenticated
@@ -53,7 +53,7 @@ class Search(View):
             "remote": search_remote,
         }
         if query:
-            results = endpoints[search_type](
+            results, search_remote = endpoints[search_type](
                 query, request.user, min_confidence, search_remote
             )
             if results:
@@ -61,25 +61,28 @@ class Search(View):
                     request.GET.get("page")
                 )
                 data["results"] = paginated
+                data["remote"] = search_remote
 
         return TemplateResponse(request, "search/{:s}.html".format(search_type), data)
 
 
 def book_search(query, _, min_confidence, search_remote=False):
     """the real business is elsewhere"""
-    if search_remote:
-        return connector_manager.search(query, min_confidence=min_confidence)
-    results = connector_manager.local_search(query, min_confidence=min_confidence)
-    if not results:
-        return None
-    return [{"results": results}]
+    # try a local-only search
+    if not search_remote:
+        results = connector_manager.local_search(query, min_confidence=min_confidence)
+        if results:
+            # gret, we found something
+            return [{"results": results}], False
+    # if there weere no local results, or the request was for remote, search all sources
+    return connector_manager.search(query, min_confidence=min_confidence), True
 
 
 def user_search(query, viewer, *_):
     """cool kids members only user search"""
     # logged out viewers can't search users
     if not viewer.is_authenticated:
-        return models.User.objects.none()
+        return models.User.objects.none(), None
 
     # use webfinger for mastodon style account@domain.com username to load the user if
     # they don't exist locally (handle_remote_webfinger will check the db)
@@ -98,7 +101,7 @@ def user_search(query, viewer, *_):
             similarity__gt=0.5,
         )
         .order_by("-similarity")[:10]
-    )
+    ), None
 
 
 def list_search(query, viewer, *_):
@@ -119,4 +122,4 @@ def list_search(query, viewer, *_):
             similarity__gt=0.1,
         )
         .order_by("-similarity")[:10]
-    )
+    ), None

bookwyrm/views/shelf.py

@@ -2,7 +2,7 @@
 from collections import namedtuple
 
 from django.db import IntegrityError
-from django.db.models import OuterRef, Subquery
+from django.db.models import OuterRef, Subquery, F
 from django.contrib.auth.decorators import login_required
 from django.core.paginator import Paginator
 from django.http import HttpResponseBadRequest, HttpResponseNotFound
@@ -69,7 +69,8 @@ class Shelf(View):
             reviews = privacy_filter(request.user, reviews)
 
         books = books.annotate(
-            rating=Subquery(reviews.values("rating")[:1])
+            rating=Subquery(reviews.values("rating")[:1]),
+            shelved_date=F("shelfbook__shelved_date"),
         ).prefetch_related("authors")
 
         paginated = Paginator(

bookwyrm/views/status.py

@@ -150,7 +150,7 @@ def find_mentions(content):
 def format_links(content):
     """detect and format links"""
     return re.sub(
-        r'([^(href=")]|^|\()(https?:\/\/(%s([\w\.\-_\/+&\?=:;,])*))' % regex.DOMAIN,
+        r'([^(href=")]|^|\()(https?:\/\/(%s([\w\.\-_\/+&\?=:;,@#])*))' % regex.DOMAIN,
         r'\g<1><a href="\g<2>">\g<3></a>',
         content,
     )

bw-dev

@@ -38,8 +38,21 @@ function makeitblack {
     docker-compose run --rm web black celerywyrm bookwyrm
 }
 
+function awscommand {
+    # expose env vars
+    export AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}
+    export AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}
+    export AWS_DEFAULT_REGION=${AWS_S3_REGION_NAME}
+    # first arg is mountpoint, second is the whole aws command
+    docker run --rm -it -v $1\
+        -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e AWS_DEFAULT_REGION\
+        amazon/aws-cli $2
+}
+
 CMD=$1
-shift
+if [ -n "$CMD" ]; then
+    shift
+fi
 
 # show commands as they're executed
 set -x
@@ -56,9 +69,12 @@ case "$CMD" in
         ;;
     resetdb)
         clean
-        docker-compose up --build -d
+        # Start just the DB so no one else is using it
+        docker-compose up --build -d db
         execdb dropdb -U ${POSTGRES_USER} ${POSTGRES_DB}
         execdb createdb -U ${POSTGRES_USER} ${POSTGRES_DB}
+        # Now start up web so we can run the migrations
+        docker-compose up --build -d web
         initdb
         clean
         ;;
@@ -113,7 +129,52 @@ case "$CMD" in
     generate_preview_images)
         runweb python manage.py generate_preview_images $@
         ;;
+    copy_media_to_s3)
+        awscommand "bookwyrm_media_volume:/images"\
+            "s3 cp /images s3://${AWS_STORAGE_BUCKET_NAME}/images\
+            --endpoint-url ${AWS_S3_ENDPOINT_URL}\
+            --recursive --acl public-read"
+        ;;
+    set_cors_to_s3)
+        awscommand "$(pwd):/bw"\
+            "s3api put-bucket-cors\
+            --bucket ${AWS_STORAGE_BUCKET_NAME}\
+            --endpoint-url ${AWS_S3_ENDPOINT_URL}\
+            --cors-configuration file:///bw/$@"
+        ;;
+    runweb)
+        runweb "$@"
+        ;;
+    rundb)
+        rundb "$@"
+        ;;
     *)
-        echo "Unrecognised command. Try: build, clean, up, initdb, resetdb, makemigrations, migrate, bash, shell, dbshell, restart_celery, test, pytest, test_report, black, populate_streams, generate_preview_images, generateimages"
+        set +x # No need to echo echo
+        echo "Unrecognised command. Try:"
+        echo "    up [container]"
+        echo "    run"
+        echo "    initdb"
+        echo "    resetdb"
+        echo "    makemigrations [migration]"
+        echo "    migrate [migration]"
+        echo "    bash"
+        echo "    shell"
+        echo "    dbshell"
+        echo "    restart_celery"
+        echo "    test [path]"
+        echo "    pytest [path]"
+        echo "    collectstatic"
+        echo "    makemessages [locale]"
+        echo "    compilemessages [locale]"
+        echo "    build"
+        echo "    clean"
+        echo "    black"
+        echo "    populate_streams"
+        echo "    generate_preview_images [--all]"
+        echo "    generateimages"
+        echo "    copy_media_to_s3"
+        echo "    set_cors_to_s3 [cors file]"
+        echo "    runweb [command]"
+        echo "    rundb [command]"
         ;;
 esac

requirements.txt

@@ -15,6 +15,8 @@ requests==2.22.0
 responses==0.10.14
 django-rename-app==0.1.2
 pytz>=2021.1
+boto3==1.17.88
+django-storages==1.11.1
 
 # Dev
 black==21.4b0