Merge pull request #1768 from bookwyrm-social/shelf-button-cache

Cache queries in feed view

.env.dev.example

@@ -26,15 +26,15 @@ POSTGRES_HOST=db
 MAX_STREAM_LENGTH=200
 REDIS_ACTIVITY_HOST=redis_activity
 REDIS_ACTIVITY_PORT=6379
-#REDIS_ACTIVITY_PASSWORD=redispassword345
+REDIS_ACTIVITY_PASSWORD=redispassword345
 
 # Redis as celery broker
 REDIS_BROKER_PORT=6379
-#REDIS_BROKER_PASSWORD=redispassword123
+REDIS_BROKER_PASSWORD=redispassword123
 
 FLOWER_PORT=8888
-#FLOWER_USER=mouse
+FLOWER_USER=mouse
-#FLOWER_PASSWORD=changeme
+FLOWER_PASSWORD=changeme
 
 EMAIL_HOST=smtp.mailgun.org
 EMAIL_PORT=587

.github/workflows/django-tests.yml

@@ -46,6 +46,8 @@ jobs:
         POSTGRES_HOST: 127.0.0.1
         CELERY_BROKER: ""
         REDIS_BROKER_PORT: 6379
+        REDIS_BROKER_PASSWORD: beep
+        USE_LOCAL_CACHE: true
         FLOWER_PORT: 8888
         EMAIL_HOST: "smtp.mailgun.org"
         EMAIL_PORT: 587

bookwyrm/models/author.py

@@ -1,6 +1,8 @@
 """ database schema for info about authors """
 import re
 from django.contrib.postgres.indexes import GinIndex
+from django.core.cache import cache
+from django.core.cache.utils import make_template_fragment_key
 from django.db import models
 
 from bookwyrm import activitypub
@@ -34,6 +36,17 @@ class Author(BookDataModel):
     )
     bio = fields.HtmlField(null=True, blank=True)
 
+    def save(self, *args, **kwargs):
+        """clear related template caches"""
+        # clear template caches
+        if self.id:
+            cache_keys = [
+                make_template_fragment_key("titleby", [book])
+                for book in self.book_set.values_list("id", flat=True)
+            ]
+            cache.delete_many(cache_keys)
+        return super().save(*args, **kwargs)
+
     @property
     def isni_link(self):
         """generate the url from the isni id"""

bookwyrm/models/book.py

@@ -3,6 +3,8 @@ import re
 
 from django.contrib.postgres.search import SearchVectorField
 from django.contrib.postgres.indexes import GinIndex
+from django.core.cache import cache
+from django.core.cache.utils import make_template_fragment_key
 from django.db import models, transaction
 from django.db.models import Prefetch
 from django.dispatch import receiver
@@ -185,6 +187,11 @@ class Book(BookDataModel):
         """can't be abstract for query reasons, but you shouldn't USE it"""
         if not isinstance(self, Edition) and not isinstance(self, Work):
             raise ValueError("Books should be added as Editions or Works")
+
+        # clear template caches
+        cache_key = make_template_fragment_key("titleby", [self.id])
+        cache.delete(cache_key)
+
         return super().save(*args, **kwargs)
 
     def get_remote_id(self):

bookwyrm/models/relationship.py

@@ -1,5 +1,7 @@
 """ defines relationships between users """
 from django.apps import apps
+from django.core.cache import cache
+from django.core.cache.utils import make_template_fragment_key
 from django.db import models, transaction, IntegrityError
 from django.db.models import Q
 
@@ -36,6 +38,20 @@ class UserRelationship(BookWyrmModel):
         """the remote user needs to recieve direct broadcasts"""
         return [u for u in [self.user_subject, self.user_object] if not u.local]
 
+    def save(self, *args, **kwargs):
+        """clear the template cache"""
+        # invalidate the template cache
+        cache_keys = [
+            make_template_fragment_key(
+                "follow_button", [self.user_subject.id, self.user_object.id]
+            ),
+            make_template_fragment_key(
+                "follow_button", [self.user_object.id, self.user_subject.id]
+            ),
+        ]
+        cache.delete_many(cache_keys)
+        super().save(*args, **kwargs)
+
     class Meta:
         """relationships should be unique"""
 

bookwyrm/models/status.py

@@ -82,6 +82,7 @@ class Status(OrderedCollectionPageMixin, BookWyrmModel):
 
         if not self.reply_parent:
             self.thread_id = self.id
+
         super().save(broadcast=False, update_fields=["thread_id"])
 
     def delete(self, *args, **kwargs):  # pylint: disable=unused-argument

bookwyrm/redis_store.py

@@ -5,7 +5,10 @@ import redis
 from bookwyrm import settings
 
 r = redis.Redis(
-    host=settings.REDIS_ACTIVITY_HOST, port=settings.REDIS_ACTIVITY_PORT, db=0
+    host=settings.REDIS_ACTIVITY_HOST,
+    port=settings.REDIS_ACTIVITY_PORT,
+    password=settings.REDIS_ACTIVITY_PASSWORD,
+    db=0,
 )
 
 

bookwyrm/settings.py

@@ -119,6 +119,22 @@ STREAMS = [
     {"key": "books", "name": _("Books Timeline"), "shortname": _("Books")},
 ]
 
+# Redis cache backend
+if not env("USE_LOCAL_CACHE", False):
+    # pylint: disable=line-too-long
+    CACHES = {
+        "default": {
+            "BACKEND": "django_redis.cache.RedisCache",
+            "LOCATION": f"redis://:{REDIS_ACTIVITY_PASSWORD}@{REDIS_ACTIVITY_HOST}:{REDIS_ACTIVITY_PORT}/0",
+            "OPTIONS": {
+                "CLIENT_CLASS": "django_redis.client.DefaultClient",
+            },
+        }
+    }
+
+    SESSION_ENGINE = "django.contrib.sessions.backends.cache"
+    SESSION_CACHE_ALIAS = "default"
+
 # Database
 # https://docs.djangoproject.com/en/3.2/ref/settings/#databases
 

bookwyrm/suggested_users.py

@@ -29,6 +29,7 @@ class SuggestedUsers(RedisStore):
 
     def get_counts_from_rank(self, rank):  # pylint: disable=no-self-use
         """calculate mutuals count and shared books count from rank"""
+        # pylint: disable=c-extension-no-member
         return {
             "mutuals": math.floor(rank),
             # "shared_books": int(1 / (-1 * (rank % 1 - 1))) - 1,
@@ -112,6 +113,7 @@ def get_annotated_users(viewer, *args, **kwargs):
                 ),
                 distinct=True,
             ),
+            # pylint: disable=line-too-long
             # shared_books=Count(
             #     "shelfbook",
             #     filter=Q(

bookwyrm/templates/feed/layout.html

@@ -8,82 +8,7 @@
 <div class="columns">
     {% if user.is_authenticated %}
     <div class="column is-one-third">
-        <section class="block">
+        {% include "feed/suggested_books.html" %}
-            <h2 class="title is-4">{% trans "Your Books" %}</h2>
-            {% if not suggested_books %}
-            <p>{% trans "There are no books here right now! Try searching for a book to get started" %}</p>
-            {% else %}
-            {% with active_book=request.GET.book %}
-            <div class="tab-group">
-                <div class="tabs is-small">
-                    <ul role="tablist">
-                    {% for shelf in suggested_books %}
-                    {% if shelf.books %}
-                    {% with shelf_counter=forloop.counter %}
-                        <li>
-                            <p>
-                                {% if shelf.identifier == 'to-read' %}{% trans "To Read" %}
-                                {% elif shelf.identifier == 'reading' %}{% trans "Currently Reading" %}
-                                {% elif shelf.identifier == 'read' %}{% trans "Read" %}
-                                {% else %}{{ shelf.name }}{% endif %}
-                            </p>
-                            <div class="tabs is-small is-toggle">
-                                <ul>
-                                    {% for book in shelf.books %}
-                                    <li class="{% if active_book == book.id|stringformat:'d' %}is-active{% elif not active_book and shelf_counter == 1 and forloop.first %}is-active{% endif %}">
-                                        <a
-                                            href="{{ request.path }}?book={{ book.id }}"
-                                            id="tab_book_{{ book.id }}"
-                                            role="tab"
-                                            aria-label="{{ book.title }}"
-                                            aria-selected="{% if active_book == book.id|stringformat:'d' %}true{% elif shelf_counter == 1 and forloop.first %}true{% else %}false{% endif %}"
-                                            aria-controls="book_{{ book.id }}">
-                                            {% include 'snippets/book_cover.html' with book=book cover_class='is-h-m' %}
-                                        </a>
-                                    </li>
-                                    {% endfor %}
-                                </ul>
-                            </div>
-                        </li>
-                    {% endwith %}
-                    {% endif %}
-                    {% endfor %}
-                    </ul>
-                </div>
-                {% for shelf in suggested_books %}
-                {% with shelf_counter=forloop.counter %}
-                {% for book in shelf.books %}
-                <div
-                    class="suggested-tabs card"
-                    role="tabpanel"
-                    id="book_{{ book.id }}"
-                    {% if active_book and active_book == book.id|stringformat:'d' %}{% elif not active_book and shelf_counter == 1 and forloop.first %}{% else %} hidden{% endif %}
-                    aria-labelledby="tab_book_{{ book.id }}">
-
-                    <div class="card-header">
-                        <div class="card-header-title">
-                            <div>
-                                <p class="mb-2">{% include 'snippets/book_titleby.html' with book=book %}</p>
-                                {% include 'snippets/shelve_button/shelve_button.html' with book=book %}
-                            </div>
-                        </div>
-                        <div class="card-header-icon is-hidden-tablet">
-                            {% trans "Close" as button_text %}
-                            {% include 'snippets/toggle/toggle_button.html' with label=button_text controls_text="book" controls_uid=book.id class="delete" nonbutton=True pressed=True %}
-                        </div>
-                    </div>
-                    <div class="card-content">
-                    {% include 'snippets/create_status.html' with book=book %}
-                    </div>
-                </div>
-                {% endfor %}
-                {% endwith %}
-                {% endfor %}
-            </div>
-            {% endwith %}
-            {% endif %}
-        </section>
-
         {% if goal %}
         <section class="block">
             <div class="block">

bookwyrm/templates/feed/suggested_books.html

@@ -0,0 +1,83 @@
+{% load i18n %}
+{% load cache %}
+{% load bookwyrm_tags %}
+
+{# 6 month cache #}
+{% cache 15552000 suggested_books request.user.id %}
+{% suggested_books as suggested_books %}
+<section class="block">
+    <h2 class="title is-4">{% trans "Your Books" %}</h2>
+    {% if not suggested_books %}
+    <p>{% trans "There are no books here right now! Try searching for a book to get started" %}</p>
+    {% else %}
+    {% with active_book=request.GET.book %}
+    <div class="tab-group">
+        <div class="tabs is-small">
+            <ul role="tablist">
+            {% for shelf in suggested_books %}
+            {% if shelf.books %}
+            {% with shelf_counter=forloop.counter %}
+                <li>
+                    <p>
+                        {% if shelf.identifier == 'to-read' %}{% trans "To Read" %}
+                        {% elif shelf.identifier == 'reading' %}{% trans "Currently Reading" %}
+                        {% elif shelf.identifier == 'read' %}{% trans "Read" %}
+                        {% else %}{{ shelf.name }}{% endif %}
+                    </p>
+                    <div class="tabs is-small is-toggle">
+                        <ul>
+                            {% for book in shelf.books %}
+                            <li class="{% if active_book == book.id|stringformat:'d' %}is-active{% elif not active_book and shelf_counter == 1 and forloop.first %}is-active{% endif %}">
+                                <a
+                                    href="{{ request.path }}?book={{ book.id }}"
+                                    id="tab_book_{{ book.id }}"
+                                    role="tab"
+                                    aria-label="{{ book.title }}"
+                                    aria-selected="{% if active_book == book.id|stringformat:'d' %}true{% elif shelf_counter == 1 and forloop.first %}true{% else %}false{% endif %}"
+                                    aria-controls="book_{{ book.id }}">
+                                    {% include 'snippets/book_cover.html' with book=book cover_class='is-h-m' %}
+                                </a>
+                            </li>
+                            {% endfor %}
+                        </ul>
+                    </div>
+                </li>
+            {% endwith %}
+            {% endif %}
+            {% endfor %}
+            </ul>
+        </div>
+        {% for shelf in suggested_books %}
+        {% with shelf_counter=forloop.counter %}
+        {% for book in shelf.books %}
+        <div
+            class="suggested-tabs card"
+            role="tabpanel"
+            id="book_{{ book.id }}"
+            {% if active_book and active_book == book.id|stringformat:'d' %}{% elif not active_book and shelf_counter == 1 and forloop.first %}{% else %} hidden{% endif %}
+            aria-labelledby="tab_book_{{ book.id }}">
+
+            <div class="card-header">
+                <div class="card-header-title">
+                    <div>
+                        <p class="mb-2">{% include 'snippets/book_titleby.html' with book=book %}</p>
+                        {% include 'snippets/shelve_button/shelve_button.html' with book=book %}
+                    </div>
+                </div>
+                <div class="card-header-icon is-hidden-tablet">
+                    {% trans "Close" as button_text %}
+                    {% include 'snippets/toggle/toggle_button.html' with label=button_text controls_text="book" controls_uid=book.id class="delete" nonbutton=True pressed=True %}
+                </div>
+            </div>
+            <div class="card-content">
+            {% include 'snippets/create_status.html' with book=book %}
+            </div>
+        </div>
+        {% endfor %}
+        {% endwith %}
+        {% endfor %}
+    </div>
+    {% endwith %}
+    {% endif %}
+</section>
+{% endcache %}

bookwyrm/templates/snippets/book_titleby.html

@@ -1,7 +1,11 @@
 {% load i18n %}
 {% load utilities %}
+{% load cache %}
 {% spaceless %}
 
+{# 6 month cache #}
+{% cache 15552000 titleby book.id %}
+
 {% if book.authors.exists %}
 {% blocktrans trimmed with path=book.local_path title=book|book_title %}
 <a href="{{ path }}">{{ title }}</a> by
@@ -10,4 +14,6 @@
 {% else %}
 <a href="{{ book.local_path }}">{{ book|book_title }}</a>
 {% endif %}
+
+{% endcache %}
 {% endspaceless %}

bookwyrm/templates/snippets/follow_button.html

@@ -1,4 +1,8 @@
 {% load i18n %}
+{% load cache %}
+
+{# 6 month cache #}
+{% cache 15552000 follow_button request.user.id user.id %}
 {% if request.user == user or not request.user.is_authenticated %}
 {% elif user in request.user.blocks.all %}
 {% include 'snippets/block_button.html' with blocks=True %}
@@ -42,3 +46,4 @@
     {% endif %}
 </div>
 {% endif %}
+{% endcache %}

bookwyrm/templates/snippets/shelve_button/shelve_button.html

@@ -1,7 +1,10 @@
 {% load bookwyrm_tags %}
 {% load utilities %}
+{% load cache %}
 
 {% if request.user.is_authenticated %}
+{# 6 month cache #}
+{% cache 15552000 shelve_button request.user.id book.id %}
 
 {% with book.id|uuid as uuid %}
 {% active_shelf book as active_shelf %}
@@ -32,4 +35,5 @@
 {% include 'snippets/reading_modals/progress_update_modal.html' with book=active_shelf.book id=modal_id readthrough=readthrough class="" %}
 
 {% endwith %}
+{% endcache %}
 {% endif %}

bookwyrm/templates/snippets/status/headers/generatednote.html

@@ -1,3 +1,7 @@
+{% load cache %}
+
+{# Three day cache #}
+{% cache 259200 generated_note_header status.id %}
 {% if status.content == 'wants to read' %}
     {% include 'snippets/status/headers/to_read.html' with book=status.mention_books.first %}
 {% elif status.content == 'finished reading' %}
@@ -7,3 +11,4 @@
 {% else %}
     {{ status.content }}
 {% endif %}
+{% endcache %}

bookwyrm/templates/snippets/status/layout.html

@@ -1,6 +1,7 @@
 {% extends 'components/card.html' %}
 {% load i18n %}
 {% load utilities %}
+{% load cache %}
 
 {% block card-header %}
 <div
@@ -30,6 +31,7 @@
 {# nothing here #}
 {% elif request.user.is_authenticated %}
 
+    {% cache 259200 interact request.user.id status.id %}
     <div class="card-footer-item">
         {% trans "Reply" as button_text %}
         {% include 'snippets/toggle/toggle_button.html' with controls_text="show_comment" controls_uid=status.id text=button_text icon_with_text="comment" class="is-small is-light is-transparent toggle-button" focus="id_content_reply" %}
@@ -45,8 +47,10 @@
         {% include 'snippets/status/status_options.html' with class="is-small is-light is-transparent" right=True %}
     </div>
     {% endif %}
+    {% endcache %}
 
 {% else %}
+
     <div class="card-footer-item">
         <a href="{% url 'login' %}">
             <span class="icon icon-comment is-small" title="{% trans 'Reply' %}">

bookwyrm/templatetags/bookwyrm_tags.py

@@ -3,6 +3,7 @@ from django import template
 from django.db.models import Avg
 
 from bookwyrm import models
+from bookwyrm.views.feed import get_suggested_books
 
 
 register = template.Library()
@@ -115,3 +116,11 @@ def mutuals_count(context, user):
     if not viewer.is_authenticated:
         return None
     return user.followers.filter(followers=viewer).count()
+
+
+@register.simple_tag(takes_context=True)
+def suggested_books(context):
+    """get books for suggested books panel"""
+    # this happens here instead of in the view so that the template snippet can
+    # be cached in the template
+    return get_suggested_books(context["request"].user)

bookwyrm/views/feed.py

@@ -223,7 +223,6 @@ def feed_page_data(user):
 
     goal = models.AnnualGoal.objects.filter(user=user, year=timezone.now().year).first()
     return {
-        "suggested_books": get_suggested_books(user),
         "goal": goal,
         "goal_form": forms.GoalForm(),
     }

bookwyrm/views/interaction.py

@@ -1,6 +1,8 @@
 """ boosts and favs """
-from django.db import IntegrityError
 from django.contrib.auth.decorators import login_required
+from django.core.cache import cache
+from django.core.cache.utils import make_template_fragment_key
+from django.db import IntegrityError
 from django.http import HttpResponse, HttpResponseBadRequest, HttpResponseNotFound
 from django.shortcuts import redirect
 from django.utils.decorators import method_decorator
@@ -17,6 +19,7 @@ class Favorite(View):
 
     def post(self, request, status_id):
         """create a like"""
+        clear_cache(request.user.id, status_id)
         status = models.Status.objects.get(id=status_id)
         try:
             models.Favorite.objects.create(status=status, user=request.user)
@@ -43,6 +46,7 @@ class Unfavorite(View):
             return HttpResponseNotFound()
 
         favorite.delete()
+        clear_cache(request.user.id, status_id)
         if is_api_request(request):
             return HttpResponse()
         return redirect(request.headers.get("Referer", "/"))
@@ -70,6 +74,7 @@ class Boost(View):
             privacy=status.privacy,
             user=request.user,
         )
+        clear_cache(request.user.id, status_id)
         if is_api_request(request):
             return HttpResponse()
         return redirect(request.headers.get("Referer", "/"))
@@ -87,6 +92,13 @@ class Unboost(View):
         ).first()
 
         boost.delete()
+        clear_cache(request.user.id, status_id)
         if is_api_request(request):
             return HttpResponse()
         return redirect(request.headers.get("Referer", "/"))
+
+
+def clear_cache(user_id, status_id):
+    """clear template cache"""
+    cache_key = make_template_fragment_key("interact", [user_id, status_id])
+    cache.delete(cache_key)

bookwyrm/views/landing/landing.py

@@ -1,6 +1,8 @@
 """ non-interactive pages """
 from django.template.response import TemplateResponse
 from django.views import View
+from django.utils.decorators import method_decorator
+from django.views.decorators.cache import cache_page
 
 from bookwyrm import forms
 from bookwyrm.views import helpers
@@ -31,6 +33,7 @@ class Home(View):
 class Landing(View):
     """preview of recently reviewed books"""
 
+    @method_decorator(cache_page(60 * 60), name="dispatch")
     def get(self, request):
         """tiled book activity page"""
         data = {

bookwyrm/views/reading.py

@@ -1,5 +1,7 @@
 """ the good stuff! the books! """
 from django.contrib.auth.decorators import login_required
+from django.core.cache import cache
+from django.core.cache.utils import make_template_fragment_key
 from django.db import transaction
 from django.http import HttpResponse, HttpResponseBadRequest, HttpResponseNotFound
 from django.shortcuts import get_object_or_404, redirect
@@ -44,6 +46,13 @@ class ReadingStatus(View):
         if not identifier:
             return HttpResponseBadRequest()
 
+        # invalidate the template cache
+        cache_keys = [
+            make_template_fragment_key("shelve_button", [request.user.id, book_id]),
+            make_template_fragment_key("suggested_books", [request.user.id]),
+        ]
+        cache.delete_many(cache_keys)
+
         desired_shelf = get_object_or_404(
             models.Shelf, identifier=identifier, user=request.user
         )

celerywyrm/settings.py

@@ -3,11 +3,15 @@
 # pylint: disable=unused-wildcard-import
 from bookwyrm.settings import *
 
-CELERY_BROKER_URL = "redis://:{}@redis_broker:{}/0".format(
+REDIS_BROKER_PASSWORD = requests.utils.quote(env("REDIS_BROKER_PASSWORD", None))
-    requests.utils.quote(env("REDIS_BROKER_PASSWORD", "")), env("REDIS_BROKER_PORT")
+REDIS_BROKER_HOST = env("REDIS_BROKER_HOST", "redis_broker")
+REDIS_BROKER_PORT = env("REDIS_BROKER_PORT", 6379)
+
+CELERY_BROKER_URL = (
+    f"redis://:{REDIS_BROKER_PASSWORD}@{REDIS_BROKER_HOST}:{REDIS_BROKER_PORT}/0"
 )
-CELERY_RESULT_BACKEND = "redis://:{}@redis_broker:{}/0".format(
+CELERY_RESULT_BACKEND = (
-    requests.utils.quote(env("REDIS_BROKER_PASSWORD", "")), env("REDIS_BROKER_PORT")
+    f"redis://:{REDIS_BROKER_PASSWORD}@{REDIS_BROKER_HOST}:{REDIS_BROKER_PORT}/0"
 )
 
 CELERY_DEFAULT_QUEUE = "low_priority"

docker-compose.yml

@@ -38,16 +38,17 @@ services:
       - 8000:8000
   redis_activity:
     image: redis
-    command: ["redis-server", "--appendonly", "yes"]
+    command: redis-server --requirepass ${REDIS_ACTIVITY_PASSWORD} --appendonly yes --port ${REDIS_ACTIVITY_PORT}
     env_file: .env
     networks:
       - main
     restart: on-failure
     volumes:
+      - ./redis.conf:/etc/redis/redis.conf
       - redis_activity_data:/data
   redis_broker:
     image: redis
-    command: ["redis-server", "--appendonly", "yes"]
+    command: redis-server --requirepass ${REDIS_BROKER_PASSWORD} --appendonly yes --port ${REDIS_BROKER_PORT}
     env_file: .env
     ports:
       - 6379:6379
@@ -55,6 +56,7 @@ services:
       - main
     restart: on-failure
     volumes:
+      - ./redis.conf:/etc/redis/redis.conf
       - redis_broker_data:/data
   celery_worker:
     env_file: .env

redis.conf

@@ -0,0 +1,9 @@
+bind 127.0.0.1 ::1
+protected-mode yes
+port 6379
+
+rename-command FLUSHDB ""
+rename-command FLUSHALL ""
+rename-command DEBUG ""
+rename-command CONFIG ""
+rename-command SHUTDOWN ""

requirements.txt

@@ -17,6 +17,7 @@ django-rename-app==0.1.2
 pytz>=2021.1
 boto3==1.17.88
 django-storages==1.11.1
+django-redis==5.2.0
 
 # Dev
 black==21.4b0