rafaelcaricio/moviewyrm
1
0
Fork 0
forked from mirrors/bookwyrm
Code Issues 4 Pull requests Projects Releases Wiki Activity

Merge pull request #1768 from bookwyrm-social/shelf-button-cache

Browse source 
Cache queries in feed view
This commit is contained in:
Mouse Reeve 2022-01-06 10:42:31 -08:00 committed by GitHub
commit 32ac4111aa
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
25 changed files with 267 additions and 127 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
.env.dev.example
View file

@ -26,15 +26,15 @@ POSTGRES_HOST=db
MAX_STREAM_LENGTH=200
REDIS_ACTIVITY_HOST=redis_activity
REDIS_ACTIVITY_PORT=6379
#REDIS_ACTIVITY_PASSWORD=redispassword345
REDIS_ACTIVITY_PASSWORD=redispassword345
# Redis as celery broker
REDIS_BROKER_PORT=6379
#REDIS_BROKER_PASSWORD=redispassword123
REDIS_BROKER_PASSWORD=redispassword123
FLOWER_PORT=8888
#FLOWER_USER=mouse
#FLOWER_PASSWORD=changeme
FLOWER_USER=mouse
FLOWER_PASSWORD=changeme
EMAIL_HOST=smtp.mailgun.org
EMAIL_PORT=587

2
.github/workflows/django-tests.yml vendored
View file

@ -46,6 +46,8 @@ jobs:
POSTGRES_HOST: 127.0.0.1
CELERY_BROKER: ""
REDIS_BROKER_PORT: 6379
REDIS_BROKER_PASSWORD: beep
USE_LOCAL_CACHE: true
FLOWER_PORT: 8888
EMAIL_HOST: "smtp.mailgun.org"
EMAIL_PORT: 587

13
bookwyrm/models/author.py
View file

@ -1,6 +1,8 @@
""" database schema for info about authors """
import re
from django.contrib.postgres.indexes import GinIndex
from django.core.cache import cache
from django.core.cache.utils import make_template_fragment_key
from django.db import models
from bookwyrm import activitypub
@ -34,6 +36,17 @@ class Author(BookDataModel):
)
bio = fields.HtmlField(null=True, blank=True)
def save(self, *args, **kwargs):
"""clear related template caches"""
# clear template caches
if self.id:
cache_keys = [
make_template_fragment_key("titleby", [book])
for book in self.book_set.values_list("id", flat=True)
]
cache.delete_many(cache_keys)
return super().save(*args, **kwargs)
@property
def isni_link(self):
"""generate the url from the isni id"""

7
bookwyrm/models/book.py
View file

@ -3,6 +3,8 @@ import re
from django.contrib.postgres.search import SearchVectorField
from django.contrib.postgres.indexes import GinIndex
from django.core.cache import cache
from django.core.cache.utils import make_template_fragment_key
from django.db import models, transaction
from django.db.models import Prefetch
from django.dispatch import receiver
@ -185,6 +187,11 @@ class Book(BookDataModel):
"""can't be abstract for query reasons, but you shouldn't USE it"""
if not isinstance(self, Edition) and not isinstance(self, Work):
raise ValueError("Books should be added as Editions or Works")
# clear template caches
cache_key = make_template_fragment_key("titleby", [self.id])
cache.delete(cache_key)
return super().save(*args, **kwargs)
def get_remote_id(self):

16
bookwyrm/models/relationship.py
View file

@ -1,5 +1,7 @@
""" defines relationships between users """
from django.apps import apps
from django.core.cache import cache
from django.core.cache.utils import make_template_fragment_key
from django.db import models, transaction, IntegrityError
from django.db.models import Q
@ -36,6 +38,20 @@ class UserRelationship(BookWyrmModel):
"""the remote user needs to recieve direct broadcasts"""
return [u for u in [self.user_subject, self.user_object] if not u.local]
def save(self, *args, **kwargs):
"""clear the template cache"""
# invalidate the template cache
cache_keys = [
make_template_fragment_key(
"follow_button", [self.user_subject.id, self.user_object.id]
),
make_template_fragment_key(
"follow_button", [self.user_object.id, self.user_subject.id]
),
]
cache.delete_many(cache_keys)
super().save(*args, **kwargs)
class Meta:
"""relationships should be unique"""

1
bookwyrm/models/status.py
View file

@ -82,6 +82,7 @@ class Status(OrderedCollectionPageMixin, BookWyrmModel):
if not self.reply_parent:
self.thread_id = self.id
super().save(broadcast=False, update_fields=["thread_id"])
def delete(self, *args, **kwargs): # pylint: disable=unused-argument

5
bookwyrm/redis_store.py
View file

@ -5,7 +5,10 @@ import redis
from bookwyrm import settings
r = redis.Redis(
host=settings.REDIS_ACTIVITY_HOST, port=settings.REDIS_ACTIVITY_PORT, db=0
host=settings.REDIS_ACTIVITY_HOST,
port=settings.REDIS_ACTIVITY_PORT,
password=settings.REDIS_ACTIVITY_PASSWORD,
db=0,
)

16
bookwyrm/settings.py
View file

@ -119,6 +119,22 @@ STREAMS = [
{"key": "books", "name": _("Books Timeline"), "shortname": _("Books")},
]
# Redis cache backend
if not env("USE_LOCAL_CACHE", False):
# pylint: disable=line-too-long
CACHES = {
"default": {
"BACKEND": "django_redis.cache.RedisCache",
"LOCATION": f"redis://:{REDIS_ACTIVITY_PASSWORD}@{REDIS_ACTIVITY_HOST}:{REDIS_ACTIVITY_PORT}/0",
"OPTIONS": {
"CLIENT_CLASS": "django_redis.client.DefaultClient",
},
}
}
SESSION_ENGINE = "django.contrib.sessions.backends.cache"
SESSION_CACHE_ALIAS = "default"
# Database
# https://docs.djangoproject.com/en/3.2/ref/settings/#databases

22
bookwyrm/suggested_users.py
View file

@ -29,6 +29,7 @@ class SuggestedUsers(RedisStore):
def get_counts_from_rank(self, rank): # pylint: disable=no-self-use
"""calculate mutuals count and shared books count from rank"""
# pylint: disable=c-extension-no-member
return {
"mutuals": math.floor(rank),
# "shared_books": int(1 / (-1 * (rank % 1 - 1))) - 1,
@ -112,16 +113,17 @@ def get_annotated_users(viewer, *args, **kwargs):
),
distinct=True,
),
# shared_books=Count(
# "shelfbook",
# filter=Q(
# ~Q(id=viewer.id),
# shelfbook__book__parent_work__in=[
# s.book.parent_work for s in viewer.shelfbook_set.all()
# ],
# ),
# distinct=True,
# ),
# pylint: disable=line-too-long
# shared_books=Count(
# "shelfbook",
# filter=Q(
# ~Q(id=viewer.id),
# shelfbook__book__parent_work__in=[
# s.book.parent_work for s in viewer.shelfbook_set.all()
# ],
# ),
# distinct=True,
# ),
)
)

77
bookwyrm/templates/feed/layout.html
View file

@ -8,82 +8,7 @@
<div class="columns">
{% if user.is_authenticated %}
<div class="column is-one-third">
<section class="block">
<h2 class="title is-4">{% trans "Your Books" %}</h2>
{% if not suggested_books %}
<p>{% trans "There are no books here right now! Try searching for a book to get started" %}</p>
{% else %}
{% with active_book=request.GET.book %}
<div class="tab-group">
<div class="tabs is-small">
<ul role="tablist">
{% for shelf in suggested_books %}
{% if shelf.books %}
{% with shelf_counter=forloop.counter %}
<li>
<p>
{% if shelf.identifier == 'to-read' %}{% trans "To Read" %}
{% elif shelf.identifier == 'reading' %}{% trans "Currently Reading" %}
{% elif shelf.identifier == 'read' %}{% trans "Read" %}
{% else %}{{ shelf.name }}{% endif %}
</p>
<div class="tabs is-small is-toggle">
<ul>
{% for book in shelf.books %}
<li class="{% if active_book == book.id|stringformat:'d' %}is-active{% elif not active_book and shelf_counter == 1 and forloop.first %}is-active{% endif %}">
<a
href="{{ request.path }}?book={{ book.id }}"
id="tab_book_{{ book.id }}"
role="tab"
aria-label="{{ book.title }}"
aria-selected="{% if active_book == book.id|stringformat:'d' %}true{% elif shelf_counter == 1 and forloop.first %}true{% else %}false{% endif %}"
aria-controls="book_{{ book.id }}">
{% include 'snippets/book_cover.html' with book=book cover_class='is-h-m' %}
</a>
</li>
{% endfor %}
</ul>
</div>
</li>
{% endwith %}
{% endif %}
{% endfor %}
</ul>
</div>
{% for shelf in suggested_books %}
{% with shelf_counter=forloop.counter %}
{% for book in shelf.books %}
<div
class="suggested-tabs card"
role="tabpanel"
id="book_{{ book.id }}"
{% if active_book and active_book == book.id|stringformat:'d' %}{% elif not active_book and shelf_counter == 1 and forloop.first %}{% else %} hidden{% endif %}
aria-labelledby="tab_book_{{ book.id }}">
<div class="card-header">
<div class="card-header-title">
<div>
<p class="mb-2">{% include 'snippets/book_titleby.html' with book=book %}</p>
{% include 'snippets/shelve_button/shelve_button.html' with book=book %}
</div>
</div>
<div class="card-header-icon is-hidden-tablet">
{% trans "Close" as button_text %}
{% include 'snippets/toggle/toggle_button.html' with label=button_text controls_text="book" controls_uid=book.id class="delete" nonbutton=True pressed=True %}
</div>
</div>
<div class="card-content">
{% include 'snippets/create_status.html' with book=book %}
</div>
</div>
{% endfor %}
{% endwith %}
{% endfor %}
</div>
{% endwith %}
{% endif %}
</section>
{% include "feed/suggested_books.html" %}
{% if goal %}
<section class="block">
<div class="block">

83
bookwyrm/templates/feed/suggested_books.html Normal file
View file

@ -0,0 +1,83 @@
{% load i18n %}
{% load cache %}
{% load bookwyrm_tags %}
{# 6 month cache #}
{% cache 15552000 suggested_books request.user.id %}
{% suggested_books as suggested_books %}
<section class="block">
<h2 class="title is-4">{% trans "Your Books" %}</h2>
{% if not suggested_books %}
<p>{% trans "There are no books here right now! Try searching for a book to get started" %}</p>
{% else %}
{% with active_book=request.GET.book %}
<div class="tab-group">
<div class="tabs is-small">
<ul role="tablist">
{% for shelf in suggested_books %}
{% if shelf.books %}
{% with shelf_counter=forloop.counter %}
<li>
<p>
{% if shelf.identifier == 'to-read' %}{% trans "To Read" %}
{% elif shelf.identifier == 'reading' %}{% trans "Currently Reading" %}
{% elif shelf.identifier == 'read' %}{% trans "Read" %}
{% else %}{{ shelf.name }}{% endif %}
</p>
<div class="tabs is-small is-toggle">
<ul>
{% for book in shelf.books %}
<li class="{% if active_book == book.id|stringformat:'d' %}is-active{% elif not active_book and shelf_counter == 1 and forloop.first %}is-active{% endif %}">
<a
href="{{ request.path }}?book={{ book.id }}"
id="tab_book_{{ book.id }}"
role="tab"
aria-label="{{ book.title }}"
aria-selected="{% if active_book == book.id|stringformat:'d' %}true{% elif shelf_counter == 1 and forloop.first %}true{% else %}false{% endif %}"
aria-controls="book_{{ book.id }}">
{% include 'snippets/book_cover.html' with book=book cover_class='is-h-m' %}
</a>
</li>
{% endfor %}
</ul>
</div>
</li>
{% endwith %}
{% endif %}
{% endfor %}
</ul>
</div>
{% for shelf in suggested_books %}
{% with shelf_counter=forloop.counter %}
{% for book in shelf.books %}
<div
class="suggested-tabs card"
role="tabpanel"
id="book_{{ book.id }}"
{% if active_book and active_book == book.id|stringformat:'d' %}{% elif not active_book and shelf_counter == 1 and forloop.first %}{% else %} hidden{% endif %}
aria-labelledby="tab_book_{{ book.id }}">
<div class="card-header">
<div class="card-header-title">
<div>
<p class="mb-2">{% include 'snippets/book_titleby.html' with book=book %}</p>
{% include 'snippets/shelve_button/shelve_button.html' with book=book %}
</div>
</div>
<div class="card-header-icon is-hidden-tablet">
{% trans "Close" as button_text %}
{% include 'snippets/toggle/toggle_button.html' with label=button_text controls_text="book" controls_uid=book.id class="delete" nonbutton=True pressed=True %}
</div>
</div>
<div class="card-content">
{% include 'snippets/create_status.html' with book=book %}
</div>
</div>
{% endfor %}
{% endwith %}
{% endfor %}
</div>
{% endwith %}
{% endif %}
</section>
{% endcache %}

6
bookwyrm/templates/snippets/book_titleby.html
View file

@ -1,7 +1,11 @@
{% load i18n %}
{% load utilities %}
{% load cache %}
{% spaceless %}
{# 6 month cache #}
{% cache 15552000 titleby book.id %}
{% if book.authors.exists %}
{% blocktrans trimmed with path=book.local_path title=book|book_title %}
<a href="{{ path }}">{{ title }}</a> by
@ -10,4 +14,6 @@
{% else %}
<a href="{{ book.local_path }}">{{ book|book_title }}</a>
{% endif %}
{% endcache %}
{% endspaceless %}

5
bookwyrm/templates/snippets/follow_button.html
View file

@ -1,4 +1,8 @@
{% load i18n %}
{% load cache %}
{# 6 month cache #}
{% cache 15552000 follow_button request.user.id user.id %}
{% if request.user == user or not request.user.is_authenticated %}
{% elif user in request.user.blocks.all %}
{% include 'snippets/block_button.html' with blocks=True %}
@ -42,3 +46,4 @@
{% endif %}
</div>
{% endif %}
{% endcache %}

4
bookwyrm/templates/snippets/shelve_button/shelve_button.html
View file

@ -1,7 +1,10 @@
{% load bookwyrm_tags %}
{% load utilities %}
{% load cache %}
{% if request.user.is_authenticated %}
{# 6 month cache #}
{% cache 15552000 shelve_button request.user.id book.id %}
{% with book.id|uuid as uuid %}
{% active_shelf book as active_shelf %}
@ -32,4 +35,5 @@
{% include 'snippets/reading_modals/progress_update_modal.html' with book=active_shelf.book id=modal_id readthrough=readthrough class="" %}
{% endwith %}
{% endcache %}
{% endif %}

5
bookwyrm/templates/snippets/status/headers/generatednote.html
View file

@ -1,3 +1,7 @@
{% load cache %}
{# Three day cache #}
{% cache 259200 generated_note_header status.id %}
{% if status.content == 'wants to read' %}
{% include 'snippets/status/headers/to_read.html' with book=status.mention_books.first %}
{% elif status.content == 'finished reading' %}
@ -7,3 +11,4 @@
{% else %}
{{ status.content }}
{% endif %}
{% endcache %}

60
bookwyrm/templates/snippets/status/layout.html
View file

@ -1,6 +1,7 @@
{% extends 'components/card.html' %}
{% load i18n %}
{% load utilities %}
{% load cache %}
{% block card-header %}
<div
@ -30,38 +31,41 @@
{# nothing here #}
{% elif request.user.is_authenticated %}
<div class="card-footer-item">
{% trans "Reply" as button_text %}
{% include 'snippets/toggle/toggle_button.html' with controls_text="show_comment" controls_uid=status.id text=button_text icon_with_text="comment" class="is-small is-light is-transparent toggle-button" focus="id_content_reply" %}
</div>
<div class="card-footer-item">
{% include 'snippets/boost_button.html' with status=status %}
</div>
<div class="card-footer-item">
{% include 'snippets/fav_button.html' with status=status %}
</div>
{% if not moderation_mode %}
<div class="card-footer-item">
{% include 'snippets/status/status_options.html' with class="is-small is-light is-transparent" right=True %}
</div>
{% endif %}
{% cache 259200 interact request.user.id status.id %}
<div class="card-footer-item">
{% trans "Reply" as button_text %}
{% include 'snippets/toggle/toggle_button.html' with controls_text="show_comment" controls_uid=status.id text=button_text icon_with_text="comment" class="is-small is-light is-transparent toggle-button" focus="id_content_reply" %}
</div>
<div class="card-footer-item">
{% include 'snippets/boost_button.html' with status=status %}
</div>
<div class="card-footer-item">
{% include 'snippets/fav_button.html' with status=status %}
</div>
{% if not moderation_mode %}
<div class="card-footer-item">
{% include 'snippets/status/status_options.html' with class="is-small is-light is-transparent" right=True %}
</div>
{% endif %}
{% endcache %}
{% else %}
<div class="card-footer-item">
<a href="{% url 'login' %}">
<span class="icon icon-comment is-small" title="{% trans 'Reply' %}">
<span class="is-sr-only">{% trans "Reply" %}</span>
</span>
<span class="icon icon-boost is-small ml-4" title="{% trans 'Boost status' %}">
<span class="is-sr-only">{% trans "Boost status" %}</span>
</span>
<div class="card-footer-item">
<a href="{% url 'login' %}">
<span class="icon icon-comment is-small" title="{% trans 'Reply' %}">
<span class="is-sr-only">{% trans "Reply" %}</span>
</span>
<span class="icon icon-heart is-small ml-4" title="{% trans 'Like status' %}">
<span class="is-sr-only">{% trans "Like status" %}</span>
</span>
</a>
</div>
<span class="icon icon-boost is-small ml-4" title="{% trans 'Boost status' %}">
<span class="is-sr-only">{% trans "Boost status" %}</span>
</span>
<span class="icon icon-heart is-small ml-4" title="{% trans 'Like status' %}">
<span class="is-sr-only">{% trans "Like status" %}</span>
</span>
</a>
</div>
{% endif %}
{% endblock %}

9
bookwyrm/templatetags/bookwyrm_tags.py
View file

@ -3,6 +3,7 @@ from django import template
from django.db.models import Avg
from bookwyrm import models
from bookwyrm.views.feed import get_suggested_books
register = template.Library()
@ -115,3 +116,11 @@ def mutuals_count(context, user):
if not viewer.is_authenticated:
return None
return user.followers.filter(followers=viewer).count()
@register.simple_tag(takes_context=True)
def suggested_books(context):
"""get books for suggested books panel"""
# this happens here instead of in the view so that the template snippet can
# be cached in the template
return get_suggested_books(context["request"].user)

1
bookwyrm/views/feed.py
View file

@ -223,7 +223,6 @@ def feed_page_data(user):
goal = models.AnnualGoal.objects.filter(user=user, year=timezone.now().year).first()
return {
"suggested_books": get_suggested_books(user),
"goal": goal,
"goal_form": forms.GoalForm(),
}

14
bookwyrm/views/interaction.py
View file

@ -1,6 +1,8 @@
""" boosts and favs """
from django.db import IntegrityError
from django.contrib.auth.decorators import login_required
from django.core.cache import cache
from django.core.cache.utils import make_template_fragment_key
from django.db import IntegrityError
from django.http import HttpResponse, HttpResponseBadRequest, HttpResponseNotFound
from django.shortcuts import redirect
from django.utils.decorators import method_decorator
@ -17,6 +19,7 @@ class Favorite(View):
def post(self, request, status_id):
"""create a like"""
clear_cache(request.user.id, status_id)
status = models.Status.objects.get(id=status_id)
try:
models.Favorite.objects.create(status=status, user=request.user)
@ -43,6 +46,7 @@ class Unfavorite(View):
return HttpResponseNotFound()
favorite.delete()
clear_cache(request.user.id, status_id)
if is_api_request(request):
return HttpResponse()
return redirect(request.headers.get("Referer", "/"))
@ -70,6 +74,7 @@ class Boost(View):
privacy=status.privacy,
user=request.user,
)
clear_cache(request.user.id, status_id)
if is_api_request(request):
return HttpResponse()
return redirect(request.headers.get("Referer", "/"))
@ -87,6 +92,13 @@ class Unboost(View):
).first()
boost.delete()
clear_cache(request.user.id, status_id)
if is_api_request(request):
return HttpResponse()
return redirect(request.headers.get("Referer", "/"))
def clear_cache(user_id, status_id):
"""clear template cache"""
cache_key = make_template_fragment_key("interact", [user_id, status_id])
cache.delete(cache_key)

3
bookwyrm/views/landing/landing.py
View file

@ -1,6 +1,8 @@
""" non-interactive pages """
from django.template.response import TemplateResponse
from django.views import View
from django.utils.decorators import method_decorator
from django.views.decorators.cache import cache_page
from bookwyrm import forms
from bookwyrm.views import helpers
@ -31,6 +33,7 @@ class Home(View):
class Landing(View):
"""preview of recently reviewed books"""
@method_decorator(cache_page(60 * 60), name="dispatch")
def get(self, request):
"""tiled book activity page"""
data = {

9
bookwyrm/views/reading.py
View file

@ -1,5 +1,7 @@
""" the good stuff! the books! """
from django.contrib.auth.decorators import login_required
from django.core.cache import cache
from django.core.cache.utils import make_template_fragment_key
from django.db import transaction
from django.http import HttpResponse, HttpResponseBadRequest, HttpResponseNotFound
from django.shortcuts import get_object_or_404, redirect
@ -44,6 +46,13 @@ class ReadingStatus(View):
if not identifier:
return HttpResponseBadRequest()
# invalidate the template cache
cache_keys = [
make_template_fragment_key("shelve_button", [request.user.id, book_id]),
make_template_fragment_key("suggested_books", [request.user.id]),
]
cache.delete_many(cache_keys)
desired_shelf = get_object_or_404(
models.Shelf, identifier=identifier, user=request.user
)

12
celerywyrm/settings.py
View file

@ -3,11 +3,15 @@
# pylint: disable=unused-wildcard-import
from bookwyrm.settings import *
CELERY_BROKER_URL = "redis://:{}@redis_broker:{}/0".format(
requests.utils.quote(env("REDIS_BROKER_PASSWORD", "")), env("REDIS_BROKER_PORT")
REDIS_BROKER_PASSWORD = requests.utils.quote(env("REDIS_BROKER_PASSWORD", None))
REDIS_BROKER_HOST = env("REDIS_BROKER_HOST", "redis_broker")
REDIS_BROKER_PORT = env("REDIS_BROKER_PORT", 6379)
CELERY_BROKER_URL = (
f"redis://:{REDIS_BROKER_PASSWORD}@{REDIS_BROKER_HOST}:{REDIS_BROKER_PORT}/0"
)
CELERY_RESULT_BACKEND = "redis://:{}@redis_broker:{}/0".format(
requests.utils.quote(env("REDIS_BROKER_PASSWORD", "")), env("REDIS_BROKER_PORT")
CELERY_RESULT_BACKEND = (
f"redis://:{REDIS_BROKER_PASSWORD}@{REDIS_BROKER_HOST}:{REDIS_BROKER_PORT}/0"
)
CELERY_DEFAULT_QUEUE = "low_priority"

6
docker-compose.yml
View file

@ -38,16 +38,17 @@ services:
- 8000:8000
redis_activity:
image: redis
command: ["redis-server", "--appendonly", "yes"]
command: redis-server --requirepass ${REDIS_ACTIVITY_PASSWORD} --appendonly yes --port ${REDIS_ACTIVITY_PORT}
env_file: .env
networks:
- main
restart: on-failure
volumes:
- ./redis.conf:/etc/redis/redis.conf
- redis_activity_data:/data
redis_broker:
image: redis
command: ["redis-server", "--appendonly", "yes"]
command: redis-server --requirepass ${REDIS_BROKER_PASSWORD} --appendonly yes --port ${REDIS_BROKER_PORT}
env_file: .env
ports:
- 6379:6379
@ -55,6 +56,7 @@ services:
- main
restart: on-failure
volumes:
- ./redis.conf:/etc/redis/redis.conf
- redis_broker_data:/data
celery_worker:
env_file: .env

9
redis.conf Normal file
View file

@ -0,0 +1,9 @@
bind 127.0.0.1 ::1
protected-mode yes
port 6379
rename-command FLUSHDB ""
rename-command FLUSHALL ""
rename-command DEBUG ""
rename-command CONFIG ""
rename-command SHUTDOWN ""

1
requirements.txt
View file

@ -17,6 +17,7 @@ django-rename-app==0.1.2
pytz>=2021.1
boto3==1.17.88
django-storages==1.11.1
django-redis==5.2.0
# Dev
black==21.4b0