From d93da4e86d771f88d654339b3563a852dc69c767 Mon Sep 17 00:00:00 2001 From: Ross Chapman Date: Mon, 27 Nov 2023 15:03:59 -0800 Subject: [PATCH 02/17] Checkpoint --- bookwyrm/book_search.py | 20 ++++++++----- bookwyrm/settings.py | 2 +- .../templates/shelf/search_my_books_form.html | 25 ++++++++++++++++ bookwyrm/templates/shelf/shelf.html | 1 + bookwyrm/tests/views/test_search.py | 2 +- bookwyrm/views/search.py | 6 ++-- bookwyrm/views/shelf/shelf.py | 29 +++++++++++++------ docker-compose.yml | 2 ++ 8 files changed, 66 insertions(+), 21 deletions(-) create mode 100644 bookwyrm/templates/shelf/search_my_books_form.html diff --git a/bookwyrm/book_search.py b/bookwyrm/book_search.py index ceb228f40..6ab33fd5a 100644 --- a/bookwyrm/book_search.py +++ b/bookwyrm/book_search.py @@ -43,6 +43,7 @@ def search( min_confidence: float = 0, filters: Optional[list[Any]] = None, return_first: bool = False, + books = None ) -> Union[Optional[models.Edition], QuerySet[models.Edition]]: """search your local database""" filters = filters or [] @@ -54,17 +55,16 @@ def search( # first, try searching unique identifiers # unique identifiers never have spaces, title/author usually do if not " " in query: - results = search_identifiers(query, *filters, return_first=return_first) + results = search_identifiers(query, *filters, return_first=return_first, books=books) # if there were no identifier results... if not results: # then try searching title/author results = search_title_author( - query, min_confidence, *filters, return_first=return_first + query, min_confidence, *filters, return_first=return_first, books=books ) return results - def isbn_search(query): """search your local database""" if not query: @@ -98,8 +98,9 @@ def format_search_result(search_result): def search_identifiers( - query, *filters, return_first=False + query, *filters, return_first=False, books=None, ) -> Union[Optional[models.Edition], QuerySet[models.Edition]]: + books = books or models.Edition """tries remote_id, isbn; defined as dedupe fields on the model""" if connectors.maybe_isbn(query): # Oh did you think the 'S' in ISBN stood for 'standard'? @@ -111,7 +112,7 @@ def search_identifiers( for f in models.Edition._meta.get_fields() if hasattr(f, "deduplication_field") and f.deduplication_field ] - results = models.Edition.objects.filter( + results = books.filter( *filters, reduce(operator.or_, (Q(**f) for f in or_filters)) ).distinct() @@ -121,12 +122,17 @@ def search_identifiers( def search_title_author( - query, min_confidence, *filters, return_first=False + query, + min_confidence, + *filters, + return_first=False, + books=None, ) -> QuerySet[models.Edition]: """searches for title and author""" + books = books or models.Edition.objects query = SearchQuery(query, config="simple") | SearchQuery(query, config="english") results = ( - models.Edition.objects.filter(*filters, search_vector=query) + books.filter(*filters, search_vector=query) .annotate(rank=SearchRank(F("search_vector"), query)) .filter(rank__gt=min_confidence) .order_by("-rank") diff --git a/bookwyrm/settings.py b/bookwyrm/settings.py index 4cecc4df6..49e4e2116 100644 --- a/bookwyrm/settings.py +++ b/bookwyrm/settings.py @@ -140,7 +140,7 @@ TEMPLATES = [ }, ] -LOG_LEVEL = env("LOG_LEVEL", "INFO").upper() +LOG_LEVEL = env("LOG_LEVEL", "DEBUG").upper() # Override aspects of the default handler to our taste # See https://docs.djangoproject.com/en/3.2/topics/logging/#default-logging-configuration # for a reference to the defaults we're overriding diff --git a/bookwyrm/templates/shelf/search_my_books_form.html b/bookwyrm/templates/shelf/search_my_books_form.html new file mode 100644 index 000000000..ca9cd2991 --- /dev/null +++ b/bookwyrm/templates/shelf/search_my_books_form.html @@ -0,0 +1,25 @@ +{% load i18n %} +{% load utilities %} + +
+
+
+ {% trans "Search for a book" as my_shelves_search_placeholder %} + +
+
+ +
+
+ +
+
+
diff --git a/bookwyrm/templates/shelf/shelf.html b/bookwyrm/templates/shelf/shelf.html index a2410ef95..b84ae185f 100644 --- a/bookwyrm/templates/shelf/shelf.html +++ b/bookwyrm/templates/shelf/shelf.html @@ -123,6 +123,7 @@ {% endif %} {% endwith %} + {% include 'shelf/search_my_books_form.html' with user=user query=query %} {% if is_self and shelf.id %} diff --git a/bookwyrm/tests/views/test_search.py b/bookwyrm/tests/views/test_search.py index 28f8268e3..021d502f0 100644 --- a/bookwyrm/tests/views/test_search.py +++ b/bookwyrm/tests/views/test_search.py @@ -9,7 +9,7 @@ from django.test import TestCase from django.test.client import RequestFactory from bookwyrm import models, views -from bookwyrm.book_search import SearchResult +from bookwyrm.book_search import SearchResult, search from bookwyrm.settings import DOMAIN from bookwyrm.tests.validate_html import validate_html diff --git a/bookwyrm/views/search.py b/bookwyrm/views/search.py index 743f33f59..f8eddd1a6 100644 --- a/bookwyrm/views/search.py +++ b/bookwyrm/views/search.py @@ -51,7 +51,7 @@ class Search(View): def api_book_search(request): """Return books via API response""" query = request.GET.get("q") - query = isbn_check(query) + query = isbn_check_and_format(query) min_confidence = request.GET.get("min_confidence", 0) # only return local book results via json so we don't cascade book_results = search(query, min_confidence=min_confidence) @@ -64,7 +64,7 @@ def book_search(request): """the real business is elsewhere""" query = request.GET.get("q") # check if query is isbn - query = isbn_check(query) + query = isbn_check_and_format(query) min_confidence = request.GET.get("min_confidence", 0) search_remote = request.GET.get("remote", False) and request.user.is_authenticated @@ -159,7 +159,7 @@ def list_search(request): return TemplateResponse(request, "search/list.html", data) -def isbn_check(query): +def isbn_check_and_format(query): """isbn10 or isbn13 check, if so remove separators""" if query: su_num = re.sub(r"(?<=\d)\D(?=\d|[xX])", "", query) diff --git a/bookwyrm/views/shelf/shelf.py b/bookwyrm/views/shelf/shelf.py index dbbcc2d3a..600dfb731 100644 --- a/bookwyrm/views/shelf/shelf.py +++ b/bookwyrm/views/shelf/shelf.py @@ -1,7 +1,7 @@ """ shelf views """ from collections import namedtuple -from django.db.models import OuterRef, Subquery, F, Max +from django.db.models import OuterRef, Subquery, F, Max, QuerySet from django.contrib.auth.decorators import login_required from django.core.paginator import Paginator from django.http import HttpResponseBadRequest @@ -15,6 +15,10 @@ from bookwyrm import forms, models from bookwyrm.activitypub import ActivitypubResponse from bookwyrm.settings import PAGE_LENGTH from bookwyrm.views.helpers import is_api_request, get_user_from_username +from bookwyrm.book_search import search + +import logging +logger = logging.getLogger(__name__) # pylint: disable=no-self-use @@ -32,6 +36,8 @@ class Shelf(View): else: shelves = models.Shelf.privacy_filter(request.user).filter(user=user).all() + shelves_search_query = request.GET.get("shelves_q") + # get the shelf and make sure the logged in user should be able to see it if shelf_identifier: shelf = get_object_or_404(user.shelf_set, identifier=shelf_identifier) @@ -42,14 +48,17 @@ class Shelf(View): FakeShelf = namedtuple( "Shelf", ("identifier", "name", "user", "books", "privacy") ) - books = ( - models.Edition.viewer_aware_objects(request.user) - .filter( - # privacy is ensured because the shelves are already filtered above - shelfbook__shelf__in=shelves - ) - .distinct() - ) + if shelves_search_query: + logger.debug("AAAAAAAAAAAA") + all_books = models.Edition.viewer_aware_objects(request.user).filter( + # privacy is ensured because the shelves are already filtered above + shelfbook__shelf__in=shelves + ).distinct() + books = search(shelves_search_query, books=all_books) + else: + logger.debug("BBBBBBBBB") + books = shelf.books + shelf = FakeShelf("all", _("All books"), user, books, "public") if is_api_request(request) and shelf_identifier: @@ -103,6 +112,8 @@ class Shelf(View): "page_range": paginated.get_elided_page_range( page.number, on_each_side=2, on_ends=1 ), + "has_shelves_query": bool(shelves_search_query), + "shelves_search_query": shelves_search_query } return TemplateResponse(request, "shelf/shelf.html", data) diff --git a/docker-compose.yml b/docker-compose.yml index 4d4037681..bb0acbdd1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -21,6 +21,8 @@ services: - pgdata:/var/lib/postgresql/data networks: - main + ports: + - "5432:5432" web: build: . env_file: .env From b27ed847d575a7d504b04792e45bd0c73a2dffa4 Mon Sep 17 00:00:00 2001 From: Ross Chapman Date: Tue, 5 Dec 2023 16:36:58 -0800 Subject: [PATCH 03/17] Fixes result set passed to template --- bookwyrm/book_search.py | 6 ++++-- bookwyrm/settings.py | 2 +- bookwyrm/views/shelf/shelf.py | 18 +++++++++++------- 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/bookwyrm/book_search.py b/bookwyrm/book_search.py index 6ab33fd5a..db7a51426 100644 --- a/bookwyrm/book_search.py +++ b/bookwyrm/book_search.py @@ -14,6 +14,7 @@ from bookwyrm import connectors from bookwyrm.settings import MEDIA_FULL_URL + @overload def search( query: str, @@ -43,7 +44,7 @@ def search( min_confidence: float = 0, filters: Optional[list[Any]] = None, return_first: bool = False, - books = None + books: Optional[QuerySet[models.Edition]] = None ) -> Union[Optional[models.Edition], QuerySet[models.Edition]]: """search your local database""" filters = filters or [] @@ -59,6 +60,7 @@ def search( # if there were no identifier results... if not results: + print('SSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSS') # then try searching title/author results = search_title_author( query, min_confidence, *filters, return_first=return_first, books=books @@ -100,7 +102,7 @@ def format_search_result(search_result): def search_identifiers( query, *filters, return_first=False, books=None, ) -> Union[Optional[models.Edition], QuerySet[models.Edition]]: - books = books or models.Edition + books = books or models.Edition.objects """tries remote_id, isbn; defined as dedupe fields on the model""" if connectors.maybe_isbn(query): # Oh did you think the 'S' in ISBN stood for 'standard'? diff --git a/bookwyrm/settings.py b/bookwyrm/settings.py index 49e4e2116..c5f5f52f4 100644 --- a/bookwyrm/settings.py +++ b/bookwyrm/settings.py @@ -194,7 +194,7 @@ LOGGING = { # Add a bookwyrm-specific logger "bookwyrm": { "handlers": ["console"], - "level": LOG_LEVEL, + "level": DEBUG, }, }, } diff --git a/bookwyrm/views/shelf/shelf.py b/bookwyrm/views/shelf/shelf.py index 600dfb731..159cc1541 100644 --- a/bookwyrm/views/shelf/shelf.py +++ b/bookwyrm/views/shelf/shelf.py @@ -43,21 +43,23 @@ class Shelf(View): shelf = get_object_or_404(user.shelf_set, identifier=shelf_identifier) shelf.raise_visible_to_user(request.user) books = shelf.books + if shelves_search_query: + books = search(shelves_search_query, books=books) else: # this is a constructed "all books" view, with a fake "shelf" obj FakeShelf = namedtuple( "Shelf", ("identifier", "name", "user", "books", "privacy") ) - if shelves_search_query: - logger.debug("AAAAAAAAAAAA") - all_books = models.Edition.viewer_aware_objects(request.user).filter( + + books = models.Edition.viewer_aware_objects(request.user).filter( # privacy is ensured because the shelves are already filtered above shelfbook__shelf__in=shelves ).distinct() - books = search(shelves_search_query, books=all_books) - else: - logger.debug("BBBBBBBBB") - books = shelf.books + + # TODO: [COMMENT] + if shelves_search_query: + books = search(shelves_search_query, books=books) + books = models.Edition.objects.filter(pk__in=books) shelf = FakeShelf("all", _("All books"), user, books, "public") @@ -81,6 +83,8 @@ class Shelf(View): "start_date" ) + # import pdb; pdb.set_trace() + books = books.annotate(shelved_date=Max("shelfbook__shelved_date")) books = books.annotate( rating=Subquery(reviews.values("rating")[:1]), From 979162da10c68cae5ad694d5b3f74d96d84e9acf Mon Sep 17 00:00:00 2001 From: Ross Chapman Date: Tue, 5 Dec 2023 19:33:59 -0800 Subject: [PATCH 04/17] Uses filters, fixes for any shelf --- bookwyrm/book_search.py | 1 - .../templates/shelf/search_filter_field.html | 9 +++++++ bookwyrm/templates/shelf/search_filters.html | 5 ++++ .../templates/shelf/search_my_books_form.html | 25 ------------------- bookwyrm/templates/shelf/shelf.html | 5 +++- bookwyrm/views/shelf/shelf.py | 13 +++------- 6 files changed, 21 insertions(+), 37 deletions(-) create mode 100644 bookwyrm/templates/shelf/search_filter_field.html create mode 100644 bookwyrm/templates/shelf/search_filters.html delete mode 100644 bookwyrm/templates/shelf/search_my_books_form.html diff --git a/bookwyrm/book_search.py b/bookwyrm/book_search.py index db7a51426..b3e66cdd6 100644 --- a/bookwyrm/book_search.py +++ b/bookwyrm/book_search.py @@ -60,7 +60,6 @@ def search( # if there were no identifier results... if not results: - print('SSSSSSSSSSSSSSSSSSSSSSSSSSSSSSSS') # then try searching title/author results = search_title_author( query, min_confidence, *filters, return_first=return_first, books=books diff --git a/bookwyrm/templates/shelf/search_filter_field.html b/bookwyrm/templates/shelf/search_filter_field.html new file mode 100644 index 000000000..591e6aaa9 --- /dev/null +++ b/bookwyrm/templates/shelf/search_filter_field.html @@ -0,0 +1,9 @@ +{% extends 'snippets/filters_panel/filter_field.html' %} +{% load i18n %} + +{% block filter %} +
+ + +
+{% endblock %} diff --git a/bookwyrm/templates/shelf/search_filters.html b/bookwyrm/templates/shelf/search_filters.html new file mode 100644 index 000000000..840eec57e --- /dev/null +++ b/bookwyrm/templates/shelf/search_filters.html @@ -0,0 +1,5 @@ +{% extends 'snippets/filters_panel/filters_panel.html' %} + +{% block filter_fields %} + {% include 'shelf/search_filter_field.html' %} +{% endblock %} diff --git a/bookwyrm/templates/shelf/search_my_books_form.html b/bookwyrm/templates/shelf/search_my_books_form.html deleted file mode 100644 index ca9cd2991..000000000 --- a/bookwyrm/templates/shelf/search_my_books_form.html +++ /dev/null @@ -1,25 +0,0 @@ -{% load i18n %} -{% load utilities %} - -
-
-
- {% trans "Search for a book" as my_shelves_search_placeholder %} - -
-
- -
-
- -
-
-
diff --git a/bookwyrm/templates/shelf/shelf.html b/bookwyrm/templates/shelf/shelf.html index b84ae185f..ea5d2343c 100644 --- a/bookwyrm/templates/shelf/shelf.html +++ b/bookwyrm/templates/shelf/shelf.html @@ -121,10 +121,13 @@ {% endblocktrans %} {% endif %} + {% endif %} + {% endwith %} - {% include 'shelf/search_my_books_form.html' with user=user query=query %} + + {% include 'shelf/search_filters.html' with user=user query=query %} {% if is_self and shelf.id %}
diff --git a/bookwyrm/views/shelf/shelf.py b/bookwyrm/views/shelf/shelf.py index 159cc1541..be205597b 100644 --- a/bookwyrm/views/shelf/shelf.py +++ b/bookwyrm/views/shelf/shelf.py @@ -43,8 +43,6 @@ class Shelf(View): shelf = get_object_or_404(user.shelf_set, identifier=shelf_identifier) shelf.raise_visible_to_user(request.user) books = shelf.books - if shelves_search_query: - books = search(shelves_search_query, books=books) else: # this is a constructed "all books" view, with a fake "shelf" obj FakeShelf = namedtuple( @@ -56,11 +54,6 @@ class Shelf(View): shelfbook__shelf__in=shelves ).distinct() - # TODO: [COMMENT] - if shelves_search_query: - books = search(shelves_search_query, books=books) - books = models.Edition.objects.filter(pk__in=books) - shelf = FakeShelf("all", _("All books"), user, books, "public") if is_api_request(request) and shelf_identifier: @@ -83,8 +76,6 @@ class Shelf(View): "start_date" ) - # import pdb; pdb.set_trace() - books = books.annotate(shelved_date=Max("shelfbook__shelved_date")) books = books.annotate( rating=Subquery(reviews.values("rating")[:1]), @@ -99,6 +90,9 @@ class Shelf(View): books = sort_books(books, request.GET.get("sort")) + if shelves_search_query: + books = search(shelves_search_query, books=books) + paginated = Paginator( books, PAGE_LENGTH, @@ -116,7 +110,6 @@ class Shelf(View): "page_range": paginated.get_elided_page_range( page.number, on_each_side=2, on_ends=1 ), - "has_shelves_query": bool(shelves_search_query), "shelves_search_query": shelves_search_query } From c65e165aeb1d36921da1ff110c695c1bfa890f5a Mon Sep 17 00:00:00 2001 From: Ross Chapman Date: Tue, 5 Dec 2023 19:37:29 -0800 Subject: [PATCH 05/17] Hides filter if shelf empty --- bookwyrm/templates/shelf/search_filter_field.html | 2 +- bookwyrm/templates/shelf/shelf.html | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/bookwyrm/templates/shelf/search_filter_field.html b/bookwyrm/templates/shelf/search_filter_field.html index 591e6aaa9..5641bae85 100644 --- a/bookwyrm/templates/shelf/search_filter_field.html +++ b/bookwyrm/templates/shelf/search_filter_field.html @@ -4,6 +4,6 @@ {% block filter %}
- +
{% endblock %} diff --git a/bookwyrm/templates/shelf/shelf.html b/bookwyrm/templates/shelf/shelf.html index ea5d2343c..4fb78ba6f 100644 --- a/bookwyrm/templates/shelf/shelf.html +++ b/bookwyrm/templates/shelf/shelf.html @@ -127,7 +127,10 @@ {% endwith %} - {% include 'shelf/search_filters.html' with user=user query=query %} + {% if books|length > 0 %} + {% include 'shelf/search_filters.html' with user=user query=query %} + {% endif %} +
{% if is_self and shelf.id %}
From 0f6e567b21b9cfbae86c7bfdff82eb0b3e673cc1 Mon Sep 17 00:00:00 2001 From: Ross Chapman Date: Tue, 5 Dec 2023 19:49:38 -0800 Subject: [PATCH 06/17] Clean up --- bookwyrm/settings.py | 4 ++-- bookwyrm/templates/shelf/shelf.html | 3 --- bookwyrm/tests/views/test_search.py | 2 +- bookwyrm/views/shelf/shelf.py | 4 ---- docker-compose.yml | 2 -- 5 files changed, 3 insertions(+), 12 deletions(-) diff --git a/bookwyrm/settings.py b/bookwyrm/settings.py index c5f5f52f4..4cecc4df6 100644 --- a/bookwyrm/settings.py +++ b/bookwyrm/settings.py @@ -140,7 +140,7 @@ TEMPLATES = [ }, ] -LOG_LEVEL = env("LOG_LEVEL", "DEBUG").upper() +LOG_LEVEL = env("LOG_LEVEL", "INFO").upper() # Override aspects of the default handler to our taste # See https://docs.djangoproject.com/en/3.2/topics/logging/#default-logging-configuration # for a reference to the defaults we're overriding @@ -194,7 +194,7 @@ LOGGING = { # Add a bookwyrm-specific logger "bookwyrm": { "handlers": ["console"], - "level": DEBUG, + "level": LOG_LEVEL, }, }, } diff --git a/bookwyrm/templates/shelf/shelf.html b/bookwyrm/templates/shelf/shelf.html index 4fb78ba6f..14b748e54 100644 --- a/bookwyrm/templates/shelf/shelf.html +++ b/bookwyrm/templates/shelf/shelf.html @@ -121,11 +121,8 @@ {% endblocktrans %} {% endif %} - {% endif %} - {% endwith %} - {% if books|length > 0 %} {% include 'shelf/search_filters.html' with user=user query=query %} diff --git a/bookwyrm/tests/views/test_search.py b/bookwyrm/tests/views/test_search.py index 021d502f0..28f8268e3 100644 --- a/bookwyrm/tests/views/test_search.py +++ b/bookwyrm/tests/views/test_search.py @@ -9,7 +9,7 @@ from django.test import TestCase from django.test.client import RequestFactory from bookwyrm import models, views -from bookwyrm.book_search import SearchResult, search +from bookwyrm.book_search import SearchResult from bookwyrm.settings import DOMAIN from bookwyrm.tests.validate_html import validate_html diff --git a/bookwyrm/views/shelf/shelf.py b/bookwyrm/views/shelf/shelf.py index be205597b..aad90da82 100644 --- a/bookwyrm/views/shelf/shelf.py +++ b/bookwyrm/views/shelf/shelf.py @@ -17,10 +17,6 @@ from bookwyrm.settings import PAGE_LENGTH from bookwyrm.views.helpers import is_api_request, get_user_from_username from bookwyrm.book_search import search -import logging -logger = logging.getLogger(__name__) - - # pylint: disable=no-self-use class Shelf(View): """shelf page""" diff --git a/docker-compose.yml b/docker-compose.yml index bb0acbdd1..4d4037681 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -21,8 +21,6 @@ services: - pgdata:/var/lib/postgresql/data networks: - main - ports: - - "5432:5432" web: build: . env_file: .env From aac8aa1adfd668e95595600d5b059b1b89655507 Mon Sep 17 00:00:00 2001 From: Ross Chapman Date: Wed, 6 Dec 2023 11:36:15 -0800 Subject: [PATCH 07/17] Fixes formatting --- bookwyrm/book_search.py | 19 ++++++++++++++----- bookwyrm/views/shelf/shelf.py | 18 ++++++++++++------ 2 files changed, 26 insertions(+), 11 deletions(-) diff --git a/bookwyrm/book_search.py b/bookwyrm/book_search.py index b3e66cdd6..669b7d377 100644 --- a/bookwyrm/book_search.py +++ b/bookwyrm/book_search.py @@ -14,7 +14,6 @@ from bookwyrm import connectors from bookwyrm.settings import MEDIA_FULL_URL - @overload def search( query: str, @@ -44,7 +43,7 @@ def search( min_confidence: float = 0, filters: Optional[list[Any]] = None, return_first: bool = False, - books: Optional[QuerySet[models.Edition]] = None + books: Optional[QuerySet[models.Edition]] = None, ) -> Union[Optional[models.Edition], QuerySet[models.Edition]]: """search your local database""" filters = filters or [] @@ -56,7 +55,9 @@ def search( # first, try searching unique identifiers # unique identifiers never have spaces, title/author usually do if not " " in query: - results = search_identifiers(query, *filters, return_first=return_first, books=books) + results = search_identifiers( + query, *filters, return_first=return_first, books=books + ) # if there were no identifier results... if not results: @@ -66,6 +67,7 @@ def search( ) return results + def isbn_search(query): """search your local database""" if not query: @@ -99,10 +101,17 @@ def format_search_result(search_result): def search_identifiers( - query, *filters, return_first=False, books=None, + query, + *filters, + return_first=False, + books=None, ) -> Union[Optional[models.Edition], QuerySet[models.Edition]]: + """search Editions by deduplication fields + + Best for cases when we can assume someone is searching for an exact match on + commonly unique data identifiers like isbn or specific library ids. + """ books = books or models.Edition.objects - """tries remote_id, isbn; defined as dedupe fields on the model""" if connectors.maybe_isbn(query): # Oh did you think the 'S' in ISBN stood for 'standard'? normalized_isbn = query.strip().upper().rjust(10, "0") diff --git a/bookwyrm/views/shelf/shelf.py b/bookwyrm/views/shelf/shelf.py index aad90da82..0617fcc56 100644 --- a/bookwyrm/views/shelf/shelf.py +++ b/bookwyrm/views/shelf/shelf.py @@ -1,7 +1,7 @@ """ shelf views """ from collections import namedtuple -from django.db.models import OuterRef, Subquery, F, Max, QuerySet +from django.db.models import OuterRef, Subquery, F, Max from django.contrib.auth.decorators import login_required from django.core.paginator import Paginator from django.http import HttpResponseBadRequest @@ -17,10 +17,12 @@ from bookwyrm.settings import PAGE_LENGTH from bookwyrm.views.helpers import is_api_request, get_user_from_username from bookwyrm.book_search import search + # pylint: disable=no-self-use class Shelf(View): """shelf page""" + # pylint: disable=R0914 def get(self, request, username, shelf_identifier=None): """display a shelf""" user = get_user_from_username(request.user, username) @@ -45,10 +47,14 @@ class Shelf(View): "Shelf", ("identifier", "name", "user", "books", "privacy") ) - books = models.Edition.viewer_aware_objects(request.user).filter( - # privacy is ensured because the shelves are already filtered above - shelfbook__shelf__in=shelves - ).distinct() + books = ( + models.Edition.viewer_aware_objects(request.user) + .filter( + # privacy is ensured because the shelves are already filtered above + shelfbook__shelf__in=shelves + ) + .distinct() + ) shelf = FakeShelf("all", _("All books"), user, books, "public") @@ -106,7 +112,7 @@ class Shelf(View): "page_range": paginated.get_elided_page_range( page.number, on_each_side=2, on_ends=1 ), - "shelves_search_query": shelves_search_query + "shelves_search_query": shelves_search_query, } return TemplateResponse(request, "shelf/shelf.html", data) From 4a4046a704d9d2abfa72b51f1da33e6f6d0c908a Mon Sep 17 00:00:00 2001 From: Ross Chapman Date: Thu, 14 Dec 2023 11:30:01 -0800 Subject: [PATCH 08/17] Shows message if empty and renames "search" to "filter" --- bookwyrm/templates/shelf/search_filter_field.html | 9 --------- bookwyrm/templates/shelf/shelf.html | 6 ++++-- bookwyrm/templates/shelf/shelves_filter_field.html | 9 +++++++++ .../{search_filters.html => shelves_filters.html} | 2 +- bookwyrm/views/shelf/shelf.py | 11 +++++++---- 5 files changed, 21 insertions(+), 16 deletions(-) delete mode 100644 bookwyrm/templates/shelf/search_filter_field.html create mode 100644 bookwyrm/templates/shelf/shelves_filter_field.html rename bookwyrm/templates/shelf/{search_filters.html => shelves_filters.html} (66%) diff --git a/bookwyrm/templates/shelf/search_filter_field.html b/bookwyrm/templates/shelf/search_filter_field.html deleted file mode 100644 index 5641bae85..000000000 --- a/bookwyrm/templates/shelf/search_filter_field.html +++ /dev/null @@ -1,9 +0,0 @@ -{% extends 'snippets/filters_panel/filter_field.html' %} -{% load i18n %} - -{% block filter %} -
- - -
-{% endblock %} diff --git a/bookwyrm/templates/shelf/shelf.html b/bookwyrm/templates/shelf/shelf.html index 91d9ee99f..60b822322 100644 --- a/bookwyrm/templates/shelf/shelf.html +++ b/bookwyrm/templates/shelf/shelf.html @@ -101,18 +101,20 @@ {% plural %} {{ formatted_count }} books {% endblocktrans %} - {% if books.has_other_pages %} {% blocktrans trimmed with start=books.start_index end=books.end_index %} (showing {{ start }}-{{ end }}) {% endblocktrans %} {% endif %} + {% if shelves_filter_msg %} + - {{ shelves_filter_msg }} "{{ shelves_filter_query }}" + {% endif %} {% endif %} {% endwith %} {% if books|length > 0 %} - {% include 'shelf/search_filters.html' with user=user query=query %} + {% include 'shelf/shelves_filters.html' with user=user query=query %} {% endif %}
diff --git a/bookwyrm/templates/shelf/shelves_filter_field.html b/bookwyrm/templates/shelf/shelves_filter_field.html new file mode 100644 index 000000000..707f033ea --- /dev/null +++ b/bookwyrm/templates/shelf/shelves_filter_field.html @@ -0,0 +1,9 @@ +{% extends 'snippets/filters_panel/filter_field.html' %} +{% load i18n %} + +{% block filter %} +
+ + +
+{% endblock %} diff --git a/bookwyrm/templates/shelf/search_filters.html b/bookwyrm/templates/shelf/shelves_filters.html similarity index 66% rename from bookwyrm/templates/shelf/search_filters.html rename to bookwyrm/templates/shelf/shelves_filters.html index 840eec57e..ad7fc3dbc 100644 --- a/bookwyrm/templates/shelf/search_filters.html +++ b/bookwyrm/templates/shelf/shelves_filters.html @@ -1,5 +1,5 @@ {% extends 'snippets/filters_panel/filters_panel.html' %} {% block filter_fields %} - {% include 'shelf/search_filter_field.html' %} + {% include 'shelf/shelves_filter_field.html' %} {% endblock %} diff --git a/bookwyrm/views/shelf/shelf.py b/bookwyrm/views/shelf/shelf.py index 0617fcc56..2b9ab176a 100644 --- a/bookwyrm/views/shelf/shelf.py +++ b/bookwyrm/views/shelf/shelf.py @@ -34,7 +34,8 @@ class Shelf(View): else: shelves = models.Shelf.privacy_filter(request.user).filter(user=user).all() - shelves_search_query = request.GET.get("shelves_q") + shelves_filter_query = request.GET.get("filter") + shelves_filter_msg = "" # get the shelf and make sure the logged in user should be able to see it if shelf_identifier: @@ -92,8 +93,9 @@ class Shelf(View): books = sort_books(books, request.GET.get("sort")) - if shelves_search_query: - books = search(shelves_search_query, books=books) + if shelves_filter_query: + books = search(shelves_filter_query, books=books) or books + shelves_filter_msg = "We couldn't find any books that matched" paginated = Paginator( books, @@ -112,7 +114,8 @@ class Shelf(View): "page_range": paginated.get_elided_page_range( page.number, on_each_side=2, on_ends=1 ), - "shelves_search_query": shelves_search_query, + "shelves_filter_query": shelves_filter_query, + "shelves_filter_msg": shelves_filter_msg, } return TemplateResponse(request, "shelf/shelf.html", data) From bd3acdbf31887801138324574726cff2e6c21a6f Mon Sep 17 00:00:00 2001 From: Ross Chapman Date: Thu, 14 Dec 2023 12:33:27 -0800 Subject: [PATCH 09/17] Puts string in template --- bookwyrm/templates/shelf/shelf.html | 4 ++-- bookwyrm/views/shelf/shelf.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/bookwyrm/templates/shelf/shelf.html b/bookwyrm/templates/shelf/shelf.html index 60b822322..d224b0a4f 100644 --- a/bookwyrm/templates/shelf/shelf.html +++ b/bookwyrm/templates/shelf/shelf.html @@ -106,8 +106,8 @@ (showing {{ start }}-{{ end }}) {% endblocktrans %} {% endif %} - {% if shelves_filter_msg %} - - {{ shelves_filter_msg }} "{{ shelves_filter_query }}" + {% if show_shelves_filter_msg %} + - {% trans "We couldn't find any books that matched" %} "{{ shelves_filter_query }}" {% endif %} {% endif %} diff --git a/bookwyrm/views/shelf/shelf.py b/bookwyrm/views/shelf/shelf.py index 2b9ab176a..eae57b409 100644 --- a/bookwyrm/views/shelf/shelf.py +++ b/bookwyrm/views/shelf/shelf.py @@ -35,7 +35,7 @@ class Shelf(View): shelves = models.Shelf.privacy_filter(request.user).filter(user=user).all() shelves_filter_query = request.GET.get("filter") - shelves_filter_msg = "" + show_shelves_filter_msg = False # get the shelf and make sure the logged in user should be able to see it if shelf_identifier: @@ -95,7 +95,7 @@ class Shelf(View): if shelves_filter_query: books = search(shelves_filter_query, books=books) or books - shelves_filter_msg = "We couldn't find any books that matched" + show_shelves_filter_msg = True paginated = Paginator( books, @@ -115,7 +115,7 @@ class Shelf(View): page.number, on_each_side=2, on_ends=1 ), "shelves_filter_query": shelves_filter_query, - "shelves_filter_msg": shelves_filter_msg, + "show_shelves_filter_msg": show_shelves_filter_msg, } return TemplateResponse(request, "shelf/shelf.html", data) From 44d21d1ba474ae71b05fd0195f9a7ae7c73719cc Mon Sep 17 00:00:00 2001 From: Ross Chapman Date: Thu, 14 Dec 2023 13:04:45 -0800 Subject: [PATCH 10/17] Updates view logic --- bookwyrm/templates/shelf/shelf.html | 14 +++++++------- bookwyrm/views/shelf/shelf.py | 4 +--- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/bookwyrm/templates/shelf/shelf.html b/bookwyrm/templates/shelf/shelf.html index d224b0a4f..faadd84ab 100644 --- a/bookwyrm/templates/shelf/shelf.html +++ b/bookwyrm/templates/shelf/shelf.html @@ -106,16 +106,11 @@ (showing {{ start }}-{{ end }}) {% endblocktrans %} {% endif %} - {% if show_shelves_filter_msg %} - - {% trans "We couldn't find any books that matched" %} "{{ shelves_filter_query }}" - {% endif %} {% endif %} {% endwith %} - {% if books|length > 0 %} - {% include 'shelf/shelves_filters.html' with user=user query=query %} - {% endif %} + {% include 'shelf/shelves_filters.html' with user=user query=query %} {% if is_self and shelf.id %} @@ -215,7 +210,12 @@ {% else %} -

{% trans "This shelf is empty." %}

+

{% if shelves_filter_query %} + {% trans "We couldn't find any books that matched" %} "{{ shelves_filter_query }}" + {% else %} + {% trans "This shelf is empty." %} + {% endif %} +

{% endif %} diff --git a/bookwyrm/views/shelf/shelf.py b/bookwyrm/views/shelf/shelf.py index eae57b409..17e17433f 100644 --- a/bookwyrm/views/shelf/shelf.py +++ b/bookwyrm/views/shelf/shelf.py @@ -94,8 +94,7 @@ class Shelf(View): books = sort_books(books, request.GET.get("sort")) if shelves_filter_query: - books = search(shelves_filter_query, books=books) or books - show_shelves_filter_msg = True + books = search(shelves_filter_query, books=books) paginated = Paginator( books, @@ -115,7 +114,6 @@ class Shelf(View): page.number, on_each_side=2, on_ends=1 ), "shelves_filter_query": shelves_filter_query, - "show_shelves_filter_msg": show_shelves_filter_msg, } return TemplateResponse(request, "shelf/shelf.html", data) From fb369584442c64e70e3ec41c2f39aed240741666 Mon Sep 17 00:00:00 2001 From: Ross Chapman Date: Thu, 14 Dec 2023 13:47:51 -0800 Subject: [PATCH 11/17] Removes unused variable --- bookwyrm/views/shelf/shelf.py | 1 - 1 file changed, 1 deletion(-) diff --git a/bookwyrm/views/shelf/shelf.py b/bookwyrm/views/shelf/shelf.py index 17e17433f..aac8085d0 100644 --- a/bookwyrm/views/shelf/shelf.py +++ b/bookwyrm/views/shelf/shelf.py @@ -35,7 +35,6 @@ class Shelf(View): shelves = models.Shelf.privacy_filter(request.user).filter(user=user).all() shelves_filter_query = request.GET.get("filter") - show_shelves_filter_msg = False # get the shelf and make sure the logged in user should be able to see it if shelf_identifier: From a4172214d16f2515b72c238df47f15229b9578a4 Mon Sep 17 00:00:00 2001 From: Ross Chapman Date: Fri, 15 Dec 2023 13:17:23 -0800 Subject: [PATCH 12/17] Updates size of filters panel label --- bookwyrm/views/shelf/shelf.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bookwyrm/views/shelf/shelf.py b/bookwyrm/views/shelf/shelf.py index aac8085d0..a39512fe6 100644 --- a/bookwyrm/views/shelf/shelf.py +++ b/bookwyrm/views/shelf/shelf.py @@ -113,6 +113,7 @@ class Shelf(View): page.number, on_each_side=2, on_ends=1 ), "shelves_filter_query": shelves_filter_query, + "size": "small", } return TemplateResponse(request, "shelf/shelf.html", data) From b728bb43232651d968be7aa477c2c62570d4b427 Mon Sep 17 00:00:00 2001 From: Ross Chapman Date: Sat, 16 Dec 2023 12:05:35 -0800 Subject: [PATCH 13/17] Uses block trans --- bookwyrm/templates/shelf/shelf.html | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/bookwyrm/templates/shelf/shelf.html b/bookwyrm/templates/shelf/shelf.html index faadd84ab..71f4bc088 100644 --- a/bookwyrm/templates/shelf/shelf.html +++ b/bookwyrm/templates/shelf/shelf.html @@ -210,12 +210,17 @@ {% else %} -

{% if shelves_filter_query %} - {% trans "We couldn't find any books that matched" %} "{{ shelves_filter_query }}" - {% else %} - {% trans "This shelf is empty." %} - {% endif %} -

+

+ + {% if shelves_filter_query %} + {% blocktrans trimmed %} + We couldn't find any books that matched {{ shelves_filter_query }} + {% endblocktrans %} + {% else %} + {% trans "This shelf is empty." %} + {% endif %} + +

{% endif %} From 381490e31d80670e8bb790a97b46b30611e32b9e Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Tue, 2 Jan 2024 10:35:30 -0800 Subject: [PATCH 14/17] Adds management command to clear all deleted user data --- .../commands/erase_deleted_user_data.py | 40 +++++++++++++++++++ bookwyrm/models/user.py | 14 +++++++ bw-dev | 5 ++- 3 files changed, 58 insertions(+), 1 deletion(-) create mode 100644 bookwyrm/management/commands/erase_deleted_user_data.py diff --git a/bookwyrm/management/commands/erase_deleted_user_data.py b/bookwyrm/management/commands/erase_deleted_user_data.py new file mode 100644 index 000000000..fd3c790ce --- /dev/null +++ b/bookwyrm/management/commands/erase_deleted_user_data.py @@ -0,0 +1,40 @@ +""" Erase any data stored about deleted users """ +import sys +from django.core.management.base import BaseCommand, CommandError +from bookwyrm import models +from bookwyrm.models.user import erase_user_data + +# pylint: disable=missing-function-docstring +class Command(BaseCommand): + """command-line options""" + + help = "Remove Two Factor Authorisation from user" + + def add_arguments(self, parser): # pylint: disable=no-self-use + parser.add_argument( + "--dryrun", + action="store_true", + help="Preview users to be cleared without altering the database", + ) + + def handle(self, *args, **options): # pylint: disable=unused-argument + + # Check for anything fishy + bad_state = models.User.objects.filter(is_deleted=True, is_active=True) + if bad_state.exists(): + raise CommandError( + f"{bad_state.count()} user(s) marked as both active and deleted" + ) + + deleted_users = models.User.objects.filter(is_deleted=True) + self.stdout.write(f"Found {deleted_users.count()} deleted users") + if options["dryrun"]: + self.stdout.write("\n".join(u.username for u in deleted_users[:5])) + if deleted_users.count() > 5: + self.stdout.write("... and more") + sys.exit() + + self.stdout.write("Erasing user data:") + for user_id in deleted_users.values_list("id", flat=True): + erase_user_data.delay(user_id) + self.stdout.write(".", ending="") diff --git a/bookwyrm/models/user.py b/bookwyrm/models/user.py index 75ca1d527..89fd39b73 100644 --- a/bookwyrm/models/user.py +++ b/bookwyrm/models/user.py @@ -523,6 +523,20 @@ class KeyPair(ActivitypubMixin, BookWyrmModel): return super().save(*args, **kwargs) +@app.task(queue=MISC) +def erase_user_data(user_id): + """Erase any custom data about this user asynchronously + This is for deleted historical user data that pre-dates data + being cleared automatically""" + user = User.objects.get(id=user_id) + user.erase_user_data() + user.save( + broadcast=False, + update_fields=["email", "avatar", "preview_image", "summary", "name"], + ) + user.erase_user_statuses(broadcast=False) + + @app.task(queue=MISC) def set_remote_server(user_id, allow_external_connections=False): """figure out the user's remote server in the background""" diff --git a/bw-dev b/bw-dev index 27c20fe45..5a36f78e0 100755 --- a/bw-dev +++ b/bw-dev @@ -246,6 +246,9 @@ case "$CMD" in remove_remote_user_preview_images) runweb python manage.py remove_remote_user_preview_images ;; + erase_deleted_user_data) + runweb python manage.py erase_deleted_user_data "$@" + ;; copy_media_to_s3) awscommand "bookwyrm_media_volume:/images"\ "s3 cp /images s3://${AWS_STORAGE_BUCKET_NAME}/images\ @@ -297,7 +300,7 @@ case "$CMD" in echo "Unrecognised command. Try:" echo " setup" echo " up [container]" - echo " down" + echo " down" echo " service_ports_web" echo " initdb" echo " resetdb" From 01db77a74584c6df18bc8dc624cacf8dbebb30f6 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Tue, 2 Jan 2024 18:29:55 -0800 Subject: [PATCH 15/17] Adds success message --- bookwyrm/management/commands/erase_deleted_user_data.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bookwyrm/management/commands/erase_deleted_user_data.py b/bookwyrm/management/commands/erase_deleted_user_data.py index fd3c790ce..40c3f042b 100644 --- a/bookwyrm/management/commands/erase_deleted_user_data.py +++ b/bookwyrm/management/commands/erase_deleted_user_data.py @@ -38,3 +38,6 @@ class Command(BaseCommand): for user_id in deleted_users.values_list("id", flat=True): erase_user_data.delay(user_id) self.stdout.write(".", ending="") + + self.stdout.write("") + self.stdout.write("Tasks created successfully") From f267fc32355542776828e7894fe1f87c5ca68b75 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 5 Jan 2024 17:42:04 +0000 Subject: [PATCH 16/17] Bump pycryptodome from 3.16.0 to 3.19.1 Bumps [pycryptodome](https://github.com/Legrandin/pycryptodome) from 3.16.0 to 3.19.1. - [Release notes](https://github.com/Legrandin/pycryptodome/releases) - [Changelog](https://github.com/Legrandin/pycryptodome/blob/master/Changelog.rst) - [Commits](https://github.com/Legrandin/pycryptodome/compare/v3.16.0...v3.19.1) --- updated-dependencies: - dependency-name: pycryptodome dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 0b09f3c19..6509effc7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,7 +16,7 @@ libsass==0.22.0 Markdown==3.4.1 Pillow==10.0.1 psycopg2==2.9.5 -pycryptodome==3.16.0 +pycryptodome==3.19.1 python-dateutil==2.8.2 redis==4.5.4 requests==2.31.0 From 83ff880603745adf2ef92b8ca489f9d2a7ea1d88 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sun, 7 Jan 2024 08:31:48 -0800 Subject: [PATCH 17/17] Revert "Don't show notification for user follow request if the user is inactive" --- bookwyrm/templates/notifications/item.html | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/bookwyrm/templates/notifications/item.html b/bookwyrm/templates/notifications/item.html index a1329d31e..ac60ad46f 100644 --- a/bookwyrm/templates/notifications/item.html +++ b/bookwyrm/templates/notifications/item.html @@ -10,9 +10,7 @@ {% elif notification.notification_type == 'FOLLOW' %} {% include 'notifications/items/follow.html' %} {% elif notification.notification_type == 'FOLLOW_REQUEST' %} - {% if notification.related_users.0.is_active %} - {% include 'notifications/items/follow_request.html' %} - {% endif %} + {% include 'notifications/items/follow_request.html' %} {% elif notification.notification_type == 'IMPORT' %} {% include 'notifications/items/import.html' %} {% elif notification.notification_type == 'USER_IMPORT' %}