From 6301656a0ec2367a4450c4ab8da57f4136c23c0f Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Tue, 10 Aug 2021 13:46:20 -0700 Subject: [PATCH 001/191] Fixes setting book results from title/author search --- bookwyrm/models/import_job.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bookwyrm/models/import_job.py b/bookwyrm/models/import_job.py index f29938469..81c75b4b2 100644 --- a/bookwyrm/models/import_job.py +++ b/bookwyrm/models/import_job.py @@ -71,6 +71,7 @@ class ImportItem(models.Model): index = models.IntegerField() data = models.JSONField() book = models.ForeignKey(Book, on_delete=models.SET_NULL, null=True, blank=True) + book_guess = models.ForeignKey(Book, on_delete=models.SET_NULL, null=True, blank=True) fail_reason = models.TextField(null=True) def resolve(self): @@ -80,7 +81,7 @@ class ImportItem(models.Model): else: # don't fall back on title/author search is isbn is present. # you're too likely to mismatch - self.get_book_from_title_author() + self.book = self.get_book_from_title_author() def get_book_from_isbn(self): """search by isbn""" From ebabbf475a0496861d7a8858627ffa4a6ab82f2c Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Tue, 10 Aug 2021 13:48:09 -0700 Subject: [PATCH 002/191] Translate error messages --- bookwyrm/importers/importer.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bookwyrm/importers/importer.py b/bookwyrm/importers/importer.py index d5f1449ca..48629a01f 100644 --- a/bookwyrm/importers/importer.py +++ b/bookwyrm/importers/importer.py @@ -3,6 +3,7 @@ import csv import logging from django.utils import timezone +from django.utils.translation import gettext_lazy as _ from bookwyrm import models from bookwyrm.models import ImportJob, ImportItem @@ -71,7 +72,7 @@ def import_data(source, job_id): item.resolve() except Exception as err: # pylint: disable=broad-except logger.exception(err) - item.fail_reason = "Error loading book" + item.fail_reason = _("Error loading book") item.save() continue @@ -83,7 +84,7 @@ def import_data(source, job_id): source, job.user, item, job.include_reviews, job.privacy ) else: - item.fail_reason = "Could not find a match for book" + item.fail_reason = _("Could not find a match for book") item.save() finally: job.complete = True From fa396d4bc8c3419497eeeb9246b583aa33c2e420 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Tue, 10 Aug 2021 13:54:52 -0700 Subject: [PATCH 003/191] Save best-guess search results on import --- bookwyrm/importers/importer.py | 3 ++- .../migrations/0083_importitem_book_guess.py | 25 +++++++++++++++++++ bookwyrm/models/import_job.py | 18 ++++++++++--- 3 files changed, 41 insertions(+), 5 deletions(-) create mode 100644 bookwyrm/migrations/0083_importitem_book_guess.py diff --git a/bookwyrm/importers/importer.py b/bookwyrm/importers/importer.py index 48629a01f..39d88e100 100644 --- a/bookwyrm/importers/importer.py +++ b/bookwyrm/importers/importer.py @@ -76,9 +76,10 @@ def import_data(source, job_id): item.save() continue - if item.book: + if item.book or item.book_guess: item.save() + if item.book: # shelves book and handles reviews handle_imported_book( source, job.user, item, job.include_reviews, job.privacy diff --git a/bookwyrm/migrations/0083_importitem_book_guess.py b/bookwyrm/migrations/0083_importitem_book_guess.py new file mode 100644 index 000000000..fa4f94ab3 --- /dev/null +++ b/bookwyrm/migrations/0083_importitem_book_guess.py @@ -0,0 +1,25 @@ +# Generated by Django 3.2.4 on 2021-08-10 20:49 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ("bookwyrm", "0082_auto_20210806_2324"), + ] + + operations = [ + migrations.AddField( + model_name="importitem", + name="book_guess", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="book_guess", + to="bookwyrm.book", + ), + ), + ] diff --git a/bookwyrm/models/import_job.py b/bookwyrm/models/import_job.py index 81c75b4b2..53368bce2 100644 --- a/bookwyrm/models/import_job.py +++ b/bookwyrm/models/import_job.py @@ -71,7 +71,13 @@ class ImportItem(models.Model): index = models.IntegerField() data = models.JSONField() book = models.ForeignKey(Book, on_delete=models.SET_NULL, null=True, blank=True) - book_guess = models.ForeignKey(Book, on_delete=models.SET_NULL, null=True, blank=True) + book_guess = models.ForeignKey( + Book, + on_delete=models.SET_NULL, + null=True, + blank=True, + related_name="book_guess", + ) fail_reason = models.TextField(null=True) def resolve(self): @@ -81,7 +87,11 @@ class ImportItem(models.Model): else: # don't fall back on title/author search is isbn is present. # you're too likely to mismatch - self.book = self.get_book_from_title_author() + book, confidence = self.get_book_from_title_author() + if confidence > 0.999: + self.book = book + else: + self.book_guess = book def get_book_from_isbn(self): """search by isbn""" @@ -97,12 +107,12 @@ class ImportItem(models.Model): """search by title and author""" search_term = construct_search_term(self.title, self.author) search_result = connector_manager.first_search_result( - search_term, min_confidence=0.999 + search_term, min_confidence=0.1 ) if search_result: # raises ConnectorException return search_result.connector.get_or_create_book(search_result.key) - return None + return None, 0 @property def title(self): From ef1896da18b78f53b388cb9cd5df12c57b204bb0 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Tue, 10 Aug 2021 14:02:22 -0700 Subject: [PATCH 004/191] Return confidence rating --- bookwyrm/models/import_job.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/bookwyrm/models/import_job.py b/bookwyrm/models/import_job.py index 53368bce2..53192ce37 100644 --- a/bookwyrm/models/import_job.py +++ b/bookwyrm/models/import_job.py @@ -85,7 +85,7 @@ class ImportItem(models.Model): if self.isbn: self.book = self.get_book_from_isbn() else: - # don't fall back on title/author search is isbn is present. + # don't fall back on title/author search if isbn is present. # you're too likely to mismatch book, confidence = self.get_book_from_title_author() if confidence > 0.999: @@ -111,7 +111,10 @@ class ImportItem(models.Model): ) if search_result: # raises ConnectorException - return search_result.connector.get_or_create_book(search_result.key) + return ( + search_result.connector.get_or_create_book(search_result.key), + search_result.confidence, + ) return None, 0 @property From f25835feebb8e28faa1ce810e8f836eceb781aa4 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Tue, 7 Sep 2021 09:39:42 -0700 Subject: [PATCH 005/191] Fixes error breaking registration --- bookwyrm/activitystreams.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bookwyrm/activitystreams.py b/bookwyrm/activitystreams.py index 6dedd7176..f59eaf2e6 100644 --- a/bookwyrm/activitystreams.py +++ b/bookwyrm/activitystreams.py @@ -355,7 +355,7 @@ def populate_streams_on_account_create(sender, instance, created, *args, **kwarg if not created or not instance.local: return - for stream in streams.values(): + for stream in streams: populate_stream_task.delay(stream, instance.id) From 5a224b5aa4bc9cf60f9d06aa38cefe41384cec15 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Tue, 7 Sep 2021 10:09:28 -0700 Subject: [PATCH 006/191] Use atomic when creating new users --- bookwyrm/models/user.py | 67 ++++++++++++++++++++++------------------- 1 file changed, 36 insertions(+), 31 deletions(-) diff --git a/bookwyrm/models/user.py b/bookwyrm/models/user.py index 0745dffa2..7756846f0 100644 --- a/bookwyrm/models/user.py +++ b/bookwyrm/models/user.py @@ -274,30 +274,46 @@ class User(OrderedCollectionPageMixin, AbstractUser): transaction.on_commit(lambda: set_remote_server.delay(self.id)) return - # populate fields for local users - link = site_link() - self.remote_id = f"{link}/user/{self.localname}" - self.followers_url = f"{self.remote_id}/followers" - self.inbox = f"{self.remote_id}/inbox" - self.shared_inbox = f"{link}/inbox" - self.outbox = f"{self.remote_id}/outbox" + with transaction.atomic(): + # populate fields for local users + link = site_link() + self.remote_id = f"{link}/user/{self.localname}" + self.followers_url = f"{self.remote_id}/followers" + self.inbox = f"{self.remote_id}/inbox" + self.shared_inbox = f"{link}/inbox" + self.outbox = f"{self.remote_id}/outbox" - # an id needs to be set before we can proceed with related models + # an id needs to be set before we can proceed with related models + super().save(*args, **kwargs) + + # make users editors by default + try: + self.groups.add(Group.objects.get(name="editor")) + except Group.DoesNotExist: + # this should only happen in tests + pass + + # create keys and shelves for new local users + self.key_pair = KeyPair.objects.create( + remote_id=f"{self.remote_id}/#main-key" + ) + self.save(broadcast=False, update_fields=["key_pair"]) + + self.create_shelves() + + def delete(self, *args, **kwargs): + """deactivate rather than delete a user""" + self.is_active = False + # skip the logic in this class's save() super().save(*args, **kwargs) - # make users editors by default - try: - self.groups.add(Group.objects.get(name="editor")) - except Group.DoesNotExist: - # this should only happen in tests - pass - - # create keys and shelves for new local users - self.key_pair = KeyPair.objects.create( - remote_id="%s/#main-key" % self.remote_id - ) - self.save(broadcast=False, update_fields=["key_pair"]) + @property + def local_path(self): + """this model doesn't inherit bookwyrm model, so here we are""" + return "/user/%s" % (self.localname or self.username) + def create_shelves(self): + """default shelves for a new user""" shelves = [ { "name": "To Read", @@ -321,17 +337,6 @@ class User(OrderedCollectionPageMixin, AbstractUser): editable=False, ).save(broadcast=False) - def delete(self, *args, **kwargs): - """deactivate rather than delete a user""" - self.is_active = False - # skip the logic in this class's save() - super().save(*args, **kwargs) - - @property - def local_path(self): - """this model doesn't inherit bookwyrm model, so here we are""" - return "/user/%s" % (self.localname or self.username) - class KeyPair(ActivitypubMixin, BookWyrmModel): """public and private keys for a user""" From f4b43af600380b45e003a562efab13c088ace40e Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Tue, 7 Sep 2021 10:24:09 -0700 Subject: [PATCH 007/191] Creates test files for activitystreams --- bookwyrm/tests/activitystreams/__init__.py | 1 + .../test_activitystreams.py | 0 .../test_activitystreams_tasks.py | 32 +++++++++++++++++++ 3 files changed, 33 insertions(+) create mode 100644 bookwyrm/tests/activitystreams/__init__.py rename bookwyrm/tests/{ => activitystreams}/test_activitystreams.py (100%) create mode 100644 bookwyrm/tests/activitystreams/test_activitystreams_tasks.py diff --git a/bookwyrm/tests/activitystreams/__init__.py b/bookwyrm/tests/activitystreams/__init__.py new file mode 100644 index 000000000..b6e690fd5 --- /dev/null +++ b/bookwyrm/tests/activitystreams/__init__.py @@ -0,0 +1 @@ +from . import * diff --git a/bookwyrm/tests/test_activitystreams.py b/bookwyrm/tests/activitystreams/test_activitystreams.py similarity index 100% rename from bookwyrm/tests/test_activitystreams.py rename to bookwyrm/tests/activitystreams/test_activitystreams.py diff --git a/bookwyrm/tests/activitystreams/test_activitystreams_tasks.py b/bookwyrm/tests/activitystreams/test_activitystreams_tasks.py new file mode 100644 index 000000000..313177001 --- /dev/null +++ b/bookwyrm/tests/activitystreams/test_activitystreams_tasks.py @@ -0,0 +1,32 @@ +""" testing activitystreams """ +from unittest.mock import patch +from django.test import TestCase +from bookwyrm import activitystreams, models + + +class Activitystreams(TestCase): + """using redis to build activity streams""" + + def setUp(self): + """use a test csv""" + with patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"), patch( + "bookwyrm.activitystreams.populate_stream_task.delay" + ): + self.local_user = models.User.objects.create_user( + "mouse", "mouse@mouse.mouse", "password", local=True, localname="mouse" + ) + work = models.Work.objects.create(title="test work") + self.book = models.Edition.objects.create(title="test book", parent_work=work) + + class TestStream(activitystreams.ActivityStream): + """test stream, don't have to do anything here""" + + key = "test" + + self.test_stream = TestStream() + + def test_add_book_statuses_task(self): + """statuses related to a book""" + with patch("bookwyrm.activitystreams.BooksStream") as mock: + activitystreams.add_book_statuses_task(self.local_user.id, self.book.id) + self.assertTrue(mock.called) From a4041911c6f94c6333bea79d3f8c0c9b0293feeb Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Tue, 7 Sep 2021 11:01:27 -0700 Subject: [PATCH 008/191] Adds tests more activitystreams tasks --- .../activitystreams/test_activitystreams.py | 68 ------- .../test_activitystreams_tasks.py | 171 +++++++++++++++++- 2 files changed, 163 insertions(+), 76 deletions(-) diff --git a/bookwyrm/tests/activitystreams/test_activitystreams.py b/bookwyrm/tests/activitystreams/test_activitystreams.py index add5875f7..4e91c5bea 100644 --- a/bookwyrm/tests/activitystreams/test_activitystreams.py +++ b/bookwyrm/tests/activitystreams/test_activitystreams.py @@ -290,71 +290,3 @@ class Activitystreams(TestCase): # yes book, yes audience result = activitystreams.BooksStream().get_statuses_for_user(self.local_user) self.assertEqual(list(result), [status]) - - @patch("bookwyrm.activitystreams.LocalStream.remove_object_from_related_stores") - @patch("bookwyrm.activitystreams.BooksStream.remove_object_from_related_stores") - def test_boost_to_another_timeline(self, *_): - """add a boost and deduplicate the boosted status on the timeline""" - status = models.Status.objects.create(user=self.local_user, content="hi") - with patch("bookwyrm.activitystreams.handle_boost_task.delay"): - boost = models.Boost.objects.create( - boosted_status=status, - user=self.another_user, - ) - with patch( - "bookwyrm.activitystreams.HomeStream.remove_object_from_related_stores" - ) as mock: - activitystreams.handle_boost_task(boost.id) - - self.assertTrue(mock.called) - call_args = mock.call_args - self.assertEqual(call_args[0][0], status) - self.assertEqual( - call_args[1]["stores"], ["{:d}-home".format(self.another_user.id)] - ) - - @patch("bookwyrm.activitystreams.LocalStream.remove_object_from_related_stores") - @patch("bookwyrm.activitystreams.BooksStream.remove_object_from_related_stores") - def test_boost_to_following_timeline(self, *_): - """add a boost and deduplicate the boosted status on the timeline""" - self.local_user.following.add(self.another_user) - status = models.Status.objects.create(user=self.local_user, content="hi") - with patch("bookwyrm.activitystreams.handle_boost_task.delay"): - boost = models.Boost.objects.create( - boosted_status=status, - user=self.another_user, - ) - with patch( - "bookwyrm.activitystreams.HomeStream.remove_object_from_related_stores" - ) as mock: - activitystreams.handle_boost_task(boost.id) - self.assertTrue(mock.called) - call_args = mock.call_args - self.assertEqual(call_args[0][0], status) - self.assertTrue( - "{:d}-home".format(self.another_user.id) in call_args[1]["stores"] - ) - self.assertTrue( - "{:d}-home".format(self.local_user.id) in call_args[1]["stores"] - ) - - @patch("bookwyrm.activitystreams.LocalStream.remove_object_from_related_stores") - @patch("bookwyrm.activitystreams.BooksStream.remove_object_from_related_stores") - def test_boost_to_same_timeline(self, *_): - """add a boost and deduplicate the boosted status on the timeline""" - status = models.Status.objects.create(user=self.local_user, content="hi") - with patch("bookwyrm.activitystreams.handle_boost_task.delay"): - boost = models.Boost.objects.create( - boosted_status=status, - user=self.local_user, - ) - with patch( - "bookwyrm.activitystreams.HomeStream.remove_object_from_related_stores" - ) as mock: - activitystreams.handle_boost_task(boost.id) - self.assertTrue(mock.called) - call_args = mock.call_args - self.assertEqual(call_args[0][0], status) - self.assertEqual( - call_args[1]["stores"], ["{:d}-home".format(self.local_user.id)] - ) diff --git a/bookwyrm/tests/activitystreams/test_activitystreams_tasks.py b/bookwyrm/tests/activitystreams/test_activitystreams_tasks.py index 313177001..a51549ca6 100644 --- a/bookwyrm/tests/activitystreams/test_activitystreams_tasks.py +++ b/bookwyrm/tests/activitystreams/test_activitystreams_tasks.py @@ -15,18 +15,173 @@ class Activitystreams(TestCase): self.local_user = models.User.objects.create_user( "mouse", "mouse@mouse.mouse", "password", local=True, localname="mouse" ) + self.another_user = models.User.objects.create_user( + "nutria", + "nutria@nutria.nutria", + "password", + local=True, + localname="nutria", + ) work = models.Work.objects.create(title="test work") self.book = models.Edition.objects.create(title="test book", parent_work=work) - - class TestStream(activitystreams.ActivityStream): - """test stream, don't have to do anything here""" - - key = "test" - - self.test_stream = TestStream() + self.status = models.Status.objects.create(content="hi", user=self.local_user) def test_add_book_statuses_task(self): """statuses related to a book""" - with patch("bookwyrm.activitystreams.BooksStream") as mock: + with patch("bookwyrm.activitystreams.BooksStream.add_book_statuses") as mock: activitystreams.add_book_statuses_task(self.local_user.id, self.book.id) self.assertTrue(mock.called) + args = mock.call_args[0] + self.assertEqual(args[0], self.local_user) + self.assertEqual(args[1], self.book) + + def test_remove_book_statuses_task(self): + """remove stauses related to a book""" + with patch("bookwyrm.activitystreams.BooksStream.remove_book_statuses") as mock: + activitystreams.remove_book_statuses_task(self.local_user.id, self.book.id) + self.assertTrue(mock.called) + args = mock.call_args[0] + self.assertEqual(args[0], self.local_user) + self.assertEqual(args[1], self.book) + + def test_populate_stream_task(self): + """populate a given stream""" + with patch("bookwyrm.activitystreams.BooksStream.populate_streams") as mock: + activitystreams.populate_stream_task("books", self.local_user.id) + self.assertTrue(mock.called) + args = mock.call_args[0] + self.assertEqual(args[0], self.local_user) + + with patch("bookwyrm.activitystreams.HomeStream.populate_streams") as mock: + activitystreams.populate_stream_task("home", self.local_user.id) + self.assertTrue(mock.called) + args = mock.call_args[0] + self.assertEqual(args[0], self.local_user) + + def test_remove_status_task(self): + """remove a status from all streams""" + with patch( + "bookwyrm.activitystreams.ActivityStream.remove_object_from_related_stores" + ) as mock: + activitystreams.remove_status_task(self.status.id) + self.assertEqual(mock.call_count, 3) + args = mock.call_args[0] + self.assertEqual(args[0], self.status) + + def test_add_status_task(self): + """add a status to all streams""" + with patch("bookwyrm.activitystreams.ActivityStream.add_status") as mock: + activitystreams.add_status_task(self.status.id) + self.assertEqual(mock.call_count, 3) + args = mock.call_args[0] + self.assertEqual(args[0], self.status) + + def test_remove_user_statuses_task(self): + """remove all statuses by a user from another users' feeds""" + with patch( + "bookwyrm.activitystreams.ActivityStream.remove_user_statuses" + ) as mock: + activitystreams.remove_user_statuses_task( + self.local_user.id, self.another_user.id + ) + self.assertEqual(mock.call_count, 3) + args = mock.call_args[0] + self.assertEqual(args[0], self.local_user) + self.assertEqual(args[1], self.another_user) + + with patch("bookwyrm.activitystreams.HomeStream.remove_user_statuses") as mock: + activitystreams.remove_user_statuses_task( + self.local_user.id, self.another_user.id, stream_list=["home"] + ) + self.assertEqual(mock.call_count, 1) + args = mock.call_args[0] + self.assertEqual(args[0], self.local_user) + self.assertEqual(args[1], self.another_user) + + def test_add_user_statuses_task(self): + """add a user's statuses to another users feeds""" + with patch("bookwyrm.activitystreams.ActivityStream.add_user_statuses") as mock: + activitystreams.add_user_statuses_task( + self.local_user.id, self.another_user.id + ) + self.assertEqual(mock.call_count, 3) + args = mock.call_args[0] + self.assertEqual(args[0], self.local_user) + self.assertEqual(args[1], self.another_user) + + with patch("bookwyrm.activitystreams.HomeStream.add_user_statuses") as mock: + activitystreams.add_user_statuses_task( + self.local_user.id, self.another_user.id, stream_list=["home"] + ) + self.assertEqual(mock.call_count, 1) + args = mock.call_args[0] + self.assertEqual(args[0], self.local_user) + self.assertEqual(args[1], self.another_user) + + @patch("bookwyrm.activitystreams.LocalStream.remove_object_from_related_stores") + @patch("bookwyrm.activitystreams.BooksStream.remove_object_from_related_stores") + def test_boost_to_another_timeline(self, *_): + """add a boost and deduplicate the boosted status on the timeline""" + status = models.Status.objects.create(user=self.local_user, content="hi") + with patch("bookwyrm.activitystreams.handle_boost_task.delay"): + boost = models.Boost.objects.create( + boosted_status=status, + user=self.another_user, + ) + with patch( + "bookwyrm.activitystreams.HomeStream.remove_object_from_related_stores" + ) as mock: + activitystreams.handle_boost_task(boost.id) + + self.assertTrue(mock.called) + call_args = mock.call_args + self.assertEqual(call_args[0][0], status) + self.assertEqual( + call_args[1]["stores"], ["{:d}-home".format(self.another_user.id)] + ) + + @patch("bookwyrm.activitystreams.LocalStream.remove_object_from_related_stores") + @patch("bookwyrm.activitystreams.BooksStream.remove_object_from_related_stores") + def test_boost_to_following_timeline(self, *_): + """add a boost and deduplicate the boosted status on the timeline""" + self.local_user.following.add(self.another_user) + status = models.Status.objects.create(user=self.local_user, content="hi") + with patch("bookwyrm.activitystreams.handle_boost_task.delay"): + boost = models.Boost.objects.create( + boosted_status=status, + user=self.another_user, + ) + with patch( + "bookwyrm.activitystreams.HomeStream.remove_object_from_related_stores" + ) as mock: + activitystreams.handle_boost_task(boost.id) + self.assertTrue(mock.called) + call_args = mock.call_args + self.assertEqual(call_args[0][0], status) + self.assertTrue( + "{:d}-home".format(self.another_user.id) in call_args[1]["stores"] + ) + self.assertTrue( + "{:d}-home".format(self.local_user.id) in call_args[1]["stores"] + ) + + @patch("bookwyrm.activitystreams.LocalStream.remove_object_from_related_stores") + @patch("bookwyrm.activitystreams.BooksStream.remove_object_from_related_stores") + def test_boost_to_same_timeline(self, *_): + """add a boost and deduplicate the boosted status on the timeline""" + status = models.Status.objects.create(user=self.local_user, content="hi") + with patch("bookwyrm.activitystreams.handle_boost_task.delay"): + boost = models.Boost.objects.create( + boosted_status=status, + user=self.local_user, + ) + with patch( + "bookwyrm.activitystreams.HomeStream.remove_object_from_related_stores" + ) as mock: + activitystreams.handle_boost_task(boost.id) + self.assertTrue(mock.called) + call_args = mock.call_args + self.assertEqual(call_args[0][0], status) + self.assertEqual( + call_args[1]["stores"], ["{:d}-home".format(self.local_user.id)] + ) From d147c6ac2f84d35f78e443139362c7f5eb356bec Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Tue, 7 Sep 2021 11:09:52 -0700 Subject: [PATCH 009/191] Separate tests into more files --- .../activitystreams/test_abstractstream.py | 145 +++++++++ .../activitystreams/test_activitystreams.py | 292 ------------------ .../tests/activitystreams/test_booksstream.py | 55 ++++ .../tests/activitystreams/test_homestream.py | 67 ++++ .../tests/activitystreams/test_localstream.py | 139 +++++++++ ...activitystreams_tasks.py => test_tasks.py} | 0 6 files changed, 406 insertions(+), 292 deletions(-) create mode 100644 bookwyrm/tests/activitystreams/test_abstractstream.py delete mode 100644 bookwyrm/tests/activitystreams/test_activitystreams.py create mode 100644 bookwyrm/tests/activitystreams/test_booksstream.py create mode 100644 bookwyrm/tests/activitystreams/test_homestream.py create mode 100644 bookwyrm/tests/activitystreams/test_localstream.py rename bookwyrm/tests/activitystreams/{test_activitystreams_tasks.py => test_tasks.py} (100%) diff --git a/bookwyrm/tests/activitystreams/test_abstractstream.py b/bookwyrm/tests/activitystreams/test_abstractstream.py new file mode 100644 index 000000000..f96742865 --- /dev/null +++ b/bookwyrm/tests/activitystreams/test_abstractstream.py @@ -0,0 +1,145 @@ +""" testing activitystreams """ +from unittest.mock import patch +from django.test import TestCase +from bookwyrm import activitystreams, models + + +@patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay") +@patch("bookwyrm.activitystreams.add_status_task.delay") +@patch("bookwyrm.activitystreams.add_book_statuses_task.delay") +@patch("bookwyrm.suggested_users.rerank_suggestions_task.delay") +@patch("bookwyrm.activitystreams.populate_stream_task.delay") +class Activitystreams(TestCase): + """using redis to build activity streams""" + + def setUp(self): + """use a test csv""" + with patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"), patch( + "bookwyrm.activitystreams.populate_stream_task.delay" + ): + self.local_user = models.User.objects.create_user( + "mouse", "mouse@mouse.mouse", "password", local=True, localname="mouse" + ) + self.another_user = models.User.objects.create_user( + "nutria", + "nutria@nutria.nutria", + "password", + local=True, + localname="nutria", + ) + with patch("bookwyrm.models.user.set_remote_server.delay"): + self.remote_user = models.User.objects.create_user( + "rat", + "rat@rat.com", + "ratword", + local=False, + remote_id="https://example.com/users/rat", + inbox="https://example.com/users/rat/inbox", + outbox="https://example.com/users/rat/outbox", + ) + work = models.Work.objects.create(title="test work") + self.book = models.Edition.objects.create(title="test book", parent_work=work) + + class TestStream(activitystreams.ActivityStream): + """test stream, don't have to do anything here""" + + key = "test" + + self.test_stream = TestStream() + + def test_activitystream_class_ids(self, *_): + """the abstract base class for stream objects""" + self.assertEqual( + self.test_stream.stream_id(self.local_user), + "{}-test".format(self.local_user.id), + ) + self.assertEqual( + self.test_stream.unread_id(self.local_user), + "{}-test-unread".format(self.local_user.id), + ) + + def test_abstractstream_get_audience(self, *_): + """get a list of users that should see a status""" + status = models.Status.objects.create( + user=self.remote_user, content="hi", privacy="public" + ) + users = self.test_stream.get_audience(status) + # remote users don't have feeds + self.assertFalse(self.remote_user in users) + self.assertTrue(self.local_user in users) + self.assertTrue(self.another_user in users) + + def test_abstractstream_get_audience_direct(self, *_): + """get a list of users that should see a status""" + status = models.Status.objects.create( + user=self.remote_user, + content="hi", + privacy="direct", + ) + status.mention_users.add(self.local_user) + users = self.test_stream.get_audience(status) + self.assertEqual(users, []) + + status = models.Comment.objects.create( + user=self.remote_user, + content="hi", + privacy="direct", + book=self.book, + ) + status.mention_users.add(self.local_user) + users = self.test_stream.get_audience(status) + self.assertTrue(self.local_user in users) + self.assertFalse(self.another_user in users) + self.assertFalse(self.remote_user in users) + + def test_abstractstream_get_audience_followers_remote_user(self, *_): + """get a list of users that should see a status""" + status = models.Status.objects.create( + user=self.remote_user, + content="hi", + privacy="followers", + ) + users = self.test_stream.get_audience(status) + self.assertFalse(users.exists()) + + def test_abstractstream_get_audience_followers_self(self, *_): + """get a list of users that should see a status""" + status = models.Comment.objects.create( + user=self.local_user, + content="hi", + privacy="direct", + book=self.book, + ) + users = self.test_stream.get_audience(status) + self.assertTrue(self.local_user in users) + self.assertFalse(self.another_user in users) + self.assertFalse(self.remote_user in users) + + def test_abstractstream_get_audience_followers_with_mention(self, *_): + """get a list of users that should see a status""" + status = models.Comment.objects.create( + user=self.remote_user, + content="hi", + privacy="direct", + book=self.book, + ) + status.mention_users.add(self.local_user) + + users = self.test_stream.get_audience(status) + self.assertTrue(self.local_user in users) + self.assertFalse(self.another_user in users) + self.assertFalse(self.remote_user in users) + + def test_abstractstream_get_audience_followers_with_relationship(self, *_): + """get a list of users that should see a status""" + self.remote_user.followers.add(self.local_user) + status = models.Comment.objects.create( + user=self.remote_user, + content="hi", + privacy="direct", + book=self.book, + ) + users = self.test_stream.get_audience(status) + self.assertFalse(self.local_user in users) + self.assertFalse(self.another_user in users) + self.assertFalse(self.remote_user in users) diff --git a/bookwyrm/tests/activitystreams/test_activitystreams.py b/bookwyrm/tests/activitystreams/test_activitystreams.py deleted file mode 100644 index 4e91c5bea..000000000 --- a/bookwyrm/tests/activitystreams/test_activitystreams.py +++ /dev/null @@ -1,292 +0,0 @@ -""" testing activitystreams """ -from unittest.mock import patch -from django.test import TestCase -from bookwyrm import activitystreams, models - - -@patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay") -@patch("bookwyrm.activitystreams.add_status_task.delay") -@patch("bookwyrm.activitystreams.add_book_statuses_task.delay") -@patch("bookwyrm.suggested_users.rerank_suggestions_task.delay") -@patch("bookwyrm.activitystreams.populate_stream_task.delay") -# pylint: disable=too-many-public-methods -class Activitystreams(TestCase): - """using redis to build activity streams""" - - def setUp(self): - """use a test csv""" - with patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"), patch( - "bookwyrm.activitystreams.populate_stream_task.delay" - ): - self.local_user = models.User.objects.create_user( - "mouse", "mouse@mouse.mouse", "password", local=True, localname="mouse" - ) - self.another_user = models.User.objects.create_user( - "nutria", - "nutria@nutria.nutria", - "password", - local=True, - localname="nutria", - ) - with patch("bookwyrm.models.user.set_remote_server.delay"): - self.remote_user = models.User.objects.create_user( - "rat", - "rat@rat.com", - "ratword", - local=False, - remote_id="https://example.com/users/rat", - inbox="https://example.com/users/rat/inbox", - outbox="https://example.com/users/rat/outbox", - ) - work = models.Work.objects.create(title="test work") - self.book = models.Edition.objects.create(title="test book", parent_work=work) - - class TestStream(activitystreams.ActivityStream): - """test stream, don't have to do anything here""" - - key = "test" - - self.test_stream = TestStream() - - def test_activitystream_class_ids(self, *_): - """the abstract base class for stream objects""" - self.assertEqual( - self.test_stream.stream_id(self.local_user), - "{}-test".format(self.local_user.id), - ) - self.assertEqual( - self.test_stream.unread_id(self.local_user), - "{}-test-unread".format(self.local_user.id), - ) - - def test_abstractstream_get_audience(self, *_): - """get a list of users that should see a status""" - status = models.Status.objects.create( - user=self.remote_user, content="hi", privacy="public" - ) - users = self.test_stream.get_audience(status) - # remote users don't have feeds - self.assertFalse(self.remote_user in users) - self.assertTrue(self.local_user in users) - self.assertTrue(self.another_user in users) - - def test_abstractstream_get_audience_direct(self, *_): - """get a list of users that should see a status""" - status = models.Status.objects.create( - user=self.remote_user, - content="hi", - privacy="direct", - ) - status.mention_users.add(self.local_user) - users = self.test_stream.get_audience(status) - self.assertEqual(users, []) - - status = models.Comment.objects.create( - user=self.remote_user, - content="hi", - privacy="direct", - book=self.book, - ) - status.mention_users.add(self.local_user) - users = self.test_stream.get_audience(status) - self.assertTrue(self.local_user in users) - self.assertFalse(self.another_user in users) - self.assertFalse(self.remote_user in users) - - def test_abstractstream_get_audience_followers_remote_user(self, *_): - """get a list of users that should see a status""" - status = models.Status.objects.create( - user=self.remote_user, - content="hi", - privacy="followers", - ) - users = self.test_stream.get_audience(status) - self.assertFalse(users.exists()) - - def test_abstractstream_get_audience_followers_self(self, *_): - """get a list of users that should see a status""" - status = models.Comment.objects.create( - user=self.local_user, - content="hi", - privacy="direct", - book=self.book, - ) - users = self.test_stream.get_audience(status) - self.assertTrue(self.local_user in users) - self.assertFalse(self.another_user in users) - self.assertFalse(self.remote_user in users) - - def test_abstractstream_get_audience_followers_with_mention(self, *_): - """get a list of users that should see a status""" - status = models.Comment.objects.create( - user=self.remote_user, - content="hi", - privacy="direct", - book=self.book, - ) - status.mention_users.add(self.local_user) - - users = self.test_stream.get_audience(status) - self.assertTrue(self.local_user in users) - self.assertFalse(self.another_user in users) - self.assertFalse(self.remote_user in users) - - def test_abstractstream_get_audience_followers_with_relationship(self, *_): - """get a list of users that should see a status""" - self.remote_user.followers.add(self.local_user) - status = models.Comment.objects.create( - user=self.remote_user, - content="hi", - privacy="direct", - book=self.book, - ) - users = self.test_stream.get_audience(status) - self.assertFalse(self.local_user in users) - self.assertFalse(self.another_user in users) - self.assertFalse(self.remote_user in users) - - def test_homestream_get_audience(self, *_): - """get a list of users that should see a status""" - status = models.Status.objects.create( - user=self.remote_user, content="hi", privacy="public" - ) - users = activitystreams.HomeStream().get_audience(status) - self.assertFalse(users.exists()) - - def test_homestream_get_audience_with_mentions(self, *_): - """get a list of users that should see a status""" - status = models.Status.objects.create( - user=self.remote_user, content="hi", privacy="public" - ) - status.mention_users.add(self.local_user) - users = activitystreams.HomeStream().get_audience(status) - self.assertFalse(self.local_user in users) - self.assertFalse(self.another_user in users) - - def test_homestream_get_audience_with_relationship(self, *_): - """get a list of users that should see a status""" - self.remote_user.followers.add(self.local_user) - status = models.Status.objects.create( - user=self.remote_user, content="hi", privacy="public" - ) - users = activitystreams.HomeStream().get_audience(status) - self.assertTrue(self.local_user in users) - self.assertFalse(self.another_user in users) - - def test_localstream_get_audience_remote_status(self, *_): - """get a list of users that should see a status""" - status = models.Status.objects.create( - user=self.remote_user, content="hi", privacy="public" - ) - users = activitystreams.LocalStream().get_audience(status) - self.assertEqual(users, []) - - def test_localstream_get_audience_local_status(self, *_): - """get a list of users that should see a status""" - status = models.Status.objects.create( - user=self.local_user, content="hi", privacy="public" - ) - users = activitystreams.LocalStream().get_audience(status) - self.assertTrue(self.local_user in users) - self.assertTrue(self.another_user in users) - - def test_localstream_get_audience_unlisted(self, *_): - """get a list of users that should see a status""" - status = models.Status.objects.create( - user=self.local_user, content="hi", privacy="unlisted" - ) - users = activitystreams.LocalStream().get_audience(status) - self.assertEqual(users, []) - - def test_localstream_get_audience_books_no_book(self, *_): - """get a list of users that should see a status""" - status = models.Status.objects.create( - user=self.local_user, content="hi", privacy="public" - ) - audience = activitystreams.BooksStream().get_audience(status) - # no books, no audience - self.assertEqual(audience, []) - - def test_localstream_get_audience_books_mention_books(self, *_): - """get a list of users that should see a status""" - status = models.Status.objects.create( - user=self.local_user, content="hi", privacy="public" - ) - status.mention_books.add(self.book) - status.save(broadcast=False) - models.ShelfBook.objects.create( - user=self.local_user, - shelf=self.local_user.shelf_set.first(), - book=self.book, - ) - # yes book, yes audience - audience = activitystreams.BooksStream().get_audience(status) - self.assertTrue(self.local_user in audience) - - def test_localstream_get_audience_books_book_field(self, *_): - """get a list of users that should see a status""" - status = models.Comment.objects.create( - user=self.local_user, content="hi", privacy="public", book=self.book - ) - models.ShelfBook.objects.create( - user=self.local_user, - shelf=self.local_user.shelf_set.first(), - book=self.book, - ) - # yes book, yes audience - audience = activitystreams.BooksStream().get_audience(status) - self.assertTrue(self.local_user in audience) - - def test_localstream_get_audience_books_alternate_edition(self, *_): - """get a list of users that should see a status""" - alt_book = models.Edition.objects.create( - title="hi", parent_work=self.book.parent_work - ) - status = models.Comment.objects.create( - user=self.remote_user, content="hi", privacy="public", book=alt_book - ) - models.ShelfBook.objects.create( - user=self.local_user, - shelf=self.local_user.shelf_set.first(), - book=self.book, - ) - # yes book, yes audience - audience = activitystreams.BooksStream().get_audience(status) - self.assertTrue(self.local_user in audience) - - def test_localstream_get_audience_books_non_public(self, *_): - """get a list of users that should see a status""" - alt_book = models.Edition.objects.create( - title="hi", parent_work=self.book.parent_work - ) - status = models.Comment.objects.create( - user=self.remote_user, content="hi", privacy="unlisted", book=alt_book - ) - models.ShelfBook.objects.create( - user=self.local_user, - shelf=self.local_user.shelf_set.first(), - book=self.book, - ) - # yes book, yes audience - audience = activitystreams.BooksStream().get_audience(status) - self.assertEqual(audience, []) - - def test_get_statuses_for_user_books(self, *_): - """create a stream for a user""" - alt_book = models.Edition.objects.create( - title="hi", parent_work=self.book.parent_work - ) - status = models.Status.objects.create( - user=self.local_user, content="hi", privacy="public" - ) - status = models.Comment.objects.create( - user=self.remote_user, content="hi", privacy="public", book=alt_book - ) - models.ShelfBook.objects.create( - user=self.local_user, - shelf=self.local_user.shelf_set.first(), - book=self.book, - ) - # yes book, yes audience - result = activitystreams.BooksStream().get_statuses_for_user(self.local_user) - self.assertEqual(list(result), [status]) diff --git a/bookwyrm/tests/activitystreams/test_booksstream.py b/bookwyrm/tests/activitystreams/test_booksstream.py new file mode 100644 index 000000000..5730dccf7 --- /dev/null +++ b/bookwyrm/tests/activitystreams/test_booksstream.py @@ -0,0 +1,55 @@ +""" testing activitystreams """ +from unittest.mock import patch +from django.test import TestCase +from bookwyrm import activitystreams, models + + +@patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay") +@patch("bookwyrm.activitystreams.add_status_task.delay") +@patch("bookwyrm.activitystreams.add_book_statuses_task.delay") +@patch("bookwyrm.suggested_users.rerank_suggestions_task.delay") +@patch("bookwyrm.activitystreams.populate_stream_task.delay") +class Activitystreams(TestCase): + """using redis to build activity streams""" + + def setUp(self): + """use a test csv""" + with patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"), patch( + "bookwyrm.activitystreams.populate_stream_task.delay" + ): + self.local_user = models.User.objects.create_user( + "mouse", "mouse@mouse.mouse", "password", local=True, localname="mouse" + ) + with patch("bookwyrm.models.user.set_remote_server.delay"): + self.remote_user = models.User.objects.create_user( + "rat", + "rat@rat.com", + "ratword", + local=False, + remote_id="https://example.com/users/rat", + inbox="https://example.com/users/rat/inbox", + outbox="https://example.com/users/rat/outbox", + ) + work = models.Work.objects.create(title="test work") + self.book = models.Edition.objects.create(title="test book", parent_work=work) + + + def test_get_statuses_for_user_books(self, *_): + """create a stream for a user""" + alt_book = models.Edition.objects.create( + title="hi", parent_work=self.book.parent_work + ) + status = models.Status.objects.create( + user=self.local_user, content="hi", privacy="public" + ) + status = models.Comment.objects.create( + user=self.remote_user, content="hi", privacy="public", book=alt_book + ) + models.ShelfBook.objects.create( + user=self.local_user, + shelf=self.local_user.shelf_set.first(), + book=self.book, + ) + # yes book, yes audience + result = activitystreams.BooksStream().get_statuses_for_user(self.local_user) + self.assertEqual(list(result), [status]) diff --git a/bookwyrm/tests/activitystreams/test_homestream.py b/bookwyrm/tests/activitystreams/test_homestream.py new file mode 100644 index 000000000..42a73f3cf --- /dev/null +++ b/bookwyrm/tests/activitystreams/test_homestream.py @@ -0,0 +1,67 @@ +""" testing activitystreams """ +from unittest.mock import patch +from django.test import TestCase +from bookwyrm import activitystreams, models + + +@patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay") +@patch("bookwyrm.activitystreams.add_status_task.delay") +@patch("bookwyrm.activitystreams.add_book_statuses_task.delay") +@patch("bookwyrm.suggested_users.rerank_suggestions_task.delay") +@patch("bookwyrm.activitystreams.populate_stream_task.delay") +class Activitystreams(TestCase): + """using redis to build activity streams""" + + def setUp(self): + """use a test csv""" + with patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"), patch( + "bookwyrm.activitystreams.populate_stream_task.delay" + ): + self.local_user = models.User.objects.create_user( + "mouse", "mouse@mouse.mouse", "password", local=True, localname="mouse" + ) + self.another_user = models.User.objects.create_user( + "nutria", + "nutria@nutria.nutria", + "password", + local=True, + localname="nutria", + ) + with patch("bookwyrm.models.user.set_remote_server.delay"): + self.remote_user = models.User.objects.create_user( + "rat", + "rat@rat.com", + "ratword", + local=False, + remote_id="https://example.com/users/rat", + inbox="https://example.com/users/rat/inbox", + outbox="https://example.com/users/rat/outbox", + ) + + def test_homestream_get_audience(self, *_): + """get a list of users that should see a status""" + status = models.Status.objects.create( + user=self.remote_user, content="hi", privacy="public" + ) + users = activitystreams.HomeStream().get_audience(status) + self.assertFalse(users.exists()) + + def test_homestream_get_audience_with_mentions(self, *_): + """get a list of users that should see a status""" + status = models.Status.objects.create( + user=self.remote_user, content="hi", privacy="public" + ) + status.mention_users.add(self.local_user) + users = activitystreams.HomeStream().get_audience(status) + self.assertFalse(self.local_user in users) + self.assertFalse(self.another_user in users) + + def test_homestream_get_audience_with_relationship(self, *_): + """get a list of users that should see a status""" + self.remote_user.followers.add(self.local_user) + status = models.Status.objects.create( + user=self.remote_user, content="hi", privacy="public" + ) + users = activitystreams.HomeStream().get_audience(status) + self.assertTrue(self.local_user in users) + self.assertFalse(self.another_user in users) diff --git a/bookwyrm/tests/activitystreams/test_localstream.py b/bookwyrm/tests/activitystreams/test_localstream.py new file mode 100644 index 000000000..e6b05557f --- /dev/null +++ b/bookwyrm/tests/activitystreams/test_localstream.py @@ -0,0 +1,139 @@ +""" testing activitystreams """ +from unittest.mock import patch +from django.test import TestCase +from bookwyrm import activitystreams, models + + +@patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay") +@patch("bookwyrm.activitystreams.add_status_task.delay") +@patch("bookwyrm.activitystreams.add_book_statuses_task.delay") +@patch("bookwyrm.suggested_users.rerank_suggestions_task.delay") +@patch("bookwyrm.activitystreams.populate_stream_task.delay") +class Activitystreams(TestCase): + """using redis to build activity streams""" + + def setUp(self): + """use a test csv""" + with patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"), patch( + "bookwyrm.activitystreams.populate_stream_task.delay" + ): + self.local_user = models.User.objects.create_user( + "mouse", "mouse@mouse.mouse", "password", local=True, localname="mouse" + ) + self.another_user = models.User.objects.create_user( + "nutria", + "nutria@nutria.nutria", + "password", + local=True, + localname="nutria", + ) + with patch("bookwyrm.models.user.set_remote_server.delay"): + self.remote_user = models.User.objects.create_user( + "rat", + "rat@rat.com", + "ratword", + local=False, + remote_id="https://example.com/users/rat", + inbox="https://example.com/users/rat/inbox", + outbox="https://example.com/users/rat/outbox", + ) + work = models.Work.objects.create(title="test work") + self.book = models.Edition.objects.create(title="test book", parent_work=work) + + def test_localstream_get_audience_remote_status(self, *_): + """get a list of users that should see a status""" + status = models.Status.objects.create( + user=self.remote_user, content="hi", privacy="public" + ) + users = activitystreams.LocalStream().get_audience(status) + self.assertEqual(users, []) + + def test_localstream_get_audience_local_status(self, *_): + """get a list of users that should see a status""" + status = models.Status.objects.create( + user=self.local_user, content="hi", privacy="public" + ) + users = activitystreams.LocalStream().get_audience(status) + self.assertTrue(self.local_user in users) + self.assertTrue(self.another_user in users) + + def test_localstream_get_audience_unlisted(self, *_): + """get a list of users that should see a status""" + status = models.Status.objects.create( + user=self.local_user, content="hi", privacy="unlisted" + ) + users = activitystreams.LocalStream().get_audience(status) + self.assertEqual(users, []) + + def test_localstream_get_audience_books_no_book(self, *_): + """get a list of users that should see a status""" + status = models.Status.objects.create( + user=self.local_user, content="hi", privacy="public" + ) + audience = activitystreams.BooksStream().get_audience(status) + # no books, no audience + self.assertEqual(audience, []) + + def test_localstream_get_audience_books_mention_books(self, *_): + """get a list of users that should see a status""" + status = models.Status.objects.create( + user=self.local_user, content="hi", privacy="public" + ) + status.mention_books.add(self.book) + status.save(broadcast=False) + models.ShelfBook.objects.create( + user=self.local_user, + shelf=self.local_user.shelf_set.first(), + book=self.book, + ) + # yes book, yes audience + audience = activitystreams.BooksStream().get_audience(status) + self.assertTrue(self.local_user in audience) + + def test_localstream_get_audience_books_book_field(self, *_): + """get a list of users that should see a status""" + status = models.Comment.objects.create( + user=self.local_user, content="hi", privacy="public", book=self.book + ) + models.ShelfBook.objects.create( + user=self.local_user, + shelf=self.local_user.shelf_set.first(), + book=self.book, + ) + # yes book, yes audience + audience = activitystreams.BooksStream().get_audience(status) + self.assertTrue(self.local_user in audience) + + def test_localstream_get_audience_books_alternate_edition(self, *_): + """get a list of users that should see a status""" + alt_book = models.Edition.objects.create( + title="hi", parent_work=self.book.parent_work + ) + status = models.Comment.objects.create( + user=self.remote_user, content="hi", privacy="public", book=alt_book + ) + models.ShelfBook.objects.create( + user=self.local_user, + shelf=self.local_user.shelf_set.first(), + book=self.book, + ) + # yes book, yes audience + audience = activitystreams.BooksStream().get_audience(status) + self.assertTrue(self.local_user in audience) + + def test_localstream_get_audience_books_non_public(self, *_): + """get a list of users that should see a status""" + alt_book = models.Edition.objects.create( + title="hi", parent_work=self.book.parent_work + ) + status = models.Comment.objects.create( + user=self.remote_user, content="hi", privacy="unlisted", book=alt_book + ) + models.ShelfBook.objects.create( + user=self.local_user, + shelf=self.local_user.shelf_set.first(), + book=self.book, + ) + # yes book, yes audience + audience = activitystreams.BooksStream().get_audience(status) + self.assertEqual(audience, []) diff --git a/bookwyrm/tests/activitystreams/test_activitystreams_tasks.py b/bookwyrm/tests/activitystreams/test_tasks.py similarity index 100% rename from bookwyrm/tests/activitystreams/test_activitystreams_tasks.py rename to bookwyrm/tests/activitystreams/test_tasks.py From 82ad1abe9aa9a7a3be5c7501557249db578f2f3e Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Tue, 7 Sep 2021 11:53:41 -0700 Subject: [PATCH 010/191] more activitystreams tests --- .../tests/activitystreams/test_signals.py | 69 +++++++++++++++++++ bookwyrm/tests/activitystreams/test_tasks.py | 6 +- 2 files changed, 74 insertions(+), 1 deletion(-) create mode 100644 bookwyrm/tests/activitystreams/test_signals.py diff --git a/bookwyrm/tests/activitystreams/test_signals.py b/bookwyrm/tests/activitystreams/test_signals.py new file mode 100644 index 000000000..ef1ac97c9 --- /dev/null +++ b/bookwyrm/tests/activitystreams/test_signals.py @@ -0,0 +1,69 @@ +""" testing activitystreams """ +from unittest.mock import patch +from django.test import TestCase +from bookwyrm import activitystreams, models + + +@patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay") +class ActivitystreamsSignals(TestCase): + """using redis to build activity streams""" + + def setUp(self): + """use a test csv""" + with patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"), patch( + "bookwyrm.activitystreams.populate_stream_task.delay" + ): + self.local_user = models.User.objects.create_user( + "mouse", "mouse@mouse.mouse", "password", local=True, localname="mouse" + ) + with patch("bookwyrm.models.user.set_remote_server.delay"): + self.remote_user = models.User.objects.create_user( + "rat", + "rat@rat.com", + "ratword", + local=False, + remote_id="https://example.com/users/rat", + inbox="https://example.com/users/rat/inbox", + outbox="https://example.com/users/rat/outbox", + ) + work = models.Work.objects.create(title="test work") + self.book = models.Edition.objects.create(title="test book", parent_work=work) + + + def test_add_status_on_create_ignore(self, _): + """a new statuses has entered""" + activitystreams.add_status_on_create(models.User, self.local_user, False) + + def test_add_status_on_create_deleted(self, _): + """a new statuses has entered""" + with patch("bookwyrm.activitystreams.remove_status_task.delay"): + status = models.Status.objects.create( + user=self.remote_user, content="hi", privacy="public", deleted=True + ) + with patch("bookwyrm.activitystreams.remove_status_task.delay") as mock: + activitystreams.add_status_on_create(models.Status, status, False) + self.assertEqual(mock.call_count, 1) + args = mock.call_args[0] + self.assertEqual(args[0], status.id) + + def test_add_status_on_create_created(self, _): + """a new statuses has entered""" + status = models.Status.objects.create( + user=self.remote_user, content="hi", privacy="public" + ) + with patch("bookwyrm.activitystreams.add_status_task.delay") as mock: + activitystreams.add_status_on_create_command(models.Status, status, False) + self.assertEqual(mock.call_count, 1) + args = mock.call_args[0] + self.assertEqual(args[0], status.id) + + def test_populate_streams_on_account_create(self): + """create streams for a user""" + with patch("bookwyrm.activitystreams.populate_stream_task") as mock: + activitystreams.populate_streams_on_account_create( + models.User, self.local_user, True + ) + self.assertEqual(mock.call_count, 3) + args = mock.call_args[0] + self.assertEqual(args[0], "home") + self.assertEqual(args[1], self.local_user.id) diff --git a/bookwyrm/tests/activitystreams/test_tasks.py b/bookwyrm/tests/activitystreams/test_tasks.py index a51549ca6..287bf6bc0 100644 --- a/bookwyrm/tests/activitystreams/test_tasks.py +++ b/bookwyrm/tests/activitystreams/test_tasks.py @@ -24,7 +24,8 @@ class Activitystreams(TestCase): ) work = models.Work.objects.create(title="test work") self.book = models.Edition.objects.create(title="test book", parent_work=work) - self.status = models.Status.objects.create(content="hi", user=self.local_user) + with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): + self.status = models.Status.objects.create(content="hi", user=self.local_user) def test_add_book_statuses_task(self): """statuses related to a book""" @@ -120,6 +121,7 @@ class Activitystreams(TestCase): @patch("bookwyrm.activitystreams.LocalStream.remove_object_from_related_stores") @patch("bookwyrm.activitystreams.BooksStream.remove_object_from_related_stores") + @patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay") def test_boost_to_another_timeline(self, *_): """add a boost and deduplicate the boosted status on the timeline""" status = models.Status.objects.create(user=self.local_user, content="hi") @@ -142,6 +144,7 @@ class Activitystreams(TestCase): @patch("bookwyrm.activitystreams.LocalStream.remove_object_from_related_stores") @patch("bookwyrm.activitystreams.BooksStream.remove_object_from_related_stores") + @patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay") def test_boost_to_following_timeline(self, *_): """add a boost and deduplicate the boosted status on the timeline""" self.local_user.following.add(self.another_user) @@ -167,6 +170,7 @@ class Activitystreams(TestCase): @patch("bookwyrm.activitystreams.LocalStream.remove_object_from_related_stores") @patch("bookwyrm.activitystreams.BooksStream.remove_object_from_related_stores") + @patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay") def test_boost_to_same_timeline(self, *_): """add a boost and deduplicate the boosted status on the timeline""" status = models.Status.objects.create(user=self.local_user, content="hi") From b717c2fcd81dc594f943b63345d7800f6b12676a Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Tue, 7 Sep 2021 11:57:18 -0700 Subject: [PATCH 011/191] Fixes mock arg on test --- bookwyrm/tests/activitystreams/test_signals.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bookwyrm/tests/activitystreams/test_signals.py b/bookwyrm/tests/activitystreams/test_signals.py index ef1ac97c9..ed5b4a169 100644 --- a/bookwyrm/tests/activitystreams/test_signals.py +++ b/bookwyrm/tests/activitystreams/test_signals.py @@ -57,7 +57,7 @@ class ActivitystreamsSignals(TestCase): args = mock.call_args[0] self.assertEqual(args[0], status.id) - def test_populate_streams_on_account_create(self): + def test_populate_streams_on_account_create(self, _): """create streams for a user""" with patch("bookwyrm.activitystreams.populate_stream_task") as mock: activitystreams.populate_streams_on_account_create( From 6bc4bf45038eca21b23ff9c7bb7b6d170b1d0417 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Tue, 7 Sep 2021 12:02:29 -0700 Subject: [PATCH 012/191] Fixes mock --- bookwyrm/tests/activitystreams/test_signals.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bookwyrm/tests/activitystreams/test_signals.py b/bookwyrm/tests/activitystreams/test_signals.py index ed5b4a169..2cf26edd4 100644 --- a/bookwyrm/tests/activitystreams/test_signals.py +++ b/bookwyrm/tests/activitystreams/test_signals.py @@ -59,7 +59,7 @@ class ActivitystreamsSignals(TestCase): def test_populate_streams_on_account_create(self, _): """create streams for a user""" - with patch("bookwyrm.activitystreams.populate_stream_task") as mock: + with patch("bookwyrm.activitystreams.populate_stream_task.delay") as mock: activitystreams.populate_streams_on_account_create( models.User, self.local_user, True ) From da3bc1e59160b0f3bd2bb3e5ae6de166c7dc84ba Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Tue, 7 Sep 2021 12:05:34 -0700 Subject: [PATCH 013/191] Python formatting --- bookwyrm/tests/activitystreams/test_booksstream.py | 1 - bookwyrm/tests/activitystreams/test_signals.py | 3 +-- bookwyrm/tests/activitystreams/test_tasks.py | 4 +++- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/bookwyrm/tests/activitystreams/test_booksstream.py b/bookwyrm/tests/activitystreams/test_booksstream.py index 5730dccf7..d6d94b738 100644 --- a/bookwyrm/tests/activitystreams/test_booksstream.py +++ b/bookwyrm/tests/activitystreams/test_booksstream.py @@ -33,7 +33,6 @@ class Activitystreams(TestCase): work = models.Work.objects.create(title="test work") self.book = models.Edition.objects.create(title="test book", parent_work=work) - def test_get_statuses_for_user_books(self, *_): """create a stream for a user""" alt_book = models.Edition.objects.create( diff --git a/bookwyrm/tests/activitystreams/test_signals.py b/bookwyrm/tests/activitystreams/test_signals.py index 2cf26edd4..1c94cc9f5 100644 --- a/bookwyrm/tests/activitystreams/test_signals.py +++ b/bookwyrm/tests/activitystreams/test_signals.py @@ -29,7 +29,6 @@ class ActivitystreamsSignals(TestCase): work = models.Work.objects.create(title="test work") self.book = models.Edition.objects.create(title="test book", parent_work=work) - def test_add_status_on_create_ignore(self, _): """a new statuses has entered""" activitystreams.add_status_on_create(models.User, self.local_user, False) @@ -65,5 +64,5 @@ class ActivitystreamsSignals(TestCase): ) self.assertEqual(mock.call_count, 3) args = mock.call_args[0] - self.assertEqual(args[0], "home") + self.assertEqual(args[0], "books") self.assertEqual(args[1], self.local_user.id) diff --git a/bookwyrm/tests/activitystreams/test_tasks.py b/bookwyrm/tests/activitystreams/test_tasks.py index 287bf6bc0..f4c85e1bf 100644 --- a/bookwyrm/tests/activitystreams/test_tasks.py +++ b/bookwyrm/tests/activitystreams/test_tasks.py @@ -25,7 +25,9 @@ class Activitystreams(TestCase): work = models.Work.objects.create(title="test work") self.book = models.Edition.objects.create(title="test book", parent_work=work) with patch("bookwyrm.models.activitypub_mixin.broadcast_task.delay"): - self.status = models.Status.objects.create(content="hi", user=self.local_user) + self.status = models.Status.objects.create( + content="hi", user=self.local_user + ) def test_add_book_statuses_task(self): """statuses related to a book""" From 75cc2ee164ef6c4e29730e8e5e122ff31c525649 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Tue, 7 Sep 2021 13:11:44 -0700 Subject: [PATCH 014/191] Split authentication views into login and register --- bookwyrm/tests/views/test_authentication.py | 16 ++-- bookwyrm/views/__init__.py | 4 +- bookwyrm/views/invite.py | 2 +- bookwyrm/views/login.py | 80 +++++++++++++++++++ .../views/{authentication.py => register.py} | 73 +---------------- 5 files changed, 92 insertions(+), 83 deletions(-) create mode 100644 bookwyrm/views/login.py rename bookwyrm/views/{authentication.py => register.py} (61%) diff --git a/bookwyrm/tests/views/test_authentication.py b/bookwyrm/tests/views/test_authentication.py index 95a4d9a0b..74f3c0902 100644 --- a/bookwyrm/tests/views/test_authentication.py +++ b/bookwyrm/tests/views/test_authentication.py @@ -63,7 +63,7 @@ class AuthenticationViews(TestCase): request = self.factory.post("", form.data) request.user = self.anonymous_user - with patch("bookwyrm.views.authentication.login"): + with patch("bookwyrm.views.login.login"): result = view(request) self.assertEqual(result.url, "/") self.assertEqual(result.status_code, 302) @@ -77,7 +77,7 @@ class AuthenticationViews(TestCase): request = self.factory.post("", form.data) request.user = self.anonymous_user - with patch("bookwyrm.views.authentication.login"): + with patch("bookwyrm.views.login.login"): result = view(request) self.assertEqual(result.url, "/") self.assertEqual(result.status_code, 302) @@ -91,7 +91,7 @@ class AuthenticationViews(TestCase): request = self.factory.post("", form.data) request.user = self.anonymous_user - with patch("bookwyrm.views.authentication.login"): + with patch("bookwyrm.views.login.login"): result = view(request) self.assertEqual(result.url, "/") self.assertEqual(result.status_code, 302) @@ -105,7 +105,7 @@ class AuthenticationViews(TestCase): request = self.factory.post("", form.data) request.user = self.anonymous_user - with patch("bookwyrm.views.authentication.login"): + with patch("bookwyrm.views.login.login"): result = view(request) result.render() self.assertEqual(result.status_code, 200) @@ -126,7 +126,7 @@ class AuthenticationViews(TestCase): "email": "aa@bb.cccc", }, ) - with patch("bookwyrm.views.authentication.login"): + with patch("bookwyrm.views.login.login"): response = view(request) self.assertEqual(models.User.objects.count(), 2) self.assertEqual(response.status_code, 302) @@ -151,7 +151,7 @@ class AuthenticationViews(TestCase): "email": "aa@bb.cccc", }, ) - with patch("bookwyrm.views.authentication.login"): + with patch("bookwyrm.views.login.login"): response = view(request) self.assertEqual(response.status_code, 302) nutria = models.User.objects.get(localname="nutria") @@ -169,7 +169,7 @@ class AuthenticationViews(TestCase): "register/", {"localname": "nutria ", "password": "mouseword", "email": "aa@bb.ccc"}, ) - with patch("bookwyrm.views.authentication.login"): + with patch("bookwyrm.views.login.login"): response = view(request) self.assertEqual(models.User.objects.count(), 2) self.assertEqual(response.status_code, 302) @@ -248,7 +248,7 @@ class AuthenticationViews(TestCase): "invite_code": "testcode", }, ) - with patch("bookwyrm.views.authentication.login"): + with patch("bookwyrm.views.login.login"): response = view(request) self.assertEqual(models.User.objects.count(), 2) self.assertEqual(response.status_code, 302) diff --git a/bookwyrm/views/__init__.py b/bookwyrm/views/__init__.py index ca52800c4..841026a54 100644 --- a/bookwyrm/views/__init__.py +++ b/bookwyrm/views/__init__.py @@ -1,7 +1,5 @@ """ make sure all our nice views are available """ from .announcements import Announcements, Announcement, delete_announcement -from .authentication import Login, Register, Logout -from .authentication import ConfirmEmail, ConfirmEmailCode, resend_link from .author import Author, EditAuthor from .block import Block, unblock from .books import Book, EditBook, ConfirmEditBook @@ -28,11 +26,13 @@ from .landing import About, Home, Landing from .list import Lists, SavedLists, List, Curate, UserLists from .list import save_list, unsave_list from .list import delete_list +from .login import Login, Logout from .notifications import Notifications from .outbox import Outbox from .reading import edit_readthrough, create_readthrough from .reading import delete_readthrough, delete_progressupdate from .reading import ReadingStatus +from .register import Register, ConfirmEmail, ConfirmEmailCode, resend_link from .reports import Report, Reports, make_report, resolve_report, suspend_user from .rss_feed import RssFeed from .password import PasswordResetRequest, PasswordReset, ChangePassword diff --git a/bookwyrm/views/invite.py b/bookwyrm/views/invite.py index 005d57cf8..6a9bfedb9 100644 --- a/bookwyrm/views/invite.py +++ b/bookwyrm/views/invite.py @@ -83,7 +83,7 @@ class Invite(View): } return TemplateResponse(request, "invite.html", data) - # post handling is in views.authentication.Register + # post handling is in views.register.Register class ManageInviteRequests(View): diff --git a/bookwyrm/views/login.py b/bookwyrm/views/login.py new file mode 100644 index 000000000..b213590fb --- /dev/null +++ b/bookwyrm/views/login.py @@ -0,0 +1,80 @@ +""" class views for login/register views """ +from django.contrib.auth import authenticate, login, logout +from django.contrib.auth.decorators import login_required +from django.shortcuts import redirect +from django.template.response import TemplateResponse +from django.utils import timezone +from django.utils.decorators import method_decorator +from django.utils.translation import gettext_lazy as _ +from django.views.decorators.csrf import csrf_exempt +from django.views import View + +from bookwyrm import forms, models +from bookwyrm.settings import DOMAIN + + +# pylint: disable=no-self-use +@method_decorator(csrf_exempt, name="dispatch") +class Login(View): + """authenticate an existing user""" + + def get(self, request, confirmed=None): + """login page""" + if request.user.is_authenticated: + return redirect("/") + # send user to the login page + data = { + "show_confirmed_email": confirmed, + "login_form": forms.LoginForm(), + "register_form": forms.RegisterForm(), + } + return TemplateResponse(request, "login.html", data) + + def post(self, request): + """authentication action""" + if request.user.is_authenticated: + return redirect("/") + login_form = forms.LoginForm(request.POST) + + localname = login_form.data["localname"] + if "@" in localname: # looks like an email address to me + try: + username = models.User.objects.get(email=localname).username + except models.User.DoesNotExist: # maybe it's a full username? + username = localname + else: + username = "%s@%s" % (localname, DOMAIN) + password = login_form.data["password"] + + # perform authentication + user = authenticate(request, username=username, password=password) + if user is not None: + # successful login + login(request, user) + user.last_active_date = timezone.now() + user.save(broadcast=False, update_fields=["last_active_date"]) + if request.POST.get("first_login"): + return redirect("get-started-profile") + return redirect(request.GET.get("next", "/")) + + # maybe the user is pending email confirmation + if models.User.objects.filter( + username=username, is_active=False, deactivation_reason="pending" + ).exists(): + return redirect("confirm-email") + + # login errors + login_form.non_field_errors = _("Username or password are incorrect") + register_form = forms.RegisterForm() + data = {"login_form": login_form, "register_form": register_form} + return TemplateResponse(request, "login.html", data) + + +@method_decorator(login_required, name="dispatch") +class Logout(View): + """log out""" + + def get(self, request): + """done with this place! outa here!""" + logout(request) + return redirect("/") diff --git a/bookwyrm/views/authentication.py b/bookwyrm/views/register.py similarity index 61% rename from bookwyrm/views/authentication.py rename to bookwyrm/views/register.py index 70f51864d..334b29687 100644 --- a/bookwyrm/views/authentication.py +++ b/bookwyrm/views/register.py @@ -1,13 +1,8 @@ """ class views for login/register views """ -from django.contrib.auth import authenticate, login, logout -from django.contrib.auth.decorators import login_required +from django.contrib.auth import login from django.core.exceptions import PermissionDenied from django.shortcuts import get_object_or_404, redirect from django.template.response import TemplateResponse -from django.utils import timezone -from django.utils.decorators import method_decorator -from django.utils.translation import gettext_lazy as _ -from django.views.decorators.csrf import csrf_exempt from django.views.decorators.http import require_POST from django.views import View @@ -16,72 +11,6 @@ from bookwyrm.settings import DOMAIN # pylint: disable=no-self-use -@method_decorator(csrf_exempt, name="dispatch") -class Login(View): - """authenticate an existing user""" - - def get(self, request, confirmed=None): - """login page""" - if request.user.is_authenticated: - return redirect("/") - # send user to the login page - data = { - "show_confirmed_email": confirmed, - "login_form": forms.LoginForm(), - "register_form": forms.RegisterForm(), - } - return TemplateResponse(request, "login.html", data) - - def post(self, request): - """authentication action""" - if request.user.is_authenticated: - return redirect("/") - login_form = forms.LoginForm(request.POST) - - localname = login_form.data["localname"] - if "@" in localname: # looks like an email address to me - try: - username = models.User.objects.get(email=localname).username - except models.User.DoesNotExist: # maybe it's a full username? - username = localname - else: - username = "%s@%s" % (localname, DOMAIN) - password = login_form.data["password"] - - # perform authentication - user = authenticate(request, username=username, password=password) - if user is not None: - # successful login - login(request, user) - user.last_active_date = timezone.now() - user.save(broadcast=False, update_fields=["last_active_date"]) - if request.POST.get("first_login"): - return redirect("get-started-profile") - return redirect(request.GET.get("next", "/")) - - # maybe the user is pending email confirmation - if models.User.objects.filter( - username=username, is_active=False, deactivation_reason="pending" - ).exists(): - return redirect("confirm-email") - - # login errors - login_form.non_field_errors = _("Username or password are incorrect") - register_form = forms.RegisterForm() - data = {"login_form": login_form, "register_form": register_form} - return TemplateResponse(request, "login.html", data) - - -@method_decorator(login_required, name="dispatch") -class Logout(View): - """log out""" - - def get(self, request): - """done with this place! outa here!""" - logout(request) - return redirect("/") - - class Register(View): """register a user""" From ec501dfee9fe615767d7d139d7237bb8f9932c16 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Tue, 7 Sep 2021 13:21:40 -0700 Subject: [PATCH 015/191] Make sure passwords aren't exposed in error reporting --- bookwyrm/views/login.py | 5 ++++- bookwyrm/views/register.py | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/bookwyrm/views/login.py b/bookwyrm/views/login.py index b213590fb..1ca65f2ff 100644 --- a/bookwyrm/views/login.py +++ b/bookwyrm/views/login.py @@ -6,8 +6,9 @@ from django.template.response import TemplateResponse from django.utils import timezone from django.utils.decorators import method_decorator from django.utils.translation import gettext_lazy as _ -from django.views.decorators.csrf import csrf_exempt from django.views import View +from django.views.decorators.csrf import csrf_exempt +from django.views.decorators.debug import sensitive_variables, sensitive_post_parameters from bookwyrm import forms, models from bookwyrm.settings import DOMAIN @@ -30,6 +31,8 @@ class Login(View): } return TemplateResponse(request, "login.html", data) + @sensitive_variables("password") + @sensitive_post_parameters("password") def post(self, request): """authentication action""" if request.user.is_authenticated: diff --git a/bookwyrm/views/register.py b/bookwyrm/views/register.py index 334b29687..1ffa16ec6 100644 --- a/bookwyrm/views/register.py +++ b/bookwyrm/views/register.py @@ -3,8 +3,9 @@ from django.contrib.auth import login from django.core.exceptions import PermissionDenied from django.shortcuts import get_object_or_404, redirect from django.template.response import TemplateResponse -from django.views.decorators.http import require_POST from django.views import View +from django.views.decorators.http import require_POST +from django.views.decorators.debug import sensitive_variables, sensitive_post_parameters from bookwyrm import emailing, forms, models from bookwyrm.settings import DOMAIN @@ -14,6 +15,8 @@ from bookwyrm.settings import DOMAIN class Register(View): """register a user""" + @sensitive_variables("password") + @sensitive_post_parameters("password") def post(self, request): """join the server""" settings = models.SiteSettings.get() From 37dcae4558ac3ae66bb5019c2869890fd72b5954 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Tue, 7 Sep 2021 13:51:59 -0700 Subject: [PATCH 016/191] Simply celery settings --- bookwyrm/settings.py | 11 ----------- bookwyrm/tasks.py | 4 ++-- celerywyrm/settings.py | 11 ++++++++++- redis.conf | 9 +++++++++ 4 files changed, 21 insertions(+), 14 deletions(-) create mode 100644 redis.conf diff --git a/bookwyrm/settings.py b/bookwyrm/settings.py index c1f900794..9450ba81f 100644 --- a/bookwyrm/settings.py +++ b/bookwyrm/settings.py @@ -13,17 +13,6 @@ VERSION = "0.0.1" PAGE_LENGTH = env("PAGE_LENGTH", 15) DEFAULT_LANGUAGE = env("DEFAULT_LANGUAGE", "English") -# celery -CELERY_BROKER = "redis://:{}@redis_broker:{}/0".format( - requests.utils.quote(env("REDIS_BROKER_PASSWORD", "")), env("REDIS_BROKER_PORT") -) -CELERY_RESULT_BACKEND = "redis://:{}@redis_broker:{}/0".format( - requests.utils.quote(env("REDIS_BROKER_PASSWORD", "")), env("REDIS_BROKER_PORT") -) -CELERY_ACCEPT_CONTENT = ["application/json"] -CELERY_TASK_SERIALIZER = "json" -CELERY_RESULT_SERIALIZER = "json" - # email EMAIL_BACKEND = env("EMAIL_BACKEND", "django.core.mail.backends.smtp.EmailBackend") EMAIL_HOST = env("EMAIL_HOST") diff --git a/bookwyrm/tasks.py b/bookwyrm/tasks.py index 6d1992a77..b860e0184 100644 --- a/bookwyrm/tasks.py +++ b/bookwyrm/tasks.py @@ -2,10 +2,10 @@ import os from celery import Celery -from bookwyrm import settings +from celerywyrm import settings # set the default Django settings module for the 'celery' program. os.environ.setdefault("DJANGO_SETTINGS_MODULE", "celerywyrm.settings") app = Celery( - "tasks", broker=settings.CELERY_BROKER, backend=settings.CELERY_RESULT_BACKEND + "tasks", broker=settings.CELERY_BROKER_URL, backend=settings.CELERY_RESULT_BACKEND ) diff --git a/celerywyrm/settings.py b/celerywyrm/settings.py index 107a39572..a67ab4113 100644 --- a/celerywyrm/settings.py +++ b/celerywyrm/settings.py @@ -1,9 +1,18 @@ """ bookwyrm settings and configuration """ +# pylint: disable=wildcard-import +# pylint: disable=unused-wildcard-import from bookwyrm.settings import * -CELERY_BROKER_URL = CELERY_BROKER +CELERY_BROKER_URL = "redis://:{}@redis_broker:{}/0".format( + requests.utils.quote(env("REDIS_BROKER_PASSWORD", "")), env("REDIS_BROKER_PORT") +) +CELERY_RESULT_BACKEND = "redis://:{}@redis_broker:{}/0".format( + requests.utils.quote(env("REDIS_BROKER_PASSWORD", "")), env("REDIS_BROKER_PORT") +) + CELERY_ACCEPT_CONTENT = ["json"] CELERY_TASK_SERIALIZER = "json" +CELERY_RESULT_SERIALIZER = "json" FLOWER_PORT = env("FLOWER_PORT") INSTALLED_APPS = INSTALLED_APPS + [ diff --git a/redis.conf b/redis.conf new file mode 100644 index 000000000..2a417579f --- /dev/null +++ b/redis.conf @@ -0,0 +1,9 @@ +bind 127.0.0.1 ::1 +protected-mode yes +port 6379 + +rename-command FLUSHDB "" +rename-command FLUSHALL "" +rename-command DEBUG "" +rename-command CONFIG "" +rename-command SHUTDOWN "" From 47ba2478b6b08f99649df6d152970f87f4a2450d Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Tue, 7 Sep 2021 15:03:15 -0700 Subject: [PATCH 017/191] Split out test files --- bookwyrm/tests/views/test_login.py | 110 ++++++++++++++++++ ...est_authentication.py => test_register.py} | 88 +------------- bookwyrm/views/login.py | 2 +- bookwyrm/views/register.py | 3 +- 4 files changed, 119 insertions(+), 84 deletions(-) create mode 100644 bookwyrm/tests/views/test_login.py rename bookwyrm/tests/views/{test_authentication.py => test_register.py} (76%) diff --git a/bookwyrm/tests/views/test_login.py b/bookwyrm/tests/views/test_login.py new file mode 100644 index 000000000..c37eaa514 --- /dev/null +++ b/bookwyrm/tests/views/test_login.py @@ -0,0 +1,110 @@ +""" test for app action functionality """ +from unittest.mock import patch + +from django.contrib.auth.models import AnonymousUser +from django.template.response import TemplateResponse +from django.test import TestCase +from django.test.client import RequestFactory + +from bookwyrm import forms, models, views + + +# pylint: disable=too-many-public-methods +@patch("bookwyrm.suggested_users.rerank_suggestions_task.delay") +@patch("bookwyrm.activitystreams.populate_stream_task.delay") +class LoginViews(TestCase): + """login and password management""" + + def setUp(self): + """we need basic test data and mocks""" + self.factory = RequestFactory() + with patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"), patch( + "bookwyrm.activitystreams.populate_stream_task.delay" + ): + self.local_user = models.User.objects.create_user( + "mouse@your.domain.here", + "mouse@mouse.com", + "password", + local=True, + localname="mouse", + ) + self.anonymous_user = AnonymousUser + self.anonymous_user.is_authenticated = False + + models.SiteSettings.objects.create(id=1, require_confirm_email=False) + + def test_login_get(self, *_): + """there are so many views, this just makes sure it LOADS""" + login = views.Login.as_view() + request = self.factory.get("") + request.user = self.anonymous_user + + result = login(request) + self.assertIsInstance(result, TemplateResponse) + result.render() + self.assertEqual(result.status_code, 200) + + request.user = self.local_user + result = login(request) + self.assertEqual(result.url, "/") + self.assertEqual(result.status_code, 302) + + def test_login_post_localname(self, *_): + """there are so many views, this just makes sure it LOADS""" + view = views.Login.as_view() + form = forms.LoginForm() + form.data["localname"] = "mouse@mouse.com" + form.data["password"] = "password" + request = self.factory.post("", form.data) + request.user = self.anonymous_user + + with patch("bookwyrm.views.login.login"): + result = view(request) + self.assertEqual(result.url, "/") + self.assertEqual(result.status_code, 302) + + def test_login_post_username(self, *_): + """there are so many views, this just makes sure it LOADS""" + view = views.Login.as_view() + form = forms.LoginForm() + form.data["localname"] = "mouse@your.domain.here" + form.data["password"] = "password" + request = self.factory.post("", form.data) + request.user = self.anonymous_user + + with patch("bookwyrm.views.login.login"): + result = view(request) + self.assertEqual(result.url, "/") + self.assertEqual(result.status_code, 302) + + def test_login_post_email(self, *_): + """there are so many views, this just makes sure it LOADS""" + view = views.Login.as_view() + form = forms.LoginForm() + form.data["localname"] = "mouse" + form.data["password"] = "password" + request = self.factory.post("", form.data) + request.user = self.anonymous_user + + with patch("bookwyrm.views.login.login"): + result = view(request) + self.assertEqual(result.url, "/") + self.assertEqual(result.status_code, 302) + + def test_login_post_invalid_credentials(self, *_): + """there are so many views, this just makes sure it LOADS""" + view = views.Login.as_view() + form = forms.LoginForm() + form.data["localname"] = "mouse" + form.data["password"] = "passsword1" + request = self.factory.post("", form.data) + request.user = self.anonymous_user + + with patch("bookwyrm.views.login.login"): + result = view(request) + result.render() + self.assertEqual(result.status_code, 200) + self.assertEqual( + result.context_data["login_form"].non_field_errors, + "Username or password are incorrect", + ) diff --git a/bookwyrm/tests/views/test_authentication.py b/bookwyrm/tests/views/test_register.py similarity index 76% rename from bookwyrm/tests/views/test_authentication.py rename to bookwyrm/tests/views/test_register.py index 74f3c0902..45e748807 100644 --- a/bookwyrm/tests/views/test_authentication.py +++ b/bookwyrm/tests/views/test_register.py @@ -8,14 +8,14 @@ from django.template.response import TemplateResponse from django.test import TestCase from django.test.client import RequestFactory -from bookwyrm import forms, models, views +from bookwyrm import models, views from bookwyrm.settings import DOMAIN # pylint: disable=too-many-public-methods @patch("bookwyrm.suggested_users.rerank_suggestions_task.delay") @patch("bookwyrm.activitystreams.populate_stream_task.delay") -class AuthenticationViews(TestCase): +class RegisterViews(TestCase): """login and password management""" def setUp(self): @@ -38,82 +38,6 @@ class AuthenticationViews(TestCase): id=1, require_confirm_email=False ) - def test_login_get(self, *_): - """there are so many views, this just makes sure it LOADS""" - login = views.Login.as_view() - request = self.factory.get("") - request.user = self.anonymous_user - - result = login(request) - self.assertIsInstance(result, TemplateResponse) - result.render() - self.assertEqual(result.status_code, 200) - - request.user = self.local_user - result = login(request) - self.assertEqual(result.url, "/") - self.assertEqual(result.status_code, 302) - - def test_login_post_localname(self, *_): - """there are so many views, this just makes sure it LOADS""" - view = views.Login.as_view() - form = forms.LoginForm() - form.data["localname"] = "mouse@mouse.com" - form.data["password"] = "password" - request = self.factory.post("", form.data) - request.user = self.anonymous_user - - with patch("bookwyrm.views.login.login"): - result = view(request) - self.assertEqual(result.url, "/") - self.assertEqual(result.status_code, 302) - - def test_login_post_username(self, *_): - """there are so many views, this just makes sure it LOADS""" - view = views.Login.as_view() - form = forms.LoginForm() - form.data["localname"] = "mouse@your.domain.here" - form.data["password"] = "password" - request = self.factory.post("", form.data) - request.user = self.anonymous_user - - with patch("bookwyrm.views.login.login"): - result = view(request) - self.assertEqual(result.url, "/") - self.assertEqual(result.status_code, 302) - - def test_login_post_email(self, *_): - """there are so many views, this just makes sure it LOADS""" - view = views.Login.as_view() - form = forms.LoginForm() - form.data["localname"] = "mouse" - form.data["password"] = "password" - request = self.factory.post("", form.data) - request.user = self.anonymous_user - - with patch("bookwyrm.views.login.login"): - result = view(request) - self.assertEqual(result.url, "/") - self.assertEqual(result.status_code, 302) - - def test_login_post_invalid_credentials(self, *_): - """there are so many views, this just makes sure it LOADS""" - view = views.Login.as_view() - form = forms.LoginForm() - form.data["localname"] = "mouse" - form.data["password"] = "passsword1" - request = self.factory.post("", form.data) - request.user = self.anonymous_user - - with patch("bookwyrm.views.login.login"): - result = view(request) - result.render() - self.assertEqual(result.status_code, 200) - self.assertEqual( - result.context_data["login_form"].non_field_errors, - "Username or password are incorrect", - ) - def test_register(self, *_): """create a user""" view = views.Register.as_view() @@ -126,7 +50,7 @@ class AuthenticationViews(TestCase): "email": "aa@bb.cccc", }, ) - with patch("bookwyrm.views.login.login"): + with patch("bookwyrm.views.register.login"): response = view(request) self.assertEqual(models.User.objects.count(), 2) self.assertEqual(response.status_code, 302) @@ -151,7 +75,7 @@ class AuthenticationViews(TestCase): "email": "aa@bb.cccc", }, ) - with patch("bookwyrm.views.login.login"): + with patch("bookwyrm.views.register.login"): response = view(request) self.assertEqual(response.status_code, 302) nutria = models.User.objects.get(localname="nutria") @@ -169,7 +93,7 @@ class AuthenticationViews(TestCase): "register/", {"localname": "nutria ", "password": "mouseword", "email": "aa@bb.ccc"}, ) - with patch("bookwyrm.views.login.login"): + with patch("bookwyrm.views.register.login"): response = view(request) self.assertEqual(models.User.objects.count(), 2) self.assertEqual(response.status_code, 302) @@ -248,7 +172,7 @@ class AuthenticationViews(TestCase): "invite_code": "testcode", }, ) - with patch("bookwyrm.views.login.login"): + with patch("bookwyrm.views.register.login"): response = view(request) self.assertEqual(models.User.objects.count(), 2) self.assertEqual(response.status_code, 302) diff --git a/bookwyrm/views/login.py b/bookwyrm/views/login.py index 1ca65f2ff..97d541690 100644 --- a/bookwyrm/views/login.py +++ b/bookwyrm/views/login.py @@ -32,7 +32,7 @@ class Login(View): return TemplateResponse(request, "login.html", data) @sensitive_variables("password") - @sensitive_post_parameters("password") + @method_decorator(sensitive_post_parameters("password")) def post(self, request): """authentication action""" if request.user.is_authenticated: diff --git a/bookwyrm/views/register.py b/bookwyrm/views/register.py index 1ffa16ec6..1ecb97b16 100644 --- a/bookwyrm/views/register.py +++ b/bookwyrm/views/register.py @@ -3,6 +3,7 @@ from django.contrib.auth import login from django.core.exceptions import PermissionDenied from django.shortcuts import get_object_or_404, redirect from django.template.response import TemplateResponse +from django.utils.decorators import method_decorator from django.views import View from django.views.decorators.http import require_POST from django.views.decorators.debug import sensitive_variables, sensitive_post_parameters @@ -16,7 +17,7 @@ class Register(View): """register a user""" @sensitive_variables("password") - @sensitive_post_parameters("password") + @method_decorator(sensitive_post_parameters("password")) def post(self, request): """join the server""" settings = models.SiteSettings.get() From d76f2ab95c1530983ca2b41536961081a26b76e3 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Tue, 7 Sep 2021 15:13:10 -0700 Subject: [PATCH 018/191] Remove stray file --- redis.conf | 9 --------- 1 file changed, 9 deletions(-) delete mode 100644 redis.conf diff --git a/redis.conf b/redis.conf deleted file mode 100644 index 2a417579f..000000000 --- a/redis.conf +++ /dev/null @@ -1,9 +0,0 @@ -bind 127.0.0.1 ::1 -protected-mode yes -port 6379 - -rename-command FLUSHDB "" -rename-command FLUSHALL "" -rename-command DEBUG "" -rename-command CONFIG "" -rename-command SHUTDOWN "" From c0b662cbd7551eeef31300f4575ac15e8b8e9528 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Tue, 7 Sep 2021 16:06:54 -0700 Subject: [PATCH 019/191] Adds priority queues to celery --- celerywyrm/celery.py | 2 +- celerywyrm/settings.py | 16 ++++++++++++++++ docker-compose.yml | 2 +- 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/celerywyrm/celery.py b/celerywyrm/celery.py index de5e56304..0b4ff7bd3 100644 --- a/celerywyrm/celery.py +++ b/celerywyrm/celery.py @@ -3,7 +3,7 @@ from __future__ import absolute_import, unicode_literals import os from celery import Celery -from . import settings +from . import settings # pylint: disable=unused-import # set the default Django settings module for the 'celery' program. diff --git a/celerywyrm/settings.py b/celerywyrm/settings.py index a67ab4113..48d134468 100644 --- a/celerywyrm/settings.py +++ b/celerywyrm/settings.py @@ -10,6 +10,22 @@ CELERY_RESULT_BACKEND = "redis://:{}@redis_broker:{}/0".format( requests.utils.quote(env("REDIS_BROKER_PASSWORD", "")), env("REDIS_BROKER_PORT") ) +CELERY_TASK_ROUTES = ([ + # high - should really happen ASAP + ("bookwyrm.emailing.*", {"queue": "high_priority"}), + # medium - should really happen + ("bookwyrm.activitypub.base_activity.*", {"queue": "medium_priority"}), + ("bookwyrm.views.inbox.*", {"queue": "medium_priority"}), + ("bookwyrm.broadcast.*", {"queue": "medium_priority"}), + ("bookwyrm.activitystreams.*", {"queue": "medium_priority"}), + # low - no rush + ("bookwyrm.connectors.abstract_connector.*", {"queue": "low_priority"}), + ("bookwyrm.goodreads_import.*", {"queue": "low_priority"}), + ("bookwyrm.models.user.*", {"queue": "low_priority"}), + ("bookwyrm.suggested_users.*", {"queue": "low_priority"}), + ("bookwyrm.preview_images.*", {"queue": "low_priority"}), +]) + CELERY_ACCEPT_CONTENT = ["json"] CELERY_TASK_SERIALIZER = "json" CELERY_RESULT_SERIALIZER = "json" diff --git a/docker-compose.yml b/docker-compose.yml index 49d02e706..5d24a4b75 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -63,7 +63,7 @@ services: build: . networks: - main - command: celery -A celerywyrm worker -l info + command: celery -A celerywyrm worker -l info -Q high_priority,medium_priority,low_priority volumes: - .:/app - static_volume:/app/static From 7b2e4eebd2b4dabb8c079583f2d2d26002ac9492 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Tue, 7 Sep 2021 16:33:18 -0700 Subject: [PATCH 020/191] Remove unused code from celery --- celerywyrm/celery.py | 10 ---------- celerywyrm/settings.py | 16 +--------------- 2 files changed, 1 insertion(+), 25 deletions(-) diff --git a/celerywyrm/celery.py b/celerywyrm/celery.py index 0b4ff7bd3..3a0969e5d 100644 --- a/celerywyrm/celery.py +++ b/celerywyrm/celery.py @@ -19,13 +19,3 @@ app.config_from_object("django.conf:settings", namespace="CELERY") # Load task modules from all registered Django app configs. app.autodiscover_tasks() -app.autodiscover_tasks(["bookwyrm"], related_name="activitypub.base_activity") -app.autodiscover_tasks(["bookwyrm"], related_name="activitystreams") -app.autodiscover_tasks(["bookwyrm"], related_name="broadcast") -app.autodiscover_tasks(["bookwyrm"], related_name="connectors.abstract_connector") -app.autodiscover_tasks(["bookwyrm"], related_name="emailing") -app.autodiscover_tasks(["bookwyrm"], related_name="goodreads_import") -app.autodiscover_tasks(["bookwyrm"], related_name="preview_images") -app.autodiscover_tasks(["bookwyrm"], related_name="models.user") -app.autodiscover_tasks(["bookwyrm"], related_name="suggested_users") -app.autodiscover_tasks(["bookwyrm"], related_name="views.inbox") diff --git a/celerywyrm/settings.py b/celerywyrm/settings.py index 48d134468..55af4ce4d 100644 --- a/celerywyrm/settings.py +++ b/celerywyrm/settings.py @@ -10,21 +10,7 @@ CELERY_RESULT_BACKEND = "redis://:{}@redis_broker:{}/0".format( requests.utils.quote(env("REDIS_BROKER_PASSWORD", "")), env("REDIS_BROKER_PORT") ) -CELERY_TASK_ROUTES = ([ - # high - should really happen ASAP - ("bookwyrm.emailing.*", {"queue": "high_priority"}), - # medium - should really happen - ("bookwyrm.activitypub.base_activity.*", {"queue": "medium_priority"}), - ("bookwyrm.views.inbox.*", {"queue": "medium_priority"}), - ("bookwyrm.broadcast.*", {"queue": "medium_priority"}), - ("bookwyrm.activitystreams.*", {"queue": "medium_priority"}), - # low - no rush - ("bookwyrm.connectors.abstract_connector.*", {"queue": "low_priority"}), - ("bookwyrm.goodreads_import.*", {"queue": "low_priority"}), - ("bookwyrm.models.user.*", {"queue": "low_priority"}), - ("bookwyrm.suggested_users.*", {"queue": "low_priority"}), - ("bookwyrm.preview_images.*", {"queue": "low_priority"}), -]) +CELERY_DEFAULT_QUEUE="low_priority" CELERY_ACCEPT_CONTENT = ["json"] CELERY_TASK_SERIALIZER = "json" From de3f18655cb1b793013847f33f7dbdf09bb4ab8d Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Tue, 7 Sep 2021 16:33:43 -0700 Subject: [PATCH 021/191] Set priorities on tasks --- bookwyrm/activitypub/base_activity.py | 2 +- bookwyrm/activitystreams.py | 18 +++++++++--------- bookwyrm/connectors/connector_manager.py | 2 +- bookwyrm/emailing.py | 2 +- bookwyrm/importers/importer.py | 2 +- bookwyrm/models/activitypub_mixin.py | 2 +- bookwyrm/models/user.py | 4 ++-- bookwyrm/preview_images.py | 6 +++--- bookwyrm/suggested_users.py | 8 ++++---- bookwyrm/views/inbox.py | 2 +- 10 files changed, 24 insertions(+), 24 deletions(-) diff --git a/bookwyrm/activitypub/base_activity.py b/bookwyrm/activitypub/base_activity.py index 52b1b1f27..4f7b55d50 100644 --- a/bookwyrm/activitypub/base_activity.py +++ b/bookwyrm/activitypub/base_activity.py @@ -213,7 +213,7 @@ class ActivityObject: return data -@app.task +@app.task(queue="medium_priority") @transaction.atomic def set_related_field( model_name, origin_model_name, related_field_name, related_remote_id, data diff --git a/bookwyrm/activitystreams.py b/bookwyrm/activitystreams.py index f59eaf2e6..00ee5c775 100644 --- a/bookwyrm/activitystreams.py +++ b/bookwyrm/activitystreams.py @@ -395,7 +395,7 @@ def remove_statuses_on_unshelve(sender, instance, *args, **kwargs): # ---- TASKS -@app.task +@app.task(priority="low_priority") def add_book_statuses_task(user_id, book_id): """add statuses related to a book on shelve""" user = models.User.objects.get(id=user_id) @@ -403,7 +403,7 @@ def add_book_statuses_task(user_id, book_id): BooksStream().add_book_statuses(user, book) -@app.task +@app.task(priority="low_priority") def remove_book_statuses_task(user_id, book_id): """remove statuses about a book from a user's books feed""" user = models.User.objects.get(id=user_id) @@ -411,7 +411,7 @@ def remove_book_statuses_task(user_id, book_id): BooksStream().remove_book_statuses(user, book) -@app.task +@app.task(priority="medium_priority") def populate_stream_task(stream, user_id): """background task for populating an empty activitystream""" user = models.User.objects.get(id=user_id) @@ -419,7 +419,7 @@ def populate_stream_task(stream, user_id): stream.populate_streams(user) -@app.task +@app.task(priority="medium_priority") def remove_status_task(status_ids): """remove a status from any stream it might be in""" # this can take an id or a list of ids @@ -432,7 +432,7 @@ def remove_status_task(status_ids): stream.remove_object_from_related_stores(status) -@app.task +@app.task(priority="medium_priority") def add_status_task(status_id, increment_unread=False): """remove a status from any stream it might be in""" status = models.Status.objects.get(id=status_id) @@ -440,7 +440,7 @@ def add_status_task(status_id, increment_unread=False): stream.add_status(status, increment_unread=increment_unread) -@app.task +@app.task(priority="medium_priority") def remove_user_statuses_task(viewer_id, user_id, stream_list=None): """remove all statuses by a user from a viewer's stream""" stream_list = [streams[s] for s in stream_list] if stream_list else streams.values() @@ -450,9 +450,9 @@ def remove_user_statuses_task(viewer_id, user_id, stream_list=None): stream.remove_user_statuses(viewer, user) -@app.task +@app.task(priority="medium_priority") def add_user_statuses_task(viewer_id, user_id, stream_list=None): - """remove all statuses by a user from a viewer's stream""" + """add all statuses by a user to a viewer's stream""" stream_list = [streams[s] for s in stream_list] if stream_list else streams.values() viewer = models.User.objects.get(id=viewer_id) user = models.User.objects.get(id=user_id) @@ -460,7 +460,7 @@ def add_user_statuses_task(viewer_id, user_id, stream_list=None): stream.add_user_statuses(viewer, user) -@app.task +@app.task(priority="medium_priority") def handle_boost_task(boost_id): """remove the original post and other, earlier boosts""" instance = models.Status.objects.get(id=boost_id) diff --git a/bookwyrm/connectors/connector_manager.py b/bookwyrm/connectors/connector_manager.py index 1a615c9b2..798e90aaf 100644 --- a/bookwyrm/connectors/connector_manager.py +++ b/bookwyrm/connectors/connector_manager.py @@ -119,7 +119,7 @@ def get_or_create_connector(remote_id): return load_connector(connector_info) -@app.task +@app.task(priority="low_priority") def load_more_data(connector_id, book_id): """background the work of getting all 10,000 editions of LoTR""" connector_info = models.Connector.objects.get(id=connector_id) diff --git a/bookwyrm/emailing.py b/bookwyrm/emailing.py index fff3985ef..4f43c69e6 100644 --- a/bookwyrm/emailing.py +++ b/bookwyrm/emailing.py @@ -64,7 +64,7 @@ def format_email(email_name, data): return (subject, html_content, text_content) -@app.task +@app.task(queue="high_priority") def send_email(recipient, subject, html_content, text_content): """use a task to send the email""" email = EmailMultiAlternatives( diff --git a/bookwyrm/importers/importer.py b/bookwyrm/importers/importer.py index d5f1449ca..976141711 100644 --- a/bookwyrm/importers/importer.py +++ b/bookwyrm/importers/importer.py @@ -61,7 +61,7 @@ class Importer: job.save() -@app.task +@app.task(priority="low_priority") def import_data(source, job_id): """does the actual lookup work in a celery task""" job = ImportJob.objects.get(id=job_id) diff --git a/bookwyrm/models/activitypub_mixin.py b/bookwyrm/models/activitypub_mixin.py index f287b752f..ed51158bc 100644 --- a/bookwyrm/models/activitypub_mixin.py +++ b/bookwyrm/models/activitypub_mixin.py @@ -502,7 +502,7 @@ def unfurl_related_field(related_field, sort_field=None): return related_field.remote_id -@app.task +@app.task(queue="medium_priority") def broadcast_task(sender_id, activity, recipients): """the celery task for broadcast""" user_model = apps.get_model("bookwyrm.User", require_ready=True) diff --git a/bookwyrm/models/user.py b/bookwyrm/models/user.py index 7756846f0..d2a2c276c 100644 --- a/bookwyrm/models/user.py +++ b/bookwyrm/models/user.py @@ -425,7 +425,7 @@ class AnnualGoal(BookWyrmModel): } -@app.task +@app.task(priority="low_priority") def set_remote_server(user_id): """figure out the user's remote server in the background""" user = User.objects.get(id=user_id) @@ -464,7 +464,7 @@ def get_or_create_remote_server(domain): return server -@app.task +@app.task(priority="low_priority") def get_remote_reviews(outbox): """ingest reviews by a new remote bookwyrm user""" outbox_page = outbox + "?page=true&type=Review" diff --git a/bookwyrm/preview_images.py b/bookwyrm/preview_images.py index 4f85bb56e..5e9ad2f24 100644 --- a/bookwyrm/preview_images.py +++ b/bookwyrm/preview_images.py @@ -352,7 +352,7 @@ def save_and_cleanup(image, instance=None): # pylint: disable=invalid-name -@app.task +@app.task(priority="low_priority") def generate_site_preview_image_task(): """generate preview_image for the website""" if not settings.ENABLE_PREVIEW_IMAGES: @@ -377,7 +377,7 @@ def generate_site_preview_image_task(): # pylint: disable=invalid-name -@app.task +@app.task(priority="low_priority") def generate_edition_preview_image_task(book_id): """generate preview_image for a book""" if not settings.ENABLE_PREVIEW_IMAGES: @@ -402,7 +402,7 @@ def generate_edition_preview_image_task(book_id): save_and_cleanup(image, instance=book) -@app.task +@app.task(priority="low_priority") def generate_user_preview_image_task(user_id): """generate preview_image for a book""" if not settings.ENABLE_PREVIEW_IMAGES: diff --git a/bookwyrm/suggested_users.py b/bookwyrm/suggested_users.py index 9c42d79d8..808fc021d 100644 --- a/bookwyrm/suggested_users.py +++ b/bookwyrm/suggested_users.py @@ -194,27 +194,27 @@ def add_new_user(sender, instance, created, update_fields=None, **kwargs): remove_user_task.delay(instance.id) -@app.task +@app.task(priority="low_priority") def rerank_suggestions_task(user_id): """do the hard work in celery""" suggested_users.rerank_user_suggestions(user_id) -@app.task +@app.task(priority="low_priority") def rerank_user_task(user_id, update_only=False): """do the hard work in celery""" user = models.User.objects.get(id=user_id) suggested_users.rerank_obj(user, update_only=update_only) -@app.task +@app.task(priority="low_priority") def remove_user_task(user_id): """do the hard work in celery""" user = models.User.objects.get(id=user_id) suggested_users.remove_object_from_related_stores(user) -@app.task +@app.task(priority="medium_priority") def remove_suggestion_task(user_id, suggested_user_id): """remove a specific user from a specific user's suggestions""" suggested_user = models.User.objects.get(id=suggested_user_id) diff --git a/bookwyrm/views/inbox.py b/bookwyrm/views/inbox.py index ff5fa46da..e255b4fa2 100644 --- a/bookwyrm/views/inbox.py +++ b/bookwyrm/views/inbox.py @@ -93,7 +93,7 @@ def is_blocked_activity(activity_json): return models.FederatedServer.is_blocked(actor) -@app.task +@app.task(queue="medium_priority") def activity_task(activity_json): """do something with this json we think is legit""" # lets see if the activitypub module can make sense of this json From 26adf2d974940cebae61451b3d9dc4e22c903488 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Tue, 7 Sep 2021 17:04:10 -0700 Subject: [PATCH 022/191] Fixes queue for import task --- bookwyrm/importers/importer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bookwyrm/importers/importer.py b/bookwyrm/importers/importer.py index 976141711..0968cdd79 100644 --- a/bookwyrm/importers/importer.py +++ b/bookwyrm/importers/importer.py @@ -61,7 +61,7 @@ class Importer: job.save() -@app.task(priority="low_priority") +@app.task(queue="low_priority") def import_data(source, job_id): """does the actual lookup work in a celery task""" job = ImportJob.objects.get(id=job_id) From aa91361fe4a42286ea4add274c831525967a373e Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Tue, 7 Sep 2021 17:09:44 -0700 Subject: [PATCH 023/191] Fixes celery kwarg for queue --- bookwyrm/activitystreams.py | 16 ++++++++-------- bookwyrm/connectors/connector_manager.py | 2 +- bookwyrm/models/user.py | 4 ++-- bookwyrm/preview_images.py | 6 +++--- bookwyrm/suggested_users.py | 8 ++++---- 5 files changed, 18 insertions(+), 18 deletions(-) diff --git a/bookwyrm/activitystreams.py b/bookwyrm/activitystreams.py index 00ee5c775..c32dfa35d 100644 --- a/bookwyrm/activitystreams.py +++ b/bookwyrm/activitystreams.py @@ -395,7 +395,7 @@ def remove_statuses_on_unshelve(sender, instance, *args, **kwargs): # ---- TASKS -@app.task(priority="low_priority") +@app.task(queue="low_priority") def add_book_statuses_task(user_id, book_id): """add statuses related to a book on shelve""" user = models.User.objects.get(id=user_id) @@ -403,7 +403,7 @@ def add_book_statuses_task(user_id, book_id): BooksStream().add_book_statuses(user, book) -@app.task(priority="low_priority") +@app.task(queue="low_priority") def remove_book_statuses_task(user_id, book_id): """remove statuses about a book from a user's books feed""" user = models.User.objects.get(id=user_id) @@ -411,7 +411,7 @@ def remove_book_statuses_task(user_id, book_id): BooksStream().remove_book_statuses(user, book) -@app.task(priority="medium_priority") +@app.task(queue="medium_priority") def populate_stream_task(stream, user_id): """background task for populating an empty activitystream""" user = models.User.objects.get(id=user_id) @@ -419,7 +419,7 @@ def populate_stream_task(stream, user_id): stream.populate_streams(user) -@app.task(priority="medium_priority") +@app.task(queue="medium_priority") def remove_status_task(status_ids): """remove a status from any stream it might be in""" # this can take an id or a list of ids @@ -432,7 +432,7 @@ def remove_status_task(status_ids): stream.remove_object_from_related_stores(status) -@app.task(priority="medium_priority") +@app.task(queue="medium_priority") def add_status_task(status_id, increment_unread=False): """remove a status from any stream it might be in""" status = models.Status.objects.get(id=status_id) @@ -440,7 +440,7 @@ def add_status_task(status_id, increment_unread=False): stream.add_status(status, increment_unread=increment_unread) -@app.task(priority="medium_priority") +@app.task(queue="medium_priority") def remove_user_statuses_task(viewer_id, user_id, stream_list=None): """remove all statuses by a user from a viewer's stream""" stream_list = [streams[s] for s in stream_list] if stream_list else streams.values() @@ -450,7 +450,7 @@ def remove_user_statuses_task(viewer_id, user_id, stream_list=None): stream.remove_user_statuses(viewer, user) -@app.task(priority="medium_priority") +@app.task(queue="medium_priority") def add_user_statuses_task(viewer_id, user_id, stream_list=None): """add all statuses by a user to a viewer's stream""" stream_list = [streams[s] for s in stream_list] if stream_list else streams.values() @@ -460,7 +460,7 @@ def add_user_statuses_task(viewer_id, user_id, stream_list=None): stream.add_user_statuses(viewer, user) -@app.task(priority="medium_priority") +@app.task(queue="medium_priority") def handle_boost_task(boost_id): """remove the original post and other, earlier boosts""" instance = models.Status.objects.get(id=boost_id) diff --git a/bookwyrm/connectors/connector_manager.py b/bookwyrm/connectors/connector_manager.py index 798e90aaf..1d9588d6b 100644 --- a/bookwyrm/connectors/connector_manager.py +++ b/bookwyrm/connectors/connector_manager.py @@ -119,7 +119,7 @@ def get_or_create_connector(remote_id): return load_connector(connector_info) -@app.task(priority="low_priority") +@app.task(queue="low_priority") def load_more_data(connector_id, book_id): """background the work of getting all 10,000 editions of LoTR""" connector_info = models.Connector.objects.get(id=connector_id) diff --git a/bookwyrm/models/user.py b/bookwyrm/models/user.py index d2a2c276c..4b03f6656 100644 --- a/bookwyrm/models/user.py +++ b/bookwyrm/models/user.py @@ -425,7 +425,7 @@ class AnnualGoal(BookWyrmModel): } -@app.task(priority="low_priority") +@app.task(queue="low_priority") def set_remote_server(user_id): """figure out the user's remote server in the background""" user = User.objects.get(id=user_id) @@ -464,7 +464,7 @@ def get_or_create_remote_server(domain): return server -@app.task(priority="low_priority") +@app.task(queue="low_priority") def get_remote_reviews(outbox): """ingest reviews by a new remote bookwyrm user""" outbox_page = outbox + "?page=true&type=Review" diff --git a/bookwyrm/preview_images.py b/bookwyrm/preview_images.py index 5e9ad2f24..900a3e123 100644 --- a/bookwyrm/preview_images.py +++ b/bookwyrm/preview_images.py @@ -352,7 +352,7 @@ def save_and_cleanup(image, instance=None): # pylint: disable=invalid-name -@app.task(priority="low_priority") +@app.task(queue="low_priority") def generate_site_preview_image_task(): """generate preview_image for the website""" if not settings.ENABLE_PREVIEW_IMAGES: @@ -377,7 +377,7 @@ def generate_site_preview_image_task(): # pylint: disable=invalid-name -@app.task(priority="low_priority") +@app.task(queue="low_priority") def generate_edition_preview_image_task(book_id): """generate preview_image for a book""" if not settings.ENABLE_PREVIEW_IMAGES: @@ -402,7 +402,7 @@ def generate_edition_preview_image_task(book_id): save_and_cleanup(image, instance=book) -@app.task(priority="low_priority") +@app.task(queue="low_priority") def generate_user_preview_image_task(user_id): """generate preview_image for a book""" if not settings.ENABLE_PREVIEW_IMAGES: diff --git a/bookwyrm/suggested_users.py b/bookwyrm/suggested_users.py index 808fc021d..92902938a 100644 --- a/bookwyrm/suggested_users.py +++ b/bookwyrm/suggested_users.py @@ -194,27 +194,27 @@ def add_new_user(sender, instance, created, update_fields=None, **kwargs): remove_user_task.delay(instance.id) -@app.task(priority="low_priority") +@app.task(queue="low_priority") def rerank_suggestions_task(user_id): """do the hard work in celery""" suggested_users.rerank_user_suggestions(user_id) -@app.task(priority="low_priority") +@app.task(queue="low_priority") def rerank_user_task(user_id, update_only=False): """do the hard work in celery""" user = models.User.objects.get(id=user_id) suggested_users.rerank_obj(user, update_only=update_only) -@app.task(priority="low_priority") +@app.task(queue="low_priority") def remove_user_task(user_id): """do the hard work in celery""" user = models.User.objects.get(id=user_id) suggested_users.remove_object_from_related_stores(user) -@app.task(priority="medium_priority") +@app.task(queue="medium_priority") def remove_suggestion_task(user_id, suggested_user_id): """remove a specific user from a specific user's suggestions""" suggested_user = models.User.objects.get(id=suggested_user_id) From e608d8b8ade727975629de38c63be0598426de5f Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Tue, 7 Sep 2021 17:11:32 -0700 Subject: [PATCH 024/191] python formatting --- celerywyrm/celery.py | 2 +- celerywyrm/settings.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/celerywyrm/celery.py b/celerywyrm/celery.py index 3a0969e5d..8e45db908 100644 --- a/celerywyrm/celery.py +++ b/celerywyrm/celery.py @@ -3,7 +3,7 @@ from __future__ import absolute_import, unicode_literals import os from celery import Celery -from . import settings # pylint: disable=unused-import +from . import settings # pylint: disable=unused-import # set the default Django settings module for the 'celery' program. diff --git a/celerywyrm/settings.py b/celerywyrm/settings.py index 55af4ce4d..05ffdcabf 100644 --- a/celerywyrm/settings.py +++ b/celerywyrm/settings.py @@ -10,7 +10,7 @@ CELERY_RESULT_BACKEND = "redis://:{}@redis_broker:{}/0".format( requests.utils.quote(env("REDIS_BROKER_PASSWORD", "")), env("REDIS_BROKER_PORT") ) -CELERY_DEFAULT_QUEUE="low_priority" +CELERY_DEFAULT_QUEUE = "low_priority" CELERY_ACCEPT_CONTENT = ["json"] CELERY_TASK_SERIALIZER = "json" From e0f5f13a5ada89dd9562d55c328bd261fbb685a9 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Tue, 7 Sep 2021 18:09:43 -0700 Subject: [PATCH 025/191] Fixes guessing read dates from goodreads csv --- bookwyrm/models/import_job.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bookwyrm/models/import_job.py b/bookwyrm/models/import_job.py index 05aada161..69d5f5da7 100644 --- a/bookwyrm/models/import_job.py +++ b/bookwyrm/models/import_job.py @@ -174,6 +174,7 @@ class ImportItem(models.Model): if start_date and start_date is not None and not self.date_read: return [ReadThrough(start_date=start_date)] if self.date_read: + start_date = start_date if start_date < self.date_read else None return [ ReadThrough( start_date=start_date, From 2a93a8ea660a9074097eaccf537768b1fc835b71 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Tue, 7 Sep 2021 18:16:26 -0700 Subject: [PATCH 026/191] Adds class to status page --- bookwyrm/templates/import_status.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bookwyrm/templates/import_status.html b/bookwyrm/templates/import_status.html index db7330bb2..94449f260 100644 --- a/bookwyrm/templates/import_status.html +++ b/bookwyrm/templates/import_status.html @@ -40,7 +40,7 @@

{% trans "Failed to load" %}

{% if not job.retry %} -
+ {% csrf_token %} {% with failed_count=failed_items|length %} From d63e7243f5f53033b7781efc1cf7a533e77ff3b7 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Wed, 8 Sep 2021 07:26:51 -0700 Subject: [PATCH 027/191] Adds tooltip component --- bookwyrm/templates/components/tooltip.html | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 bookwyrm/templates/components/tooltip.html diff --git a/bookwyrm/templates/components/tooltip.html b/bookwyrm/templates/components/tooltip.html new file mode 100644 index 000000000..153ce35c5 --- /dev/null +++ b/bookwyrm/templates/components/tooltip.html @@ -0,0 +1,8 @@ +{% load i18n %} + + From e13e13d3f3c282e65bc1eb870d6185b41b0d7d00 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Wed, 8 Sep 2021 07:28:42 -0700 Subject: [PATCH 028/191] Move import templates into directory --- bookwyrm/templates/{ => import}/import.html | 1 + bookwyrm/templates/{ => import}/import_status.html | 0 bookwyrm/views/import_data.py | 4 ++-- 3 files changed, 3 insertions(+), 2 deletions(-) rename bookwyrm/templates/{ => import}/import.html (96%) rename bookwyrm/templates/{ => import}/import_status.html (100%) diff --git a/bookwyrm/templates/import.html b/bookwyrm/templates/import/import.html similarity index 96% rename from bookwyrm/templates/import.html rename to bookwyrm/templates/import/import.html index d2e407486..cac6fdceb 100644 --- a/bookwyrm/templates/import.html +++ b/bookwyrm/templates/import/import.html @@ -28,6 +28,7 @@
+ {% include 'components/tooltip.html' with controls_text="goodreads-tooltip" %}
{{ import_form.csv_file }} diff --git a/bookwyrm/templates/import_status.html b/bookwyrm/templates/import/import_status.html similarity index 100% rename from bookwyrm/templates/import_status.html rename to bookwyrm/templates/import/import_status.html diff --git a/bookwyrm/views/import_data.py b/bookwyrm/views/import_data.py index 9e2e817dc..634940a07 100644 --- a/bookwyrm/views/import_data.py +++ b/bookwyrm/views/import_data.py @@ -28,7 +28,7 @@ class Import(View): """load import page""" return TemplateResponse( request, - "import.html", + "import/import.html", { "import_form": forms.ImportForm(), "jobs": models.ImportJob.objects.filter(user=request.user).order_by( @@ -94,7 +94,7 @@ class ImportStatus(View): items = [i for i in items if not i.fail_reason] return TemplateResponse( request, - "import_status.html", + "import/import_status.html", {"job": job, "items": items, "failed_items": failed_items, "task": task}, ) From 95bdf7b787b4d8457b67857bbba2c690567ca40c Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Wed, 8 Sep 2021 07:35:49 -0700 Subject: [PATCH 029/191] Adds goodreads data tooltip --- bookwyrm/templates/import/import.html | 2 +- bookwyrm/templates/import/tooltip.html | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) create mode 100644 bookwyrm/templates/import/tooltip.html diff --git a/bookwyrm/templates/import/import.html b/bookwyrm/templates/import/import.html index cac6fdceb..cf911f55c 100644 --- a/bookwyrm/templates/import/import.html +++ b/bookwyrm/templates/import/import.html @@ -28,7 +28,7 @@
- {% include 'components/tooltip.html' with controls_text="goodreads-tooltip" %} + {% include 'import/tooltip.html' with controls_text="goodreads-tooltip" %}
{{ import_form.csv_file }} diff --git a/bookwyrm/templates/import/tooltip.html b/bookwyrm/templates/import/tooltip.html new file mode 100644 index 000000000..1cfe5a3b2 --- /dev/null +++ b/bookwyrm/templates/import/tooltip.html @@ -0,0 +1,8 @@ +{% extends 'components/tooltip.html' %} +{% load i18n %} + +{% block tooltip_content %} + +{% trans 'You can download your GoodReads data from the Import/Export page of your GoodReads account.' %} + +{% endblock %} From 1bf09459b55ffb677a54476c7dedf15ddbc7c10d Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Wed, 8 Sep 2021 08:19:54 -0700 Subject: [PATCH 030/191] Changes tooltip paradigm --- bookwyrm/templates/components/tooltip.html | 11 +++++++---- bookwyrm/templates/import/import.html | 12 ++++++++---- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/bookwyrm/templates/components/tooltip.html b/bookwyrm/templates/components/tooltip.html index 153ce35c5..35225b6f9 100644 --- a/bookwyrm/templates/components/tooltip.html +++ b/bookwyrm/templates/components/tooltip.html @@ -1,8 +1,11 @@ {% load i18n %} -