From 5cfe7eca6f3c177cc436812e45dfb03288b5fbc2 Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Wed, 3 Apr 2024 21:06:24 +0200 Subject: [PATCH 1/7] Add index for finding all statuses in a thread --- ...0_status_bookwyrm_st_thread__cf064f_idx.py | 19 +++++++++++++++++++ bookwyrm/models/status.py | 1 + 2 files changed, 20 insertions(+) create mode 100644 bookwyrm/migrations/0200_status_bookwyrm_st_thread__cf064f_idx.py diff --git a/bookwyrm/migrations/0200_status_bookwyrm_st_thread__cf064f_idx.py b/bookwyrm/migrations/0200_status_bookwyrm_st_thread__cf064f_idx.py new file mode 100644 index 000000000..daca654c7 --- /dev/null +++ b/bookwyrm/migrations/0200_status_bookwyrm_st_thread__cf064f_idx.py @@ -0,0 +1,19 @@ +# Generated by Django 3.2.25 on 2024-04-03 19:05 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("bookwyrm", "0199_status_bookwyrm_st_remote__06aeba_idx"), + ] + + operations = [ + migrations.AddIndex( + model_name="status", + index=models.Index( + fields=["thread_id"], name="bookwyrm_st_thread__cf064f_idx" + ), + ), + ] diff --git a/bookwyrm/models/status.py b/bookwyrm/models/status.py index a9c678cb5..546a8d6c8 100644 --- a/bookwyrm/models/status.py +++ b/bookwyrm/models/status.py @@ -82,6 +82,7 @@ class Status(OrderedCollectionPageMixin, BookWyrmModel): ordering = ("-published_date",) indexes = [ models.Index(fields=["remote_id"]), + models.Index(fields=["thread_id"]), ] def save(self, *args, **kwargs): From 4d5a30d9533cec6f137cf25b85fad9415e122cfe Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Wed, 3 Apr 2024 21:11:27 +0200 Subject: [PATCH 2/7] Add index for looking up KeyPair by remote id --- ..._keypair_bookwyrm_ke_remote__472927_idx.py | 19 +++++++++++++++++++ bookwyrm/models/user.py | 7 +++++++ 2 files changed, 26 insertions(+) create mode 100644 bookwyrm/migrations/0201_keypair_bookwyrm_ke_remote__472927_idx.py diff --git a/bookwyrm/migrations/0201_keypair_bookwyrm_ke_remote__472927_idx.py b/bookwyrm/migrations/0201_keypair_bookwyrm_ke_remote__472927_idx.py new file mode 100644 index 000000000..e3d27a11b --- /dev/null +++ b/bookwyrm/migrations/0201_keypair_bookwyrm_ke_remote__472927_idx.py @@ -0,0 +1,19 @@ +# Generated by Django 3.2.25 on 2024-04-03 19:10 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("bookwyrm", "0200_status_bookwyrm_st_thread__cf064f_idx"), + ] + + operations = [ + migrations.AddIndex( + model_name="keypair", + index=models.Index( + fields=["remote_id"], name="bookwyrm_ke_remote__472927_idx" + ), + ), + ] diff --git a/bookwyrm/models/user.py b/bookwyrm/models/user.py index 89fd39b73..e24143e8e 100644 --- a/bookwyrm/models/user.py +++ b/bookwyrm/models/user.py @@ -509,6 +509,13 @@ class KeyPair(ActivitypubMixin, BookWyrmModel): activity_serializer = activitypub.PublicKey serialize_reverse_fields = [("owner", "owner", "id")] + class Meta: + """indexes""" + + indexes = [ + models.Index(fields=["remote_id"]), + ] + def get_remote_id(self): # self.owner is set by the OneToOneField on User return f"{self.owner.remote_id}/#main-key" From 0501ce39cdd22a57b484b48f987dc342bbedf9e6 Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Wed, 3 Apr 2024 21:15:24 +0200 Subject: [PATCH 3/7] Add index for looking up User by username --- ...202_user_bookwyrm_us_usernam_b2546d_idx.py | 19 +++++++++++++++++++ bookwyrm/models/user.py | 7 +++++++ 2 files changed, 26 insertions(+) create mode 100644 bookwyrm/migrations/0202_user_bookwyrm_us_usernam_b2546d_idx.py diff --git a/bookwyrm/migrations/0202_user_bookwyrm_us_usernam_b2546d_idx.py b/bookwyrm/migrations/0202_user_bookwyrm_us_usernam_b2546d_idx.py new file mode 100644 index 000000000..d8666fe3f --- /dev/null +++ b/bookwyrm/migrations/0202_user_bookwyrm_us_usernam_b2546d_idx.py @@ -0,0 +1,19 @@ +# Generated by Django 3.2.25 on 2024-04-03 19:14 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("bookwyrm", "0201_keypair_bookwyrm_ke_remote__472927_idx"), + ] + + operations = [ + migrations.AddIndex( + model_name="user", + index=models.Index( + fields=["username"], name="bookwyrm_us_usernam_b2546d_idx" + ), + ), + ] diff --git a/bookwyrm/models/user.py b/bookwyrm/models/user.py index e24143e8e..8c1b79e45 100644 --- a/bookwyrm/models/user.py +++ b/bookwyrm/models/user.py @@ -198,6 +198,13 @@ class User(OrderedCollectionPageMixin, AbstractUser): hotp_secret = models.CharField(max_length=32, default=None, blank=True, null=True) hotp_count = models.IntegerField(default=0, blank=True, null=True) + class Meta(AbstractUser.Meta): + """indexes""" + + indexes = [ + models.Index(fields=["username"]), + ] + @property def active_follower_requests(self): """Follow requests from active users""" From 464a0298c6de25780020201ee2f397939d443cd3 Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Wed, 3 Apr 2024 21:23:24 +0200 Subject: [PATCH 4/7] Add index for finding active (and local) users --- ...203_user_bookwyrm_us_is_acti_972dc4_idx.py | 19 +++++++++++++++++++ bookwyrm/models/user.py | 1 + 2 files changed, 20 insertions(+) create mode 100644 bookwyrm/migrations/0203_user_bookwyrm_us_is_acti_972dc4_idx.py diff --git a/bookwyrm/migrations/0203_user_bookwyrm_us_is_acti_972dc4_idx.py b/bookwyrm/migrations/0203_user_bookwyrm_us_is_acti_972dc4_idx.py new file mode 100644 index 000000000..b07f1c8a9 --- /dev/null +++ b/bookwyrm/migrations/0203_user_bookwyrm_us_is_acti_972dc4_idx.py @@ -0,0 +1,19 @@ +# Generated by Django 3.2.25 on 2024-04-03 19:22 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("bookwyrm", "0202_user_bookwyrm_us_usernam_b2546d_idx"), + ] + + operations = [ + migrations.AddIndex( + model_name="user", + index=models.Index( + fields=["is_active", "local"], name="bookwyrm_us_is_acti_972dc4_idx" + ), + ), + ] diff --git a/bookwyrm/models/user.py b/bookwyrm/models/user.py index 8c1b79e45..0ec2c6529 100644 --- a/bookwyrm/models/user.py +++ b/bookwyrm/models/user.py @@ -203,6 +203,7 @@ class User(OrderedCollectionPageMixin, AbstractUser): indexes = [ models.Index(fields=["username"]), + models.Index(fields=["is_active", "local"]), ] @property From 321397a349cc9f22e9b976e6edb249896460b78e Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Wed, 3 Apr 2024 21:27:40 +0200 Subject: [PATCH 5/7] Specify which column DISTINCT should apply to --- bookwyrm/activitystreams.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bookwyrm/activitystreams.py b/bookwyrm/activitystreams.py index 42f99e209..5030005d7 100644 --- a/bookwyrm/activitystreams.py +++ b/bookwyrm/activitystreams.py @@ -139,14 +139,14 @@ class ActivityStream(RedisStore): | ( Q(following=status.user) & Q(following=status.reply_parent.user) ) # if the user is following both authors - ).distinct() + ) # only visible to the poster's followers and tagged users elif status.privacy == "followers": audience = audience.filter( Q(following=status.user) # if the user is following the author ) - return audience.distinct() + return audience.distinct("id") @tracer.start_as_current_span("ActivityStream.get_audience") def get_audience(self, status): From 439cb3ccaaddec6fa110bb1dc81561fd1710bac4 Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Thu, 4 Apr 2024 13:15:31 +0200 Subject: [PATCH 6/7] Remove unnecessary conversions between list and set --- bookwyrm/activitystreams.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bookwyrm/activitystreams.py b/bookwyrm/activitystreams.py index 5030005d7..b8f4ed985 100644 --- a/bookwyrm/activitystreams.py +++ b/bookwyrm/activitystreams.py @@ -156,7 +156,7 @@ class ActivityStream(RedisStore): status_author = models.User.objects.filter( is_active=True, local=True, id=status.user.id ).values_list("id", flat=True) - return list(set(list(audience) + list(status_author))) + return list(set(audience) | set(status_author)) def get_stores_for_users(self, user_ids): """convert a list of user ids into redis store ids""" @@ -191,7 +191,7 @@ class HomeStream(ActivityStream): status_author = models.User.objects.filter( is_active=True, local=True, id=status.user.id ).values_list("id", flat=True) - return list(set(list(audience) + list(status_author))) + return list(set(audience) | set(status_author)) def get_statuses_for_user(self, user): return models.Status.privacy_filter( From e1c54b2933bbfab278a2d49a5716911903f4385e Mon Sep 17 00:00:00 2001 From: Bart Schuurmans Date: Thu, 4 Apr 2024 13:47:51 +0200 Subject: [PATCH 7/7] Remove optimizations with adverse effects `if not audience` actually causes the entire query to be evaluated, before .values_list() is called. --- bookwyrm/activitystreams.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/bookwyrm/activitystreams.py b/bookwyrm/activitystreams.py index b8f4ed985..0009ac7a3 100644 --- a/bookwyrm/activitystreams.py +++ b/bookwyrm/activitystreams.py @@ -183,8 +183,6 @@ class HomeStream(ActivityStream): def get_audience(self, status): trace.get_current_span().set_attribute("stream_id", self.key) audience = super()._get_audience(status) - if not audience: - return [] # if the user is following the author audience = audience.filter(following=status.user).values_list("id", flat=True) # if the user is the post's author @@ -239,9 +237,7 @@ class BooksStream(ActivityStream): ) audience = super()._get_audience(status) - if not audience: - return models.User.objects.none() - return audience.filter(shelfbook__book__parent_work=work).distinct() + return audience.filter(shelfbook__book__parent_work=work) def get_audience(self, status): # only show public statuses on the books feed,