Emoji refactor

Emojis are now prefetched from the post, and if not, looked up
individually by shortcode, to prevent loading hundreds.

activities/middleware.py

@@ -1,27 +0,0 @@
-from time import time
-
-from activities.models import Emoji
-
-
-class EmojiDefaultsLoadingMiddleware:
-    """
-    Caches the default Emoji
-    """
-
-    refresh_interval: float = 30.0
-
-    def __init__(self, get_response):
-        self.get_response = get_response
-        self.loaded_ts: float = 0.0
-
-    def __call__(self, request):
-        # Allow test fixtures to force and lock the Emojis
-        if not getattr(Emoji, "__forced__", False):
-            if (
-                not getattr(Emoji, "locals", None)
-                or (time() - self.loaded_ts) >= self.refresh_interval
-            ):
-                Emoji.locals = Emoji.load_locals()
-                self.loaded_ts = time()
-        response = self.get_response(request)
-        return response

activities/models/emoji.py

@@ -140,10 +140,15 @@ class Emoji(StatorModel):
 
     @classmethod
     @cached(cache=TTLCache(maxsize=1000, ttl=60))
-    def for_domain(cls, domain: Domain | None) -> list["Emoji"]:
+    def get_by_domain(cls, shortcode, domain: Domain | None) -> "Emoji":
-        if not domain:
+        """
-            return list(cls.locals.values())
+        Given an emoji shortcode and optional domain, looks up the single
-        return list(cls.objects.usable(domain))
+        emoji and returns it. Raises Emoji.DoesNotExist if there isn't one.
+        """
+        if domain is None or domain.local:
+            return cls.objects.get(local=True, shortcode=shortcode)
+        else:
+            return cls.objects.get(domain=domain, shortcode=shortcode)
 
     @property
     def fullcode(self):

core/html.py

@@ -89,7 +89,11 @@ class ContentRenderer:
         html = self.linkify_mentions(html, post=post)
         html = self.linkify_hashtags(html, identity=post.author)
         if self.local:
-            html = self.imageify_emojis(html, identity=post.author)
+            html = self.imageify_emojis(
+                html,
+                identity=post.author,
+                emojis=post.emojis.all(),
+            )
         return mark_safe(html)
 
     def render_identity_summary(self, html: str, identity, strip: bool = False) -> str:
@@ -182,7 +186,9 @@ class ContentRenderer:
         )
         return linker.linkify(html)
 
-    def imageify_emojis(self, html: str, identity, include_local: bool = True):
+    def imageify_emojis(
+        self, html: str, identity, include_local: bool = True, emojis=None
+    ):
         """
         Find :emoji: in content and convert to <img>. If include_local is True,
         the local emoji will be used as a fallback for any shortcodes not defined
@@ -190,18 +196,26 @@ class ContentRenderer:
         """
         from activities.models import Emoji
 
-        emoji_set = Emoji.for_domain(identity.domain)
+        # If precached emojis were passed, prep them
-        if include_local:
+        cached_emojis = {}
-            emoji_set.extend(Emoji.for_domain(None))
+        if emojis:
-
+            for emoji in emojis:
-        possible_matches = {
+                cached_emojis[emoji.shortcode] = emoji
-            emoji.shortcode: emoji.as_html() for emoji in emoji_set if emoji.is_usable
-        }
 
         def replacer(match):
-            fullcode = match.group(1).lower()
+            shortcode = match.group(1).lower()
-            if fullcode in possible_matches:
+            if shortcode in cached_emojis:
-                return possible_matches[fullcode]
+                return cached_emojis[shortcode].as_html()
+            try:
+                emoji = Emoji.get_by_domain(shortcode, identity.domain)
+                if emoji.is_usable:
+                    return emoji.as_html()
+            except Emoji.DoesNotExist:
+                if include_local:
+                    try:
+                        return Emoji.get_by_domain(shortcode, identity.domain).as_html()
+                    except Emoji.DoesNotExist:
+                        pass
             return match.group()
 
         return Emoji.emoji_regex.sub(replacer, html)

takahe/settings.py

@@ -205,7 +205,6 @@ MIDDLEWARE = [
     "core.middleware.ConfigLoadingMiddleware",
     "api.middleware.ApiTokenMiddleware",
     "users.middleware.IdentityMiddleware",
-    "activities.middleware.EmojiDefaultsLoadingMiddleware",
 ]
 
 ROOT_URLCONF = "takahe.urls"

tests/activities/models/test_post.py

@@ -110,9 +110,7 @@ def test_linkify_mentions_remote(
 
 
 @pytest.mark.django_db
-def test_linkify_mentions_local(
+def test_linkify_mentions_local(config_system, identity, identity2, remote_identity):
-    config_system, emoji_locals, identity, identity2, remote_identity
-):
     """
     Tests that we can linkify post mentions properly for local use
     """

tests/conftest.py

@@ -3,7 +3,6 @@ import time
 import pytest
 from django.conf import settings
 
-from activities.models import Emoji
 from api.models import Application, Token
 from core.models import Config
 from stator.runner import StatorModel, StatorRunner
@@ -87,16 +86,6 @@ def client_with_identity(client, identity, user):
     return client
 
 
-@pytest.fixture
-@pytest.mark.django_db
-def emoji_locals():
-    Emoji.locals = Emoji.load_locals()
-    Emoji.__forced__ = True
-    yield Emoji.locals
-    Emoji.__forced__ = False
-    del Emoji.locals
-
-
 @pytest.fixture
 @pytest.mark.django_db
 def user() -> User:

tests/core/test_html.py

@@ -38,7 +38,7 @@ def test_sanitize_post():
 
 
 @pytest.mark.django_db
-def test_link_preservation(emoji_locals):
+def test_link_preservation():
     """
     We want to:
      - Preserve incoming links from other servers
@@ -53,6 +53,7 @@ def test_link_preservation(emoji_locals):
     fake_post = Mock()
     fake_post.mentions.all.return_value = [fake_mention]
     fake_post.author.domain.uri_domain = "example.com"
+    fake_post.emojis.all.return_value = []
 
     assert (
         renderer.render_post(