takahe/activities/models/post.py

import re
from collections.abc import Iterable
from typing import Optional

import httpx
import urlman
from asgiref.sync import async_to_sync, sync_to_async
from django.conf import settings
from django.contrib.postgres.indexes import GinIndex
from django.db import models, transaction
from django.template.defaultfilters import linebreaks_filter
from django.utils import timezone
from django.utils.safestring import mark_safe

from activities.models.fan_out import FanOut
from activities.models.hashtag import Hashtag
from core.html import sanitize_post, strip_html
from core.ld import canonicalise, format_ld_date, get_list, parse_ld_date
from stator.models import State, StateField, StateGraph, StatorModel
from users.models.identity import Identity, IdentityStates
from users.models.system_actor import SystemActor


class PostStates(StateGraph):
    new = State(try_interval=300)
    fanned_out = State(externally_progressed=True)
    deleted = State(try_interval=300)
    deleted_fanned_out = State()

    edited = State(try_interval=300)
    edited_fanned_out = State(externally_progressed=True)

    new.transitions_to(fanned_out)
    fanned_out.transitions_to(deleted)
    fanned_out.transitions_to(edited)

    deleted.transitions_to(deleted_fanned_out)
    edited.transitions_to(edited_fanned_out)
    edited_fanned_out.transitions_to(edited)
    edited_fanned_out.transitions_to(deleted)

    @classmethod
    async def targets_fan_out(cls, post: "Post", type_: str) -> None:
        # Fan out to each target
        for follow in await post.aget_targets():
            await FanOut.objects.acreate(
                identity=follow,
                type=type_,
                subject_post=post,
            )

    @classmethod
    async def handle_new(cls, instance: "Post"):
        """
        Creates all needed fan-out objects for a new Post.
        """
        post = await instance.afetch_full()
        await cls.targets_fan_out(post, FanOut.Types.post)
        await post.ensure_hashtags()
        return cls.fanned_out

    @classmethod
    async def handle_deleted(cls, instance: "Post"):
        """
        Creates all needed fan-out objects needed to delete a Post.
        """
        post = await instance.afetch_full()
        await cls.targets_fan_out(post, FanOut.Types.post_deleted)
        return cls.deleted_fanned_out

    @classmethod
    async def handle_edited(cls, instance: "Post"):
        """
        Creates all needed fan-out objects for an edited Post.
        """
        post = await instance.afetch_full()
        await cls.targets_fan_out(post, FanOut.Types.post_edited)
        await post.ensure_hashtags()
        return cls.edited_fanned_out


class PostQuerySet(models.QuerySet):
    def not_hidden(self):
        query = self.exclude(
            state__in=[PostStates.deleted, PostStates.deleted_fanned_out]
        )
        return query

    def public(self, include_replies: bool = False):
        query = self.filter(
            visibility__in=[
                Post.Visibilities.public,
                Post.Visibilities.local_only,
            ],
        )
        if not include_replies:
            return query.filter(in_reply_to__isnull=True)
        return query

    def local_public(self, include_replies: bool = False):
        query = self.filter(
            visibility__in=[
                Post.Visibilities.public,
                Post.Visibilities.local_only,
            ],
            author__local=True,
        )
        if not include_replies:
            return query.filter(in_reply_to__isnull=True)
        return query

    def unlisted(self, include_replies: bool = False):
        query = self.filter(
            visibility__in=[
                Post.Visibilities.public,
                Post.Visibilities.local_only,
                Post.Visibilities.unlisted,
            ],
        )
        if not include_replies:
            return query.filter(in_reply_to__isnull=True)
        return query

    def tagged_with(self, hashtag: str | Hashtag):
        if isinstance(hashtag, str):
            tag_q = models.Q(hashtags__contains=hashtag)
        else:
            tag_q = models.Q(hashtags__contains=hashtag.hashtag)
            if hashtag.aliases:
                for alias in hashtag.aliases:
                    tag_q |= models.Q(hashtags__contains=alias)
        return self.filter(tag_q)


class PostManager(models.Manager):
    def get_queryset(self):
        return PostQuerySet(self.model, using=self._db)

    def not_hidden(self):
        return self.get_queryset().not_hidden()

    def public(self, include_replies: bool = False):
        return self.get_queryset().public(include_replies=include_replies)

    def local_public(self, include_replies: bool = False):
        return self.get_queryset().local_public(include_replies=include_replies)

    def unlisted(self, include_replies: bool = False):
        return self.get_queryset().unlisted(include_replies=include_replies)

    def tagged_with(self, hashtag: str | Hashtag):
        return self.get_queryset().tagged_with(hashtag=hashtag)


class Post(StatorModel):
    """
    A post (status, toot) that is either local or remote.
    """

    class Visibilities(models.IntegerChoices):
        public = 0
        local_only = 4
        unlisted = 1
        followers = 2
        mentioned = 3

    # The author (attributedTo) of the post
    author = models.ForeignKey(
        "users.Identity",
        on_delete=models.CASCADE,
        related_name="posts",
    )

    # The state the post is in
    state = StateField(PostStates)

    # If it is our post or not
    local = models.BooleanField()

    # The canonical object ID
    object_uri = models.CharField(max_length=500, blank=True, null=True, unique=True)

    # Who should be able to see this Post
    visibility = models.IntegerField(
        choices=Visibilities.choices,
        default=Visibilities.public,
    )

    # The main (HTML) content
    content = models.TextField()

    # If the contents of the post are sensitive, and the summary (content
    # warning) to show if it is
    sensitive = models.BooleanField(default=False)
    summary = models.TextField(blank=True, null=True)

    # The public, web URL of this Post on the original server
    url = models.CharField(max_length=500, blank=True, null=True)

    # The Post it is replying to as an AP ID URI
    # (as otherwise we'd have to pull entire threads to use IDs)
    in_reply_to = models.CharField(max_length=500, blank=True, null=True)

    # The identities the post is directly to (who can see it if not public)
    to = models.ManyToManyField(
        "users.Identity",
        related_name="posts_to",
        blank=True,
    )

    # The identities mentioned in the post
    mentions = models.ManyToManyField(
        "users.Identity",
        related_name="posts_mentioning",
        blank=True,
    )

    # Hashtags in the post
    hashtags = models.JSONField(blank=True, null=True)

    # When the post was originally created (as opposed to when we received it)
    published = models.DateTimeField(default=timezone.now)

    # If the post has been edited after initial publication
    edited = models.DateTimeField(blank=True, null=True)

    created = models.DateTimeField(auto_now_add=True)
    updated = models.DateTimeField(auto_now=True)

    objects = PostManager()

    class Meta:
        indexes = [
            GinIndex(fields=["hashtags"], name="hashtags_gin"),
        ]

    class urls(urlman.Urls):
        view = "{self.author.urls.view}posts/{self.id}/"
        object_uri = "{self.author.actor_uri}posts/{self.id}/"
        action_like = "{view}like/"
        action_unlike = "{view}unlike/"
        action_boost = "{view}boost/"
        action_unboost = "{view}unboost/"
        action_delete = "{view}delete/"
        action_edit = "{view}edit/"
        action_reply = "/compose/?reply_to={self.id}"
        admin_edit = "/djadmin/activities/post/{self.id}/change/"

        def get_scheme(self, url):
            return "https"

        def get_hostname(self, url):
            return self.instance.author.domain.uri_domain

    def __str__(self):
        return f"{self.author} #{self.id}"

    def get_absolute_url(self):
        return self.urls.view

    def absolute_object_uri(self):
        """
        Returns an object URI that is always absolute, for sending out to
        other servers.
        """
        if self.local:
            return self.author.absolute_profile_uri() + f"posts/{self.id}/"
        else:
            return self.object_uri

    def in_reply_to_post(self) -> Optional["Post"]:
        """
        Returns the actual Post object we're replying to, if we can find it
        """
        if self.in_reply_to is None:
            return None
        return (
            Post.objects.filter(object_uri=self.in_reply_to)
            .select_related("author")
            .first()
        )

    ain_reply_to_post = sync_to_async(in_reply_to_post)

    ### Content cleanup and extraction ###

    mention_regex = re.compile(
        r"(^|[^\w\d\-_])@([\w\d\-_]+(?:@[\w\d\-_]+\.[\w\d\-_\.]+)?)"
    )

    def linkify_mentions(self, content: str, local: bool = False) -> str:
        """
        Links mentions _in the context of the post_ - as in, using the mentions
        property as the only source (as we might be doing this without other
        DB access allowed)
        """

        possible_matches = {}
        for mention in self.mentions.all():
            if local:
                url = str(mention.urls.view)
            else:
                url = mention.absolute_profile_uri()
            possible_matches[mention.username] = url
            possible_matches[f"{mention.username}@{mention.domain_id}"] = url

        collapse_name: dict[str, str] = {}

        def replacer(match):
            precursor = match.group(1)
            handle = match.group(2).lower()
            if "@" in handle:
                short_handle = handle.split("@", 1)[0]
            else:
                short_handle = handle
            if handle in possible_matches:
                if short_handle not in collapse_name:
                    collapse_name[short_handle] = handle
                elif collapse_name.get(short_handle) != handle:
                    short_handle = handle
                return f'{precursor}<a href="{possible_matches[handle]}">@{short_handle}</a>'
            else:
                return match.group()

        return mark_safe(self.mention_regex.sub(replacer, content))

    def safe_content_local(self):
        """
        Returns the content formatted for local display
        """
        return Hashtag.linkify_hashtags(
            self.linkify_mentions(sanitize_post(self.content), local=True)
        )

    def safe_content_remote(self):
        """
        Returns the content formatted for remote consumption
        """
        return self.linkify_mentions(sanitize_post(self.content))

    def safe_content_plain(self):
        """
        Returns the content formatted as plain text
        """
        return self.linkify_mentions(sanitize_post(self.content))

    ### Async helpers ###

    async def afetch_full(self) -> "Post":
        """
        Returns a version of the object with all relations pre-loaded
        """
        return (
            await Post.objects.select_related("author", "author__domain")
            .prefetch_related("mentions", "mentions__domain", "attachments")
            .aget(pk=self.pk)
        )

    ### Local creation/editing ###

    @classmethod
    def create_local(
        cls,
        author: Identity,
        content: str,
        summary: str | None = None,
        sensitive: bool = False,
        visibility: int = Visibilities.public,
        reply_to: Optional["Post"] = None,
        attachments: list | None = None,
    ) -> "Post":
        with transaction.atomic():
            # Find mentions in this post
            mentions = cls.mentions_from_content(content, author)
            if reply_to:
                mentions.add(reply_to.author)
                # Maintain local-only for replies
                if reply_to.visibility == reply_to.Visibilities.local_only:
                    visibility = reply_to.Visibilities.local_only
            # Find hashtags in this post
            hashtags = Hashtag.hashtags_from_content(content) or None
            # Strip all HTML and apply linebreaks filter
            content = linebreaks_filter(strip_html(content))
            # Make the Post object
            post = cls.objects.create(
                author=author,
                content=content,
                summary=summary or None,
                sensitive=bool(summary) or sensitive,
                local=True,
                visibility=visibility,
                hashtags=hashtags,
                in_reply_to=reply_to.object_uri if reply_to else None,
            )
            post.object_uri = post.urls.object_uri
            post.url = post.absolute_object_uri()
            post.mentions.set(mentions)
            if attachments:
                post.attachments.set(attachments)
            post.save()
        return post

    def edit_local(
        self,
        content: str,
        summary: str | None = None,
        visibility: int = Visibilities.public,
        attachments: list | None = None,
    ):
        with transaction.atomic():
            # Strip all HTML and apply linebreaks filter
            self.content = linebreaks_filter(strip_html(content))
            self.summary = summary or None
            self.sensitive = bool(summary)
            self.visibility = visibility
            self.edited = timezone.now()
            self.hashtags = Hashtag.hashtags_from_content(content) or None
            self.mentions.set(self.mentions_from_content(content, self.author))
            self.attachments.set(attachments or [])
            self.save()

    @classmethod
    def mentions_from_content(cls, content, author) -> set[Identity]:
        mention_hits = cls.mention_regex.findall(content)
        mentions = set()
        for precursor, handle in mention_hits:
            handle = handle.lower()
            if "@" in handle:
                username, domain = handle.split("@", 1)
            else:
                username = handle
                domain = author.domain_id
            identity = Identity.by_username_and_domain(
                username=username,
                domain=domain,
                fetch=True,
            )
            if identity is not None:
                mentions.add(identity)
        return mentions

    async def ensure_hashtags(self) -> None:
        """
        Ensure any of the already parsed hashtags from this Post
        have a corresponding Hashtag record.
        """
        # Ensure hashtags
        if self.hashtags:
            for hashtag in self.hashtags:
                await Hashtag.objects.aget_or_create(
                    hashtag=hashtag,
                )

    ### Actions ###

    def interact_as(self, identity, type):
        from activities.models import PostInteraction, PostInteractionStates

        interaction = PostInteraction.objects.get_or_create(
            type=type, identity=identity, post=self
        )[0]
        if interaction.state in [
            PostInteractionStates.undone,
            PostInteractionStates.undone_fanned_out,
        ]:
            interaction.transition_perform(PostInteractionStates.new)

    def uninteract_as(self, identity, type):
        from activities.models import PostInteraction, PostInteractionStates

        for interaction in PostInteraction.objects.filter(
            type=type, identity=identity, post=self
        ):
            interaction.transition_perform(PostInteractionStates.undone)

    def like_as(self, identity):
        from activities.models import PostInteraction

        self.interact_as(identity, PostInteraction.Types.like)

    def unlike_as(self, identity):
        from activities.models import PostInteraction

        self.uninteract_as(identity, PostInteraction.Types.like)

    def boost_as(self, identity):
        from activities.models import PostInteraction

        self.interact_as(identity, PostInteraction.Types.boost)

    def unboost_as(self, identity):
        from activities.models import PostInteraction

        self.uninteract_as(identity, PostInteraction.Types.boost)

    ### ActivityPub (outbound) ###

    def to_ap(self) -> dict:
        """
        Returns the AP JSON for this object
        """
        value = {
            "to": "as:Public",
            "cc": [],
            "type": "Note",
            "id": self.object_uri,
            "published": format_ld_date(self.published),
            "attributedTo": self.author.actor_uri,
            "content": self.safe_content_remote(),
            "as:sensitive": self.sensitive,
            "url": self.absolute_object_uri(),
            "tag": [],
            "attachment": [],
        }
        if self.summary:
            value["summary"] = self.summary
        if self.in_reply_to:
            value["inReplyTo"] = self.in_reply_to
        if self.edited:
            value["updated"] = format_ld_date(self.edited)
        # Mentions
        for mention in self.mentions.all():
            value["tag"].append(
                {
                    "href": mention.actor_uri,
                    "name": "@" + mention.handle,
                    "type": "Mention",
                }
            )
            value["cc"].append(mention.actor_uri)
        # Attachments
        for attachment in self.attachments.all():
            value["attachment"].append(attachment.to_ap())
        # Remove fields if they're empty
        for field in ["cc", "tag", "attachment"]:
            if not value[field]:
                del value[field]
        return value

    def to_create_ap(self):
        """
        Returns the AP JSON to create this object
        """
        object = self.to_ap()
        return {
            "to": object["to"],
            "cc": object.get("cc", []),
            "type": "Create",
            "id": self.object_uri + "#create",
            "actor": self.author.actor_uri,
            "object": object,
        }

    def to_update_ap(self):
        """
        Returns the AP JSON to update this object
        """
        object = self.to_ap()
        return {
            "to": object["to"],
            "cc": object.get("cc", []),
            "type": "Update",
            "id": self.object_uri + "#update",
            "actor": self.author.actor_uri,
            "object": object,
        }

    def to_delete_ap(self):
        """
        Returns the AP JSON to create this object
        """
        object = self.to_ap()
        return {
            "to": object["to"],
            "cc": object.get("cc", []),
            "type": "Delete",
            "id": self.object_uri + "#delete",
            "actor": self.author.actor_uri,
            "object": object,
        }

    async def aget_targets(self) -> Iterable[Identity]:
        """
        Returns a list of Identities that need to see posts and their changes
        """
        targets = set()
        async for mention in self.mentions.all():
            targets.add(mention)
        # Then, if it's not mentions only, also deliver to followers
        if self.visibility != Post.Visibilities.mentioned:
            async for follower in self.author.inbound_follows.select_related("source"):
                targets.add(follower.source)
        # If it's a reply, always include the original author if we know them
        reply_post = await self.ain_reply_to_post()
        if reply_post:
            targets.add(reply_post.author)
        # If this is a remote post or local-only, filter to only include
        # local identities
        if not self.local or self.visibility == Post.Visibilities.local_only:
            targets = {target for target in targets if target.local}
        # If it's a local post, include the author
        if self.local:
            targets.add(self.author)
        return targets

    ### ActivityPub (inbound) ###

    @classmethod
    def by_ap(cls, data, create=False, update=False) -> "Post":
        """
        Retrieves a Post instance by its ActivityPub JSON object.

        Optionally creates one if it's not present.
        Raises KeyError if it's not found and create is False.
        """
        # Do we have one with the right ID?
        created = False
        try:
            post = cls.objects.get(object_uri=data["id"])
        except cls.DoesNotExist:
            if create:
                # Resolve the author
                author = Identity.by_actor_uri(data["attributedTo"], create=create)
                post = cls.objects.create(
                    object_uri=data["id"],
                    author=author,
                    content=data["content"],
                    local=False,
                )
                created = True
            else:
                raise KeyError(f"No post with ID {data['id']}", data)
        if update or created:
            post.content = data["content"]
            post.summary = data.get("summary")
            post.sensitive = data.get("as:sensitive", False)
            post.url = data.get("url")
            post.published = parse_ld_date(data.get("published"))
            post.edited = parse_ld_date(data.get("updated"))
            post.in_reply_to = data.get("inReplyTo")
            # Mentions and hashtags
            post.hashtags = []
            for tag in get_list(data, "tag"):
                if tag["type"].lower() == "mention":
                    mention_identity = Identity.by_actor_uri(tag["href"], create=True)
                    post.mentions.add(mention_identity)
                elif tag["type"].lower() == "as:hashtag":
                    post.hashtags.append(tag["name"].lstrip("#"))
                elif tag["type"].lower() == "http://joinmastodon.org/ns#emoji":
                    # TODO: Handle incoming emoji
                    pass
                else:
                    raise ValueError(f"Unknown tag type {tag['type']}")
            # Visibility and to
            # (a post is public if it's ever to/cc as:Public, otherwise we
            # regard it as unlisted for now)
            targets = get_list(data, "to") + get_list(data, "cc")
            post.visibility = Post.Visibilities.unlisted
            for target in targets:
                if target.lower() == "as:public":
                    post.visibility = Post.Visibilities.public
            # Attachments
            # These have no IDs, so we have to wipe them each time
            post.attachments.all().delete()
            for attachment in get_list(data, "attachment"):
                if "http://joinmastodon.org/ns#focalPoint" in attachment:
                    focal_x, focal_y = attachment[
                        "http://joinmastodon.org/ns#focalPoint"
                    ]["@list"]
                else:
                    focal_x, focal_y = None, None
                post.attachments.create(
                    remote_url=attachment["url"],
                    mimetype=attachment["mediaType"],
                    name=attachment.get("name"),
                    width=attachment.get("width"),
                    height=attachment.get("height"),
                    blurhash=attachment.get("http://joinmastodon.org/ns#blurhash"),
                    focal_x=focal_x,
                    focal_y=focal_y,
                )
            post.save()
        return post

    @classmethod
    def by_object_uri(cls, object_uri, fetch=False):
        """
        Gets the post by URI - either looking up locally, or fetching
        from the other end if it's not here.
        """
        try:
            return cls.objects.get(object_uri=object_uri)
        except cls.DoesNotExist:
            if fetch:
                try:
                    response = async_to_sync(SystemActor().signed_request)(
                        method="get", uri=object_uri
                    )
                except (httpx.RequestError, httpx.ConnectError):
                    raise cls.DoesNotExist(f"Could not fetch {object_uri}")
                if response.status_code in [404, 410]:
                    raise cls.DoesNotExist(f"No post at {object_uri}")
                if response.status_code >= 500:
                    raise cls.DoesNotExist(f"Server error fetching {object_uri}")
                if response.status_code >= 400:
                    raise cls.DoesNotExist(
                        f"Error fetching post from {object_uri}: {response.status_code}",
                        {response.content},
                    )
                post = cls.by_ap(
                    canonicalise(response.json(), include_security=True),
                    create=True,
                    update=True,
                )
                # We may need to fetch the author too
                if post.author.state == IdentityStates.outdated:
                    async_to_sync(post.author.fetch_actor)()
                return post
            else:
                raise cls.DoesNotExist(f"Cannot find Post with URI {object_uri}")

    @classmethod
    def handle_create_ap(cls, data):
        """
        Handles an incoming create request
        """
        with transaction.atomic():
            # Ensure the Create actor is the Post's attributedTo
            if data["actor"] != data["object"]["attributedTo"]:
                raise ValueError("Create actor does not match its Post object", data)
            # Create it, stator will fan it out locally
            cls.by_ap(data["object"], create=True, update=True)

    @classmethod
    def handle_update_ap(cls, data):
        """
        Handles an incoming update request
        """
        with transaction.atomic():
            # Ensure the Create actor is the Post's attributedTo
            if data["actor"] != data["object"]["attributedTo"]:
                raise ValueError("Create actor does not match its Post object", data)
            # Find it and update it
            cls.by_ap(data["object"], create=False, update=True)

    @classmethod
    def handle_delete_ap(cls, data):
        """
        Handles an incoming delete request
        """
        with transaction.atomic():
            # Find our post by ID if we have one
            try:
                post = cls.by_object_uri(data["object"]["id"])
            except cls.DoesNotExist:
                # It's already been deleted
                return
            # Ensure the actor on the request authored the post
            if not post.author.actor_uri == data["actor"]:
                raise ValueError("Actor on delete does not match object")
            post.delete()

    def debug_fetch(self):
        """
        Fetches the Post from its original URL again and updates us with it
        """
        response = httpx.get(
            self.object_uri,
            headers={
                "Accept": "application/json",
                "User-Agent": settings.TAKAHE_USER_AGENT,
            },
            follow_redirects=True,
        )
        if 200 <= response.status_code < 300:
            return self.by_ap(
                canonicalise(response.json(), include_security=True),
                update=True,
            )

    ### Mastodon API ###

    def to_mastodon_json(self, interactions=None):
        reply_parent = None
        if self.in_reply_to:
            reply_parent = Post.objects.filter(object_uri=self.in_reply_to).first()
        visibility_mapping = {
            self.Visibilities.public: "public",
            self.Visibilities.unlisted: "unlisted",
            self.Visibilities.followers: "private",
            self.Visibilities.mentioned: "direct",
            self.Visibilities.local_only: "public",
        }
        value = {
            "id": self.pk,
            "uri": self.object_uri,
            "created_at": format_ld_date(self.published),
            "account": self.author.to_mastodon_json(),
            "content": self.safe_content_remote(),
            "visibility": visibility_mapping[self.visibility],
            "sensitive": self.sensitive,
            "spoiler_text": self.summary or "",
            "media_attachments": [
                attachment.to_mastodon_json() for attachment in self.attachments.all()
            ],
            "mentions": [
                {
                    "id": mention.id,
                    "username": mention.username,
                    "url": mention.absolute_profile_uri(),
                    "acct": mention.handle,
                }
                for mention in self.mentions.all()
                if mention.username
            ],
            "tags": (
                [{"name": tag, "url": "/tag/{tag}/"} for tag in self.hashtags]
                if self.hashtags
                else []
            ),
            "emojis": [],
            "reblogs_count": self.interactions.filter(type="boost").count(),
            "favourites_count": self.interactions.filter(type="like").count(),
            "replies_count": 0,
            "url": self.absolute_object_uri(),
            "in_reply_to_id": reply_parent.pk if reply_parent else None,
            "in_reply_to_account_id": reply_parent.author.pk if reply_parent else None,
            "reblog": None,
            "poll": None,
            "card": None,
            "language": None,
            "text": self.safe_content_plain(),
            "edited_at": format_ld_date(self.edited) if self.edited else None,
        }
        if interactions:
            value["favourited"] = self.pk in interactions.get("like", [])
            value["reblogged"] = self.pk in interactions.get("boost", [])
        return value