Merge branch 'main' into change_idx_post_url

api_tests/src/comment.spec.ts

@@ -860,7 +860,7 @@ test("Dont send a comment reply to a blocked community", async () => {
 
 /// Fetching a deeply nested comment can lead to stack overflow as all parent comments are also
 /// fetched recursively. Ensure that it works properly.
-test("Fetch a deeply nested comment", async () => {
+test.skip("Fetch a deeply nested comment", async () => {
   let lastComment;
   for (let i = 0; i < 50; i++) {
     let commentRes = await createComment(

crates/api/src/sitemap.rs

@@ -14,9 +14,9 @@ async fn generate_urlset(
 ) -> LemmyResult<UrlSet> {
   let urls = posts
     .into_iter()
-    .map_while(|post| {
+    .map_while(|(url, date_time)| {
-      Url::builder(post.0.to_string())
+      Url::builder(url.to_string())
-        .last_modified(post.1.into())
+        .last_modified(date_time.into())
         .build()
         .ok()
     })

crates/api_common/src/utils.rs

@@ -1110,7 +1110,7 @@ async fn proxy_image_link_internal(
 
 /// Rewrite a link to go through `/api/v3/image_proxy` endpoint. This is only for remote urls and
 /// if image_proxy setting is enabled.
-pub(crate) async fn proxy_image_link(link: Url, context: &LemmyContext) -> LemmyResult<DbUrl> {
+pub async fn proxy_image_link(link: Url, context: &LemmyContext) -> LemmyResult<DbUrl> {
   proxy_image_link_internal(
     link,
     context.settings().pictrs_config()?.image_mode(),

crates/apub/assets/mastodon/objects/note_2.json

@@ -0,0 +1,79 @@
+{
+  "@context": [
+    "https://www.w3.org/ns/activitystreams",
+    {
+      "ostatus": "http://ostatus.org#",
+      "atomUri": "ostatus:atomUri",
+      "inReplyToAtomUri": "ostatus:inReplyToAtomUri",
+      "conversation": "ostatus:conversation",
+      "sensitive": "as:sensitive",
+      "toot": "http://joinmastodon.org/ns#",
+      "votersCount": "toot:votersCount",
+      "blurhash": "toot:blurhash",
+      "focalPoint": {
+        "@container": "@list",
+        "@id": "toot:focalPoint"
+      }
+    }
+  ],
+  "id": "https://floss.social/users/kde/statuses/113306831140126616",
+  "type": "Note",
+  "summary": null,
+  "inReplyTo": "https://floss.social/users/kde/statuses/113306824627995724",
+  "published": "2024-10-14T16:57:15Z",
+  "url": "https://floss.social/@kde/113306831140126616",
+  "attributedTo": "https://floss.social/users/kde",
+  "to": ["https://www.w3.org/ns/activitystreams#Public"],
+  "cc": [
+    "https://floss.social/users/kde/followers",
+    "https://lemmy.kde.social/c/kde",
+    "https://lemmy.kde.social/c/kde/followers"
+  ],
+  "sensitive": false,
+  "atomUri": "https://floss.social/users/kde/statuses/113306831140126616",
+  "inReplyToAtomUri": "https://floss.social/users/kde/statuses/113306824627995724",
+  "conversation": "tag:floss.social,2024-10-14:objectId=71424279:objectType=Conversation",
+  "content": "<p><span class=\"h-card\" translate=\"no\"><a href=\"https://lemmy.kde.social/c/kde\" class=\"u-url mention\">@<span>kde@lemmy.kde.social</span></a></span> </p><p>We also need funding 💶 to keep the gears turning! Please support us with a donation:</p><p><a href=\"https://kde.org/donate/\" target=\"_blank\" rel=\"nofollow noopener noreferrer\" translate=\"no\"><span class=\"invisible\">https://</span><span class=\"\">kde.org/donate/</span><span class=\"invisible\"></span></a></p><p>[3/3]</p>",
+  "contentMap": {
+    "en": "<p><span class=\"h-card\" translate=\"no\"><a href=\"https://lemmy.kde.social/c/kde\" class=\"u-url mention\">@<span>kde@lemmy.kde.social</span></a></span> </p><p>We also need funding 💶 to keep the gears turning! Please support us with a donation:</p><p><a href=\"https://kde.org/donate/\" target=\"_blank\" rel=\"nofollow noopener noreferrer\" translate=\"no\"><span class=\"invisible\">https://</span><span class=\"\">kde.org/donate/</span><span class=\"invisible\"></span></a></p><p>[3/3]</p>"
+  },
+  "attachment": [
+    {
+      "type": "Document",
+      "mediaType": "image/jpeg",
+      "url": "https://cdn.masto.host/floss/media_attachments/files/113/306/826/682/985/891/original/c8d906a2f2ab2334.jpg",
+      "name": "The KDE dragons Katie and Konqi stand on either side of a pot filling up with gold coins. Donate!",
+      "blurhash": "USQv:h-W-qI-^,W;RPs=^-R%NZxbo#sDobSc",
+      "focalPoint": [0.0, 0.0],
+      "width": 1500,
+      "height": 1095
+    }
+  ],
+  "tag": [
+    {
+      "type": "Mention",
+      "href": "https://lemmy.kde.social/c/kde",
+      "name": "@kde@lemmy.kde.social"
+    }
+  ],
+  "replies": {
+    "id": "https://floss.social/users/kde/statuses/113306831140126616/replies",
+    "type": "Collection",
+    "first": {
+      "type": "CollectionPage",
+      "next": "https://floss.social/users/kde/statuses/113306831140126616/replies?only_other_accounts=true&page=true",
+      "partOf": "https://floss.social/users/kde/statuses/113306831140126616/replies",
+      "items": []
+    }
+  },
+  "likes": {
+    "id": "https://floss.social/users/kde/statuses/113306831140126616/likes",
+    "type": "Collection",
+    "totalItems": 39
+  },
+  "shares": {
+    "id": "https://floss.social/users/kde/statuses/113306831140126616/shares",
+    "type": "Collection",
+    "totalItems": 24
+  }
+}

crates/apub/src/objects/comment.rs

@@ -3,7 +3,7 @@ use crate::{
   check_apub_id_valid_with_strictness,
   fetcher::markdown_links::markdown_rewrite_remote_links,
   mentions::collect_non_local_mentions,
-  objects::{read_from_string_or_source, verify_is_remote_object},
+  objects::{append_attachments_to_comment, read_from_string_or_source, verify_is_remote_object},
   protocol::{
     objects::{note::Note, LanguageTag},
     InCommunity,
@@ -124,6 +124,7 @@ impl Object for ApubComment {
       distinguished: Some(self.distinguished),
       language,
       audience: Some(community.actor_id.into()),
+      attachment: vec![],
     };
 
     Ok(note)
@@ -181,6 +182,7 @@ impl Object for ApubComment {
     let local_site = LocalSite::read(&mut context.pool()).await.ok();
     let slur_regex = &local_site_opt_to_slur_regex(&local_site);
     let url_blocklist = get_url_blocklist(context).await?;
+    let content = append_attachments_to_comment(content, &note.attachment, context).await?;
     let content = process_markdown(&content, slur_regex, &url_blocklist, context).await?;
     let content = markdown_rewrite_remote_links(content, context).await;
     let language_id = Some(
@@ -247,13 +249,13 @@ pub(crate) mod tests {
   }
 
   async fn cleanup(
-    data: (ApubPerson, ApubCommunity, ApubPost, ApubSite),
+    (person, community, post, site): (ApubPerson, ApubCommunity, ApubPost, ApubSite),
     context: &LemmyContext,
   ) -> LemmyResult<()> {
-    Post::delete(&mut context.pool(), data.2.id).await?;
+    Post::delete(&mut context.pool(), post.id).await?;
-    Community::delete(&mut context.pool(), data.1.id).await?;
+    Community::delete(&mut context.pool(), community.id).await?;
-    Person::delete(&mut context.pool(), data.0.id).await?;
+    Person::delete(&mut context.pool(), person.id).await?;
-    Site::delete(&mut context.pool(), data.3.id).await?;
+    Site::delete(&mut context.pool(), site.id).await?;
     LocalSite::delete(&mut context.pool()).await?;
     Ok(())
   }

crates/apub/src/objects/mod.rs

@@ -1,4 +1,4 @@
-use crate::protocol::Source;
+use crate::protocol::{objects::page::Attachment, Source};
 use activitypub_federation::{
   config::Data,
   fetch::object_id::ObjectId,
@@ -46,6 +46,23 @@ pub(crate) fn read_from_string_or_source_opt(
     .map(|content| read_from_string_or_source(content, media_type, source))
 }
 
+pub(crate) async fn append_attachments_to_comment(
+  content: String,
+  attachments: &[Attachment],
+  context: &Data<LemmyContext>,
+) -> LemmyResult<String> {
+  let mut content = content;
+  // Don't modify comments with no attachments
+  if !attachments.is_empty() {
+    content += "\n";
+    for attachment in attachments {
+      content = content + "\n" + &attachment.as_markdown(context).await?;
+    }
+  }
+
+  Ok(content)
+}
+
 /// When for example a Post is made in a remote community, the community will send it back,
 /// wrapped in Announce. If we simply receive this like any other federated object, overwrite the
 /// existing, local Post. In particular, it will set the field local = false, so that the object

crates/apub/src/objects/person.rs

@@ -285,9 +285,12 @@ pub(crate) mod tests {
     Ok(())
   }
 
-  async fn cleanup(data: (ApubPerson, ApubSite), context: &LemmyContext) -> LemmyResult<()> {
+  async fn cleanup(
-    DbPerson::delete(&mut context.pool(), data.0.id).await?;
+    (person, site): (ApubPerson, ApubSite),
-    Site::delete(&mut context.pool(), data.1.id).await?;
+    context: &LemmyContext,
+  ) -> LemmyResult<()> {
+    DbPerson::delete(&mut context.pool(), person.id).await?;
+    Site::delete(&mut context.pool(), site.id).await?;
     Ok(())
   }
 }

crates/apub/src/objects/private_message.rs

@@ -186,12 +186,12 @@ mod tests {
   }
 
   async fn cleanup(
-    data: (ApubPerson, ApubPerson, ApubSite),
+    (person1, person2, site): (ApubPerson, ApubPerson, ApubSite),
     context: &Data<LemmyContext>,
   ) -> LemmyResult<()> {
-    Person::delete(&mut context.pool(), data.0.id).await?;
+    Person::delete(&mut context.pool(), person1.id).await?;
-    Person::delete(&mut context.pool(), data.1.id).await?;
+    Person::delete(&mut context.pool(), person2.id).await?;
-    Site::delete(&mut context.pool(), data.2.id).await?;
+    Site::delete(&mut context.pool(), site.id).await?;
     Ok(())
   }
 

crates/apub/src/protocol/objects/mod.rs

@@ -145,7 +145,8 @@ mod tests {
   #[test]
   fn test_parse_objects_mastodon() -> LemmyResult<()> {
     test_json::<Person>("assets/mastodon/objects/person.json")?;
-    test_json::<Note>("assets/mastodon/objects/note.json")?;
+    test_json::<Note>("assets/mastodon/objects/note_1.json")?;
+    test_json::<Note>("assets/mastodon/objects/note_2.json")?;
     test_json::<Page>("assets/mastodon/objects/page.json")?;
     Ok(())
   }

crates/apub/src/protocol/objects/note.rs

@@ -3,7 +3,11 @@ use crate::{
   fetcher::post_or_comment::PostOrComment,
   mentions::MentionOrValue,
   objects::{comment::ApubComment, community::ApubCommunity, person::ApubPerson, post::ApubPost},
-  protocol::{objects::LanguageTag, InCommunity, Source},
+  protocol::{
+    objects::{page::Attachment, LanguageTag},
+    InCommunity,
+    Source,
+  },
 };
 use activitypub_federation::{
   config::Data,
@@ -50,6 +54,8 @@ pub struct Note {
   pub(crate) distinguished: Option<bool>,
   pub(crate) language: Option<LanguageTag>,
   pub(crate) audience: Option<ObjectId<ApubCommunity>>,
+  #[serde(default)]
+  pub(crate) attachment: Vec<Attachment>,
 }
 
 impl Note {

crates/apub/src/protocol/objects/page.rs

@@ -19,7 +19,7 @@ use activitypub_federation::{
 };
 use chrono::{DateTime, Utc};
 use itertools::Itertools;
-use lemmy_api_common::context::LemmyContext;
+use lemmy_api_common::{context::LemmyContext, utils::proxy_image_link};
 use lemmy_utils::error::{FederationError, LemmyError, LemmyErrorType, LemmyResult};
 use serde::{de::Error, Deserialize, Deserializer, Serialize};
 use serde_with::skip_serializing_none;
@@ -93,6 +93,7 @@ pub(crate) struct Document {
   #[serde(rename = "type")]
   kind: DocumentType,
   url: Url,
+  media_type: Option<String>,
   /// Used for alt_text
   name: Option<String>,
 }
@@ -124,6 +125,24 @@ impl Attachment {
       _ => None,
     }
   }
+
+  pub(crate) async fn as_markdown(&self, context: &Data<LemmyContext>) -> LemmyResult<String> {
+    let (url, name, media_type) = match self {
+      Attachment::Image(i) => (i.url.clone(), i.name.clone(), Some(String::from("image"))),
+      Attachment::Document(d) => (d.url.clone(), d.name.clone(), d.media_type.clone()),
+      Attachment::Link(l) => (l.href.clone(), None, l.media_type.clone()),
+    };
+
+    let is_image =
+      media_type.is_some_and(|media| media.starts_with("video") || media.starts_with("image"));
+
+    if is_image {
+      let url = proxy_image_link(url, context).await?;
+      Ok(format!("![{}]({url})", name.unwrap_or_default()))
+    } else {
+      Ok(format!("[{url}]({url})"))
+    }
+  }
 }
 
 #[derive(Clone, Debug, Deserialize, Serialize)]

crates/db_views/src/custom_emoji_view.rs

@@ -76,16 +76,16 @@ impl CustomEmojiView {
   fn from_tuple_to_vec(items: Vec<CustomEmojiTuple>) -> Vec<Self> {
     let mut result = Vec::new();
     let mut hash: HashMap<CustomEmojiId, Vec<CustomEmojiKeyword>> = HashMap::new();
-    for item in &items {
+    for (emoji, keyword) in &items {
-      let emoji_id: CustomEmojiId = item.0.id;
+      let emoji_id: CustomEmojiId = emoji.id;
       if let std::collections::hash_map::Entry::Vacant(e) = hash.entry(emoji_id) {
         e.insert(Vec::new());
         result.push(CustomEmojiView {
-          custom_emoji: item.0.clone(),
+          custom_emoji: emoji.clone(),
           keywords: Vec::new(),
         })
       }
-      if let Some(item_keyword) = &item.1 {
+      if let Some(item_keyword) = &keyword {
         if let Some(keywords) = hash.get_mut(&emoji_id) {
           keywords.push(item_keyword.clone())
         }

crates/utils/src/utils/markdown/image_links.rs

@@ -42,13 +42,10 @@ pub fn markdown_rewrite_image_links(mut src: String) -> (String, Vec<Url>) {
 pub fn markdown_handle_title(src: &str, start: usize, end: usize) -> (&str, Option<&str>) {
   let content = src.get(start..end).unwrap_or_default();
   // necessary for custom emojis which look like `![name](url "title")`
-  let (url, extra) = if content.contains(' ') {
+  match content.split_once(' ') {
-    let split = content.split_once(' ').expect("split is valid");
+    Some((a, b)) => (a, Some(b)),
-    (split.0, Some(split.1))
+    _ => (content, None),
-  } else {
+  }
-    (content, None)
-  };
-  (url, extra)
 }
 
 pub fn markdown_find_links(src: &str) -> Vec<(usize, usize)> {
@@ -61,9 +58,9 @@ fn find_urls<T: NodeValue + UrlAndTitle>(src: &str) -> Vec<(usize, usize)> {
   let mut links_offsets = vec![];
   ast.walk(|node, _depth| {
     if let Some(image) = node.cast::<T>() {
-      let node_offsets = node.srcmap.expect("srcmap is none").get_byte_offsets();
+      let (_, node_offset) = node.srcmap.expect("srcmap is none").get_byte_offsets();
-      let start_offset = node_offsets.1 - image.url_len() - 1 - image.title_len();
+      let start_offset = node_offset - image.url_len() - 1 - image.title_len();
-      let end_offset = node_offsets.1 - 1;
+      let end_offset = node_offset - 1;
 
       links_offsets.push((start_offset, end_offset));
     }

src/scheduled_tasks.rs

@@ -393,10 +393,10 @@ async fn active_counts(pool: &mut DbPool<'_>) {
         ("6 months", "half_year"),
       ];
 
-      for i in &intervals {
+      for (full_form, abbr) in &intervals {
         let update_site_stmt = format!(
       "update site_aggregates set users_active_{} = (select * from site_aggregates_activity('{}')) where site_id = 1",
-      i.1, i.0
+      abbr, full_form
     );
         sql_query(update_site_stmt)
           .execute(&mut conn)
@@ -404,7 +404,7 @@ async fn active_counts(pool: &mut DbPool<'_>) {
           .inspect_err(|e| error!("Failed to update site stats: {e}"))
           .ok();
 
-        let update_community_stmt = format!("update community_aggregates ca set users_active_{} = mv.count_ from community_aggregates_activity('{}') mv where ca.community_id = mv.community_id_", i.1, i.0);
+        let update_community_stmt = format!("update community_aggregates ca set users_active_{} = mv.count_ from community_aggregates_activity('{}') mv where ca.community_id = mv.community_id_", abbr, full_form);
         sql_query(update_community_stmt)
           .execute(&mut conn)
           .await