Wrap attributedTo for group to fix Peertube federation (#5509)

* Wrap attributedTo for group to fix Peertube federation

* Combine AttributedTo

* Add func to truncate community description

* Fix endless recursion

* Add tests

* Cleaner wrapper for attributedTo

* Address comments

Cargo.lock

@@ -2900,6 +2900,7 @@ dependencies = [
  "tokio",
  "tracing",
  "ts-rs",
+ "unicode-segmentation",
  "url",
  "urlencoding",
  "uuid",
@@ -5515,6 +5516,12 @@ version = "0.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "52ea75f83c0137a9b98608359a5f1af8144876eb67bcb1ce837368e906a9f524"
 
+[[package]]
+name = "unicode-segmentation"
+version = "1.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493"
+
 [[package]]
 name = "unicode-width"
 version = "0.1.13"

crates/apub/assets/peertube/objects/group.json

@@ -64,7 +64,13 @@
       "url": "https://tilvids.com/lazy-static/banners/1a8d6881-30c8-47cb-8576-7af62d869c45.jpg"
     }
   ],
-  "summary": "I'm Nick, and I like to tinker with Linux stuff. I'll bumble through distro reviews, tutorials, and general helpful tidbits and impressions onLinux desktop environments, applications, and news. \n\nYou might see a bit of Linux gaming here and there, and some more personal opinion pieces, but in the end, it's more or less all about Linux and FOSS !\n\nIf you want to stay up to snuff, follow me on Mastodon: https://mastodon.social/@thelinuxEXP \n\nIf you can, consider supporting the channel here: \nhttps://www.patreon.com/thelinuxexperiment",
+  "summary": "I'm Nick, and I like to tinker with Linux stuff. I'll bumble through distro reviews, tutorials, and general helpful tidbits and impressions on Linux desktop environments, applications, and news. \n\nYou might see a bit of Linux gaming here and there, and some more personal opinion pieces, but in the end, it's more or less all about Linux and FOSS !\n\nIf you want to stay up to snuff, follow me on Mastodon: https://mastodon.social/@thelinuxEXP \n\nIf you can, consider supporting the channel here: \nhttps://www.patreon.com/thelinuxexperiment",
   "support": "Support the channel on Patreon: \nhttps://www.patreon.com/thelinuxexperiment\n\nSupport on Liberapay:\nhttps://liberapay.com/TheLinuxExperiment/",
-  "postingRestrictedToMods": true
+  "postingRestrictedToMods": true,
+  "attributedTo": [
+    {
+      "type": "Person",
+      "id": "https://tilvids.com/accounts/thelinuxexperiment"
+    }
+  ]
 }

crates/apub/src/activities/community/update.rs

@@ -10,7 +10,7 @@ use crate::{
   activity_lists::AnnouncableActivities,
   insert_received_activity,
   objects::{community::ApubCommunity, person::ApubPerson, read_from_string_or_source_opt},
-  protocol::{activities::community::update::UpdateCommunity, InCommunity},
+  protocol::{activities::community::update::UpdateCommunity, objects::AttributedTo, InCommunity},
 };
 use activitypub_federation::{
   config::Data,
@@ -112,7 +112,7 @@ impl ActivityHandler for UpdateCommunity {
           .unwrap_or(self.object.inbox)
           .into(),
       ),
-      moderators_url: self.object.attributed_to.map(Into::into),
+      moderators_url: self.object.attributed_to.and_then(AttributedTo::url),
       posting_restricted_to_mods: self.object.posting_restricted_to_mods,
       featured_url: self.object.featured.map(Into::into),
       ..Default::default()

crates/apub/src/collections/community_moderators.rs

@@ -1,5 +1,5 @@
 use crate::{
-  objects::{community::ApubCommunity, person::ApubPerson},
+  objects::{community::ApubCommunity, handle_community_moderators, person::ApubPerson},
   protocol::collections::group_moderators::GroupModerators,
 };
 use activitypub_federation::{
@@ -10,10 +10,6 @@ use activitypub_federation::{
   traits::Collection,
 };
 use lemmy_api_common::{context::LemmyContext, utils::generate_moderators_url};
-use lemmy_db_schema::{
-  source::community::{CommunityActions, CommunityModeratorForm},
-  traits::Joinable,
-};
 use lemmy_db_views::structs::CommunityModeratorView;
 use lemmy_utils::error::{LemmyError, LemmyResult};
 use url::Url;
@@ -55,34 +51,7 @@ impl Collection for ApubCommunityModerators {
     owner: &Self::Owner,
     data: &Data<Self::DataType>,
   ) -> LemmyResult<Self> {
-    let community_id = owner.id;
-    let current_moderators =
-      CommunityModeratorView::for_community(&mut data.pool(), community_id).await?;
-    // Remove old mods from database which arent in the moderators collection anymore
-    for mod_user in &current_moderators {
-      let mod_id = ObjectId::from(mod_user.moderator.ap_id.clone());
-      if !apub.ordered_items.contains(&mod_id) {
-        let community_moderator_form =
-          CommunityModeratorForm::new(mod_user.community.id, mod_user.moderator.id);
-        CommunityActions::leave(&mut data.pool(), &community_moderator_form).await?;
-      }
-    }
-
-    // Add new mods to database which have been added to moderators collection
-    for mod_id in apub.ordered_items {
-      // Ignore errors as mod accounts might be deleted or instances unavailable.
-      let mod_user: Option<ApubPerson> = mod_id.dereference(data).await.ok();
-      if let Some(mod_user) = mod_user {
-        if !current_moderators
-          .iter()
-          .map(|c| c.moderator.ap_id.clone())
-          .any(|x| x == mod_user.ap_id)
-        {
-          let community_moderator_form = CommunityModeratorForm::new(owner.id, mod_user.id);
-          CommunityActions::join(&mut data.pool(), &community_moderator_form).await?;
-        }
-      }
-    }
+    handle_community_moderators(&apub.ordered_items, owner, data).await?;
 
     // This return value is unused, so just set an empty vec
     Ok(ApubCommunityModerators(()))
@@ -100,12 +69,12 @@ mod tests {
   };
   use lemmy_db_schema::{
     source::{
-      community::Community,
+      community::{Community, CommunityActions, CommunityModeratorForm},
       instance::Instance,
       person::{Person, PersonInsertForm},
       site::Site,
     },
-    traits::Crud,
+    traits::{Crud, Joinable},
   };
   use pretty_assertions::assert_eq;
   use serial_test::serial;

crates/apub/src/objects/community.rs

@@ -1,15 +1,20 @@
+use super::{handle_community_moderators, person::ApubPerson};
 use crate::{
   activities::GetActorType,
-  fetcher::markdown_links::markdown_rewrite_remote_links_opt,
+  fetcher::{
+    markdown_links::markdown_rewrite_remote_links_opt,
+    user_or_community::PersonOrGroupType,
+  },
   objects::{instance::fetch_instance_actor_for_object, read_from_string_or_source_opt},
   protocol::{
-    objects::{group::Group, LanguageTag},
+    objects::{group::Group, AttributedTo, LanguageTag},
     ImageObject,
     Source,
   },
 };
 use activitypub_federation::{
   config::Data,
+  fetch::object_id::ObjectId,
   kinds::actor::GroupType,
   protocol::values::MediaTypeHtml,
   traits::{Actor, Object},
@@ -42,7 +47,7 @@ use lemmy_db_schema::{
 use lemmy_utils::{
   error::{LemmyError, LemmyResult},
   spawn_try_task,
-  utils::markdown::markdown_to_html,
+  utils::{markdown::markdown_to_html, validation::truncate_description},
 };
 use std::ops::Deref;
 use url::Url;
@@ -120,7 +125,9 @@ impl Object for ApubCommunity {
       published: Some(self.published),
       updated: self.updated,
       posting_restricted_to_mods: Some(self.posting_restricted_to_mods),
-      attributed_to: Some(generate_moderators_url(&self.ap_id)?.into()),
+      attributed_to: Some(AttributedTo::Lemmy(
+        generate_moderators_url(&self.ap_id)?.into(),
+      )),
       manually_approves_followers: Some(self.visibility == CommunityVisibility::Private),
       discoverable: Some(self.visibility != CommunityVisibility::Unlisted),
     };
@@ -172,7 +179,7 @@ impl Object for ApubCommunity {
       banner,
       sidebar,
       removed,
-      description: group.summary,
+      description: group.summary.as_deref().map(truncate_description),
       followers_url: group.followers.clone().map(Into::into),
       inbox_url: Some(
         group
@@ -181,7 +188,7 @@ impl Object for ApubCommunity {
           .unwrap_or(group.inbox)
           .into(),
       ),
-      moderators_url: group.attributed_to.clone().map(Into::into),
+      moderators_url: group.attributed_to.clone().and_then(AttributedTo::url),
       posting_restricted_to_mods: group.posting_restricted_to_mods,
       featured_url: group.featured.clone().map(Into::into),
       visibility,
@@ -213,7 +220,21 @@ impl Object for ApubCommunity {
         featured.dereference(&community_, &context_).await.ok();
       }
       if let Some(moderators) = group.attributed_to {
-        moderators.dereference(&community_, &context_).await.ok();
+        if let AttributedTo::Lemmy(l) = moderators {
+          l.moderators()
+            .dereference(&community_, &context_)
+            .await
+            .ok();
+        } else if let AttributedTo::Peertube(p) = moderators {
+          let new_mods = p
+            .iter()
+            .filter(|p| p.kind == PersonOrGroupType::Person)
+            .map(|p| ObjectId::<ApubPerson>::from(p.id.clone().into_inner()))
+            .collect();
+          handle_community_moderators(&new_mods, &community_, &context_)
+            .await
+            .ok();
+        }
       }
       Ok(())
     });

crates/apub/src/objects/mod.rs

@@ -1,8 +1,19 @@
 use crate::protocol::{objects::page::Attachment, Source};
-use activitypub_federation::{config::Data, protocol::values::MediaTypeMarkdownOrHtml};
+use activitypub_federation::{
+  config::Data,
+  fetch::object_id::ObjectId,
+  protocol::values::MediaTypeMarkdownOrHtml,
+};
+use community::ApubCommunity;
 use html2md::parse_html;
 use lemmy_api_common::context::LemmyContext;
+use lemmy_db_schema::{
+  source::community::{CommunityActions, CommunityModeratorForm},
+  traits::Joinable,
+};
+use lemmy_db_views::structs::CommunityModeratorView;
 use lemmy_utils::error::LemmyResult;
+use person::ApubPerson;
 
 pub mod comment;
 pub mod community;
@@ -54,3 +65,38 @@ pub(crate) async fn append_attachments_to_comment(
 
   Ok(content)
 }
+
+pub(crate) async fn handle_community_moderators(
+  new_mods: &Vec<ObjectId<ApubPerson>>,
+  community: &ApubCommunity,
+  context: &Data<LemmyContext>,
+) -> LemmyResult<()> {
+  let community_id = community.id;
+  let current_moderators =
+    CommunityModeratorView::for_community(&mut context.pool(), community_id).await?;
+  // Remove old mods from database which arent in the moderators collection anymore
+  for mod_user in &current_moderators {
+    let mod_id = ObjectId::from(mod_user.moderator.ap_id.clone());
+    if !new_mods.contains(&mod_id) {
+      let community_moderator_form =
+        CommunityModeratorForm::new(mod_user.community.id, mod_user.moderator.id);
+      CommunityActions::leave(&mut context.pool(), &community_moderator_form).await?;
+    }
+  }
+
+  // Add new mods to database which have been added to moderators collection
+  for mod_id in new_mods {
+    // Ignore errors as mod accounts might be deleted or instances unavailable.
+    let mod_user: Option<ApubPerson> = mod_id.dereference(context).await.ok();
+    if let Some(mod_user) = mod_user {
+      if !current_moderators
+        .iter()
+        .any(|x| x.moderator.ap_id == mod_user.ap_id)
+      {
+        let community_moderator_form = CommunityModeratorForm::new(community.id, mod_user.id);
+        CommunityActions::join(&mut context.pool(), &community_moderator_form).await?;
+      }
+    }
+  }
+  Ok(())
+}

crates/apub/src/objects/post.rs

@@ -5,7 +5,8 @@ use crate::{
   objects::read_from_string_or_source_opt,
   protocol::{
     objects::{
-      page::{Attachment, AttributedTo, Hashtag, HashtagType, Page, PageType},
+      page::{Attachment, Hashtag, HashtagType, Page, PageType},
+      AttributedTo,
       LanguageTag,
     },
     ImageObject,
@@ -249,7 +250,11 @@ impl Object for ApubPost {
     let url = if let Some(url) = url {
       is_url_blocked(&url, &url_blocklist)?;
       is_valid_url(&url)?;
-      to_local_url(url.as_str(), context).await.or(Some(url))
+      if page.kind != PageType::Video {
+        to_local_url(url.as_str(), context).await.or(Some(url))
+      } else {
+        Some(url)
+      }
     } else {
       None
     };

crates/apub/src/protocol/objects/group.rs

@@ -3,12 +3,11 @@ use crate::{
   collections::{
     community_featured::ApubCommunityFeatured,
     community_follower::ApubCommunityFollower,
-    community_moderators::ApubCommunityModerators,
     community_outbox::ApubCommunityOutbox,
   },
   objects::community::ApubCommunity,
   protocol::{
-    objects::{Endpoints, LanguageTag},
+    objects::{AttributedTo, Endpoints, LanguageTag},
     ImageObject,
     Source,
   },
@@ -65,7 +64,7 @@ pub struct Group {
   // lemmy extension
   pub(crate) sensitive: Option<bool>,
   #[serde(deserialize_with = "deserialize_skip_error", default)]
-  pub(crate) attributed_to: Option<CollectionId<ApubCommunityModerators>>,
+  pub(crate) attributed_to: Option<AttributedTo>,
   // lemmy extension
   pub(crate) posting_restricted_to_mods: Option<bool>,
   pub(crate) outbox: CollectionId<ApubCommunityOutbox>,

crates/apub/src/protocol/objects/mod.rs

@@ -1,11 +1,18 @@
+use crate::{
+  collections::community_moderators::ApubCommunityModerators,
+  fetcher::user_or_community::{PersonOrGroupType, UserOrCommunity},
+  objects::person::ApubPerson,
+};
+use activitypub_federation::fetch::{collection_id::CollectionId, object_id::ObjectId};
 use lemmy_db_schema::{
   impls::actor_language::UNDETERMINED_ID,
-  newtypes::LanguageId,
+  newtypes::{DbUrl, LanguageId},
   source::language::Language,
   utils::DbPool,
 };
 use lemmy_utils::error::LemmyResult;
 use serde::{Deserialize, Serialize};
+use std::ops::Deref;
 use url::Url;
 
 pub(crate) mod group;
@@ -99,6 +106,57 @@ impl LanguageTag {
   }
 }
 
+#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
+pub(crate) struct PersonOrGroupModerators(Url);
+
+impl Deref for PersonOrGroupModerators {
+  type Target = Url;
+
+  fn deref(&self) -> &Self::Target {
+    &self.0
+  }
+}
+
+impl From<DbUrl> for PersonOrGroupModerators {
+  fn from(value: DbUrl) -> Self {
+    PersonOrGroupModerators(value.into())
+  }
+}
+
+impl PersonOrGroupModerators {
+  pub(crate) fn creator(&self) -> ObjectId<ApubPerson> {
+    self.deref().clone().into()
+  }
+
+  pub(crate) fn moderators(&self) -> CollectionId<ApubCommunityModerators> {
+    self.deref().clone().into()
+  }
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)]
+#[serde(untagged)]
+pub(crate) enum AttributedTo {
+  Lemmy(PersonOrGroupModerators),
+  Peertube(Vec<AttributedToPeertube>),
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)]
+#[serde(rename_all = "camelCase")]
+pub(crate) struct AttributedToPeertube {
+  #[serde(rename = "type")]
+  pub kind: PersonOrGroupType,
+  pub id: ObjectId<UserOrCommunity>,
+}
+
+impl AttributedTo {
+  pub(crate) fn url(self) -> Option<DbUrl> {
+    match self {
+      AttributedTo::Lemmy(l) => Some(l.moderators().into()),
+      AttributedTo::Peertube(_) => None,
+    }
+  }
+}
+
 #[cfg(test)]
 mod tests {
   use crate::protocol::{

crates/apub/src/protocol/objects/page.rs

@@ -1,7 +1,12 @@
 use crate::{
-  fetcher::user_or_community::{PersonOrGroupType, UserOrCommunity},
+  fetcher::user_or_community::PersonOrGroupType,
   objects::{community::ApubCommunity, person::ApubPerson, post::ApubPost},
-  protocol::{objects::LanguageTag, ImageObject, InCommunity, Source},
+  protocol::{
+    objects::{AttributedTo, LanguageTag},
+    ImageObject,
+    InCommunity,
+    Source,
+  },
 };
 use activitypub_federation::{
   config::Data,
@@ -146,21 +151,6 @@ impl Attachment {
   }
 }
 
-#[derive(Clone, Debug, Deserialize, Serialize)]
-#[serde(untagged)]
-pub(crate) enum AttributedTo {
-  Lemmy(ObjectId<ApubPerson>),
-  Peertube([AttributedToPeertube; 2]),
-}
-
-#[derive(Clone, Debug, Deserialize, Serialize)]
-#[serde(rename_all = "camelCase")]
-pub(crate) struct AttributedToPeertube {
-  #[serde(rename = "type")]
-  pub kind: PersonOrGroupType,
-  pub id: ObjectId<UserOrCommunity>,
-}
-
 #[derive(Clone, Debug, Deserialize, Serialize)]
 pub struct Hashtag {
   pub(crate) href: Url,
@@ -177,7 +167,7 @@ pub enum HashtagType {
 impl Page {
   pub(crate) fn creator(&self) -> LemmyResult<ObjectId<ApubPerson>> {
     match &self.attributed_to {
-      AttributedTo::Lemmy(l) => Ok(l.clone()),
+      AttributedTo::Lemmy(l) => Ok(l.creator()),
       AttributedTo::Peertube(p) => p
         .iter()
         .find(|a| a.kind == PersonOrGroupType::Person)

crates/utils/Cargo.toml

@@ -93,6 +93,7 @@ markdown-it-ruby = "1.0.1"
 markdown-it-footnote = "0.2.0"
 moka = { workspace = true, optional = true }
 git-version = "0.3.9"
+unicode-segmentation = "1.9.0"
 
 [dev-dependencies]
 pretty_assertions = { workspace = true }

crates/utils/src/utils/validation.rs

@@ -3,6 +3,7 @@ use clearurls::UrlCleaner;
 use itertools::Itertools;
 use regex::{Regex, RegexBuilder, RegexSet};
 use std::sync::LazyLock;
+use unicode_segmentation::UnicodeSegmentation;
 use url::{ParseError, Url};
 
 // From here: https://github.com/vector-im/element-android/blob/develop/matrix-sdk-android/src/main/java/org/matrix/android/sdk/api/MatrixPatterns.kt#L35
@@ -348,6 +349,45 @@ pub fn build_url_str_without_scheme(url_str: &str) -> LemmyResult<String> {
   Ok(out)
 }
 
+// Shorten a string to n chars, being mindful of unicode grapheme
+// boundaries
+pub fn truncate_for_db(text: &str, len: usize) -> String {
+  if text.chars().count() <= len {
+    text.to_string()
+  } else {
+    let offset = text
+      .char_indices()
+      .nth(len)
+      .unwrap_or(text.char_indices().last().unwrap_or_default());
+    let graphemes: Vec<(usize, _)> = text.grapheme_indices(true).collect();
+    let mut index = 0;
+    // Walk the string backwards and find the first char within our length
+    for idx in (0..graphemes.len()).rev() {
+      if let Some(grapheme) = graphemes.get(idx) {
+        if grapheme.0 < offset.0 {
+          index = idx;
+          break;
+        }
+      }
+    }
+    let grapheme = graphemes.get(index).unwrap_or(&(0, ""));
+    let grapheme_count = grapheme.1.chars().count();
+    // `take` isn't inclusive, so if the last grapheme can fit we add its char
+    // length
+    let char_count = if grapheme_count + grapheme.0 <= len {
+      grapheme.0 + grapheme_count
+    } else {
+      grapheme.0
+    };
+
+    text.chars().take(char_count).collect::<String>()
+  }
+}
+
+pub fn truncate_description(text: &str) -> String {
+  truncate_for_db(text, SITE_DESCRIPTION_MAX_LENGTH)
+}
+
 #[cfg(test)]
 mod tests {
 
@@ -368,6 +408,7 @@ mod tests {
       is_valid_url,
       site_name_length_check,
       site_or_community_description_length_check,
+      truncate_for_db,
       BIO_MAX_LENGTH,
       SITE_DESCRIPTION_MAX_LENGTH,
       SITE_NAME_MAX_LENGTH,
@@ -683,4 +724,14 @@ Line3",
     assert!(check_urls_are_valid(&vec!["https://example .com".to_string()]).is_err());
     Ok(())
   }
+
+  #[test]
+  fn test_truncate() -> LemmyResult<()> {
+    assert_eq!("Hell", truncate_for_db("Hello", 4));
+    assert_eq!("word", truncate_for_db("word", 10));
+    assert_eq!("Wales: ", truncate_for_db("Wales: 🏴󠁧󠁢󠁷󠁬󠁳󠁿", 10));
+    assert_eq!("Wales: 🏴󠁧󠁢󠁷󠁬󠁳󠁿", truncate_for_db("Wales: 🏴󠁧󠁢󠁷󠁬󠁳󠁿", 14));
+
+    Ok(())
+  }
 }