Add tests for lotide federation, make lotide groups fetchable (#2035)

* Add tests for lotide federation, make lotide groups fetchable

* Accept posts using Note type (and better error messages for tests)

crates/apub/assets/lotide/objects/group.json

@@ -0,0 +1,28 @@
+{
+  "publicKey": {
+    "id": "https://narwhal.city//communities/12#main-key",
+    "owner": "https://narwhal.city/communities/12",
+    "publicKeyPem": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAtktBbjovDSQmjZo1SIGK\n1TP1FKuIj8JlFgY6iGrAA5IBUN8PPKRzvo0U0FDvF+7SsUx+yiY0JrU1KzWcJxRr\nCfTrjNzaKeMS4E6ZU9czf8D157JUJQtkgikObxwU84eY5K+jic1ZgGv2eX77E6f/\nBZFO8StdS73g8a1vxPEsJVBn/VEVdsD9fg3uvhwFN7UrUKoKGf+1h2PajeX1aPZb\ntD3ql3Xff2IZFZu6Euj80OezozQ6/AqZx+qW6HfjvSf30C8ZGYU1PSF6MczY+Sg6\n6nyPMfmbKykYgWqfRMZ/NKaldsIjN8nMRDCfHASt6+pNmZgWh9HvSaFiSFKIn3Xj\nXwIDAQAB\n-----END PUBLIC KEY-----\n",
+    "signatureAlgorithm": "http://www.w3.org/2001/04/xmldsig-more#rsa-sha256"
+  },
+  "featured": "https://narwhal.city/communities/12/featured",
+  "inbox": "https://narwhal.city/communities/12/inbox",
+  "outbox": "https://narwhal.city/communities/12/outbox",
+  "followers": "https://narwhal.city/communities/12/followers",
+  "preferredUsername": "Iotide",
+  "summary": "This is for talking about lotide\r\n\r\n\r\nI accidentally called it iotide because I misread the text when I made it lol",
+  "@context": [
+    "https://www.w3.org/ns/activitystreams",
+    "https://w3id.org/security/v1",
+    {
+      "featured": {
+        "@id": "toot:featured",
+        "@type": "@id"
+      },
+      "toot": "http://joinmastodon.org/ns#"
+    }
+  ],
+  "id": "https://narwhal.city/communities/12",
+  "type": "Group",
+  "name": "Iotide"
+}

crates/apub/assets/lotide/objects/note.json

@@ -0,0 +1,19 @@
+{
+  "source": {
+    "mediaType": "text/markdown",
+    "content": "ed: now featuring Bob Dylan and RNG"
+  },
+  "attributedTo": "https://narwhal.city/users/3",
+  "content": "<p>ed: now featuring Bob Dylan and RNG</p>\n",
+  "@context": "https://www.w3.org/ns/activitystreams",
+  "inReplyTo": "https://narwhal.city/posts/9",
+  "to": "https://narwhal.city/users/1",
+  "cc": [
+    "https://www.w3.org/ns/activitystreams#Public",
+    "https://narwhal.city/communities/4"
+  ],
+  "id": "https://narwhal.city/comments/3",
+  "type": "Note",
+  "mediaType": "text/html",
+  "published": "2020-12-31T06:47:24.470801+00:00"
+}

crates/apub/assets/lotide/objects/page.json

@@ -0,0 +1,12 @@
+{
+  "@context": "https://www.w3.org/ns/activitystreams",
+  "id": "https://narwhal.city/posts/9",
+  "type": "Page",
+  "name": "What's Dylan Grillin'? (reupload)",
+  "to": "https://narwhal.city/communities/4",
+  "attributedTo": "https://narwhal.city/users/1",
+  "published": "2020-12-30T07:29:19.460932+00:00",
+  "url": "https://www.youtube.com/watch?v=ZI4LGTXscR4",
+  "summary": "What's Dylan Grillin'? (reupload)",
+  "cc": "https://www.w3.org/ns/activitystreams#Public"
+}

crates/apub/assets/lotide/objects/person.json

@@ -0,0 +1,22 @@
+{
+  "publicKey": {
+    "id": "https://narwhal.city//users/3#main-key",
+    "owner": "https://narwhal.city/users/3",
+    "publicKeyPem": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAvC+ZURasjlyX1o4FqMWB\npAppKWU2zPV7cUokKsnKo9m2PKw+53mmVUMQ66LtN80l/WCK/hy7r2lDKvpyt3YO\nnEsNcSCYLaYnTLDNkE2u14kx8jKOFiyRKKVKCNA32b+XvM+rLDmfaNOeBsB92mVR\nVmIz+WO+0FVPtg1MQMKWIoe6SgKW8SHpz/qVeggYNMKp/b2ai7Of0KTSbYIcqFR2\nT8g/6L5Mmjz4zKIn+a5GFmBNTMTCsJTxa5yOjPwefh/9SrukWt01N5KLrIpmApms\nRoJSsBWh0xo7N+v23PaFHEkaJ2zCtT5zkzITa8bUfHoIc3rM6Ipa1uFlnmrnUIZE\nUQIDAQAB\n-----END PUBLIC KEY-----\n",
+    "signatureAlgorithm": "http://www.w3.org/2001/04/xmldsig-more#rsa-sha256"
+  },
+  "inbox": "https://narwhal.city/users/3/inbox",
+  "outbox": "https://narwhal.city/users/3/outbox",
+  "preferredUsername": "57H",
+  "endpoints": {
+    "sharedInbox": "https://narwhal.city/inbox"
+  },
+  "summary": "",
+  "@context": [
+    "https://www.w3.org/ns/activitystreams",
+    "https://w3id.org/security/v1"
+  ],
+  "id": "https://narwhal.city/users/3",
+  "type": "Person",
+  "name": "57H"
+}

crates/apub/src/collections/community_moderators.rs

@@ -177,7 +177,7 @@ mod tests {
     let new_mod = parse_lemmy_person(&context).await;
 
     let json: GroupModerators =
-      file_to_json_object("assets/lemmy/collections/group_moderators.json");
+      file_to_json_object("assets/lemmy/collections/group_moderators.json").unwrap();
     let url = Url::parse("https://enterprise.lemmy.ml/c/tenforward").unwrap();
     let mut request_counter = 0;
     let community_context = CommunityContext {

crates/apub/src/objects/comment.rs

@@ -227,7 +227,7 @@ pub(crate) mod tests {
   ) -> (ApubPerson, ApubCommunity, ApubPost) {
     let person = parse_lemmy_person(context).await;
     let community = parse_lemmy_community(context).await;
-    let post_json = file_to_json_object("assets/lemmy/objects/page.json");
+    let post_json = file_to_json_object("assets/lemmy/objects/page.json").unwrap();
     ApubPost::verify(&post_json, url, context, &mut 0)
       .await
       .unwrap();
@@ -252,7 +252,7 @@ pub(crate) mod tests {
     let url = Url::parse("https://enterprise.lemmy.ml/comment/38741").unwrap();
     let data = prepare_comment_test(&url, &context).await;
 
-    let json: Note = file_to_json_object("assets/lemmy/objects/note.json");
+    let json: Note = file_to_json_object("assets/lemmy/objects/note.json").unwrap();
     let mut request_counter = 0;
     ApubComment::verify(&json, &url, &context, &mut request_counter)
       .await
@@ -286,14 +286,14 @@ pub(crate) mod tests {
     let pleroma_url =
       Url::parse("https://queer.hacktivis.me/objects/8d4973f4-53de-49cd-8c27-df160e16a9c2")
         .unwrap();
-    let person_json = file_to_json_object("assets/pleroma/objects/person.json");
+    let person_json = file_to_json_object("assets/pleroma/objects/person.json").unwrap();
     ApubPerson::verify(&person_json, &pleroma_url, &context, &mut 0)
       .await
       .unwrap();
     ApubPerson::from_apub(person_json, &context, &mut 0)
       .await
       .unwrap();
-    let json = file_to_json_object("assets/pleroma/objects/note.json");
+    let json = file_to_json_object("assets/pleroma/objects/note.json").unwrap();
     let mut request_counter = 0;
     ApubComment::verify(&json, &pleroma_url, &context, &mut request_counter)
       .await

crates/apub/src/objects/community.rs

@@ -103,9 +103,9 @@ impl ApubObject for ApubCommunity {
       inbox: self.inbox_url.clone().into(),
       outbox: ObjectId::new(generate_outbox_url(&self.actor_id)?),
       followers: self.followers_url.clone().into(),
-      endpoints: Endpoints {
-        shared_inbox: self.shared_inbox_url.clone().map(|s| s.into()),
-      },
+      endpoints: self.shared_inbox_url.clone().map(|s| Endpoints {
+        shared_inbox: s.into(),
+      }),
       public_key: self.get_public_key()?,
       published: Some(convert_datetime(self.published)),
       updated: self.updated.map(convert_datetime),
@@ -225,7 +225,7 @@ pub(crate) mod tests {
   use serial_test::serial;
 
   pub(crate) async fn parse_lemmy_community(context: &LemmyContext) -> ApubCommunity {
-    let mut json: Group = file_to_json_object("assets/lemmy/objects/group.json");
+    let mut json: Group = file_to_json_object("assets/lemmy/objects/group.json").unwrap();
     // change these links so they dont fetch over the network
     json.moderators = None;
     json.outbox =

crates/apub/src/objects/mod.rs

@@ -90,9 +90,11 @@ pub(crate) mod tests {
     LemmyContext::create(pool, chat_server, client, activity_queue, settings, secret)
   }
 
-  pub(crate) fn file_to_json_object<T: DeserializeOwned>(path: &str) -> T {
+  pub(crate) fn file_to_json_object<T: DeserializeOwned>(
+    path: &str,
+  ) -> serde_json::error::Result<T> {
     let file = File::open(path).unwrap();
     let reader = BufReader::new(file);
-    serde_json::from_reader(reader).unwrap()
+    serde_json::from_reader(reader)
   }
 }

crates/apub/src/objects/person.rs

@@ -106,9 +106,9 @@ impl ApubObject for ApubPerson {
       matrix_user_id: self.matrix_user_id.clone(),
       published: Some(convert_datetime(self.published)),
       outbox: generate_outbox_url(&self.actor_id)?.into(),
-      endpoints: Endpoints {
-        shared_inbox: self.shared_inbox_url.clone().map(|s| s.into()),
-      },
+      endpoints: self.shared_inbox_url.clone().map(|s| Endpoints {
+        shared_inbox: s.into(),
+      }),
       public_key: self.get_public_key()?,
       updated: self.updated.map(convert_datetime),
       unparsed: Default::default(),
@@ -167,7 +167,7 @@ impl ApubObject for ApubPerson {
       public_key: person.public_key.public_key_pem,
       last_refreshed_at: Some(naive_now()),
       inbox_url: Some(person.inbox.into()),
-      shared_inbox_url: Some(person.endpoints.shared_inbox.map(|s| s.into())),
+      shared_inbox_url: Some(person.endpoints.map(|e| e.shared_inbox.into())),
       matrix_user_id: Some(person.matrix_user_id),
     };
     let person = blocking(context.pool(), move |conn| {
@@ -209,7 +209,7 @@ pub(crate) mod tests {
   use serial_test::serial;
 
   pub(crate) async fn parse_lemmy_person(context: &LemmyContext) -> ApubPerson {
-    let json = file_to_json_object("assets/lemmy/objects/person.json");
+    let json = file_to_json_object("assets/lemmy/objects/person.json").unwrap();
     let url = Url::parse("https://enterprise.lemmy.ml/u/picard").unwrap();
     let mut request_counter = 0;
     ApubPerson::verify(&json, &url, context, &mut request_counter)
@@ -243,7 +243,7 @@ pub(crate) mod tests {
     let client = reqwest::Client::new().into();
     let manager = create_activity_queue(client);
     let context = init_context(manager.queue_handle().clone());
-    let json = file_to_json_object("assets/pleroma/objects/person.json");
+    let json = file_to_json_object("assets/pleroma/objects/person.json").unwrap();
     let url = Url::parse("https://queer.hacktivis.me/users/lanodan").unwrap();
     let mut request_counter = 0;
     ApubPerson::verify(&json, &url, &context, &mut request_counter)

crates/apub/src/objects/post.rs

@@ -2,12 +2,15 @@ use crate::{
   activities::{verify_is_public, verify_person_in_community},
   check_is_apub_id_valid,
   protocol::{
-    objects::{page::Page, tombstone::Tombstone},
+    objects::{
+      page::{Page, PageType},
+      tombstone::Tombstone,
+    },
     ImageObject,
     Source,
   },
 };
-use activitystreams_kinds::{object::PageType, public};
+use activitystreams_kinds::public;
 use chrono::NaiveDateTime;
 use lemmy_api_common::blocking;
 use lemmy_apub_lib::{
@@ -222,7 +225,7 @@ mod tests {
     let community = parse_lemmy_community(&context).await;
     let person = parse_lemmy_person(&context).await;
 
-    let json = file_to_json_object("assets/lemmy/objects/page.json");
+    let json = file_to_json_object("assets/lemmy/objects/page.json").unwrap();
     let url = Url::parse("https://enterprise.lemmy.ml/post/55143").unwrap();
     let mut request_counter = 0;
     ApubPost::verify(&json, &url, &context, &mut request_counter)

crates/apub/src/objects/private_message.rs

@@ -171,14 +171,14 @@ mod tests {
   use serial_test::serial;
 
   async fn prepare_comment_test(url: &Url, context: &LemmyContext) -> (ApubPerson, ApubPerson) {
-    let lemmy_person = file_to_json_object("assets/lemmy/objects/person.json");
+    let lemmy_person = file_to_json_object("assets/lemmy/objects/person.json").unwrap();
     ApubPerson::verify(&lemmy_person, url, context, &mut 0)
       .await
       .unwrap();
     let person1 = ApubPerson::from_apub(lemmy_person, context, &mut 0)
       .await
       .unwrap();
-    let pleroma_person = file_to_json_object("assets/pleroma/objects/person.json");
+    let pleroma_person = file_to_json_object("assets/pleroma/objects/person.json").unwrap();
     let pleroma_url = Url::parse("https://queer.hacktivis.me/users/lanodan").unwrap();
     ApubPerson::verify(&pleroma_person, &pleroma_url, context, &mut 0)
       .await
@@ -202,7 +202,7 @@ mod tests {
     let context = init_context(manager.queue_handle().clone());
     let url = Url::parse("https://enterprise.lemmy.ml/private_message/1621").unwrap();
     let data = prepare_comment_test(&url, &context).await;
-    let json: ChatMessage = file_to_json_object("assets/lemmy/objects/chat_message.json");
+    let json: ChatMessage = file_to_json_object("assets/lemmy/objects/chat_message.json").unwrap();
     let mut request_counter = 0;
     ApubPrivateMessage::verify(&json, &url, &context, &mut request_counter)
       .await
@@ -232,7 +232,7 @@ mod tests {
     let url = Url::parse("https://enterprise.lemmy.ml/private_message/1621").unwrap();
     let data = prepare_comment_test(&url, &context).await;
     let pleroma_url = Url::parse("https://queer.hacktivis.me/objects/2").unwrap();
-    let json = file_to_json_object("assets/pleroma/objects/chat_message.json");
+    let json = file_to_json_object("assets/pleroma/objects/chat_message.json").unwrap();
     let mut request_counter = 0;
     ApubPrivateMessage::verify(&json, &pleroma_url, &context, &mut request_counter)
       .await

crates/apub/src/protocol/activities/create_or_update/mod.rs

@@ -26,13 +26,17 @@ mod tests {
 
     file_to_json_object::<WithContext<CreateOrUpdateComment>>(
       "assets/pleroma/activities/create_note.json",
-    );
+    )
+    .unwrap();
     file_to_json_object::<WithContext<CreateOrUpdateComment>>(
       "assets/smithereen/activities/create_note.json",
-    );
-    file_to_json_object::<CreateOrUpdateComment>("assets/mastodon/activities/create_note.json");
+    )
+    .unwrap();
+    file_to_json_object::<CreateOrUpdateComment>("assets/mastodon/activities/create_note.json")
+      .unwrap();
 
-    file_to_json_object::<CreateOrUpdatePost>("assets/lotide/activities/create_page.json");
-    file_to_json_object::<CreateOrUpdateComment>("assets/lotide/activities/create_note_reply.json");
+    file_to_json_object::<CreateOrUpdatePost>("assets/lotide/activities/create_page.json").unwrap();
+    file_to_json_object::<CreateOrUpdateComment>("assets/lotide/activities/create_note_reply.json")
+      .unwrap();
   }
 }

crates/apub/src/protocol/activities/following/mod.rs

@@ -25,6 +25,7 @@ mod tests {
       "assets/lemmy/activities/following/undo_follow.json",
     );
 
-    file_to_json_object::<WithContext<FollowCommunity>>("assets/pleroma/activities/follow.json");
+    file_to_json_object::<WithContext<FollowCommunity>>("assets/pleroma/activities/follow.json")
+      .unwrap();
   }
 }

crates/apub/src/protocol/mod.rs

@@ -45,13 +45,17 @@ pub(crate) mod tests {
   use serde::{de::DeserializeOwned, Serialize};
   use std::collections::HashMap;
 
+  /// Check that json deserialize -> serialize -> deserialize gives identical file as initial one.
+  /// Ensures that there are no breaking changes in sent data.
   pub(crate) fn test_parse_lemmy_item<T: Serialize + DeserializeOwned + std::fmt::Debug>(
     path: &str,
   ) -> T {
-    let parsed = file_to_json_object::<T>(path);
+    // parse file as T
+    let parsed = file_to_json_object::<T>(path).unwrap();
 
-    // ensure that no field is ignored when parsing
-    let raw = file_to_json_object::<HashMap<String, serde_json::Value>>(path);
+    // parse file into hashmap, which ensures that every field is included
+    let raw = file_to_json_object::<HashMap<String, serde_json::Value>>(path).unwrap();
+    // assert that all fields are identical, otherwise print diff
     assert_json_include!(actual: &parsed, expected: raw);
     parsed
   }

crates/apub/src/protocol/objects/group.rs

@@ -43,7 +43,7 @@ pub struct Group {
   pub(crate) inbox: Url,
   pub(crate) outbox: ObjectId<ApubCommunityOutbox>,
   pub(crate) followers: Url,
-  pub(crate) endpoints: Endpoints,
+  pub(crate) endpoints: Option<Endpoints>,
   pub(crate) public_key: PublicKey,
   pub(crate) published: Option<DateTime<FixedOffset>>,
   pub(crate) updated: Option<DateTime<FixedOffset>>,
@@ -87,7 +87,7 @@ impl Group {
       banner: Some(self.image.map(|i| i.url.into())),
       followers_url: Some(self.followers.into()),
       inbox_url: Some(self.inbox.into()),
-      shared_inbox_url: Some(self.endpoints.shared_inbox.map(|s| s.into())),
+      shared_inbox_url: Some(self.endpoints.map(|e| e.shared_inbox.into())),
     }
   }
 }

crates/apub/src/protocol/objects/mod.rs

@@ -11,8 +11,7 @@ pub(crate) mod tombstone;
 #[derive(Clone, Debug, Deserialize, Serialize)]
 #[serde(rename_all = "camelCase")]
 pub struct Endpoints {
-  #[serde(skip_serializing_if = "Option::is_none")]
-  pub shared_inbox: Option<Url>,
+  pub shared_inbox: Url,
 }
 
 #[cfg(test)]
@@ -27,21 +26,39 @@ mod tests {
   };
 
   #[actix_rt::test]
-  async fn test_parse_object() {
+  async fn test_parse_object_lemmy() {
     test_parse_lemmy_item::<Person>("assets/lemmy/objects/person.json");
     test_parse_lemmy_item::<Group>("assets/lemmy/objects/group.json");
     test_parse_lemmy_item::<Page>("assets/lemmy/objects/page.json");
     test_parse_lemmy_item::<Note>("assets/lemmy/objects/note.json");
     test_parse_lemmy_item::<ChatMessage>("assets/lemmy/objects/chat_message.json");
+  }
 
-    file_to_json_object::<WithContext<Person>>("assets/pleroma/objects/person.json");
-    file_to_json_object::<WithContext<Note>>("assets/pleroma/objects/note.json");
-    file_to_json_object::<WithContext<ChatMessage>>("assets/pleroma/objects/chat_message.json");
+  #[actix_rt::test]
+  async fn test_parse_object_pleroma() {
+    file_to_json_object::<WithContext<Person>>("assets/pleroma/objects/person.json").unwrap();
+    file_to_json_object::<WithContext<Note>>("assets/pleroma/objects/note.json").unwrap();
+    file_to_json_object::<WithContext<ChatMessage>>("assets/pleroma/objects/chat_message.json")
+      .unwrap();
+  }
 
-    file_to_json_object::<WithContext<Person>>("assets/smithereen/objects/person.json");
-    file_to_json_object::<Note>("assets/smithereen/objects/note.json");
+  #[actix_rt::test]
+  async fn test_parse_object_smithereen() {
+    file_to_json_object::<WithContext<Person>>("assets/smithereen/objects/person.json").unwrap();
+    file_to_json_object::<Note>("assets/smithereen/objects/note.json").unwrap();
+  }
 
-    file_to_json_object::<Person>("assets/mastodon/objects/person.json");
-    file_to_json_object::<Note>("assets/mastodon/objects/note.json");
+  #[actix_rt::test]
+  async fn test_parse_object_mastodon() {
+    file_to_json_object::<Person>("assets/mastodon/objects/person.json").unwrap();
+    file_to_json_object::<Note>("assets/mastodon/objects/note.json").unwrap();
+  }
+
+  #[actix_rt::test]
+  async fn test_parse_object_lotide() {
+    file_to_json_object::<WithContext<Group>>("assets/lotide/objects/group.json").unwrap();
+    file_to_json_object::<WithContext<Person>>("assets/lotide/objects/person.json").unwrap();
+    file_to_json_object::<WithContext<Note>>("assets/lotide/objects/note.json").unwrap();
+    file_to_json_object::<WithContext<Page>>("assets/lotide/objects/page.json").unwrap();
   }
 }

crates/apub/src/protocol/objects/page.rs

@@ -2,7 +2,6 @@ use crate::{
   objects::{community::ApubCommunity, person::ApubPerson, post::ApubPost},
   protocol::{ImageObject, Source, Unparsed},
 };
-use activitystreams_kinds::object::PageType;
 use chrono::{DateTime, FixedOffset};
 use lemmy_apub_lib::{
   data::Data,
@@ -16,6 +15,12 @@ use serde::{Deserialize, Serialize};
 use serde_with::skip_serializing_none;
 use url::Url;
 
+#[derive(Clone, Debug, Deserialize, Serialize)]
+pub enum PageType {
+  Page,
+  Note,
+}
+
 #[skip_serializing_none]
 #[derive(Clone, Debug, Deserialize, Serialize)]
 #[serde(rename_all = "camelCase")]

crates/apub/src/protocol/objects/person.rs

@@ -35,7 +35,7 @@ pub struct Person {
   pub(crate) inbox: Url,
   /// mandatory field in activitypub, currently empty in lemmy
   pub(crate) outbox: Url,
-  pub(crate) endpoints: Endpoints,
+  pub(crate) endpoints: Option<Endpoints>,
   pub(crate) public_key: PublicKey,
   pub(crate) published: Option<DateTime<FixedOffset>>,
   pub(crate) updated: Option<DateTime<FixedOffset>>,