From 023c9f4fcdc4af195eabcd8f07f1d789fdaf9bb8 Mon Sep 17 00:00:00 2001 From: Nutomic Date: Thu, 4 Jan 2024 17:42:18 +0100 Subject: [PATCH] Fix fetching of community posts (fixes #4283) (#4293) * Fix fetching of community posts (fixes #4283) Also use spawn_try_task to fetch community outbox, mods etc to avoid delay/timeout when fetching new community. * prettier * fix test * fix api test * prettier * add delay * Update run-federation-test.sh * fix test --- api_tests/src/community.spec.ts | 43 ++++++++++++++++--- api_tests/src/shared.ts | 2 + .../apub/src/collections/community_outbox.rs | 14 +++--- crates/apub/src/objects/community.rs | 30 +++++-------- 4 files changed, 59 insertions(+), 30 deletions(-) diff --git a/api_tests/src/community.spec.ts b/api_tests/src/community.spec.ts index b4a58bb7b..03c941abe 100644 --- a/api_tests/src/community.spec.ts +++ b/api_tests/src/community.spec.ts @@ -31,6 +31,7 @@ import { searchPostLocal, resolveBetaCommunity, longDelay, + delay, } from "./shared"; import { EditSite } from "lemmy-js-client"; @@ -377,7 +378,9 @@ test("User blocks instance, communities are hidden", async () => { test("Community follower count is federated", async () => { // Follow the beta community from alpha - let resolved = await resolveBetaCommunity(alpha); + let community = await createCommunity(beta); + let community_id = community.community_view.community.actor_id; + let resolved = await resolveCommunity(alpha, community_id); if (!resolved.community) { throw "Missing beta community"; } @@ -385,7 +388,7 @@ test("Community follower count is federated", async () => { await followCommunity(alpha, true, resolved.community.community.id); let followed = ( await waitUntil( - () => resolveBetaCommunity(alpha), + () => resolveCommunity(alpha, community_id), c => c.community?.subscribed === "Subscribed", ) ).community; @@ -394,7 +397,7 @@ test("Community follower count is federated", async () => { expect(followed?.counts.subscribers).toBe(1); // Follow the community from gamma - resolved = await resolveBetaCommunity(gamma); + resolved = await resolveCommunity(gamma, community_id); if (!resolved.community) { throw "Missing beta community"; } @@ -402,7 +405,7 @@ test("Community follower count is federated", async () => { await followCommunity(gamma, true, resolved.community.community.id); followed = ( await waitUntil( - () => resolveBetaCommunity(gamma), + () => resolveCommunity(gamma, community_id), c => c.community?.subscribed === "Subscribed", ) ).community; @@ -411,7 +414,7 @@ test("Community follower count is federated", async () => { expect(followed?.counts?.subscribers).toBe(2); // Follow the community from delta - resolved = await resolveBetaCommunity(delta); + resolved = await resolveCommunity(delta, community_id); if (!resolved.community) { throw "Missing beta community"; } @@ -419,7 +422,7 @@ test("Community follower count is federated", async () => { await followCommunity(delta, true, resolved.community.community.id); followed = ( await waitUntil( - () => resolveBetaCommunity(delta), + () => resolveCommunity(delta, community_id), c => c.community?.subscribed === "Subscribed", ) ).community; @@ -480,3 +483,31 @@ test("Dont receive community activities after unsubscribe", async () => { let postResBeta = searchPostLocal(beta, postRes.post_view.post); expect((await postResBeta).posts.length).toBe(0); }); + +test("Fetch community, includes posts", async () => { + let communityRes = await createCommunity(alpha); + expect(communityRes.community_view.community.name).toBeDefined(); + expect(communityRes.community_view.counts.subscribers).toBe(1); + + let postRes = await createPost( + alpha, + communityRes.community_view.community.id, + ); + expect(postRes.post_view.post).toBeDefined(); + + let resolvedCommunity = await waitUntil( + () => + resolveCommunity(beta, communityRes.community_view.community.actor_id), + c => c.community?.community.id != undefined, + ); + let betaCommunity = resolvedCommunity.community; + expect(betaCommunity?.community.actor_id).toBe( + communityRes.community_view.community.actor_id, + ); + + await longDelay(); + + let post_listing = await getPosts(beta, "All", betaCommunity?.community.id); + expect(post_listing.posts.length).toBe(1); + expect(post_listing.posts[0].post.ap_id).toBe(postRes.post_view.post.ap_id); +}); diff --git a/api_tests/src/shared.ts b/api_tests/src/shared.ts index fe51fb046..9a2d45075 100644 --- a/api_tests/src/shared.ts +++ b/api_tests/src/shared.ts @@ -805,10 +805,12 @@ export async function listCommentReports( export function getPosts( api: LemmyHttp, listingType?: ListingType, + community_id?: number, ): Promise { let form: GetPosts = { type_: listingType, limit: 50, + community_id, }; return api.getPosts(form); } diff --git a/crates/apub/src/collections/community_outbox.rs b/crates/apub/src/collections/community_outbox.rs index 854db9349..8e319403d 100644 --- a/crates/apub/src/collections/community_outbox.rs +++ b/crates/apub/src/collections/community_outbox.rs @@ -96,11 +96,15 @@ impl Collection for ApubCommunityOutbox { // process items in parallel, to avoid long delay from fetch_site_metadata() and other processing join_all(outbox_activities.into_iter().map(|activity| { async { - // use separate request counter for each item, otherwise there will be problems with - // parallel processing - let verify = activity.verify(data).await; - if verify.is_ok() { - activity.receive(data).await.ok(); + // Receiving announce requires at least one local community follower for anti spam purposes. + // This won't be the case for newly fetched communities, so we extract the inner activity + // and handle it directly to bypass this check. + let inner = activity.object.object(data).await.map(TryInto::try_into); + if let Ok(Ok(AnnouncableActivities::CreateOrUpdatePost(inner))) = inner { + let verify = inner.verify(data).await; + if verify.is_ok() { + inner.receive(data).await.ok(); + } } } })) diff --git a/crates/apub/src/objects/community.rs b/crates/apub/src/objects/community.rs index ced4b85d7..3cbf352cd 100644 --- a/crates/apub/src/objects/community.rs +++ b/crates/apub/src/objects/community.rs @@ -28,9 +28,8 @@ use lemmy_db_schema::{ traits::{ApubActor, Crud}, }; use lemmy_db_views_actor::structs::CommunityFollowerView; -use lemmy_utils::{error::LemmyError, utils::markdown::markdown_to_html}; +use lemmy_utils::{error::LemmyError, spawn_try_task, utils::markdown::markdown_to_html}; use std::ops::Deref; -use tracing::debug; use url::Url; #[derive(Clone, Debug)] @@ -142,21 +141,16 @@ impl Object for ApubCommunity { // Fetching mods and outbox is not necessary for Lemmy to work, so ignore errors. Besides, // we need to ignore these errors so that tests can work entirely offline. - let fetch_outbox = group.outbox.dereference(&community, context); - let fetch_followers = group.followers.dereference(&community, context); - - if let Some(moderators) = group.attributed_to { - let fetch_moderators = moderators.dereference(&community, context); - // Fetch mods, outbox and followers in parallel - let res = tokio::join!(fetch_outbox, fetch_moderators, fetch_followers); - res.0.map_err(|e| debug!("{}", e)).ok(); - res.1.map_err(|e| debug!("{}", e)).ok(); - res.2.map_err(|e| debug!("{}", e)).ok(); - } else { - let res = tokio::join!(fetch_outbox, fetch_followers); - res.0.map_err(|e| debug!("{}", e)).ok(); - res.1.map_err(|e| debug!("{}", e)).ok(); - } + let community_ = community.clone(); + let context_ = context.reset_request_count(); + spawn_try_task(async move { + group.outbox.dereference(&community_, &context_).await?; + group.followers.dereference(&community_, &context_).await?; + if let Some(moderators) = group.attributed_to { + moderators.dereference(&community_, &context_).await?; + } + Ok(()) + }); Ok(community) } @@ -241,8 +235,6 @@ pub(crate) mod tests { let url = Url::parse("https://enterprise.lemmy.ml/c/tenforward")?; ApubCommunity::verify(&json, &url, &context2).await?; let community = ApubCommunity::from_json(json, &context2).await?; - // this makes requests to the (intentionally broken) outbox and followers collections - assert_eq!(context2.request_count(), 2); Ok(community) }