Fix fetching of community posts (fixes #4283) (#4293)

* Fix fetching of community posts (fixes #4283)

Also use spawn_try_task to fetch community outbox, mods etc to avoid
delay/timeout when fetching new community.

* prettier

* fix test

* fix api test

* prettier

* add delay

* Update run-federation-test.sh

* fix test
This commit is contained in:
Nutomic 2024-01-04 17:42:18 +01:00 committed by GitHub
parent 952c162dac
commit 023c9f4fcd
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 59 additions and 30 deletions

View file

@ -31,6 +31,7 @@ import {
searchPostLocal, searchPostLocal,
resolveBetaCommunity, resolveBetaCommunity,
longDelay, longDelay,
delay,
} from "./shared"; } from "./shared";
import { EditSite } from "lemmy-js-client"; import { EditSite } from "lemmy-js-client";
@ -377,7 +378,9 @@ test("User blocks instance, communities are hidden", async () => {
test("Community follower count is federated", async () => { test("Community follower count is federated", async () => {
// Follow the beta community from alpha // Follow the beta community from alpha
let resolved = await resolveBetaCommunity(alpha); let community = await createCommunity(beta);
let community_id = community.community_view.community.actor_id;
let resolved = await resolveCommunity(alpha, community_id);
if (!resolved.community) { if (!resolved.community) {
throw "Missing beta community"; throw "Missing beta community";
} }
@ -385,7 +388,7 @@ test("Community follower count is federated", async () => {
await followCommunity(alpha, true, resolved.community.community.id); await followCommunity(alpha, true, resolved.community.community.id);
let followed = ( let followed = (
await waitUntil( await waitUntil(
() => resolveBetaCommunity(alpha), () => resolveCommunity(alpha, community_id),
c => c.community?.subscribed === "Subscribed", c => c.community?.subscribed === "Subscribed",
) )
).community; ).community;
@ -394,7 +397,7 @@ test("Community follower count is federated", async () => {
expect(followed?.counts.subscribers).toBe(1); expect(followed?.counts.subscribers).toBe(1);
// Follow the community from gamma // Follow the community from gamma
resolved = await resolveBetaCommunity(gamma); resolved = await resolveCommunity(gamma, community_id);
if (!resolved.community) { if (!resolved.community) {
throw "Missing beta community"; throw "Missing beta community";
} }
@ -402,7 +405,7 @@ test("Community follower count is federated", async () => {
await followCommunity(gamma, true, resolved.community.community.id); await followCommunity(gamma, true, resolved.community.community.id);
followed = ( followed = (
await waitUntil( await waitUntil(
() => resolveBetaCommunity(gamma), () => resolveCommunity(gamma, community_id),
c => c.community?.subscribed === "Subscribed", c => c.community?.subscribed === "Subscribed",
) )
).community; ).community;
@ -411,7 +414,7 @@ test("Community follower count is federated", async () => {
expect(followed?.counts?.subscribers).toBe(2); expect(followed?.counts?.subscribers).toBe(2);
// Follow the community from delta // Follow the community from delta
resolved = await resolveBetaCommunity(delta); resolved = await resolveCommunity(delta, community_id);
if (!resolved.community) { if (!resolved.community) {
throw "Missing beta community"; throw "Missing beta community";
} }
@ -419,7 +422,7 @@ test("Community follower count is federated", async () => {
await followCommunity(delta, true, resolved.community.community.id); await followCommunity(delta, true, resolved.community.community.id);
followed = ( followed = (
await waitUntil( await waitUntil(
() => resolveBetaCommunity(delta), () => resolveCommunity(delta, community_id),
c => c.community?.subscribed === "Subscribed", c => c.community?.subscribed === "Subscribed",
) )
).community; ).community;
@ -480,3 +483,31 @@ test("Dont receive community activities after unsubscribe", async () => {
let postResBeta = searchPostLocal(beta, postRes.post_view.post); let postResBeta = searchPostLocal(beta, postRes.post_view.post);
expect((await postResBeta).posts.length).toBe(0); expect((await postResBeta).posts.length).toBe(0);
}); });
test("Fetch community, includes posts", async () => {
let communityRes = await createCommunity(alpha);
expect(communityRes.community_view.community.name).toBeDefined();
expect(communityRes.community_view.counts.subscribers).toBe(1);
let postRes = await createPost(
alpha,
communityRes.community_view.community.id,
);
expect(postRes.post_view.post).toBeDefined();
let resolvedCommunity = await waitUntil(
() =>
resolveCommunity(beta, communityRes.community_view.community.actor_id),
c => c.community?.community.id != undefined,
);
let betaCommunity = resolvedCommunity.community;
expect(betaCommunity?.community.actor_id).toBe(
communityRes.community_view.community.actor_id,
);
await longDelay();
let post_listing = await getPosts(beta, "All", betaCommunity?.community.id);
expect(post_listing.posts.length).toBe(1);
expect(post_listing.posts[0].post.ap_id).toBe(postRes.post_view.post.ap_id);
});

View file

@ -805,10 +805,12 @@ export async function listCommentReports(
export function getPosts( export function getPosts(
api: LemmyHttp, api: LemmyHttp,
listingType?: ListingType, listingType?: ListingType,
community_id?: number,
): Promise<GetPostsResponse> { ): Promise<GetPostsResponse> {
let form: GetPosts = { let form: GetPosts = {
type_: listingType, type_: listingType,
limit: 50, limit: 50,
community_id,
}; };
return api.getPosts(form); return api.getPosts(form);
} }

View file

@ -96,11 +96,15 @@ impl Collection for ApubCommunityOutbox {
// process items in parallel, to avoid long delay from fetch_site_metadata() and other processing // process items in parallel, to avoid long delay from fetch_site_metadata() and other processing
join_all(outbox_activities.into_iter().map(|activity| { join_all(outbox_activities.into_iter().map(|activity| {
async { async {
// use separate request counter for each item, otherwise there will be problems with // Receiving announce requires at least one local community follower for anti spam purposes.
// parallel processing // This won't be the case for newly fetched communities, so we extract the inner activity
let verify = activity.verify(data).await; // and handle it directly to bypass this check.
if verify.is_ok() { let inner = activity.object.object(data).await.map(TryInto::try_into);
activity.receive(data).await.ok(); if let Ok(Ok(AnnouncableActivities::CreateOrUpdatePost(inner))) = inner {
let verify = inner.verify(data).await;
if verify.is_ok() {
inner.receive(data).await.ok();
}
} }
} }
})) }))

View file

@ -28,9 +28,8 @@ use lemmy_db_schema::{
traits::{ApubActor, Crud}, traits::{ApubActor, Crud},
}; };
use lemmy_db_views_actor::structs::CommunityFollowerView; use lemmy_db_views_actor::structs::CommunityFollowerView;
use lemmy_utils::{error::LemmyError, utils::markdown::markdown_to_html}; use lemmy_utils::{error::LemmyError, spawn_try_task, utils::markdown::markdown_to_html};
use std::ops::Deref; use std::ops::Deref;
use tracing::debug;
use url::Url; use url::Url;
#[derive(Clone, Debug)] #[derive(Clone, Debug)]
@ -142,21 +141,16 @@ impl Object for ApubCommunity {
// Fetching mods and outbox is not necessary for Lemmy to work, so ignore errors. Besides, // Fetching mods and outbox is not necessary for Lemmy to work, so ignore errors. Besides,
// we need to ignore these errors so that tests can work entirely offline. // we need to ignore these errors so that tests can work entirely offline.
let fetch_outbox = group.outbox.dereference(&community, context); let community_ = community.clone();
let fetch_followers = group.followers.dereference(&community, context); let context_ = context.reset_request_count();
spawn_try_task(async move {
if let Some(moderators) = group.attributed_to { group.outbox.dereference(&community_, &context_).await?;
let fetch_moderators = moderators.dereference(&community, context); group.followers.dereference(&community_, &context_).await?;
// Fetch mods, outbox and followers in parallel if let Some(moderators) = group.attributed_to {
let res = tokio::join!(fetch_outbox, fetch_moderators, fetch_followers); moderators.dereference(&community_, &context_).await?;
res.0.map_err(|e| debug!("{}", e)).ok(); }
res.1.map_err(|e| debug!("{}", e)).ok(); Ok(())
res.2.map_err(|e| debug!("{}", e)).ok(); });
} else {
let res = tokio::join!(fetch_outbox, fetch_followers);
res.0.map_err(|e| debug!("{}", e)).ok();
res.1.map_err(|e| debug!("{}", e)).ok();
}
Ok(community) Ok(community)
} }
@ -241,8 +235,6 @@ pub(crate) mod tests {
let url = Url::parse("https://enterprise.lemmy.ml/c/tenforward")?; let url = Url::parse("https://enterprise.lemmy.ml/c/tenforward")?;
ApubCommunity::verify(&json, &url, &context2).await?; ApubCommunity::verify(&json, &url, &context2).await?;
let community = ApubCommunity::from_json(json, &context2).await?; let community = ApubCommunity::from_json(json, &context2).await?;
// this makes requests to the (intentionally broken) outbox and followers collections
assert_eq!(context2.request_count(), 2);
Ok(community) Ok(community)
} }