From 63ea99d38ade254bd428b5d031be0feec123315f Mon Sep 17 00:00:00 2001 From: Nutomic Date: Tue, 19 Nov 2024 15:48:44 +0100 Subject: [PATCH 1/8] Guess image mime type from file extension (fixes #5196) (#5212) * Guess image mime type from file extension (fixes #5196) * Mime check fixes. (#5213) * Mime check fixes. * Adding back comment. --------- Co-authored-by: Dessalines --- Cargo.lock | 19 ++++++++++++++++++- crates/api_common/Cargo.toml | 1 + crates/api_common/src/request.rs | 20 ++++++++++++++------ 3 files changed, 33 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 54857b3a0..b62cf624c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,6 +1,6 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 +version = 4 [[package]] name = "accept-language" @@ -2515,6 +2515,7 @@ dependencies = [ "lemmy_db_views_moderator", "lemmy_utils", "mime", + "mime_guess", "moka", "pretty_assertions", "regex", @@ -3147,6 +3148,16 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "mime_guess" +version = "2.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e" +dependencies = [ + "mime", + "unicase", +] + [[package]] name = "minimal-lexical" version = "0.2.1" @@ -5300,6 +5311,12 @@ version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" +[[package]] +name = "unicase" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e51b68083f157f853b6379db119d1c1be0e6e4dec98101079dec41f6f5cf6df" + [[package]] name = "unicode-bidi" version = "0.3.15" diff --git a/crates/api_common/Cargo.toml b/crates/api_common/Cargo.toml index 7b987cbb7..76f531ddb 100644 --- a/crates/api_common/Cargo.toml +++ b/crates/api_common/Cargo.toml @@ -64,6 +64,7 @@ actix-web = { workspace = true, optional = true } enum-map = { workspace = true } urlencoding = { workspace = true } mime = { version = "0.3.17", optional = true } +mime_guess = "2.0.5" webpage = { version = "2.0", default-features = false, features = [ "serde", ], optional = true } diff --git a/crates/api_common/src/request.rs b/crates/api_common/src/request.rs index 36010f760..cc506b896 100644 --- a/crates/api_common/src/request.rs +++ b/crates/api_common/src/request.rs @@ -23,7 +23,6 @@ use lemmy_utils::{ REQWEST_TIMEOUT, VERSION, }; -use mime::Mime; use reqwest::{ header::{CONTENT_TYPE, RANGE}, Client, @@ -64,11 +63,20 @@ pub async fn fetch_link_metadata(url: &Url, context: &LemmyContext) -> LemmyResu .await? .error_for_status()?; - let content_type: Option = response - .headers() - .get(CONTENT_TYPE) - .and_then(|h| h.to_str().ok()) - .and_then(|h| h.parse().ok()); + // In some cases servers send a wrong mime type for images, which prevents thumbnail + // generation. To avoid this we also try to guess the mime type from file extension. + let content_type = mime_guess::from_path(url.path()) + .first() + // If you can guess that its an image type, then return that first. + .filter(|guess| guess.type_() == mime::IMAGE) + // Otherwise, get the content type from the headers + .or( + response + .headers() + .get(CONTENT_TYPE) + .and_then(|h| h.to_str().ok()) + .and_then(|h| h.parse().ok()), + ); let opengraph_data = { // if the content type is not text/html, we don't need to parse it From 2848c076af78d62e90a73290a0b3aeedbc725393 Mon Sep 17 00:00:00 2001 From: Nutomic Date: Fri, 22 Nov 2024 14:33:35 +0000 Subject: [PATCH 2/8] Add helper method for caching function results (#5220) * Add helper method for caching function results * fmt --- Cargo.lock | 3 +- crates/api_common/src/utils.rs | 3 +- crates/api_crud/Cargo.toml | 1 - crates/api_crud/src/site/read.rs | 67 +++++++++++------------- crates/apub/src/lib.rs | 3 +- crates/db_schema/Cargo.toml | 1 - crates/db_schema/src/impls/local_site.rs | 10 +--- crates/utils/Cargo.toml | 50 +++++++++--------- crates/utils/src/lib.rs | 22 +++++++- 9 files changed, 85 insertions(+), 75 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b62cf624c..edc6722bf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2550,7 +2550,6 @@ dependencies = [ "lemmy_db_views", "lemmy_db_views_actor", "lemmy_utils", - "moka", "serde", "serde_json", "serde_with", @@ -2632,7 +2631,6 @@ dependencies = [ "futures-util", "i-love-jesus", "lemmy_utils", - "moka", "pretty_assertions", "regex", "rustls 0.23.16", @@ -2819,6 +2817,7 @@ dependencies = [ "markdown-it-ruby", "markdown-it-sub", "markdown-it-sup", + "moka", "pretty_assertions", "regex", "reqwest-middleware", diff --git a/crates/api_common/src/utils.rs b/crates/api_common/src/utils.rs index 48b339e65..d46e57749 100644 --- a/crates/api_common/src/utils.rs +++ b/crates/api_common/src/utils.rs @@ -60,6 +60,7 @@ use lemmy_utils::{ slurs::{build_slur_regex, remove_slurs}, validation::clean_urls_in_text, }, + CacheLock, CACHE_DURATION_FEDERATION, }; use moka::future::Cache; @@ -535,7 +536,7 @@ pub fn local_site_opt_to_slur_regex(local_site: &Option) -> Option LemmyResult { - static URL_BLOCKLIST: LazyLock> = LazyLock::new(|| { + static URL_BLOCKLIST: CacheLock = LazyLock::new(|| { Cache::builder() .max_capacity(1) .time_to_live(CACHE_DURATION_FEDERATION) diff --git a/crates/api_crud/Cargo.toml b/crates/api_crud/Cargo.toml index 723864705..3f1a00ccd 100644 --- a/crates/api_crud/Cargo.toml +++ b/crates/api_crud/Cargo.toml @@ -25,7 +25,6 @@ tracing = { workspace = true } url = { workspace = true } futures.workspace = true uuid = { workspace = true } -moka.workspace = true anyhow.workspace = true chrono.workspace = true webmention = "0.6.0" diff --git a/crates/api_crud/src/site/read.rs b/crates/api_crud/src/site/read.rs index 47fd1f154..6bee0fda6 100644 --- a/crates/api_crud/src/site/read.rs +++ b/crates/api_crud/src/site/read.rs @@ -16,11 +16,11 @@ use lemmy_db_schema::source::{ use lemmy_db_views::structs::{LocalUserView, SiteView}; use lemmy_db_views_actor::structs::{CommunityFollowerView, CommunityModeratorView, PersonView}; use lemmy_utils::{ - error::{LemmyError, LemmyErrorExt, LemmyErrorType, LemmyResult}, - CACHE_DURATION_API, + build_cache, + error::{LemmyErrorExt, LemmyErrorType, LemmyResult}, + CacheLock, VERSION, }; -use moka::future::Cache; use std::sync::LazyLock; #[tracing::instrument(skip(context))] @@ -28,41 +28,10 @@ pub async fn get_site( local_user_view: Option, context: Data, ) -> LemmyResult> { - static CACHE: LazyLock> = LazyLock::new(|| { - Cache::builder() - .max_capacity(1) - .time_to_live(CACHE_DURATION_API) - .build() - }); - // This data is independent from the user account so we can cache it across requests + static CACHE: CacheLock = LazyLock::new(build_cache); let mut site_response = CACHE - .try_get_with::<_, LemmyError>((), async { - let site_view = SiteView::read_local(&mut context.pool()).await?; - let admins = PersonView::admins(&mut context.pool()).await?; - let all_languages = Language::read_all(&mut context.pool()).await?; - let discussion_languages = SiteLanguage::read_local_raw(&mut context.pool()).await?; - let blocked_urls = LocalSiteUrlBlocklist::get_all(&mut context.pool()).await?; - let tagline = Tagline::get_random(&mut context.pool()).await.ok(); - let admin_oauth_providers = OAuthProvider::get_all(&mut context.pool()).await?; - let oauth_providers = - OAuthProvider::convert_providers_to_public(admin_oauth_providers.clone()); - - Ok(GetSiteResponse { - site_view, - admins, - version: VERSION.to_string(), - my_user: None, - all_languages, - discussion_languages, - blocked_urls, - tagline, - oauth_providers: Some(oauth_providers), - admin_oauth_providers: Some(admin_oauth_providers), - taglines: vec![], - custom_emojis: vec![], - }) - }) + .try_get_with((), read_site(&context)) .await .map_err(|e| anyhow::anyhow!("Failed to construct site response: {e}"))?; @@ -112,3 +81,29 @@ pub async fn get_site( Ok(Json(site_response)) } + +async fn read_site(context: &LemmyContext) -> LemmyResult { + let site_view = SiteView::read_local(&mut context.pool()).await?; + let admins = PersonView::admins(&mut context.pool()).await?; + let all_languages = Language::read_all(&mut context.pool()).await?; + let discussion_languages = SiteLanguage::read_local_raw(&mut context.pool()).await?; + let blocked_urls = LocalSiteUrlBlocklist::get_all(&mut context.pool()).await?; + let tagline = Tagline::get_random(&mut context.pool()).await.ok(); + let admin_oauth_providers = OAuthProvider::get_all(&mut context.pool()).await?; + let oauth_providers = OAuthProvider::convert_providers_to_public(admin_oauth_providers.clone()); + + Ok(GetSiteResponse { + site_view, + admins, + version: VERSION.to_string(), + my_user: None, + all_languages, + discussion_languages, + blocked_urls, + tagline, + oauth_providers: Some(oauth_providers), + admin_oauth_providers: Some(admin_oauth_providers), + taglines: vec![], + custom_emojis: vec![], + }) +} diff --git a/crates/apub/src/lib.rs b/crates/apub/src/lib.rs index 213f6439d..028d673c2 100644 --- a/crates/apub/src/lib.rs +++ b/crates/apub/src/lib.rs @@ -11,6 +11,7 @@ use lemmy_db_schema::{ }; use lemmy_utils::{ error::{FederationError, LemmyError, LemmyErrorType, LemmyResult}, + CacheLock, CACHE_DURATION_FEDERATION, }; use moka::future::Cache; @@ -139,7 +140,7 @@ pub(crate) async fn local_site_data_cached( // multiple times. This causes a huge number of database reads if we hit the db directly. So we // cache these values for a short time, which will already make a huge difference and ensures that // changes take effect quickly. - static CACHE: LazyLock>> = LazyLock::new(|| { + static CACHE: CacheLock> = LazyLock::new(|| { Cache::builder() .max_capacity(1) .time_to_live(CACHE_DURATION_FEDERATION) diff --git a/crates/db_schema/Cargo.toml b/crates/db_schema/Cargo.toml index c52629ce3..bd83ede29 100644 --- a/crates/db_schema/Cargo.toml +++ b/crates/db_schema/Cargo.toml @@ -79,7 +79,6 @@ uuid = { workspace = true, features = ["v4"] } i-love-jesus = { workspace = true, optional = true } anyhow = { workspace = true } diesel-bind-if-some = { workspace = true, optional = true } -moka.workspace = true derive-new.workspace = true tuplex = { workspace = true, optional = true } diff --git a/crates/db_schema/src/impls/local_site.rs b/crates/db_schema/src/impls/local_site.rs index 926814c48..bdbe4ac6c 100644 --- a/crates/db_schema/src/impls/local_site.rs +++ b/crates/db_schema/src/impls/local_site.rs @@ -5,8 +5,7 @@ use crate::{ }; use diesel::{dsl::insert_into, result::Error}; use diesel_async::RunQueryDsl; -use lemmy_utils::{error::LemmyResult, CACHE_DURATION_API}; -use moka::future::Cache; +use lemmy_utils::{build_cache, error::LemmyResult, CacheLock}; use std::sync::LazyLock; impl LocalSite { @@ -18,12 +17,7 @@ impl LocalSite { .await } pub async fn read(pool: &mut DbPool<'_>) -> LemmyResult { - static CACHE: LazyLock> = LazyLock::new(|| { - Cache::builder() - .max_capacity(1) - .time_to_live(CACHE_DURATION_API) - .build() - }); + static CACHE: CacheLock = LazyLock::new(build_cache); Ok( CACHE .try_get_with((), async { diff --git a/crates/utils/Cargo.toml b/crates/utils/Cargo.toml index cd4ea5e9b..7ed4c0476 100644 --- a/crates/utils/Cargo.toml +++ b/crates/utils/Cargo.toml @@ -23,30 +23,31 @@ workspace = true [features] full = [ - "dep:ts-rs", - "dep:diesel", - "dep:rosetta-i18n", - "dep:actix-web", - "dep:reqwest-middleware", - "dep:tracing", - "dep:actix-web", - "dep:serde_json", - "dep:anyhow", - "dep:http", - "dep:deser-hjson", - "dep:regex", - "dep:urlencoding", - "dep:doku", - "dep:url", - "dep:smart-default", - "dep:enum-map", - "dep:futures", - "dep:tokio", - "dep:html2text", - "dep:lettre", - "dep:uuid", - "dep:itertools", - "dep:markdown-it", + "ts-rs", + "diesel", + "rosetta-i18n", + "actix-web", + "reqwest-middleware", + "tracing", + "actix-web", + "serde_json", + "anyhow", + "http", + "deser-hjson", + "regex", + "urlencoding", + "doku", + "url", + "smart-default", + "enum-map", + "futures", + "tokio", + "html2text", + "lettre", + "uuid", + "itertools", + "markdown-it", + "moka", ] [package.metadata.cargo-shear] @@ -89,6 +90,7 @@ markdown-it-block-spoiler = "1.0.0" markdown-it-sub = "1.0.0" markdown-it-sup = "1.0.0" markdown-it-ruby = "1.0.0" +moka = { workspace = true, optional = true } [dev-dependencies] pretty_assertions = { workspace = true } diff --git a/crates/utils/src/lib.rs b/crates/utils/src/lib.rs index 1e0cbefbf..3367c91bb 100644 --- a/crates/utils/src/lib.rs +++ b/crates/utils/src/lib.rs @@ -42,7 +42,10 @@ macro_rules! location_info { }; } -#[cfg(feature = "full")] +cfg_if! { + if #[cfg(feature = "full")] { +use moka::future::Cache;use std::fmt::Debug;use std::hash::Hash; + /// tokio::spawn, but accepts a future that may fail and also /// * logs errors /// * attaches the spawned task to the tracing span of the caller for better logging @@ -60,3 +63,20 @@ pub fn spawn_try_task( * spawn was called */ ); } + +pub fn build_cache() -> Cache +where + K: Debug + Eq + Hash + Send + Sync + 'static, + V: Debug + Clone + Send + Sync + 'static, +{ + Cache::::builder() + .max_capacity(1) + .time_to_live(CACHE_DURATION_API) + .build() +} + +#[cfg(feature = "full")] +pub type CacheLock = std::sync::LazyLock>; + + } +} From b8dda06f5b80efeb06eb2b39d747daafbbbbbefd Mon Sep 17 00:00:00 2001 From: Nutomic Date: Fri, 22 Nov 2024 15:49:56 +0000 Subject: [PATCH 3/8] More test coverage for private community, fix some bugs (#5207) * More test coverage for private community, fix some bugs * fmt * stuff * attempted fix * fix tests * api tests passing * fix tests * apub lib --- Cargo.lock | 4 +- Cargo.toml | 2 +- api_tests/src/private_community.spec.ts | 165 ++++++++++++++++-- crates/api_common/src/build_response.rs | 35 ++-- .../apub/src/activities/community/announce.rs | 2 +- crates/apub/src/activities/community/mod.rs | 2 +- .../activities/create_or_update/comment.rs | 3 + crates/apub/src/http/community.rs | 141 +++++++++++---- crates/apub/src/http/mod.rs | 28 ++- .../src/community_follower_view.rs | 19 ++ 10 files changed, 332 insertions(+), 69 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index edc6722bf..c1b1f3673 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10,9 +10,9 @@ checksum = "8f27d075294830fcab6f66e320dab524bc6d048f4a151698e153205559113772" [[package]] name = "activitypub_federation" -version = "0.6.0-alpha2" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4877d467ddf2fac85e9ee33aba6f2560df14125b8bfa864f85ab40e9b87753a9" +checksum = "ee819cada736b6e26c59706f9e6ff89a48060e635c0546ff984d84baefc8c13a" dependencies = [ "activitystreams-kinds", "actix-web", diff --git a/Cargo.toml b/Cargo.toml index d9c868ee7..bce88af67 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -94,7 +94,7 @@ lemmy_db_views = { version = "=0.19.6-beta.7", path = "./crates/db_views" } lemmy_db_views_actor = { version = "=0.19.6-beta.7", path = "./crates/db_views_actor" } lemmy_db_views_moderator = { version = "=0.19.6-beta.7", path = "./crates/db_views_moderator" } lemmy_federate = { version = "=0.19.6-beta.7", path = "./crates/federate" } -activitypub_federation = { version = "0.6.0-alpha2", default-features = false, features = [ +activitypub_federation = { version = "0.6.1", default-features = false, features = [ "actix-web", ] } diesel = "2.2.4" diff --git a/api_tests/src/private_community.spec.ts b/api_tests/src/private_community.spec.ts index 76faf800f..65340a1dd 100644 --- a/api_tests/src/private_community.spec.ts +++ b/api_tests/src/private_community.spec.ts @@ -1,6 +1,6 @@ jest.setTimeout(120000); -import { FollowCommunity } from "lemmy-js-client"; +import { FollowCommunity, LemmyHttp } from "lemmy-js-client"; import { alpha, setupLogins, @@ -21,6 +21,9 @@ import { resolveComment, likeComment, waitUntil, + gamma, + getPosts, + getComments, } from "./shared"; beforeAll(setupLogins); @@ -47,6 +50,7 @@ test("Follow a private community", async () => { await resolveCommunity(user, community.community_view.community.actor_id) ).community; expect(betaCommunity).toBeDefined(); + expect(betaCommunity?.community.visibility).toBe("Private"); const betaCommunityId = betaCommunity!.community.id; const follow_form: FollowCommunity = { community_id: betaCommunityId, @@ -148,16 +152,7 @@ test("Only followers can view and interact with private community content", asyn follow: true, }; await user.followCommunity(follow_form); - const pendingFollows1 = await waitUntil( - () => listCommunityPendingFollows(alpha), - f => f.items.length == 1, - ); - const approve = await approveCommunityPendingFollow( - alpha, - alphaCommunityId, - pendingFollows1.items[0].person.id, - ); - expect(approve.success).toBe(true); + approveFollower(alpha, alphaCommunityId); // now user can fetch posts and comments in community (using signed fetch), and create posts await waitUntil( @@ -212,3 +207,151 @@ test("Reject follower", async () => { c => c.community_view.subscribed == "NotSubscribed", ); }); + +test("Follow a private community and receive activities", async () => { + // create private community + const community = await createCommunity(alpha, randomString(10), "Private"); + expect(community.community_view.community.visibility).toBe("Private"); + const alphaCommunityId = community.community_view.community.id; + + // follow with users from beta and gamma + const betaCommunity = ( + await resolveCommunity(beta, community.community_view.community.actor_id) + ).community; + expect(betaCommunity).toBeDefined(); + const betaCommunityId = betaCommunity!.community.id; + const follow_form_beta: FollowCommunity = { + community_id: betaCommunityId, + follow: true, + }; + await beta.followCommunity(follow_form_beta); + await approveFollower(alpha, alphaCommunityId); + + const gammaCommunityId = ( + await resolveCommunity(gamma, community.community_view.community.actor_id) + ).community!.community.id; + const follow_form_gamma: FollowCommunity = { + community_id: gammaCommunityId, + follow: true, + }; + await gamma.followCommunity(follow_form_gamma); + await approveFollower(alpha, alphaCommunityId); + + // Follow is confirmed + await waitUntil( + () => getCommunity(beta, betaCommunityId), + c => c.community_view.subscribed == "Subscribed", + ); + await waitUntil( + () => getCommunity(gamma, gammaCommunityId), + c => c.community_view.subscribed == "Subscribed", + ); + + // create a post and comment from gamma + const post = await createPost(gamma, gammaCommunityId); + const post_id = post.post_view.post.id; + expect(post_id).toBeDefined(); + const comment = await createComment(gamma, post_id); + const comment_id = comment.comment_view.comment.id; + expect(comment_id).toBeDefined(); + + // post and comment were federated to beta + let posts = await waitUntil( + () => getPosts(beta, "All", betaCommunityId), + c => c.posts.length == 1, + ); + expect(posts.posts[0].post.ap_id).toBe(post.post_view.post.ap_id); + expect(posts.posts[0].post.name).toBe(post.post_view.post.name); + let comments = await waitUntil( + () => getComments(beta, posts.posts[0].post.id), + c => c.comments.length == 1, + ); + expect(comments.comments[0].comment.ap_id).toBe( + comment.comment_view.comment.ap_id, + ); + expect(comments.comments[0].comment.content).toBe( + comment.comment_view.comment.content, + ); +}); + +test("Fetch remote content in private community", async () => { + // create private community + const community = await createCommunity(alpha, randomString(10), "Private"); + expect(community.community_view.community.visibility).toBe("Private"); + const alphaCommunityId = community.community_view.community.id; + + const betaCommunityId = ( + await resolveCommunity(beta, community.community_view.community.actor_id) + ).community!.community.id; + const follow_form_beta: FollowCommunity = { + community_id: betaCommunityId, + follow: true, + }; + await beta.followCommunity(follow_form_beta); + await approveFollower(alpha, alphaCommunityId); + + // Follow is confirmed + await waitUntil( + () => getCommunity(beta, betaCommunityId), + c => c.community_view.subscribed == "Subscribed", + ); + + // beta creates post and comment + const post = await createPost(beta, betaCommunityId); + const post_id = post.post_view.post.id; + expect(post_id).toBeDefined(); + const comment = await createComment(beta, post_id); + const comment_id = comment.comment_view.comment.id; + expect(comment_id).toBeDefined(); + + // Wait for it to federate + await waitUntil( + () => resolveComment(alpha, comment.comment_view.comment), + p => p?.comment?.comment.id != undefined, + ); + + // create gamma user + const gammaCommunityId = ( + await resolveCommunity(gamma, community.community_view.community.actor_id) + ).community!.community.id; + const follow_form: FollowCommunity = { + community_id: gammaCommunityId, + follow: true, + }; + + // cannot fetch post yet + await expect(resolvePost(gamma, post.post_view.post)).rejects.toStrictEqual( + Error("not_found"), + ); + // follow community and approve + await gamma.followCommunity(follow_form); + await approveFollower(alpha, alphaCommunityId); + + // now user can fetch posts and comments in community (using signed fetch), and create posts. + // for this to work, beta checks with alpha if gamma is really an approved follower. + let resolvedPost = await waitUntil( + () => resolvePost(gamma, post.post_view.post), + p => p?.post?.post.id != undefined, + ); + expect(resolvedPost.post?.post.ap_id).toBe(post.post_view.post.ap_id); + const resolvedComment = await waitUntil( + () => resolveComment(gamma, comment.comment_view.comment), + p => p?.comment?.comment.id != undefined, + ); + expect(resolvedComment?.comment?.comment.ap_id).toBe( + comment.comment_view.comment.ap_id, + ); +}); + +async function approveFollower(user: LemmyHttp, community_id: number) { + let pendingFollows1 = await waitUntil( + () => listCommunityPendingFollows(user), + f => f.items.length == 1, + ); + const approve = await approveCommunityPendingFollow( + alpha, + community_id, + pendingFollows1.items[0].person.id, + ); + expect(approve.success).toBe(true); +} diff --git a/crates/api_common/src/build_response.rs b/crates/api_common/src/build_response.rs index d40f4c23d..b73c0e482 100644 --- a/crates/api_common/src/build_response.rs +++ b/crates/api_common/src/build_response.rs @@ -17,8 +17,10 @@ use lemmy_db_schema::{ actor_language::CommunityLanguage, comment::Comment, comment_reply::{CommentReply, CommentReplyInsertForm}, + community::Community, person::Person, person_mention::{PersonMention, PersonMentionInsertForm}, + post::Post, }, traits::Crud, }; @@ -101,17 +103,28 @@ pub async fn send_local_notifs( let mut recipient_ids = Vec::new(); let inbox_link = format!("{}/inbox", context.settings().get_protocol_and_hostname()); - // let person = my_local_user.person; - // Read the comment view to get extra info - let comment_view = CommentView::read( - &mut context.pool(), - comment_id, - local_user_view.map(|view| &view.local_user), - ) - .await?; - let comment = comment_view.comment; - let post = comment_view.post; - let community = comment_view.community; + // When called from api code, we have local user view and can read with CommentView + // to reduce db queries. But when receiving a federated comment the user view is None, + // which means that comments inside private communities cant be read. As a workaround + // we need to read the items manually to bypass this check. + let (comment, post, community) = if let Some(local_user_view) = local_user_view { + let comment_view = CommentView::read( + &mut context.pool(), + comment_id, + Some(&local_user_view.local_user), + ) + .await?; + ( + comment_view.comment, + comment_view.post, + comment_view.community, + ) + } else { + let comment = Comment::read(&mut context.pool(), comment_id).await?; + let post = Post::read(&mut context.pool(), comment.post_id).await?; + let community = Community::read(&mut context.pool(), post.community_id).await?; + (comment, post, community) + }; // Send the local mentions for mention in mentions diff --git a/crates/apub/src/activities/community/announce.rs b/crates/apub/src/activities/community/announce.rs index b31e4b74f..950f4861d 100644 --- a/crates/apub/src/activities/community/announce.rs +++ b/crates/apub/src/activities/community/announce.rs @@ -215,7 +215,7 @@ async fn can_accept_activity_in_community( ) -> LemmyResult<()> { if let Some(community) = community { // Local only community can't federate - if community.visibility != CommunityVisibility::Public { + if community.visibility == CommunityVisibility::LocalOnly { return Err(LemmyErrorType::NotFound.into()); } if !community.local { diff --git a/crates/apub/src/activities/community/mod.rs b/crates/apub/src/activities/community/mod.rs index 59b8fadbb..93c6e5c77 100644 --- a/crates/apub/src/activities/community/mod.rs +++ b/crates/apub/src/activities/community/mod.rs @@ -42,7 +42,7 @@ pub(crate) async fn send_activity_in_community( context: &Data, ) -> LemmyResult<()> { // If community is local only, don't send anything out - if community.visibility != CommunityVisibility::Public { + if community.visibility == CommunityVisibility::LocalOnly { return Ok(()); } diff --git a/crates/apub/src/activities/create_or_update/comment.rs b/crates/apub/src/activities/create_or_update/comment.rs index 90ab0153f..9f64e805b 100644 --- a/crates/apub/src/activities/create_or_update/comment.rs +++ b/crates/apub/src/activities/create_or_update/comment.rs @@ -171,6 +171,9 @@ impl ActivityHandler for CreateOrUpdateNote { // TODO: for compatibility with other projects, it would be much better to read this from cc or // tags let mentions = scrape_text_for_mentions(&comment.content); + + // TODO: this fails in local community comment as CommentView::read() returns nothing + // without passing LocalUser send_local_notifs(mentions, comment.id, &actor, do_send_email, context, None).await?; Ok(()) } diff --git a/crates/apub/src/http/community.rs b/crates/apub/src/http/community.rs index 96a917d91..dbcc51258 100644 --- a/crates/apub/src/http/community.rs +++ b/crates/apub/src/http/community.rs @@ -6,28 +6,41 @@ use crate::{ community_moderators::ApubCommunityModerators, community_outbox::ApubCommunityOutbox, }, + fetcher::site_or_community_or_user::SiteOrCommunityOrUser, http::{check_community_fetchable, create_apub_response, create_apub_tombstone_response}, objects::community::ApubCommunity, }; use activitypub_federation::{ + actix_web::signing_actor, config::Data, + fetch::object_id::ObjectId, traits::{Collection, Object}, }; -use actix_web::{web, HttpRequest, HttpResponse}; +use actix_web::{ + web::{Path, Query}, + HttpRequest, + HttpResponse, +}; use lemmy_api_common::context::LemmyContext; -use lemmy_db_schema::{source::community::Community, traits::ApubActor}; +use lemmy_db_schema::{source::community::Community, traits::ApubActor, CommunityVisibility}; +use lemmy_db_views_actor::structs::CommunityFollowerView; use lemmy_utils::error::{LemmyErrorType, LemmyResult}; use serde::Deserialize; #[derive(Deserialize, Clone)] -pub(crate) struct CommunityQuery { +pub(crate) struct CommunityPath { community_name: String, } +#[derive(Deserialize, Clone)] +pub struct CommunityIsFollowerQuery { + is_follower: Option>, +} + /// Return the ActivityPub json representation of a local community over HTTP. #[tracing::instrument(skip_all)] pub(crate) async fn get_apub_community_http( - info: web::Path, + info: Path, context: Data, ) -> LemmyResult { let community: ApubCommunity = @@ -47,21 +60,59 @@ pub(crate) async fn get_apub_community_http( /// Returns an empty followers collection, only populating the size (for privacy). pub(crate) async fn get_apub_community_followers( - info: web::Path, + info: Path, + query: Query, context: Data, + request: HttpRequest, ) -> LemmyResult { let community = Community::read_from_name(&mut context.pool(), &info.community_name, false) .await? .ok_or(LemmyErrorType::NotFound)?; + if let Some(is_follower) = &query.is_follower { + return check_is_follower(community, is_follower, context, request).await; + } check_community_fetchable(&community)?; let followers = ApubCommunityFollower::read_local(&community.into(), &context).await?; create_apub_response(&followers) } +/// Checks if a given actor follows the private community. Returns status 200 if true. +async fn check_is_follower( + community: Community, + is_follower: &ObjectId, + context: Data, + request: HttpRequest, +) -> LemmyResult { + if community.visibility != CommunityVisibility::Private { + return Ok(HttpResponse::BadRequest().body("must be a private community")); + } + // also check for http sig so that followers are not exposed publicly + let signing_actor = signing_actor::(&request, None, &context).await?; + CommunityFollowerView::check_has_followers_from_instance( + community.id, + signing_actor.instance_id(), + &mut context.pool(), + ) + .await?; + + let instance_id = is_follower.dereference(&context).await?.instance_id(); + let has_followers = CommunityFollowerView::check_has_followers_from_instance( + community.id, + instance_id, + &mut context.pool(), + ) + .await; + if has_followers.is_ok() { + Ok(HttpResponse::Ok().finish()) + } else { + Ok(HttpResponse::NotFound().finish()) + } +} + /// Returns the community outbox, which is populated by a maximum of 20 posts (but no other /// activities like votes or comments). pub(crate) async fn get_apub_community_outbox( - info: web::Path, + info: Path, context: Data, request: HttpRequest, ) -> LemmyResult { @@ -77,7 +128,7 @@ pub(crate) async fn get_apub_community_outbox( #[tracing::instrument(skip_all)] pub(crate) async fn get_apub_community_moderators( - info: web::Path, + info: Path, context: Data, ) -> LemmyResult { let community: ApubCommunity = @@ -92,7 +143,7 @@ pub(crate) async fn get_apub_community_moderators( /// Returns collection of featured (stickied) posts. pub(crate) async fn get_apub_community_featured( - info: web::Path, + info: Path, context: Data, request: HttpRequest, ) -> LemmyResult { @@ -181,17 +232,17 @@ pub(crate) mod tests { let request = TestRequest::default().to_http_request(); // fetch invalid community - let query = CommunityQuery { + let query = CommunityPath { community_name: "asd".to_string(), }; let res = get_apub_community_http(query.into(), context.reset_request_count()).await; assert!(res.is_err()); // fetch valid community - let query = CommunityQuery { + let path = CommunityPath { community_name: community.name.clone(), }; - let res = get_apub_community_http(query.clone().into(), context.reset_request_count()).await?; + let res = get_apub_community_http(path.clone().into(), context.reset_request_count()).await?; assert_eq!(200, res.status()); let res_group: Group = decode_response(res).await?; let community: ApubCommunity = community.into(); @@ -199,20 +250,26 @@ pub(crate) mod tests { assert_eq!(group, res_group); let res = get_apub_community_featured( - query.clone().into(), + path.clone().into(), + context.reset_request_count(), + request.clone(), + ) + .await?; + assert_eq!(200, res.status()); + let query = Query(CommunityIsFollowerQuery { is_follower: None }); + let res = get_apub_community_followers( + path.clone().into(), + query, context.reset_request_count(), request.clone(), ) .await?; assert_eq!(200, res.status()); let res = - get_apub_community_followers(query.clone().into(), context.reset_request_count()).await?; + get_apub_community_moderators(path.clone().into(), context.reset_request_count()).await?; assert_eq!(200, res.status()); let res = - get_apub_community_moderators(query.clone().into(), context.reset_request_count()).await?; - assert_eq!(200, res.status()); - let res = - get_apub_community_outbox(query.into(), context.reset_request_count(), request).await?; + get_apub_community_outbox(path.into(), context.reset_request_count(), request).await?; assert_eq!(200, res.status()); Instance::delete(&mut context.pool(), instance.id).await?; @@ -227,28 +284,35 @@ pub(crate) mod tests { let request = TestRequest::default().to_http_request(); // should return tombstone - let query = CommunityQuery { + let path: Path = CommunityPath { community_name: community.name.clone(), - }; - let res = get_apub_community_http(query.clone().into(), context.reset_request_count()).await?; + } + .into(); + let res = get_apub_community_http(path.clone().into(), context.reset_request_count()).await?; assert_eq!(410, res.status()); let res_tombstone = decode_response::(res).await; assert!(res_tombstone.is_ok()); let res = get_apub_community_featured( - query.clone().into(), + path.clone().into(), + context.reset_request_count(), + request.clone(), + ) + .await; + assert!(res.is_err()); + let query = Query(CommunityIsFollowerQuery { is_follower: None }); + let res = get_apub_community_followers( + path.clone().into(), + query, context.reset_request_count(), request.clone(), ) .await; assert!(res.is_err()); let res = - get_apub_community_followers(query.clone().into(), context.reset_request_count()).await; + get_apub_community_moderators(path.clone().into(), context.reset_request_count()).await; assert!(res.is_err()); - let res = - get_apub_community_moderators(query.clone().into(), context.reset_request_count()).await; - assert!(res.is_err()); - let res = get_apub_community_outbox(query.into(), context.reset_request_count(), request).await; + let res = get_apub_community_outbox(path, context.reset_request_count(), request).await; assert!(res.is_err()); //Community::delete(&mut context.pool(), community.id).await?; @@ -263,25 +327,32 @@ pub(crate) mod tests { let (instance, community) = init(false, CommunityVisibility::LocalOnly, &context).await?; let request = TestRequest::default().to_http_request(); - let query = CommunityQuery { + let path: Path = CommunityPath { community_name: community.name.clone(), - }; - let res = get_apub_community_http(query.clone().into(), context.reset_request_count()).await; + } + .into(); + let res = get_apub_community_http(path.clone().into(), context.reset_request_count()).await; assert!(res.is_err()); let res = get_apub_community_featured( - query.clone().into(), + path.clone().into(), + context.reset_request_count(), + request.clone(), + ) + .await; + assert!(res.is_err()); + let query = Query(CommunityIsFollowerQuery { is_follower: None }); + let res = get_apub_community_followers( + path.clone().into(), + query, context.reset_request_count(), request.clone(), ) .await; assert!(res.is_err()); let res = - get_apub_community_followers(query.clone().into(), context.reset_request_count()).await; + get_apub_community_moderators(path.clone().into(), context.reset_request_count()).await; assert!(res.is_err()); - let res = - get_apub_community_moderators(query.clone().into(), context.reset_request_count()).await; - assert!(res.is_err()); - let res = get_apub_community_outbox(query.into(), context.reset_request_count(), request).await; + let res = get_apub_community_outbox(path, context.reset_request_count(), request).await; assert!(res.is_err()); Instance::delete(&mut context.pool(), instance.id).await?; diff --git a/crates/apub/src/http/mod.rs b/crates/apub/src/http/mod.rs index 8780f8789..fc2fbf0d3 100644 --- a/crates/apub/src/http/mod.rs +++ b/crates/apub/src/http/mod.rs @@ -8,6 +8,7 @@ use activitypub_federation::{ actix_web::{inbox::receive_activity, signing_actor}, config::Data, protocol::context::WithContext, + traits::Actor, FEDERATION_CONTENT_TYPE, }; use actix_web::{web, web::Bytes, HttpRequest, HttpResponse}; @@ -145,14 +146,27 @@ async fn check_community_content_fetchable( // from the fetching instance then fetching is allowed Private => { let signing_actor = signing_actor::(request, None, context).await?; - Ok( - CommunityFollowerView::check_has_followers_from_instance( - community.id, - signing_actor.instance_id(), - &mut context.pool(), + if community.local { + Ok( + CommunityFollowerView::check_has_followers_from_instance( + community.id, + signing_actor.instance_id(), + &mut context.pool(), + ) + .await?, ) - .await?, - ) + } else if let Some(followers_url) = community.followers_url.clone() { + let mut followers_url = followers_url.inner().clone(); + followers_url + .query_pairs_mut() + .append_pair("is_follower", signing_actor.id().as_str()); + let req = context.client().get(followers_url.as_str()); + let req = context.sign_request(req, Bytes::new()).await?; + context.client().execute(req).await?.error_for_status()?; + Ok(()) + } else { + Err(LemmyErrorType::NotFound.into()) + } } } } diff --git a/crates/db_views_actor/src/community_follower_view.rs b/crates/db_views_actor/src/community_follower_view.rs index 56f4cca93..c32ccb5b8 100644 --- a/crates/db_views_actor/src/community_follower_view.rs +++ b/crates/db_views_actor/src/community_follower_view.rs @@ -232,6 +232,25 @@ impl CommunityFollowerView { .then_some(()) .ok_or(diesel::NotFound) } + + pub async fn is_follower( + community_id: CommunityId, + instance_id: InstanceId, + pool: &mut DbPool<'_>, + ) -> Result<(), Error> { + let conn = &mut get_conn(pool).await?; + select(exists( + action_query(community_actions::followed) + .inner_join(person::table.on(community_actions::person_id.eq(person::id))) + .filter(community_actions::community_id.eq(community_id)) + .filter(person::instance_id.eq(instance_id)) + .filter(community_actions::follow_state.eq(CommunityFollowerState::Accepted)), + )) + .get_result::(conn) + .await? + .then_some(()) + .ok_or(diesel::NotFound) + } } #[cfg(test)] From 41bd83038962500ffe1c4ca3c80a7e9208d0a431 Mon Sep 17 00:00:00 2001 From: Dessalines Date: Mon, 25 Nov 2024 04:06:39 -0500 Subject: [PATCH 4/8] Fixing a few clippy recommendations on 1.84 nightly. (#5222) --- crates/db_schema/src/utils.rs | 4 ++-- crates/db_views/src/comment_view.rs | 13 ++++++++----- crates/db_views/src/post_view.rs | 11 +++++++---- crates/db_views_actor/src/community_view.rs | 13 ++++++++----- crates/federate/src/send.rs | 2 +- 5 files changed, 26 insertions(+), 17 deletions(-) diff --git a/crates/db_schema/src/utils.rs b/crates/db_schema/src/utils.rs index ed9ca0ab4..aa213887e 100644 --- a/crates/db_schema/src/utils.rs +++ b/crates/db_schema/src/utils.rs @@ -96,7 +96,7 @@ pub async fn get_conn<'a, 'b: 'a>(pool: &'a mut DbPool<'b>) -> Result }) } -impl<'a> Deref for DbConn<'a> { +impl Deref for DbConn<'_> { type Target = AsyncPgConnection; fn deref(&self) -> &Self::Target { @@ -107,7 +107,7 @@ impl<'a> Deref for DbConn<'a> { } } -impl<'a> DerefMut for DbConn<'a> { +impl DerefMut for DbConn<'_> { fn deref_mut(&mut self) -> &mut Self::Target { match self { DbConn::Pool(conn) => conn.deref_mut(), diff --git a/crates/db_views/src/comment_view.rs b/crates/db_views/src/comment_view.rs index 22b7b3de4..1740a36be 100644 --- a/crates/db_views/src/comment_view.rs +++ b/crates/db_views/src/comment_view.rs @@ -50,9 +50,12 @@ use lemmy_db_schema::{ ListingType, }; +type QueriesReadTypes<'a> = (CommentId, Option<&'a LocalUser>); +type QueriesListTypes<'a> = (CommentQuery<'a>, &'a Site); + fn queries<'a>() -> Queries< - impl ReadFn<'a, CommentView, (CommentId, Option<&'a LocalUser>)>, - impl ListFn<'a, CommentView, (CommentQuery<'a>, &'a Site)>, + impl ReadFn<'a, CommentView, QueriesReadTypes<'a>>, + impl ListFn<'a, CommentView, QueriesListTypes<'a>>, > { let creator_is_admin = exists( local_user::table.filter( @@ -308,10 +311,10 @@ fn queries<'a>() -> Queries< } impl CommentView { - pub async fn read<'a>( + pub async fn read( pool: &mut DbPool<'_>, comment_id: CommentId, - my_local_user: Option<&'a LocalUser>, + my_local_user: Option<&'_ LocalUser>, ) -> Result { // If a person is given, then my_vote (res.9), if None, should be 0, not null // Necessary to differentiate between other person's votes @@ -345,7 +348,7 @@ pub struct CommentQuery<'a> { pub max_depth: Option, } -impl<'a> CommentQuery<'a> { +impl CommentQuery<'_> { pub async fn list(self, site: &Site, pool: &mut DbPool<'_>) -> Result, Error> { Ok( queries() diff --git a/crates/db_views/src/post_view.rs b/crates/db_views/src/post_view.rs index 4363d2a3e..e2eac77a5 100644 --- a/crates/db_views/src/post_view.rs +++ b/crates/db_views/src/post_view.rs @@ -62,9 +62,12 @@ use lemmy_db_schema::{ use tracing::debug; use PostSortType::*; +type QueriesReadTypes<'a> = (PostId, Option<&'a LocalUser>, bool); +type QueriesListTypes<'a> = (PostQuery<'a>, &'a Site); + fn queries<'a>() -> Queries< - impl ReadFn<'a, PostView, (PostId, Option<&'a LocalUser>, bool)>, - impl ListFn<'a, PostView, (PostQuery<'a>, &'a Site)>, + impl ReadFn<'a, PostView, QueriesReadTypes<'a>>, + impl ListFn<'a, PostView, QueriesListTypes<'a>>, > { let creator_is_admin = exists( local_user::table.filter( @@ -431,10 +434,10 @@ fn queries<'a>() -> Queries< } impl PostView { - pub async fn read<'a>( + pub async fn read( pool: &mut DbPool<'_>, post_id: PostId, - my_local_user: Option<&'a LocalUser>, + my_local_user: Option<&'_ LocalUser>, is_mod_or_admin: bool, ) -> Result { queries() diff --git a/crates/db_views_actor/src/community_view.rs b/crates/db_views_actor/src/community_view.rs index da349c93e..f6ce82d37 100644 --- a/crates/db_views_actor/src/community_view.rs +++ b/crates/db_views_actor/src/community_view.rs @@ -34,9 +34,12 @@ use lemmy_db_schema::{ }; use lemmy_utils::error::{LemmyErrorType, LemmyResult}; +type QueriesReadTypes<'a> = (CommunityId, Option<&'a LocalUser>, bool); +type QueriesListTypes<'a> = (CommunityQuery<'a>, &'a Site); + fn queries<'a>() -> Queries< - impl ReadFn<'a, CommunityView, (CommunityId, Option<&'a LocalUser>, bool)>, - impl ListFn<'a, CommunityView, (CommunityQuery<'a>, &'a Site)>, + impl ReadFn<'a, CommunityView, QueriesReadTypes<'a>>, + impl ListFn<'a, CommunityView, QueriesListTypes<'a>>, > { let all_joins = |query: community::BoxedQuery<'a, Pg>, my_local_user: Option<&'a LocalUser>| { query @@ -166,10 +169,10 @@ fn queries<'a>() -> Queries< } impl CommunityView { - pub async fn read<'a>( + pub async fn read( pool: &mut DbPool<'_>, community_id: CommunityId, - my_local_user: Option<&'a LocalUser>, + my_local_user: Option<&'_ LocalUser>, is_mod_or_admin: bool, ) -> Result { queries() @@ -253,7 +256,7 @@ pub struct CommunityQuery<'a> { pub limit: Option, } -impl<'a> CommunityQuery<'a> { +impl CommunityQuery<'_> { pub async fn list(self, site: &Site, pool: &mut DbPool<'_>) -> Result, Error> { queries().list(pool, (self, site)).await } diff --git a/crates/federate/src/send.rs b/crates/federate/src/send.rs index 01d620eb0..d77ebe9f7 100644 --- a/crates/federate/src/send.rs +++ b/crates/federate/src/send.rs @@ -84,7 +84,7 @@ pub(crate) struct SendRetryTask<'a> { pub stop: CancellationToken, } -impl<'a> SendRetryTask<'a> { +impl SendRetryTask<'_> { // this function will return successfully when (a) send succeeded or (b) worker cancelled // and will return an error if an internal error occurred (send errors cause an infinite loop) pub async fn send_retry_loop(self) -> Result<()> { From ba3d574d923a69b03fbfb5bfe93485799f0de1e0 Mon Sep 17 00:00:00 2001 From: Dessalines Date: Mon, 25 Nov 2024 05:24:57 -0500 Subject: [PATCH 5/8] Add report_counts to post and comment aggregate tables. (#5219) * Add report_counts to post and comment aggregate tables. - This adds a report_count and unresolved_report_count to the post and comment aggregate tables. - Useful for front-ends wishing to show report links. - Fixes #4163 * Updating the historical counts. * Switching from bigint to smallint. * Using dullbananas create_trigger function. --- .../db_schema/replaceable_schema/triggers.sql | 38 +++++++++ crates/db_schema/src/aggregates/structs.rs | 4 + crates/db_schema/src/schema.rs | 4 + crates/db_views/src/comment_report_view.rs | 6 ++ crates/db_views/src/comment_view.rs | 2 + crates/db_views/src/post_report_view.rs | 19 +++++ crates/db_views/src/post_view.rs | 2 + .../down.sql | 8 ++ .../2024-11-21-195004_add_report_count/up.sql | 79 +++++++++++++++++++ 9 files changed, 162 insertions(+) create mode 100644 migrations/2024-11-21-195004_add_report_count/down.sql create mode 100644 migrations/2024-11-21-195004_add_report_count/up.sql diff --git a/crates/db_schema/replaceable_schema/triggers.sql b/crates/db_schema/replaceable_schema/triggers.sql index 6c55ce3d6..e5b3e22d0 100644 --- a/crates/db_schema/replaceable_schema/triggers.sql +++ b/crates/db_schema/replaceable_schema/triggers.sql @@ -384,6 +384,44 @@ END; $$); +CALL r.create_triggers ('post_report', $$ +BEGIN + UPDATE + post_aggregates AS a + SET + report_count = a.report_count + diff.report_count, unresolved_report_count = a.unresolved_report_count + diff.unresolved_report_count + FROM ( + SELECT + (post_report).post_id, coalesce(sum(count_diff), 0) AS report_count, coalesce(sum(count_diff) FILTER (WHERE NOT (post_report).resolved), 0) AS unresolved_report_count + FROM select_old_and_new_rows AS old_and_new_rows GROUP BY (post_report).post_id) AS diff +WHERE (diff.report_count, diff.unresolved_report_count) != (0, 0) + AND a.post_id = diff.post_id; + +RETURN NULL; + +END; + +$$); + +CALL r.create_triggers ('comment_report', $$ +BEGIN + UPDATE + comment_aggregates AS a + SET + report_count = a.report_count + diff.report_count, unresolved_report_count = a.unresolved_report_count + diff.unresolved_report_count + FROM ( + SELECT + (comment_report).comment_id, coalesce(sum(count_diff), 0) AS report_count, coalesce(sum(count_diff) FILTER (WHERE NOT (comment_report).resolved), 0) AS unresolved_report_count + FROM select_old_and_new_rows AS old_and_new_rows GROUP BY (comment_report).comment_id) AS diff +WHERE (diff.report_count, diff.unresolved_report_count) != (0, 0) + AND a.comment_id = diff.comment_id; + +RETURN NULL; + +END; + +$$); + -- These triggers create and update rows in each aggregates table to match its associated table's rows. -- Deleting rows and updating IDs are already handled by `CASCADE` in foreign key constraints. CREATE FUNCTION r.comment_aggregates_from_comment () diff --git a/crates/db_schema/src/aggregates/structs.rs b/crates/db_schema/src/aggregates/structs.rs index c2e54ae5c..7a97666aa 100644 --- a/crates/db_schema/src/aggregates/structs.rs +++ b/crates/db_schema/src/aggregates/structs.rs @@ -39,6 +39,8 @@ pub struct CommentAggregates { pub hot_rank: f64, #[serde(skip)] pub controversy_rank: f64, + pub report_count: i16, + pub unresolved_report_count: i16, } #[derive(PartialEq, Debug, Serialize, Deserialize, Clone)] @@ -146,6 +148,8 @@ pub struct PostAggregates { /// A rank that amplifies smaller communities #[serde(skip)] pub scaled_rank: f64, + pub report_count: i16, + pub unresolved_report_count: i16, } #[derive(PartialEq, Eq, Debug, Serialize, Deserialize, Clone)] diff --git a/crates/db_schema/src/schema.rs b/crates/db_schema/src/schema.rs index 9e80c4693..f2b186d35 100644 --- a/crates/db_schema/src/schema.rs +++ b/crates/db_schema/src/schema.rs @@ -130,6 +130,8 @@ diesel::table! { child_count -> Int4, hot_rank -> Float8, controversy_rank -> Float8, + report_count -> Int2, + unresolved_report_count -> Int2, } } @@ -777,6 +779,8 @@ diesel::table! { controversy_rank -> Float8, instance_id -> Int4, scaled_rank -> Float8, + report_count -> Int2, + unresolved_report_count -> Int2, } } diff --git a/crates/db_views/src/comment_report_view.rs b/crates/db_views/src/comment_report_view.rs index 278dc5c22..b4a23a0da 100644 --- a/crates/db_views/src/comment_report_view.rs +++ b/crates/db_views/src/comment_report_view.rs @@ -444,6 +444,8 @@ mod tests { child_count: 0, hot_rank: RANK_DEFAULT, controversy_rank: 0.0, + report_count: 2, + unresolved_report_count: 2, }, my_vote: None, resolver: None, @@ -511,6 +513,10 @@ mod tests { .updated = read_jessica_report_view_after_resolve .comment_report .updated; + expected_jessica_report_view_after_resolve + .counts + .unresolved_report_count = 1; + expected_sara_report_view.counts.unresolved_report_count = 1; expected_jessica_report_view_after_resolve.resolver = Some(Person { id: inserted_timmy.id, name: inserted_timmy.name.clone(), diff --git a/crates/db_views/src/comment_view.rs b/crates/db_views/src/comment_view.rs index 1740a36be..1037cf6ff 100644 --- a/crates/db_views/src/comment_view.rs +++ b/crates/db_views/src/comment_view.rs @@ -1068,6 +1068,8 @@ mod tests { child_count: 5, hot_rank: RANK_DEFAULT, controversy_rank: 0.0, + report_count: 0, + unresolved_report_count: 0, }, }) } diff --git a/crates/db_views/src/post_report_view.rs b/crates/db_views/src/post_report_view.rs index c6c19bf6f..9429c258f 100644 --- a/crates/db_views/src/post_report_view.rs +++ b/crates/db_views/src/post_report_view.rs @@ -232,6 +232,7 @@ mod tests { structs::LocalUserView, }; use lemmy_db_schema::{ + aggregates::structs::PostAggregates, assert_length, source::{ community::{Community, CommunityInsertForm, CommunityModerator, CommunityModeratorForm}, @@ -336,6 +337,10 @@ mod tests { let read_jessica_report_view = PostReportView::read(pool, inserted_jessica_report.id, inserted_timmy.id).await?; + // Make sure the triggers are reading the aggregates correctly. + let agg_1 = PostAggregates::read(pool, inserted_post.id).await?; + let agg_2 = PostAggregates::read(pool, inserted_post_2.id).await?; + assert_eq!( read_jessica_report_view.post_report, inserted_jessica_report @@ -346,6 +351,10 @@ mod tests { assert_eq!(read_jessica_report_view.post_creator.id, inserted_timmy.id); assert_eq!(read_jessica_report_view.my_vote, None); assert_eq!(read_jessica_report_view.resolver, None); + assert_eq!(agg_1.report_count, 1); + assert_eq!(agg_1.unresolved_report_count, 1); + assert_eq!(agg_2.report_count, 1); + assert_eq!(agg_2.unresolved_report_count, 1); // Do a batch read of timmys reports let reports = PostReportQuery::default().list(pool, &timmy_view).await?; @@ -379,6 +388,16 @@ mod tests { Some(inserted_timmy.id) ); + // Make sure the unresolved_post report got decremented in the trigger + let agg_2 = PostAggregates::read(pool, inserted_post_2.id).await?; + assert_eq!(agg_2.report_count, 1); + assert_eq!(agg_2.unresolved_report_count, 0); + + // Make sure the other unresolved report isn't changed + let agg_1 = PostAggregates::read(pool, inserted_post.id).await?; + assert_eq!(agg_1.report_count, 1); + assert_eq!(agg_1.unresolved_report_count, 1); + // Do a batch read of timmys reports // It should only show saras, which is unresolved let reports_after_resolve = PostReportQuery { diff --git a/crates/db_views/src/post_view.rs b/crates/db_views/src/post_view.rs index e2eac77a5..c6d1b036f 100644 --- a/crates/db_views/src/post_view.rs +++ b/crates/db_views/src/post_view.rs @@ -1738,6 +1738,8 @@ mod tests { community_id: inserted_post.community_id, creator_id: inserted_post.creator_id, instance_id: data.inserted_instance.id, + report_count: 0, + unresolved_report_count: 0, }, subscribed: SubscribedType::NotSubscribed, read: false, diff --git a/migrations/2024-11-21-195004_add_report_count/down.sql b/migrations/2024-11-21-195004_add_report_count/down.sql new file mode 100644 index 000000000..be418840f --- /dev/null +++ b/migrations/2024-11-21-195004_add_report_count/down.sql @@ -0,0 +1,8 @@ +ALTER TABLE post_aggregates + DROP COLUMN report_count, + DROP COLUMN unresolved_report_count; + +ALTER TABLE comment_aggregates + DROP COLUMN report_count, + DROP COLUMN unresolved_report_count; + diff --git a/migrations/2024-11-21-195004_add_report_count/up.sql b/migrations/2024-11-21-195004_add_report_count/up.sql new file mode 100644 index 000000000..c7d28e1ef --- /dev/null +++ b/migrations/2024-11-21-195004_add_report_count/up.sql @@ -0,0 +1,79 @@ +-- Adding report_count and unresolved_report_count +-- to the post and comment aggregate tables +ALTER TABLE post_aggregates + ADD COLUMN report_count smallint NOT NULL DEFAULT 0, + ADD COLUMN unresolved_report_count smallint NOT NULL DEFAULT 0; + +ALTER TABLE comment_aggregates + ADD COLUMN report_count smallint NOT NULL DEFAULT 0, + ADD COLUMN unresolved_report_count smallint NOT NULL DEFAULT 0; + +-- Update the historical counts +-- Posts +UPDATE + post_aggregates AS a +SET + report_count = cnt.count +FROM ( + SELECT + post_id, + count(*) AS count + FROM + post_report + GROUP BY + post_id) cnt +WHERE + a.post_id = cnt.post_id; + +-- The unresolved +UPDATE + post_aggregates AS a +SET + unresolved_report_count = cnt.count +FROM ( + SELECT + post_id, + count(*) AS count + FROM + post_report + WHERE + resolved = 'f' + GROUP BY + post_id) cnt +WHERE + a.post_id = cnt.post_id; + +-- Comments +UPDATE + comment_aggregates AS a +SET + report_count = cnt.count +FROM ( + SELECT + comment_id, + count(*) AS count + FROM + comment_report + GROUP BY + comment_id) cnt +WHERE + a.comment_id = cnt.comment_id; + +-- The unresolved +UPDATE + comment_aggregates AS a +SET + unresolved_report_count = cnt.count +FROM ( + SELECT + comment_id, + count(*) AS count + FROM + comment_report + WHERE + resolved = 'f' + GROUP BY + comment_id) cnt +WHERE + a.comment_id = cnt.comment_id; + From bb26a701bc976a9d62e22820c4e1bec26c533aac Mon Sep 17 00:00:00 2001 From: Nutomic Date: Mon, 25 Nov 2024 16:43:34 +0000 Subject: [PATCH 6/8] Proxy pictrs healthz endpoint (#5224) --- api_tests/src/image.spec.ts | 3 +++ crates/routes/src/images.rs | 22 +++++++++++++++++++++- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/api_tests/src/image.spec.ts b/api_tests/src/image.spec.ts index ed96451a2..7ac6e7221 100644 --- a/api_tests/src/image.spec.ts +++ b/api_tests/src/image.spec.ts @@ -41,6 +41,9 @@ afterAll(async () => { }); test("Upload image and delete it", async () => { + const healthz = await fetch(alphaUrl + "/pictrs/healthz"); + expect(healthz.status).toBe(200); + // Before running this test, you need to delete all previous images in the DB await deleteAllImages(alpha); diff --git a/crates/routes/src/images.rs b/crates/routes/src/images.rs index 54fe4ea83..fe91fa42c 100644 --- a/crates/routes/src/images.rs +++ b/crates/routes/src/images.rs @@ -38,7 +38,8 @@ pub fn config( ) // This has optional query params: /image/{filename}?format=jpg&thumbnail=256 .service(web::resource("/pictrs/image/{filename}").route(web::get().to(full_res))) - .service(web::resource("/pictrs/image/delete/{token}/{filename}").route(web::get().to(delete))); + .service(web::resource("/pictrs/image/delete/{token}/{filename}").route(web::get().to(delete))) + .service(web::resource("/pictrs/healthz").route(web::get().to(healthz))); } trait ProcessUrl { @@ -250,6 +251,25 @@ async fn delete( Ok(HttpResponse::build(convert_status(res.status())).body(BodyStream::new(res.bytes_stream()))) } +async fn healthz( + req: HttpRequest, + client: web::Data, + context: web::Data, +) -> LemmyResult { + let pictrs_config = context.settings().pictrs_config()?; + let url = format!("{}healthz", pictrs_config.url); + + let mut client_req = adapt_request(&req, &client, url); + + if let Some(addr) = req.head().peer_addr { + client_req = client_req.header("X-Forwarded-For", addr.to_string()); + } + + let res = client_req.send().await?; + + Ok(HttpResponse::build(convert_status(res.status())).body(BodyStream::new(res.bytes_stream()))) +} + pub async fn image_proxy( Query(params): Query, req: HttpRequest, From c0d7d76570fb6babf8ddea1363f62dbff19bb0a0 Mon Sep 17 00:00:00 2001 From: Nutomic Date: Mon, 25 Nov 2024 23:20:45 +0000 Subject: [PATCH 7/8] Remove getrandom dependency (#5226) * Remove getrandom dependency * fix --- Cargo.lock | 1 - Cargo.toml | 2 +- crates/api_common/Cargo.toml | 8 ++------ crates/db_schema/Cargo.toml | 2 +- 4 files changed, 4 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c1b1f3673..fa63a4b27 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2507,7 +2507,6 @@ dependencies = [ "encoding_rs", "enum-map", "futures", - "getrandom", "jsonwebtoken", "lemmy_db_schema", "lemmy_db_views", diff --git a/Cargo.toml b/Cargo.toml index bce88af67..b1553be5c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -131,7 +131,7 @@ chrono = { version = "0.4.38", features = [ ], default-features = false } serde_json = { version = "1.0.132", features = ["preserve_order"] } base64 = "0.22.1" -uuid = { version = "1.11.0", features = ["serde", "v4"] } +uuid = { version = "1.11.0", features = ["serde"] } async-trait = "0.1.83" captcha = "0.0.9" anyhow = { version = "1.0.93", features = [ diff --git a/crates/api_common/Cargo.toml b/crates/api_common/Cargo.toml index 76f531ddb..3ae14717d 100644 --- a/crates/api_common/Cargo.toml +++ b/crates/api_common/Cargo.toml @@ -36,6 +36,7 @@ full = [ "futures", "jsonwebtoken", "mime", + "moka", ] [dependencies] @@ -58,7 +59,7 @@ uuid = { workspace = true, optional = true } tokio = { workspace = true, optional = true } reqwest = { workspace = true, optional = true } ts-rs = { workspace = true, optional = true } -moka.workspace = true +moka = { workspace = true, optional = true } anyhow.workspace = true actix-web = { workspace = true, optional = true } enum-map = { workspace = true } @@ -70,11 +71,6 @@ webpage = { version = "2.0", default-features = false, features = [ ], optional = true } encoding_rs = { version = "0.8.35", optional = true } jsonwebtoken = { version = "9.3.0", optional = true } -# necessary for wasmt compilation -getrandom = { version = "0.2.15", features = ["js"] } - -[package.metadata.cargo-shear] -ignored = ["getrandom"] [dev-dependencies] serial_test = { workspace = true } diff --git a/crates/db_schema/Cargo.toml b/crates/db_schema/Cargo.toml index bd83ede29..eac9d6ddd 100644 --- a/crates/db_schema/Cargo.toml +++ b/crates/db_schema/Cargo.toml @@ -75,7 +75,7 @@ tokio = { workspace = true, optional = true } tokio-postgres = { workspace = true, optional = true } tokio-postgres-rustls = { workspace = true, optional = true } rustls = { workspace = true, optional = true } -uuid = { workspace = true, features = ["v4"] } +uuid.workspace = true i-love-jesus = { workspace = true, optional = true } anyhow = { workspace = true } diesel-bind-if-some = { workspace = true, optional = true } From 7ee7b069af92d4aa35fba218be79daa33b932e1e Mon Sep 17 00:00:00 2001 From: Dessalines Date: Tue, 26 Nov 2024 03:39:14 -0500 Subject: [PATCH 8/8] Move up clippy and cargo test in CI. (#5229) - Clippy is usually the most likely CI test to fail, and it currently takes > 5m to reach it, wasting CPU resources. --- .woodpecker.yml | 60 ++++++++++++++++++++++++------------------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/.woodpecker.yml b/.woodpecker.yml index c9ba830dd..51c14af7d 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -91,6 +91,36 @@ steps: when: - event: pull_request + cargo_clippy: + image: *rust_image + environment: + CARGO_HOME: .cargo_home + commands: + - rustup component add clippy + - cargo clippy --workspace --tests --all-targets -- -D warnings + when: *slow_check_paths + + cargo_test: + image: *rust_image + environment: + LEMMY_DATABASE_URL: postgres://lemmy:password@database:5432/lemmy + RUST_BACKTRACE: "1" + CARGO_HOME: .cargo_home + LEMMY_TEST_FAST_FEDERATION: "1" + LEMMY_CONFIG_LOCATION: ../../config/config.hjson + commands: + - cargo test --workspace --no-fail-fast + when: *slow_check_paths + + check_ts_bindings: + image: *rust_image + environment: + CARGO_HOME: .cargo_home + commands: + - ./scripts/ts_bindings_check.sh + when: + - event: pull_request + # make sure api builds with default features (used by other crates relying on lemmy api) check_api_common_default_features: image: *rust_image @@ -138,15 +168,6 @@ steps: - diff tmp.schema crates/db_schema/src/schema.rs when: *slow_check_paths - cargo_clippy: - image: *rust_image - environment: - CARGO_HOME: .cargo_home - commands: - - rustup component add clippy - - cargo clippy --workspace --tests --all-targets -- -D warnings - when: *slow_check_paths - cargo_build: image: *rust_image environment: @@ -156,27 +177,6 @@ steps: - mv target/debug/lemmy_server target/lemmy_server when: *slow_check_paths - cargo_test: - image: *rust_image - environment: - LEMMY_DATABASE_URL: postgres://lemmy:password@database:5432/lemmy - RUST_BACKTRACE: "1" - CARGO_HOME: .cargo_home - LEMMY_TEST_FAST_FEDERATION: "1" - LEMMY_CONFIG_LOCATION: ../../config/config.hjson - commands: - - cargo test --workspace --no-fail-fast - when: *slow_check_paths - - check_ts_bindings: - image: *rust_image - environment: - CARGO_HOME: .cargo_home - commands: - - ./scripts/ts_bindings_check.sh - when: - - event: pull_request - check_diesel_migration: # TODO: use willsquire/diesel-cli image when shared libraries become optional in lemmy_server image: *rust_image