Feature filter keywords 3710 (#5263)

* created new table post_keyword_block related to a person and corresponding functions in diesel

* created corresponding api functions

* converted like to ilike in filter

* Fixxed errors for adding keywords for blocking posts

* Fixxed errors in diesel migrations

* Removed debug print

* #3710: Fixxed errors caused by cargo check

* #3710: added up and down scripts for diesel

* added tmp.schema to gitignore

* updated tests

* fixxed test

* fixxed error from rebasing

* Added max number of block tags and modified max length of block tags

* formatted code

* added migration to restrict block keyword length

* #3710: Added check if Blocked keyword is already existing

* #3710: Removed deprecated any

* #3710: Updated keyword length

* #3710: Fixxed errors from master merge

* #3710: Formatting

* #3710: Removed api to block keywords for post from v3 and changed of api path in v4

* #3710: Renamed post_keyword_block table to user_post_keyword_block and replaced id as primary key with the composed key person_id and keyword. Also now use update function which updates the blocked keywords for an user with a new list instead keyword by keyword.

* #3710: Formatted code

* #3710: Removed unnecessary parts for the block API in post as this will be done over the user settings

* #3710: Modified data structure for blocking keywords to reference local user and added update function which will update at all keywords for a local user at once

* #3710: Modified code so updating blocked keywords is now done over user settings

* #3710: Fixxed merging error

* #3710: Fixxed filter query

* #3710: Formatted Code

* #3710: Added suggested changes from review: keyword block update now does delete and then insert, postview uses read from LocalKeywordBlock, and validation with min_length_check and max_length_check

* #3710: formatted code

* c

* #3710: removed BlockKeywordForPost struct

* #3710: formatted code

* #3710: formatted code

* #3710: formatted code

crates/api/src/local_user/save_settings.rs

@@ -9,6 +9,7 @@ use lemmy_api_common::{
 use lemmy_db_schema::{
   source::{
     actor_language::LocalUserLanguage,
+    keyword_block::LocalUserKeywordBlock,
     local_user::{LocalUser, LocalUserUpdateForm},
     person::{Person, PersonUpdateForm},
   },
@@ -19,7 +20,12 @@ use lemmy_db_views::structs::{LocalUserView, SiteView};
 use lemmy_email::account::send_verification_email;
 use lemmy_utils::{
   error::{LemmyErrorType, LemmyResult},
-  utils::validation::{is_valid_bio_field, is_valid_display_name, is_valid_matrix_id},
+  utils::validation::{
+    check_blocking_keywords_are_valid,
+    is_valid_bio_field,
+    is_valid_display_name,
+    is_valid_matrix_id,
+  },
 };
 use std::ops::Deref;
 
@@ -108,6 +114,20 @@ pub async fn save_user_settings(
     LocalUserLanguage::update(&mut context.pool(), discussion_languages, local_user_id).await?;
   }
 
+  if let Some(blocking_keywords) = data.blocking_keywords.clone() {
+    let trimmed_blocking_keywords = blocking_keywords
+      .iter()
+      .map(|blocking_keyword| blocking_keyword.trim().to_string())
+      .collect();
+    check_blocking_keywords_are_valid(&trimmed_blocking_keywords)?;
+    LocalUserKeywordBlock::update(
+      &mut context.pool(),
+      trimmed_blocking_keywords,
+      local_user_id,
+    )
+    .await?;
+  }
+
   let local_user_form = LocalUserUpdateForm {
     email,
     show_avatars: data.show_avatars,

crates/api_common/src/person.rs

@@ -155,6 +155,9 @@ pub struct SaveUserSettings {
   /// A list of languages you are able to see discussion in.
   #[cfg_attr(feature = "full", ts(optional))]
   pub discussion_languages: Option<Vec<LanguageId>>,
+  // A list of keywords used for blocking posts having them in title,url or body.
+  #[cfg_attr(feature = "full", ts(optional))]
+  pub blocking_keywords: Option<Vec<String>>,
   /// Open links in a new tab
   #[cfg_attr(feature = "full", ts(optional))]
   pub open_links_in_new_tab: Option<bool>,

crates/api_common/src/site.rs

@@ -455,6 +455,7 @@ pub struct MyUserInfo {
   pub community_blocks: Vec<Community>,
   pub instance_blocks: Vec<Instance>,
   pub person_blocks: Vec<Person>,
+  pub keyword_blocks: Vec<String>,
   pub discussion_languages: Vec<LanguageId>,
 }
 

crates/api_crud/src/user/my_user.rs

@@ -5,6 +5,7 @@ use lemmy_db_schema::{
     actor_language::LocalUserLanguage,
     community::CommunityActions,
     instance::InstanceActions,
+    keyword_block::LocalUserKeywordBlock,
     person::PersonActions,
   },
   traits::Blockable,
@@ -23,16 +24,24 @@ pub async fn get_my_user(
   let local_user_id = local_user_view.local_user.id;
   let pool = &mut context.pool();
 
-  let (follows, community_blocks, instance_blocks, person_blocks, moderates, discussion_languages) =
-    lemmy_db_schema::try_join_with_pool!(pool => (
-      |pool| CommunityFollowerView::for_person(pool, person_id),
-      |pool| CommunityActions::read_blocks_for_person(pool, person_id),
-      |pool| InstanceActions::read_blocks_for_person(pool, person_id),
-      |pool| PersonActions::read_blocks_for_person(pool, person_id),
-      |pool| CommunityModeratorView::for_person(pool, person_id, Some(&local_user_view.local_user)),
-      |pool| LocalUserLanguage::read(pool, local_user_id)
-    ))
-    .with_lemmy_type(LemmyErrorType::SystemErrLogin)?;
+  let (
+    follows,
+    community_blocks,
+    instance_blocks,
+    person_blocks,
+    moderates,
+    keyword_blocks,
+    discussion_languages,
+  ) = lemmy_db_schema::try_join_with_pool!(pool => (
+    |pool| CommunityFollowerView::for_person(pool, person_id),
+    |pool| CommunityActions::read_blocks_for_person(pool, person_id),
+    |pool| InstanceActions::read_blocks_for_person(pool, person_id),
+    |pool| PersonActions::read_blocks_for_person(pool, person_id),
+    |pool| CommunityModeratorView::for_person(pool, person_id, Some(&local_user_view.local_user)),
+    |pool| LocalUserKeywordBlock::read(pool, local_user_id),
+    |pool| LocalUserLanguage::read(pool, local_user_id)
+  ))
+  .with_lemmy_type(LemmyErrorType::SystemErrLogin)?;
 
   Ok(Json(MyUserInfo {
     local_user_view: local_user_view.clone(),
@@ -41,6 +50,7 @@ pub async fn get_my_user(
     community_blocks,
     instance_blocks,
     person_blocks,
+    keyword_blocks,
     discussion_languages,
   }))
 }

crates/db_schema/src/impls/keyword_block.rs

@@ -0,0 +1,59 @@
+use crate::{
+  newtypes::LocalUserId,
+  source::keyword_block::{LocalUserKeywordBlock, LocalUserKeywordBlockForm},
+  utils::{get_conn, DbPool},
+};
+use diesel::{delete, insert_into, prelude::*, result::Error, QueryDsl};
+use diesel_async::{scoped_futures::ScopedFutureExt, AsyncConnection, RunQueryDsl};
+use lemmy_db_schema_file::schema::local_user_keyword_block;
+
+impl LocalUserKeywordBlock {
+  pub async fn read(
+    pool: &mut DbPool<'_>,
+    for_local_user_id: LocalUserId,
+  ) -> Result<Vec<String>, Error> {
+    let conn = &mut get_conn(pool).await?;
+    let keyword_blocks = local_user_keyword_block::table
+      .filter(local_user_keyword_block::local_user_id.eq(for_local_user_id))
+      .load::<LocalUserKeywordBlock>(conn)
+      .await?;
+    let keywords = keyword_blocks
+      .into_iter()
+      .map(|keyword_block| keyword_block.keyword)
+      .collect();
+    Ok(keywords)
+  }
+
+  pub async fn update(
+    pool: &mut DbPool<'_>,
+    blocking_keywords: Vec<String>,
+    for_local_user_id: LocalUserId,
+  ) -> Result<usize, Error> {
+    let conn = &mut get_conn(pool).await?;
+    // No need to update if keywords unchanged
+    conn
+      .transaction::<_, Error, _>(|conn| {
+        async move {
+          delete(local_user_keyword_block::table)
+            .filter(local_user_keyword_block::local_user_id.eq(for_local_user_id))
+            .filter(local_user_keyword_block::keyword.ne_all(&blocking_keywords))
+            .execute(conn)
+            .await?;
+          let forms = blocking_keywords
+            .into_iter()
+            .map(|k| LocalUserKeywordBlockForm {
+              local_user_id: for_local_user_id,
+              keyword: k,
+            })
+            .collect::<Vec<_>>();
+          insert_into(local_user_keyword_block::table)
+            .values(forms)
+            .on_conflict_do_nothing()
+            .execute(conn)
+            .await
+        }
+        .scope_boxed()
+      })
+      .await
+  }
+}

crates/db_schema/src/impls/mod.rs

@@ -13,6 +13,7 @@ pub mod federation_blocklist;
 pub mod federation_queue_state;
 pub mod images;
 pub mod instance;
+pub mod keyword_block;
 pub mod language;
 pub mod local_site;
 pub mod local_site_rate_limit;

crates/db_schema/src/source/keyword_block.rs

@@ -0,0 +1,21 @@
+use crate::newtypes::LocalUserId;
+#[cfg(feature = "full")]
+use lemmy_db_schema_file::schema::local_user_keyword_block;
+use serde::{Deserialize, Serialize};
+
+#[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize)]
+#[cfg_attr(feature = "full", derive(Queryable, Selectable, Identifiable))]
+#[cfg_attr(feature = "full", diesel(table_name = local_user_keyword_block))]
+#[cfg_attr(feature = "full", diesel(primary_key(local_user_id, keyword)))]
+#[cfg_attr(feature = "full", diesel(check_for_backend(diesel::pg::Pg)))]
+pub struct LocalUserKeywordBlock {
+  pub local_user_id: LocalUserId,
+  pub keyword: String,
+}
+
+#[cfg_attr(feature = "full", derive(Insertable, AsChangeset))]
+#[cfg_attr(feature = "full", diesel(table_name = local_user_keyword_block))]
+pub struct LocalUserKeywordBlockForm {
+  pub local_user_id: LocalUserId,
+  pub keyword: String,
+}

crates/db_schema/src/source/mod.rs

@@ -19,6 +19,7 @@ pub mod federation_blocklist;
 pub mod federation_queue_state;
 pub mod images;
 pub mod instance;
+pub mod keyword_block;
 pub mod language;
 pub mod local_site;
 pub mod local_site_rate_limit;

crates/db_schema_file/src/schema.rs

@@ -527,6 +527,14 @@ diesel::table! {
     }
 }
 
+diesel::table! {
+    local_user_keyword_block (local_user_id, keyword) {
+        local_user_id -> Int4,
+        #[max_length = 50]
+        keyword -> Varchar,
+    }
+}
+
 diesel::table! {
     local_user_language (local_user_id, language_id) {
         local_user_id -> Int4,
@@ -1106,6 +1114,7 @@ diesel::joinable!(local_image -> local_user (local_user_id));
 diesel::joinable!(local_site -> site (site_id));
 diesel::joinable!(local_site_rate_limit -> local_site (local_site_id));
 diesel::joinable!(local_user -> person (person_id));
+diesel::joinable!(local_user_keyword_block -> local_user (local_user_id));
 diesel::joinable!(local_user_language -> language (language_id));
 diesel::joinable!(local_user_language -> local_user (local_user_id));
 diesel::joinable!(login_token -> local_user (user_id));
@@ -1212,6 +1221,7 @@ diesel::allow_tables_to_appear_in_same_query!(
   local_site_url_blocklist,
   local_user,
   local_user_language,
+  local_user_keyword_block,
   login_token,
   mod_add,
   mod_add_community,

crates/db_views/src/post/post_view.rs

@@ -36,6 +36,7 @@ use lemmy_db_schema::{
   impls::local_user::LocalUserOptionHelper,
   newtypes::{CommunityId, InstanceId, PersonId, PostId},
   source::{
+    keyword_block::LocalUserKeywordBlock,
     local_user::LocalUser,
     post::{post_actions_keys, post_keys as key, Post, PostActionsCursor},
     site::Site,
@@ -301,8 +302,6 @@ impl<'a> PostQuery<'a> {
       self
     };
 
-    let conn = &mut get_conn(pool).await?;
-
     let my_person_id = o.local_user.person_id();
     let my_local_user_id = o.local_user.local_user_id();
 
@@ -460,6 +459,22 @@ impl<'a> PostQuery<'a> {
       }
 
       query = query.filter(filter_blocked());
+      if let Some(local_user_id) = my_local_user_id {
+        let blocked_keywords: Vec<String> =
+          LocalUserKeywordBlock::read(pool, local_user_id).await?;
+        if !blocked_keywords.is_empty() {
+          for keyword in blocked_keywords {
+            let pattern = format!("%{}%", keyword);
+            query = query.filter(post::name.not_ilike(pattern.clone()));
+            query = query.filter(post::url.is_null().or(post::url.not_ilike(pattern.clone())));
+            query = query.filter(
+              post::body
+                .is_null()
+                .or(post::body.not_ilike(pattern.clone())),
+            );
+          }
+        }
+      }
     }
 
     let (limit, offset) = limit_and_offset(o.page, o.limit)?;
@@ -522,9 +537,8 @@ impl<'a> PostQuery<'a> {
 
       query.as_query()
     };
-
     debug!("Post View Query: {:?}", debug_query::<Pg, _>(&query));
-
+    let conn = &mut get_conn(pool).await?;
     Commented::new(query)
       .text("PostQuery::list")
       .text_if(
@@ -563,6 +577,7 @@ mod tests {
         CommunityUpdateForm,
       },
       instance::{Instance, InstanceActions, InstanceBanForm, InstanceBlockForm},
+      keyword_block::LocalUserKeywordBlock,
       language::Language,
       local_user::{LocalUser, LocalUserInsertForm, LocalUserUpdateForm},
       person::{Person, PersonActions, PersonBlockForm, PersonInsertForm},
@@ -594,6 +609,7 @@ mod tests {
   const POST_BY_BOT: &str = "post by bot";
   const POST: &str = "post";
   const POST_WITH_TAGS: &str = "post with tags";
+  const POST_KEYWORD_BLOCKED: &str = "blocked_keyword";
 
   fn names(post_views: &[PostView]) -> Vec<&str> {
     post_views.iter().map(|i| i.post.name.as_str()).collect()
@@ -687,6 +703,13 @@ mod tests {
       let person_block = PersonBlockForm::new(inserted_tegan_person.id, inserted_john_person.id);
       PersonActions::block(pool, &person_block).await?;
 
+      LocalUserKeywordBlock::update(
+        pool,
+        vec![POST_KEYWORD_BLOCKED.to_string()],
+        inserted_tegan_local_user.id,
+      )
+      .await?;
+
       // Two community post tags
       let tag_1 = Tag::create(
         pool,
@@ -2209,6 +2232,77 @@ mod tests {
 
     Ok(())
   }
+
+  #[test_context(Data)]
+  #[tokio::test]
+  #[serial]
+  async fn post_with_blocked_keywords(data: &mut Data) -> LemmyResult<()> {
+    let pool = &data.pool();
+    let pool = &mut pool.into();
+
+    let name_blocked = format!("post_{POST_KEYWORD_BLOCKED}");
+    let name_blocked2 = format!("post2_{POST_KEYWORD_BLOCKED}2");
+    let url = Some(Url::parse(&format!("https://google.com/{POST_KEYWORD_BLOCKED}"))?.into());
+    let body = format!("post body with {POST_KEYWORD_BLOCKED}");
+    let name_not_blocked = "post_with_name_not_blocked".to_string();
+    let name_not_blocked2 = "post_with_name_not_blocked2".to_string();
+
+    let post_name_blocked = PostInsertForm::new(
+      name_blocked.clone(),
+      data.tegan_local_user_view.person.id,
+      data.community.id,
+    );
+
+    let post_body_blocked = PostInsertForm {
+      body: Some(body),
+      ..PostInsertForm::new(
+        name_not_blocked.clone(),
+        data.tegan_local_user_view.person.id,
+        data.community.id,
+      )
+    };
+
+    let post_url_blocked = PostInsertForm {
+      url,
+      ..PostInsertForm::new(
+        name_not_blocked2.clone(),
+        data.tegan_local_user_view.person.id,
+        data.community.id,
+      )
+    };
+
+    let post_name_blocked_but_not_body_and_url = PostInsertForm {
+      body: Some("Some body".to_string()),
+      url: Some(Url::parse("https://google.com")?.into()),
+      ..PostInsertForm::new(
+        name_blocked2.clone(),
+        data.tegan_local_user_view.person.id,
+        data.community.id,
+      )
+    };
+    Post::create(pool, &post_name_blocked).await?;
+    Post::create(pool, &post_body_blocked).await?;
+    Post::create(pool, &post_url_blocked).await?;
+    Post::create(pool, &post_name_blocked_but_not_body_and_url).await?;
+
+    let post_listings = PostQuery {
+      local_user: Some(&data.tegan_local_user_view.local_user),
+      ..Default::default()
+    }
+    .list(&data.site, pool)
+    .await?;
+
+    // Should not have any of the posts
+    assert!(!names(&post_listings).contains(&name_blocked.as_str()));
+    assert!(!names(&post_listings).contains(&name_blocked2.as_str()));
+    assert!(!names(&post_listings).contains(&name_not_blocked.as_str()));
+    assert!(!names(&post_listings).contains(&name_not_blocked2.as_str()));
+
+    // Should contain not blocked posts
+    assert!(names(&post_listings).contains(&POST_BY_BOT));
+    assert!(names(&post_listings).contains(&POST));
+    Ok(())
+  }
   #[test_context(Data)]
   #[tokio::test]
   #[serial]

crates/utils/src/error.rs

@@ -10,6 +10,8 @@ use strum::{Display, EnumIter};
 #[non_exhaustive]
 // TODO: order these based on the crate they belong to (utils, federation, db, api)
 pub enum LemmyErrorType {
+  BlockKeywordTooShort,
+  BlockKeywordTooLong,
   ReportReasonRequired,
   ReportTooLong,
   NotAModerator,

crates/utils/src/utils/validation.rs

@@ -1,4 +1,4 @@
-use crate::error::{LemmyErrorExt, LemmyErrorType, LemmyResult};
+use crate::error::{LemmyErrorExt, LemmyErrorType, LemmyResult, MAX_API_PARAM_ELEMENTS};
 use clearurls::UrlCleaner;
 use itertools::Itertools;
 use regex::{Regex, RegexBuilder, RegexSet};
@@ -26,6 +26,8 @@ const ALT_TEXT_MAX_LENGTH: usize = 1500;
 const SITE_NAME_MAX_LENGTH: usize = 20;
 const SITE_NAME_MIN_LENGTH: usize = 1;
 const SITE_DESCRIPTION_MAX_LENGTH: usize = 150;
+const MIN_LENGTH_BLOCKING_KEYWORD: usize = 3;
+const MAX_LENGTH_BLOCKING_KEYWORD: usize = 50;
 const TAG_NAME_MIN_LENGTH: usize = 3;
 const TAG_NAME_MAX_LENGTH: usize = 100;
 //Invisible unicode characters, taken from https://invisible-characters.com/
@@ -319,6 +321,25 @@ pub fn check_urls_are_valid(urls: &Vec<String>) -> LemmyResult<Vec<String>> {
   Ok(unique_urls)
 }
 
+pub fn check_blocking_keywords_are_valid(blocking_keywords: &Vec<String>) -> LemmyResult<()> {
+  for keyword in blocking_keywords {
+    min_length_check(
+      keyword,
+      MIN_LENGTH_BLOCKING_KEYWORD,
+      LemmyErrorType::BlockKeywordTooShort,
+    )?;
+    max_length_check(
+      keyword,
+      MAX_LENGTH_BLOCKING_KEYWORD,
+      LemmyErrorType::BlockKeywordTooLong,
+    )?;
+  }
+  if blocking_keywords.len() >= MAX_API_PARAM_ELEMENTS {
+    Err(LemmyErrorType::TooManyItems)?
+  }
+  Ok(())
+}
+
 pub fn build_url_str_without_scheme(url_str: &str) -> LemmyResult<String> {
   // Parse and check for errors
   let mut url = Url::parse(url_str).or_else(|e| {

migrations/2024-12-29-203717_add_post_keyword_block_table/down.sql

@@ -0,0 +1,2 @@
+DROP TABLE local_user_keyword_block;
+

migrations/2024-12-29-203717_add_post_keyword_block_table/up.sql

@@ -0,0 +1,6 @@
+CREATE TABLE local_user_keyword_block (
+    local_user_id int REFERENCES local_user (id) ON UPDATE CASCADE ON DELETE CASCADE NOT NULL,
+    keyword varchar(50) NOT NULL,
+    PRIMARY KEY (local_user_id, keyword)
+);
+