FediMovies/fedimovies
1
1
Fork 0
Code Issues 3 Pull requests Projects Releases Packages Activity

Add limits.media.emoji_size_limit configuration parameter

Browse source
This commit is contained in:
silverpill 2023-03-25 21:50:10 +00:00
parent 0521f1f731
commit 5e1f441e8b
15 changed files with 80 additions and 43 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
CHANGELOG.md
View file

@ -10,6 +10,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
- Added `prune-remote-emojis` command.
- Prune remote emojis in background.
- Added `limits.media.emoji_size_limit` configuration parameter.
### Changed

8
mitra-cli/src/cli.rs
View file

@ -14,7 +14,11 @@ use mitra::ethereum::{
sync::save_current_block_number,
utils::key_to_ethereum_address,
};
use mitra::media::{remove_files, remove_media};
use mitra::media::{
remove_files,
remove_media,
MediaStorage,
};
use mitra::models::{
attachments::queries::delete_unused_attachments,
cleanup::find_orphaned_files,
@ -230,7 +234,7 @@ impl RefetchActor {
update_remote_profile(
db_client,
&config.instance(),
&config.media_dir(),
&MediaStorage::from(config),
profile,
actor,
).await?;

8
mitra-config/src/limits.rs
View file

@ -38,6 +38,7 @@ fn deserialize_file_size<'de, D>(
}
const fn default_file_size_limit() -> usize { 20_000_000 } // 20 MB
const fn default_emoji_size_limit() -> usize { 500_000 } // 500 kB
#[derive(Clone, Deserialize)]
pub struct MediaLimits {
@ -46,12 +47,19 @@ pub struct MediaLimits {
deserialize_with = "deserialize_file_size",
)]
pub file_size_limit: usize,
#[serde(
default = "default_emoji_size_limit",
deserialize_with = "deserialize_file_size",
)]
pub emoji_size_limit: usize,
}
impl Default for MediaLimits {
fn default() -> Self {
Self {
file_size_limit: default_file_size_limit(),
emoji_size_limit: default_emoji_size_limit(),
}
}
}

17
src/activitypub/actors/helpers.rs
View file

@ -12,6 +12,7 @@ use crate::activitypub::{
vocabulary::{EMOJI, HASHTAG},
};
use crate::database::DatabaseClient;
use crate::media::MediaStorage;
use crate::models::{
profiles::queries::{create_profile, update_profile},
profiles::types::{
@ -104,7 +105,7 @@ fn parse_aliases(actor: &Actor) -> Vec<String> {
async fn parse_tags(
db_client: &impl DatabaseClient,
instance: &Instance,
media_dir: &Path,
storage: &MediaStorage,
actor: &Actor,
) -> Result<Vec<Uuid>, HandlerError> {
let mut emojis = vec![];
@ -118,7 +119,7 @@ async fn parse_tags(
match handle_emoji(
db_client,
instance,
media_dir,
storage,
tag_value,
).await? {
Some(emoji) => {
@ -138,7 +139,7 @@ async fn parse_tags(
pub async fn create_remote_profile(
db_client: &mut impl DatabaseClient,
instance: &Instance,
media_dir: &Path,
storage: &MediaStorage,
actor: Actor,
) -> Result<DbActorProfile, HandlerError> {
let actor_address = actor.address()?;
@ -148,7 +149,7 @@ pub async fn create_remote_profile(
let (maybe_avatar, maybe_banner) = fetch_actor_images(
instance,
&actor,
media_dir,
&storage.media_dir,
None,
None,
).await;
@ -158,7 +159,7 @@ pub async fn create_remote_profile(
let emojis = parse_tags(
db_client,
instance,
media_dir,
storage,
&actor,
).await?;
let mut profile_data = ProfileCreateData {
@ -185,7 +186,7 @@ pub async fn create_remote_profile(
pub async fn update_remote_profile(
db_client: &mut impl DatabaseClient,
instance: &Instance,
media_dir: &Path,
storage: &MediaStorage,
profile: DbActorProfile,
actor: Actor,
) -> Result<DbActorProfile, HandlerError> {
@ -207,7 +208,7 @@ pub async fn update_remote_profile(
let (maybe_avatar, maybe_banner) = fetch_actor_images(
instance,
&actor,
media_dir,
&storage.media_dir,
profile.avatar,
profile.banner,
).await;
@ -217,7 +218,7 @@ pub async fn update_remote_profile(
let emojis = parse_tags(
db_client,
instance,
media_dir,
storage,
&actor,
).await?;
let mut profile_data = ProfileUpdateData {

3
src/activitypub/authentication.rs
View file

@ -24,6 +24,7 @@ use crate::json_signatures::{
JsonSigner,
},
};
use crate::media::MediaStorage;
use crate::models::{
profiles::queries::get_profile_by_remote_actor_id,
profiles::types::DbActorProfile,
@ -89,7 +90,7 @@ async fn get_signer(
match get_or_import_profile_by_actor_id(
db_client,
&config.instance(),
&config.media_dir(),
&MediaStorage::from(config),
signer_id,
).await {
Ok(profile) => profile,

18
src/activitypub/fetcher/helpers.rs
View file

@ -1,5 +1,4 @@
use std::collections::HashMap;
use std::path::Path;
use mitra_config::{Config, Instance};
@ -12,6 +11,7 @@ use crate::activitypub::{
};
use crate::database::{DatabaseClient, DatabaseError};
use crate::errors::ValidationError;
use crate::media::MediaStorage;
use crate::models::{
posts::helpers::get_local_post_by_id,
posts::queries::get_post_by_remote_object_id,
@ -33,7 +33,7 @@ use super::fetchers::{
pub async fn get_or_import_profile_by_actor_id(
db_client: &mut impl DatabaseClient,
instance: &Instance,
media_dir: &Path,
storage: &MediaStorage,
actor_id: &str,
) -> Result<DbActorProfile, HandlerError> {
if actor_id.starts_with(&instance.url()) {
@ -52,7 +52,7 @@ pub async fn get_or_import_profile_by_actor_id(
let profile_updated = update_remote_profile(
db_client,
instance,
media_dir,
storage,
profile,
actor,
).await?;
@ -81,7 +81,7 @@ pub async fn get_or_import_profile_by_actor_id(
let profile_updated = update_remote_profile(
db_client,
instance,
media_dir,
storage,
profile,
actor,
).await?;
@ -92,7 +92,7 @@ pub async fn get_or_import_profile_by_actor_id(
let profile = create_remote_profile(
db_client,
instance,
media_dir,
storage,
actor,
).await?;
profile
@ -109,7 +109,7 @@ pub async fn get_or_import_profile_by_actor_id(
pub async fn import_profile_by_actor_address(
db_client: &mut impl DatabaseClient,
instance: &Instance,
media_dir: &Path,
storage: &MediaStorage,
actor_address: &ActorAddress,
) -> Result<DbActorProfile, HandlerError> {
if actor_address.hostname == instance.hostname() {
@ -130,7 +130,7 @@ pub async fn import_profile_by_actor_address(
let profile = create_remote_profile(
db_client,
instance,
media_dir,
storage,
actor,
).await?;
Ok(profile)
@ -140,7 +140,7 @@ pub async fn import_profile_by_actor_address(
pub async fn get_or_import_profile_by_actor_address(
db_client: &mut impl DatabaseClient,
instance: &Instance,
media_dir: &Path,
storage: &MediaStorage,
actor_address: &ActorAddress,
) -> Result<DbActorProfile, HandlerError> {
let acct = actor_address.acct(&instance.hostname());
@ -156,7 +156,7 @@ pub async fn get_or_import_profile_by_actor_address(
import_profile_by_actor_address(
db_client,
instance,
media_dir,
storage,
actor_address,
).await?
},

3
src/activitypub/handlers/announce.rs
View file

@ -11,6 +11,7 @@ use crate::activitypub::{
};
use crate::database::{DatabaseClient, DatabaseError};
use crate::errors::ValidationError;
use crate::media::MediaStorage;
use crate::models::{
posts::queries::{
create_post,
@ -54,7 +55,7 @@ pub async fn handle_announce(
let author = get_or_import_profile_by_actor_id(
db_client,
&config.instance(),
&config.media_dir(),
&MediaStorage::from(config),
&activity.actor,
).await?;
let post_id = match parse_local_object_id(

21
src/activitypub/handlers/create.rs
View file

@ -1,5 +1,4 @@
use std::collections::HashMap;
use std::path::Path;
use chrono::Utc;
use serde_json::{Value as JsonValue};
@ -27,6 +26,7 @@ use crate::activitypub::{
};
use crate::database::{DatabaseClient, DatabaseError};
use crate::errors::ValidationError;
use crate::media::MediaStorage;
use crate::models::{
attachments::queries::create_attachment,
emojis::queries::{
@ -45,7 +45,6 @@ use crate::models::{
use crate::validators::{
emojis::{
validate_emoji_name,
EMOJI_MAX_SIZE,
EMOJI_MEDIA_TYPES,
},
posts::{
@ -242,7 +241,7 @@ pub fn get_object_links(
pub async fn handle_emoji(
db_client: &impl DatabaseClient,
instance: &Instance,
media_dir: &Path,
storage: &MediaStorage,
tag_value: JsonValue,
) -> Result<Option<DbEmoji>, HandlerError> {
let tag: EmojiTag = match serde_json::from_value(tag_value) {
@ -279,8 +278,8 @@ pub async fn handle_emoji(
instance,
&tag.icon.url,
tag.icon.media_type.as_deref(),
EMOJI_MAX_SIZE,
media_dir,
storage.emoji_size_limit,
&storage.media_dir,
).await {
Ok(file) => file,
Err(error) => {
@ -338,7 +337,7 @@ pub async fn get_object_tags(
redirects: &HashMap<String, String>,
) -> Result<(Vec<Uuid>, Vec<String>, Vec<Uuid>, Vec<Uuid>), HandlerError> {
let instance = config.instance();
let media_dir = config.media_dir();
let storage = MediaStorage::from(config);
let mut mentions = vec![];
let mut hashtags = vec![];
let mut links = vec![];
@ -383,7 +382,7 @@ pub async fn get_object_tags(
match get_or_import_profile_by_actor_id(
db_client,
&instance,
&media_dir,
&storage,
&href,
).await {
Ok(profile) => {
@ -413,7 +412,7 @@ pub async fn get_object_tags(
let profile = match get_or_import_profile_by_actor_address(
db_client,
&instance,
&media_dir,
&storage,
&actor_address,
).await {
Ok(profile) => profile,
@ -469,7 +468,7 @@ pub async fn get_object_tags(
match handle_emoji(
db_client,
&instance,
&media_dir,
&storage,
tag_value,
).await? {
Some(emoji) => {
@ -534,7 +533,7 @@ pub async fn handle_note(
redirects: &HashMap<String, String>,
) -> Result<Post, HandlerError> {
let instance = config.instance();
let media_dir = config.media_dir();
let storage = MediaStorage::from(config);
match object.object_type.as_str() {
NOTE => (),
ARTICLE | EVENT | QUESTION | PAGE | VIDEO => {
@ -550,7 +549,7 @@ pub async fn handle_note(
let author = get_or_import_profile_by_actor_id(
db_client,
&instance,
&media_dir,
&storage,
&author_id,
).await.map_err(|err| {
log::warn!("failed to import {} ({})", author_id, err);

3
src/activitypub/handlers/follow.rs
View file

@ -12,6 +12,7 @@ use crate::activitypub::{
};
use crate::database::{DatabaseClient, DatabaseError};
use crate::errors::ValidationError;
use crate::media::MediaStorage;
use crate::models::{
relationships::queries::{
create_remote_follow_request_opt,
@ -40,7 +41,7 @@ pub async fn handle_follow(
let source_profile = get_or_import_profile_by_actor_id(
db_client,
&config.instance(),
&config.media_dir(),
&MediaStorage::from(config),
&activity.actor,
).await?;
let source_actor = source_profile.actor_json

3
src/activitypub/handlers/like.rs
View file

@ -13,6 +13,7 @@ use crate::activitypub::{
};
use crate::database::{DatabaseClient, DatabaseError};
use crate::errors::ValidationError;
use crate::media::MediaStorage;
use crate::models::{
reactions::queries::create_reaction,
};
@ -36,7 +37,7 @@ pub async fn handle_like(
let author = get_or_import_profile_by_actor_id(
db_client,
&config.instance(),
&config.media_dir(),
&MediaStorage::from(config),
&activity.actor,
).await?;
let post_id = match get_post_by_object_id(

7
src/activitypub/handlers/move.rs
View file

@ -15,6 +15,7 @@ use crate::activitypub::{
};
use crate::database::{DatabaseClient, DatabaseError};
use crate::errors::ValidationError;
use crate::media::MediaStorage;
use crate::models::{
notifications::queries::create_move_notification,
profiles::helpers::find_aliases,
@ -49,7 +50,7 @@ pub async fn handle_move(
};
let instance = config.instance();
let media_dir = config.media_dir();
let storage = MediaStorage::from(config);
let old_profile = if let Ok(username) = parse_local_actor_id(
&instance.url(),
&activity.object,
@ -60,7 +61,7 @@ pub async fn handle_move(
get_or_import_profile_by_actor_id(
db_client,
&instance,
&media_dir,
&storage,
&activity.object,
).await?
};
@ -68,7 +69,7 @@ pub async fn handle_move(
let new_profile = get_or_import_profile_by_actor_id(
db_client,
&instance,
&media_dir,
&storage,
&activity.target,
).await?;
let new_actor = new_profile.actor_json.as_ref()

3
src/activitypub/handlers/update.rs
View file

@ -23,6 +23,7 @@ use crate::activitypub::{
};
use crate::database::{DatabaseClient, DatabaseError};
use crate::errors::ValidationError;
use crate::media::MediaStorage;
use crate::models::{
posts::queries::{
get_post_by_remote_object_id,
@ -110,7 +111,7 @@ async fn handle_update_person(
update_remote_profile(
db_client,
&config.instance(),
&config.media_dir(),
&MediaStorage::from(config),
profile,
activity.object,
).await?;

5
src/mastodon_api/search/helpers.rs
View file

@ -21,6 +21,7 @@ use crate::activitypub::{
use crate::database::{DatabaseClient, DatabaseError};
use crate::errors::ValidationError;
use crate::ethereum::utils::validate_ethereum_address;
use crate::media::MediaStorage;
use crate::models::{
posts::{
helpers::{can_view_post, get_local_post_by_id},
@ -124,7 +125,7 @@ async fn search_profiles_or_import(
match import_profile_by_actor_address(
db_client,
&config.instance(),
&config.media_dir(),
&MediaStorage::from(config),
&actor_address,
).await {
Ok(profile) => {
@ -204,7 +205,7 @@ async fn find_profile_by_url(
get_or_import_profile_by_actor_id(
db_client,
&config.instance(),
&config.media_dir(),
&MediaStorage::from(config),
url,
).await
.map_err(|err| log::warn!("{}", err))

7
src/mastodon_api/settings/helpers.rs
View file

@ -18,6 +18,7 @@ use crate::database::{
};
use crate::errors::ValidationError;
use crate::mastodon_api::accounts::helpers::follow_or_create_request;
use crate::media::MediaStorage;
use crate::models::{
profiles::types::DbActorProfile,
relationships::queries::{
@ -88,11 +89,12 @@ pub async fn import_follows_task(
address_list: Vec<ActorAddress>,
) -> Result<(), anyhow::Error> {
let db_client = &mut **get_database_client(db_pool).await?;
let storage = MediaStorage::from(config);
for actor_address in address_list {
let profile = match get_or_import_profile_by_actor_address(
db_client,
&config.instance(),
&config.media_dir(),
&storage,
&actor_address,
).await {
Ok(profile) => profile,
@ -129,12 +131,13 @@ pub async fn move_followers_task(
) -> Result<(), anyhow::Error> {
let db_client = &mut **get_database_client(db_pool).await?;
let instance = config.instance();
let storage = MediaStorage::from(config);
let mut remote_followers = vec![];
for follower_address in address_list {
let follower = match get_or_import_profile_by_actor_address(
db_client,
&instance,
&config.media_dir(),
&storage,
&follower_address,
).await {
Ok(profile) => profile,

16
src/media.rs
View file

@ -1,6 +1,6 @@
use std::fs::remove_file;
use std::io::Error;
use std::path::Path;
use std::path::{Path, PathBuf};
use sha2::{Digest, Sha256};
@ -87,6 +87,20 @@ pub async fn remove_media(
}
}
pub struct MediaStorage {
pub media_dir: PathBuf,
pub emoji_size_limit: usize,
}
impl From<&Config> for MediaStorage {
fn from(config: &Config) -> Self {
Self {
media_dir: config.media_dir(),
emoji_size_limit: config.limits.media.emoji_size_limit,
}
}
}
#[cfg(test)]
mod tests {
use mitra_utils::files::sniff_media_type;