Retoot reviews

This commit is contained in:
Rafael Caricio 2023-04-25 13:19:04 +02:00
parent b049b75873
commit 17d8c11726
Signed by: rafaelcaricio
GPG key ID: 3C86DBCE8E93C947
26 changed files with 292 additions and 33 deletions

View file

@ -5,7 +5,7 @@ description = "Federated micro-blogging platform and content subscription servic
license = "AGPL-3.0"
edition = "2021"
rust-version = "1.57"
rust-version = "1.68"
publish = false
default-run = "mitra"

View file

@ -3,7 +3,7 @@ name = "mitra-cli"
version = "1.22.0"
license = "AGPL-3.0"
edition = "2021"
rust-version = "1.57"
rust-version = "1.68"
[[bin]]
name = "mitractl"

View file

@ -3,7 +3,7 @@ name = "mitra-config"
version = "1.22.0"
license = "AGPL-3.0"
edition = "2021"
rust-version = "1.57"
rust-version = "1.68"
[dependencies]
mitra-utils = { path = "../mitra-utils" }

View file

@ -53,6 +53,11 @@ pub struct Config {
pub instance_short_description: String,
pub instance_description: String,
#[serde(default)]
pub tmdb_api_key: Option<String>,
#[serde(default)]
pub movie_user_password: Option<String>,
#[serde(skip)]
pub(super) instance_rsa_key: Option<RsaPrivateKey>,
@ -99,8 +104,8 @@ impl Config {
onion_proxy_url: self.federation.onion_proxy_url.clone(),
i2p_proxy_url: self.federation.i2p_proxy_url.clone(),
// Private instance doesn't send activities and sign requests
is_private: !self.federation.enabled
|| matches!(self.environment, Environment::Development),
is_private: !self.federation.enabled,
// || matches!(self.environment, Environment::Development),
fetcher_timeout: self.federation.fetcher_timeout,
deliverer_timeout: self.federation.deliverer_timeout,
}

View file

@ -3,7 +3,7 @@ name = "mitra-models"
version = "1.22.0"
license = "AGPL-3.0"
edition = "2021"
rust-version = "1.57"
rust-version = "1.68"
[dependencies]
mitra-utils = { path = "../mitra-utils" }

View file

@ -32,6 +32,22 @@ async fn create_notification(
Ok(())
}
pub async fn delete_notification(
db_client: &impl DatabaseClient,
notification_id: i32,
) -> Result<(), DatabaseError> {
db_client
.execute(
"
DELETE FROM notification
WHERE id = $1
",
&[&notification_id],
)
.await?;
Ok(())
}
pub async fn create_follow_notification(
db_client: &impl DatabaseClient,
sender_id: &Uuid,
@ -151,7 +167,7 @@ pub async fn get_notifications(
let statement = format!(
"
SELECT
notification, sender, post, post_author,
notification, sender, post, post_author, recipient,
{related_attachments},
{related_mentions},
{related_tags},
@ -164,6 +180,8 @@ pub async fn get_notifications(
ON notification.post_id = post.id
LEFT JOIN actor_profile AS post_author
ON post.author_id = post_author.id
LEFT JOIN actor_profile AS recipient
ON notification.recipient_id = recipient.id
WHERE
recipient_id = $1
AND ($2::integer IS NULL OR notification.id < $2)
@ -202,3 +220,52 @@ pub async fn get_notifications(
.await?;
Ok(notifications)
}
pub async fn get_mention_notifications(
db_client: &impl DatabaseClient,
limit: u16,
) -> Result<Vec<Notification>, DatabaseError> {
let statement = format!(
"
SELECT
notification, sender, post, post_author, recipient,
{related_attachments},
{related_mentions},
{related_tags},
{related_links},
{related_emojis}
FROM notification
JOIN actor_profile AS sender
ON notification.sender_id = sender.id
LEFT JOIN post
ON notification.post_id = post.id
LEFT JOIN actor_profile AS post_author
ON post.author_id = post_author.id
LEFT JOIN actor_profile AS recipient
ON notification.recipient_id = recipient.id
WHERE
event_type = $1
ORDER BY notification.id DESC
LIMIT $2
",
related_attachments = RELATED_ATTACHMENTS,
related_mentions = RELATED_MENTIONS,
related_tags = RELATED_TAGS,
related_links = RELATED_LINKS,
related_emojis = RELATED_EMOJIS,
);
let rows = db_client
.query(
&statement,
&[
&EventType::Mention,
&i64::from(limit),
],
)
.await?;
let mut notifications: Vec<Notification> = rows
.iter()
.map(Notification::try_from)
.collect::<Result<_, _>>()?;
Ok(notifications)
}

View file

@ -82,6 +82,7 @@ struct DbNotification {
pub struct Notification {
pub id: i32,
pub sender: DbActorProfile,
pub recipient: DbActorProfile,
pub post: Option<Post>,
pub event_type: EventType,
pub created_at: DateTime<Utc>,
@ -93,6 +94,7 @@ impl TryFrom<&Row> for Notification {
fn try_from(row: &Row) -> Result<Self, Self::Error> {
let db_notification: DbNotification = row.try_get("notification")?;
let db_sender: DbActorProfile = row.try_get("sender")?;
let db_recipient: DbActorProfile = row.try_get("recipient")?;
let maybe_db_post: Option<DbPost> = row.try_get("post")?;
let maybe_post = match maybe_db_post {
Some(db_post) => {
@ -118,6 +120,7 @@ impl TryFrom<&Row> for Notification {
let notification = Self {
id: db_notification.id,
sender: db_sender,
recipient: db_recipient,
post: maybe_post,
event_type: db_notification.event_type,
created_at: db_notification.created_at,

View file

@ -262,7 +262,7 @@ impl PostCreateData {
pub fn repost(repost_of_id: Uuid, object_id: Option<String>) -> Self {
Self {
repost_of_id: Some(repost_of_id),
object_id: object_id,
object_id,
created_at: Utc::now(),
..Default::default()
}

View file

@ -249,7 +249,7 @@ pub async fn get_user_by_name(
"
SELECT user_account, actor_profile
FROM user_account JOIN actor_profile USING (id)
WHERE actor_profile.username = $1
WHERE lower(actor_profile.username) = lower($1)
",
&[&username],
)

View file

@ -3,7 +3,7 @@ name = "mitra-utils"
version = "1.22.0"
license = "AGPL-3.0"
edition = "2021"
rust-version = "1.57"
rust-version = "1.68"
[dependencies]
# Used for HTML sanitization

View file

@ -11,7 +11,7 @@ pub fn get_hostname(url: &str) -> Result<String, ParseError> {
}
pub fn guess_protocol(hostname: &str) -> &'static str {
if hostname == "localhost" || hostname.ends_with(".example.com") {
if hostname == "localhost" {
return "http";
};
let maybe_ipv4_address = hostname.parse::<Ipv4Addr>();

View file

@ -109,6 +109,8 @@ pub struct Actor {
pub inbox: String,
pub outbox: String,
pub bot: bool,
#[serde(skip_serializing_if = "Option::is_none")]
pub followers: Option<String>,
@ -325,6 +327,7 @@ pub fn get_local_actor(user: &User, instance_url: &str) -> Result<Actor, ActorKe
preferred_username: username.to_string(),
inbox,
outbox,
bot: true,
followers: Some(followers),
following: Some(following),
subscribers: Some(subscribers),
@ -359,6 +362,7 @@ pub fn get_instance_actor(instance: &Instance) -> Result<Actor, ActorKeyError> {
preferred_username: instance.hostname(),
inbox: actor_inbox,
outbox: actor_outbox,
bot: true,
followers: None,
following: None,
subscribers: None,

View file

@ -153,6 +153,8 @@ pub async fn import_post(
db_client: &mut impl DatabaseClient,
instance: &Instance,
storage: &MediaStorage,
tmdb_api_key: Option<String>,
default_movie_user_password: Option<String>,
object_id: String,
object_received: Option<Object>,
) -> Result<Post, HandlerError> {
@ -245,7 +247,16 @@ pub async fn import_post(
// starting with the root
objects.reverse();
for object in objects {
let post = handle_note(db_client, instance, storage, object, &redirects).await?;
let post = handle_note(
db_client,
instance,
storage,
tmdb_api_key.clone(),
default_movie_user_password.clone(),
object,
&redirects,
)
.await?;
posts.push(post);
}

View file

@ -54,7 +54,18 @@ pub async fn handle_announce(
Ok(post_id) => post_id,
Err(_) => {
// Try to get remote post
let post = import_post(db_client, &instance, &storage, activity.object, None).await?;
let tmdb_api_key = config.tmdb_api_key.clone();
let default_movie_user_password = config.movie_user_password.clone();
let post = import_post(
db_client,
&instance,
&storage,
tmdb_api_key,
default_movie_user_password,
activity.object,
None,
)
.await?;
post.id
}
};

View file

@ -36,6 +36,7 @@ use crate::activitypub::{
};
use crate::errors::ValidationError;
use crate::media::MediaStorage;
use crate::tmdb::lookup_and_create_movie_user;
use crate::validators::{
emojis::{validate_emoji_name, EMOJI_MEDIA_TYPES},
posts::{
@ -306,6 +307,8 @@ pub async fn get_object_tags(
db_client: &mut impl DatabaseClient,
instance: &Instance,
storage: &MediaStorage,
api_key: Option<String>,
default_movie_user_password: Option<String>,
object: &Object,
redirects: &HashMap<String, String>,
) -> Result<(Vec<Uuid>, Vec<String>, Vec<Uuid>, Vec<Uuid>), HandlerError> {
@ -346,7 +349,33 @@ pub async fn get_object_tags(
// Try to find profile by actor ID.
if let Some(href) = tag.href {
if let Ok(username) = parse_local_actor_id(&instance.url(), &href) {
let user = get_user_by_name(db_client, &username).await?;
// Check if local Movie account exists and if not, create the movie, if valid.
let user = match get_user_by_name(db_client, &username).await {
Ok(user) => {
user
},
Err(DatabaseError::NotFound(_)) => {
if let Some(api_key) = &api_key {
log::warn!("failed to find mentioned user by name {}, checking if its a valid movie...", username);
lookup_and_create_movie_user(
instance,
db_client,
api_key,
&storage.media_dir,
&username,
default_movie_user_password.clone(),
)
.await
.map_err(|err| {
log::warn!("failed to create movie user {username}: {err}");
HandlerError::LocalObject
})?
} else {
return Err(HandlerError::LocalObject);
}
}
Err(error) => return Err(error.into()),
};
if !mentions.contains(&user.id) {
mentions.push(user.id);
};
@ -503,6 +532,8 @@ pub async fn handle_note(
db_client: &mut impl DatabaseClient,
instance: &Instance,
storage: &MediaStorage,
tmdb_api_key: Option<String>,
default_movie_user_password: Option<String>,
object: Object,
redirects: &HashMap<String, String>,
) -> Result<Post, HandlerError> {
@ -543,8 +574,16 @@ pub async fn handle_note(
return Err(ValidationError("post is empty").into());
};
let (mentions, hashtags, links, emojis) =
get_object_tags(db_client, instance, storage, &object, redirects).await?;
let (mentions, hashtags, links, emojis) = get_object_tags(
db_client,
instance,
storage,
tmdb_api_key,
default_movie_user_password,
&object,
redirects,
)
.await?;
let in_reply_to_id = match object.in_reply_to {
Some(ref object_id) => {
@ -632,10 +671,15 @@ pub async fn handle_create(
// Most likely it's a forwarded reply.
None
};
let tmdb_api_key = config.tmdb_api_key.clone();
let default_movie_user_password = config.movie_user_password.clone();
import_post(
db_client,
&config.instance(),
&MediaStorage::from(config),
tmdb_api_key,
default_movie_user_password,
object_id,
object_received,
)

View file

@ -67,8 +67,19 @@ async fn handle_update_note(
if content.is_empty() && attachments.is_empty() {
return Err(ValidationError("post is empty").into());
};
let (mentions, hashtags, links, emojis) =
get_object_tags(db_client, &instance, &storage, &object, &HashMap::new()).await?;
let tmdb_api_key = config.tmdb_api_key.clone();
let default_movie_user_password = config.movie_user_password.clone();
let (mentions, hashtags, links, emojis) = get_object_tags(
db_client,
&instance,
&storage,
tmdb_api_key,
default_movie_user_password,
&object,
&HashMap::new(),
)
.await?;
let updated_at = object.updated.unwrap_or(Utc::now());
let post_data = PostUpdateData {
content,

View file

@ -86,7 +86,7 @@ pub fn validate_object_id(object_id: &str) -> Result<(), ValidationError> {
pub fn parse_local_actor_id(instance_url: &str, actor_id: &str) -> Result<String, ValidationError> {
let url_regexp_str = format!(
"^{}/users/(?P<username>[0-9a-z_]+)$",
"^{}/users/(?P<username>[0-9a-zA-Z_]+)$",
instance_url.replace('.', r"\."),
);
let url_regexp = Regex::new(&url_regexp_str).map_err(|_| ValidationError("error"))?;

View file

@ -7,7 +7,13 @@ use mitra_models::{
posts::queries::{delete_post, find_extraneous_posts},
profiles::queries::{delete_profile, find_empty_profiles, get_profile_by_id},
};
use mitra_models::database::DatabaseError;
use mitra_models::notifications::queries::{delete_notification, get_mention_notifications};
use mitra_models::posts::queries::create_post;
use mitra_models::posts::types::PostCreateData;
use mitra_models::users::queries::get_user_by_id;
use mitra_utils::datetime::days_before_now;
use crate::activitypub::builders::announce::prepare_announce;
use crate::activitypub::queues::{
process_queued_incoming_activities, process_queued_outgoing_activities,
@ -72,3 +78,58 @@ pub async fn prune_remote_emojis(config: &Config, db_pool: &DbPool) -> Result<()
}
Ok(())
}
// Finds mention notifications and repost them
pub async fn handle_movies_mentions(config: &Config, db_pool: &DbPool) -> Result<(), anyhow::Error> {
let db_client = &mut **get_database_client(db_pool).await?;
log::debug!("Reviewing mentions..");
// for each mention notification do repost
let mut transaction = db_client.transaction().await?;
let mention_notifications = match get_mention_notifications(&transaction, 50).await {
Ok(mention_notifications) => mention_notifications,
Err(DatabaseError::DatabaseClientError(err)) => {
return Err(anyhow::anyhow!("Error in client: {err}"))
}
Err(err) => return Err(err.into()),
};
for mention_notification in mention_notifications {
log::info!("Reviewing mention notification {}", mention_notification.id);
if let Some(post_with_mention) = mention_notification.post {
// Does not repost private posts or reposts
if !post_with_mention.is_public() || post_with_mention.repost_of_id.is_some() {
continue;
}
let mut post = post_with_mention.clone();
let post_id = post.id;
let current_user = get_user_by_id(&transaction, &mention_notification.recipient.id).await?;
// Repost
let repost_data = PostCreateData::repost(post.id, None);
let mut repost = match create_post(&mut transaction, &current_user.id, repost_data).await {
Ok(repost) => repost,
Err(DatabaseError::AlreadyExists(err)) => {
log::info!("Review as Mention of {} already reposted the post with id {}", current_user.profile.username, post_id);
delete_notification(&mut transaction, mention_notification.id).await?;
continue;
}
Err(err) => return Err(err.into()),
};
post.repost_count += 1;
repost.repost_of = Some(Box::new(post));
// Federate
prepare_announce(&transaction, &config.instance(), &current_user, &repost)
.await?
.enqueue(&mut transaction)
.await?;
// Delete notification to avoid re-processing
delete_notification(&mut transaction, mention_notification.id).await?;
log::info!("Review as Mention of {} reposted with post id {}", current_user.profile.username, post_id);
}
}
Ok(transaction.commit().await?)
}

View file

@ -16,6 +16,7 @@ enum PeriodicTask {
DeleteExtraneousPosts,
DeleteEmptyProfiles,
PruneRemoteEmojis,
HandleMoviesMentions,
}
impl PeriodicTask {
@ -28,6 +29,7 @@ impl PeriodicTask {
Self::DeleteExtraneousPosts => 3600,
Self::DeleteEmptyProfiles => 3600,
Self::PruneRemoteEmojis => 3600,
Self::HandleMoviesMentions => 5,
}
}
@ -49,6 +51,7 @@ pub fn run(config: Config, db_pool: DbPool) -> () {
(PeriodicTask::IncomingActivityQueueExecutor, None),
(PeriodicTask::OutgoingActivityQueueExecutor, None),
(PeriodicTask::PruneRemoteEmojis, None),
(PeriodicTask::HandleMoviesMentions, None),
]);
if config.retention.extraneous_posts.is_some() {
scheduler_state.insert(PeriodicTask::DeleteExtraneousPosts, None);
@ -80,6 +83,7 @@ pub fn run(config: Config, db_pool: DbPool) -> () {
}
PeriodicTask::PruneRemoteEmojis => prune_remote_emojis(&config, &db_pool).await,
PeriodicTask::SubscriptionExpirationMonitor => Ok(()),
PeriodicTask::HandleMoviesMentions => handle_movies_mentions(&config, &db_pool).await,
};
task_result.unwrap_or_else(|err| {
log::error!("{:?}: {}", task, err);

View file

@ -12,6 +12,7 @@ pub mod logger;
pub mod mastodon_api;
pub mod media;
pub mod nodeinfo;
mod tmdb;
pub mod validators;
pub mod web_client;
pub mod webfinger;

View file

@ -155,7 +155,19 @@ async fn find_post_by_url(
}
Err(_) => {
instance.fetcher_timeout = SEARCH_FETCHER_TIMEOUT;
match import_post(db_client, &instance, &storage, url.to_string(), None).await {
let tmdb_api_key = config.tmdb_api_key.clone();
let default_movie_user_password = config.movie_user_password.clone();
match import_post(
db_client,
&instance,
&storage,
tmdb_api_key,
default_movie_user_password,
url.to_string(),
None,
)
.await
{
Ok(post) => Some(post),
Err(err) => {
log::warn!("{}", err);

View file

@ -15,7 +15,7 @@ use crate::webfinger::types::ActorAddress;
// See also: ACTOR_ADDRESS_RE in webfinger::types
const MENTION_SEARCH_RE: &str = r"(?m)(?P<before>^|\s|>|[\(])@(?P<mention>[^\s<]+)";
const MENTION_SEARCH_SECONDARY_RE: &str =
r"^(?P<username>[\w\.-]+)(@(?P<hostname>[\w\.-]+\w))?(?P<after>[\.,:?!\)]?)$";
r"^(?P<username>[\w\.-_]+)(@(?P<hostname>[\w\.-]+\w))?(?P<after>[\.,:?!\)]?)$";
/// Finds everything that looks like a mention
fn find_mentions(instance_hostname: &str, text: &str) -> Vec<String> {

View file

@ -28,11 +28,8 @@ pub async fn authorize_subscription(
) -> Result<HttpResponse, MastodonError> {
let db_client = &**get_database_client(&db_pool).await?;
let _current_user = get_current_user(db_client, auth.token()).await?;
// The user must have a public ethereum address,
// because subscribers should be able
// to verify that payments are actually sent to the recipient.
return Err(MastodonError::PermissionError);
// We don't have subscriptions
Err(MastodonError::PermissionError)
}
#[get("/options")]

View file

@ -7,7 +7,7 @@ use super::profiles::validate_username;
pub fn validate_local_username(username: &str) -> Result<(), ValidationError> {
validate_username(username)?;
// The username regexp should not allow domain names and IP addresses
let username_regexp = Regex::new(r"^[a-z0-9_]+$").unwrap();
let username_regexp = Regex::new(r"^[a-zA-Z0-9_]+$").unwrap();
if !username_regexp.is_match(username) {
return Err(ValidationError("invalid username"));
};

View file

@ -135,9 +135,9 @@ mod tests {
#[test]
fn test_actor_address_parse_address() {
let value = "user_1@example.com";
let value = "Matrix_1999@example.com";
let actor_address: ActorAddress = value.parse().unwrap();
assert_eq!(actor_address.username, "user_1");
assert_eq!(actor_address.username, "Matrix_1999");
assert_eq!(actor_address.hostname, "example.com");
assert_eq!(actor_address.to_string(), value);
}
@ -146,7 +146,7 @@ mod tests {
fn test_actor_address_parse_mention() {
let value = "@user_1@example.com";
let result = value.parse::<ActorAddress>();
assert_eq!(result.is_err(), true);
assert!(result.is_err());
}
#[test]
@ -164,6 +164,6 @@ mod tests {
let short_mention = "@user";
let result = ActorAddress::from_mention(short_mention);
assert_eq!(result.is_err(), true);
assert!(result.is_err());
}
}

View file

@ -11,6 +11,8 @@ use crate::activitypub::{
identifiers::{local_actor_id, local_instance_actor_id, parse_local_actor_id},
};
use crate::errors::{HttpError, ValidationError};
use crate::media::MediaStorage;
use crate::tmdb::lookup_and_create_movie_user;
use super::types::{
ActorAddress, JsonResourceDescriptor, Link, WebfingerQueryParams, JRD_CONTENT_TYPE,
@ -82,8 +84,34 @@ pub async fn webfinger_view(
db_pool: web::Data<DbPool>,
query_params: web::Query<WebfingerQueryParams>,
) -> Result<HttpResponse, HttpError> {
let db_client = &**get_database_client(&db_pool).await?;
let jrd = get_jrd(db_client, config.instance(), &query_params.resource).await?;
let db_client = &mut **get_database_client(&db_pool).await?;
let jrd = match get_jrd(db_client, config.instance(), &query_params.resource).await {
Ok(jrd) => jrd,
Err(_) => {
// Lookup the movie in TMDB and create a local user. By now we know that the local
// user for this movie does not exist.
let config: &Config = &config;
if let Some(api_key) = &config.tmdb_api_key {
let movie_account = parse_acct_uri(&query_params.resource)?;
let instance = config.instance();
let storage = MediaStorage::from(config);
lookup_and_create_movie_user(
&instance,
db_client,
api_key,
&storage.media_dir,
&movie_account.username,
config.movie_user_password.clone(),
)
.await
.map_err(|err| {
log::error!("Failed to create movie user: {}", err);
HttpError::InternalError
})?;
}
get_jrd(db_client, config.instance(), &query_params.resource).await?
}
};
let response = HttpResponse::Ok().content_type(JRD_CONTENT_TYPE).json(jrd);
Ok(response)
}