Rewrite collections to use new fetcher (#1861)

* Merge traits ToApub and FromApub into ApubObject

* Rewrite community outbox to use new fetcher

* Rewrite community moderators collection

* Rewrite tombstone
This commit is contained in:
Nutomic 2021-10-27 16:03:07 +00:00 committed by GitHub
parent d9ecabee87
commit 61189efe72
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
31 changed files with 655 additions and 668 deletions

View file

@ -1,7 +1,7 @@
#!/bin/bash
set -e
cargo +nightly fmt -- --check
cargo +nightly fmt
cargo +nightly clippy --workspace --tests --all-targets --all-features -- \
-D warnings -D deprecated -D clippy::perf -D clippy::complexity -D clippy::dbg_macro

1
Cargo.lock generated
View file

@ -1833,6 +1833,7 @@ dependencies = [
"http",
"http-signature-normalization-actix",
"itertools",
"lazy_static",
"lemmy_api_common",
"lemmy_apub_lib",
"lemmy_db_schema",

View file

@ -50,6 +50,7 @@ thiserror = "1.0.29"
background-jobs = "0.9.0"
reqwest = { version = "0.11.4", features = ["json"] }
html2md = "0.2.13"
lazy_static = "1.4.0"
[dev-dependencies]
serial_test = "0.5.1"

View file

@ -21,7 +21,7 @@ use activitystreams::{base::AnyBase, link::Mention, primitives::OneOrMany, unpar
use lemmy_api_common::{blocking, check_post_deleted_or_removed};
use lemmy_apub_lib::{
data::Data,
traits::{ActivityFields, ActivityHandler, ActorType, FromApub, ToApub},
traits::{ActivityFields, ActivityHandler, ActorType, ApubObject},
values::PublicUrl,
verify::verify_domains_match,
};
@ -77,7 +77,7 @@ impl CreateOrUpdateComment {
let create_or_update = CreateOrUpdateComment {
actor: ObjectId::new(actor.actor_id()),
to: [PublicUrl::Public],
object: comment.to_apub(context.pool()).await?,
object: comment.to_apub(context).await?,
cc: maa.ccs,
tag: maa.tags,
kind,

View file

@ -22,7 +22,7 @@ use activitystreams::{
use lemmy_api_common::blocking;
use lemmy_apub_lib::{
data::Data,
traits::{ActivityFields, ActivityHandler, ActorType, ToApub},
traits::{ActivityFields, ActivityHandler, ActorType, ApubObject},
values::PublicUrl,
};
use lemmy_db_schema::{
@ -66,7 +66,7 @@ impl UpdateCommunity {
let update = UpdateCommunity {
actor: ObjectId::new(actor.actor_id()),
to: [PublicUrl::Public],
object: community.to_apub(context.pool()).await?,
object: community.to_apub(context).await?,
cc: [ObjectId::new(community.actor_id())],
kind: UpdateType::Update,
id: id.clone(),

View file

@ -21,7 +21,7 @@ use anyhow::anyhow;
use lemmy_api_common::blocking;
use lemmy_apub_lib::{
data::Data,
traits::{ActivityFields, ActivityHandler, ActorType, FromApub, ToApub},
traits::{ActivityFields, ActivityHandler, ActorType, ApubObject},
values::PublicUrl,
verify::{verify_domains_match, verify_urls_match},
};
@ -48,6 +48,28 @@ pub struct CreateOrUpdatePost {
}
impl CreateOrUpdatePost {
pub(crate) async fn new(
post: &ApubPost,
actor: &ApubPerson,
community: &ApubCommunity,
kind: CreateOrUpdateType,
context: &LemmyContext,
) -> Result<CreateOrUpdatePost, LemmyError> {
let id = generate_activity_id(
kind.clone(),
&context.settings().get_protocol_and_hostname(),
)?;
Ok(CreateOrUpdatePost {
actor: ObjectId::new(actor.actor_id()),
to: [PublicUrl::Public],
object: post.to_apub(context).await?,
cc: [ObjectId::new(community.actor_id())],
kind,
id: id.clone(),
context: lemmy_context(),
unparsed: Default::default(),
})
}
pub async fn send(
post: &ApubPost,
actor: &ApubPerson,
@ -60,22 +82,8 @@ impl CreateOrUpdatePost {
})
.await??
.into();
let id = generate_activity_id(
kind.clone(),
&context.settings().get_protocol_and_hostname(),
)?;
let create_or_update = CreateOrUpdatePost {
actor: ObjectId::new(actor.actor_id()),
to: [PublicUrl::Public],
object: post.to_apub(context.pool()).await?,
cc: [ObjectId::new(community.actor_id())],
kind,
id: id.clone(),
context: lemmy_context(),
unparsed: Default::default(),
};
let create_or_update = CreateOrUpdatePost::new(post, actor, &community, kind, context).await?;
let id = create_or_update.id.clone();
let activity = AnnouncableActivities::CreateOrUpdatePost(Box::new(create_or_update));
send_to_community(activity, &id, actor, &community, vec![], context).await
}

View file

@ -12,7 +12,7 @@ use activitystreams::{base::AnyBase, primitives::OneOrMany, unparsed::Unparsed};
use lemmy_api_common::blocking;
use lemmy_apub_lib::{
data::Data,
traits::{ActivityFields, ActivityHandler, ActorType, FromApub, ToApub},
traits::{ActivityFields, ActivityHandler, ActorType, ApubObject},
verify::verify_domains_match,
};
use lemmy_db_schema::{source::person::Person, traits::Crud};
@ -58,7 +58,7 @@ impl CreateOrUpdatePrivateMessage {
id: id.clone(),
actor: ObjectId::new(actor.actor_id()),
to: ObjectId::new(recipient.actor_id()),
object: private_message.to_apub(context.pool()).await?,
object: private_message.to_apub(context).await?,
kind,
unparsed: Default::default(),
};

View file

@ -0,0 +1,141 @@
use crate::{
collections::CommunityContext,
context::lemmy_context,
fetcher::object_id::ObjectId,
generate_moderators_url,
objects::person::ApubPerson,
};
use activitystreams::{
base::AnyBase,
chrono::NaiveDateTime,
collection::kind::OrderedCollectionType,
primitives::OneOrMany,
url::Url,
};
use lemmy_api_common::blocking;
use lemmy_apub_lib::{traits::ApubObject, verify::verify_domains_match};
use lemmy_db_schema::{
source::community::{CommunityModerator, CommunityModeratorForm},
traits::Joinable,
};
use lemmy_db_views_actor::community_moderator_view::CommunityModeratorView;
use lemmy_utils::LemmyError;
use serde::{Deserialize, Serialize};
use serde_with::skip_serializing_none;
#[skip_serializing_none]
#[derive(Clone, Debug, Deserialize, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct GroupModerators {
#[serde(rename = "@context")]
context: OneOrMany<AnyBase>,
r#type: OrderedCollectionType,
id: Url,
ordered_items: Vec<ObjectId<ApubPerson>>,
}
#[derive(Clone, Debug)]
pub(crate) struct ApubCommunityModerators(pub(crate) Vec<CommunityModeratorView>);
#[async_trait::async_trait(?Send)]
impl ApubObject for ApubCommunityModerators {
type DataType = CommunityContext;
type TombstoneType = ();
type ApubType = GroupModerators;
fn last_refreshed_at(&self) -> Option<NaiveDateTime> {
None
}
async fn read_from_apub_id(
_object_id: Url,
data: &Self::DataType,
) -> Result<Option<Self>, LemmyError> {
// Only read from database if its a local community, otherwise fetch over http
if data.0.local {
let cid = data.0.id;
let moderators = blocking(data.1.pool(), move |conn| {
CommunityModeratorView::for_community(conn, cid)
})
.await??;
Ok(Some(ApubCommunityModerators { 0: moderators }))
} else {
Ok(None)
}
}
async fn delete(self, _data: &Self::DataType) -> Result<(), LemmyError> {
unimplemented!()
}
async fn to_apub(&self, data: &Self::DataType) -> Result<Self::ApubType, LemmyError> {
let ordered_items = self
.0
.iter()
.map(|m| ObjectId::<ApubPerson>::new(m.moderator.actor_id.clone().into_inner()))
.collect();
Ok(GroupModerators {
context: lemmy_context(),
r#type: OrderedCollectionType::OrderedCollection,
id: generate_moderators_url(&data.0.actor_id)?.into(),
ordered_items,
})
}
fn to_tombstone(&self) -> Result<Self::TombstoneType, LemmyError> {
unimplemented!()
}
async fn from_apub(
apub: &Self::ApubType,
data: &Self::DataType,
expected_domain: &Url,
request_counter: &mut i32,
) -> Result<Self, LemmyError> {
verify_domains_match(expected_domain, &apub.id)?;
let community_id = data.0.id;
let current_moderators = blocking(data.1.pool(), move |conn| {
CommunityModeratorView::for_community(conn, community_id)
})
.await??;
// Remove old mods from database which arent in the moderators collection anymore
for mod_user in &current_moderators {
let mod_id = ObjectId::new(mod_user.moderator.actor_id.clone().into_inner());
if !apub.ordered_items.contains(&mod_id) {
let community_moderator_form = CommunityModeratorForm {
community_id: mod_user.community.id,
person_id: mod_user.moderator.id,
};
blocking(data.1.pool(), move |conn| {
CommunityModerator::leave(conn, &community_moderator_form)
})
.await??;
}
}
// Add new mods to database which have been added to moderators collection
for mod_id in &apub.ordered_items {
let mod_id = ObjectId::new(mod_id.clone());
let mod_user: ApubPerson = mod_id.dereference(&data.1, request_counter).await?;
if !current_moderators
.clone()
.iter()
.map(|c| c.moderator.actor_id.clone())
.any(|x| x == mod_user.actor_id)
{
let community_moderator_form = CommunityModeratorForm {
community_id: data.0.id,
person_id: mod_user.id,
};
blocking(data.1.pool(), move |conn| {
CommunityModerator::join(conn, &community_moderator_form)
})
.await??;
}
}
// This return value is unused, so just set an empty vec
Ok(ApubCommunityModerators { 0: vec![] })
}
}

View file

@ -0,0 +1,128 @@
use crate::{
activities::{post::create_or_update::CreateOrUpdatePost, CreateOrUpdateType},
collections::CommunityContext,
context::lemmy_context,
generate_outbox_url,
objects::{person::ApubPerson, post::ApubPost},
};
use activitystreams::{
base::AnyBase,
chrono::NaiveDateTime,
collection::kind::OrderedCollectionType,
primitives::OneOrMany,
url::Url,
};
use lemmy_api_common::blocking;
use lemmy_apub_lib::{
data::Data,
traits::{ActivityHandler, ApubObject},
verify::verify_domains_match,
};
use lemmy_db_schema::{
source::{person::Person, post::Post},
traits::Crud,
};
use lemmy_utils::LemmyError;
use serde::{Deserialize, Serialize};
use serde_with::skip_serializing_none;
#[skip_serializing_none]
#[derive(Clone, Debug, Deserialize, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct GroupOutbox {
#[serde(rename = "@context")]
context: OneOrMany<AnyBase>,
r#type: OrderedCollectionType,
id: Url,
ordered_items: Vec<CreateOrUpdatePost>,
}
#[derive(Clone, Debug)]
pub(crate) struct ApubCommunityOutbox(Vec<ApubPost>);
#[async_trait::async_trait(?Send)]
impl ApubObject for ApubCommunityOutbox {
type DataType = CommunityContext;
type TombstoneType = ();
type ApubType = GroupOutbox;
fn last_refreshed_at(&self) -> Option<NaiveDateTime> {
None
}
async fn read_from_apub_id(
_object_id: Url,
data: &Self::DataType,
) -> Result<Option<Self>, LemmyError> {
// Only read from database if its a local community, otherwise fetch over http
if data.0.local {
let community_id = data.0.id;
let post_list: Vec<ApubPost> = blocking(data.1.pool(), move |conn| {
Post::list_for_community(conn, community_id)
})
.await??
.into_iter()
.map(Into::into)
.collect();
Ok(Some(ApubCommunityOutbox(post_list)))
} else {
Ok(None)
}
}
async fn delete(self, _data: &Self::DataType) -> Result<(), LemmyError> {
// do nothing (it gets deleted automatically with the community)
Ok(())
}
async fn to_apub(&self, data: &Self::DataType) -> Result<Self::ApubType, LemmyError> {
let mut ordered_items = vec![];
for post in &self.0 {
let actor = post.creator_id;
let actor: ApubPerson = blocking(data.1.pool(), move |conn| Person::read(conn, actor))
.await??
.into();
let a =
CreateOrUpdatePost::new(post, &actor, &data.0, CreateOrUpdateType::Create, &data.1).await?;
ordered_items.push(a);
}
Ok(GroupOutbox {
context: lemmy_context(),
r#type: OrderedCollectionType::OrderedCollection,
id: generate_outbox_url(&data.0.actor_id)?.into(),
ordered_items,
})
}
fn to_tombstone(&self) -> Result<Self::TombstoneType, LemmyError> {
// no tombstone for this, there is only a tombstone for the community
unimplemented!()
}
async fn from_apub(
apub: &Self::ApubType,
data: &Self::DataType,
expected_domain: &Url,
request_counter: &mut i32,
) -> Result<Self, LemmyError> {
verify_domains_match(expected_domain, &apub.id)?;
let mut outbox_activities = apub.ordered_items.clone();
if outbox_activities.len() > 20 {
outbox_activities = outbox_activities[0..20].to_vec();
}
// We intentionally ignore errors here. This is because the outbox might contain posts from old
// Lemmy versions, or from other software which we cant parse. In that case, we simply skip the
// item and only parse the ones that work.
for activity in outbox_activities {
activity
.receive(&Data::new(data.1.clone()), request_counter)
.await
.ok();
}
// This return value is unused, so just set an empty vec
Ok(ApubCommunityOutbox { 0: vec![] })
}
}

View file

@ -0,0 +1,7 @@
use crate::objects::community::ApubCommunity;
use lemmy_websocket::LemmyContext;
pub(crate) mod community_moderators;
pub(crate) mod community_outbox;
/// Put community in the data, so we dont have to read it again from the database.
pub(crate) struct CommunityContext(pub ApubCommunity, pub LemmyContext);

View file

@ -1,144 +0,0 @@
use crate::{
activities::community::announce::AnnounceActivity,
fetcher::{fetch::fetch_remote_object, object_id::ObjectId},
objects::{community::Group, person::ApubPerson},
};
use activitystreams::{
base::AnyBase,
collection::{CollectionExt, OrderedCollection},
};
use anyhow::Context;
use lemmy_api_common::blocking;
use lemmy_apub_lib::{data::Data, traits::ActivityHandler};
use lemmy_db_schema::{
source::community::{Community, CommunityModerator, CommunityModeratorForm},
traits::Joinable,
};
use lemmy_db_views_actor::community_moderator_view::CommunityModeratorView;
use lemmy_utils::{location_info, LemmyError};
use lemmy_websocket::LemmyContext;
use url::Url;
pub(crate) async fn update_community_mods(
group: &Group,
community: &Community,
context: &LemmyContext,
request_counter: &mut i32,
) -> Result<(), LemmyError> {
let new_moderators = fetch_community_mods(context, group, request_counter).await?;
let community_id = community.id;
let current_moderators = blocking(context.pool(), move |conn| {
CommunityModeratorView::for_community(conn, community_id)
})
.await??;
// Remove old mods from database which arent in the moderators collection anymore
for mod_user in &current_moderators {
if !new_moderators.contains(&mod_user.moderator.actor_id.clone().into()) {
let community_moderator_form = CommunityModeratorForm {
community_id: mod_user.community.id,
person_id: mod_user.moderator.id,
};
blocking(context.pool(), move |conn| {
CommunityModerator::leave(conn, &community_moderator_form)
})
.await??;
}
}
// Add new mods to database which have been added to moderators collection
for mod_id in new_moderators {
let mod_id = ObjectId::new(mod_id);
let mod_user: ApubPerson = mod_id.dereference(context, request_counter).await?;
if !current_moderators
.clone()
.iter()
.map(|c| c.moderator.actor_id.clone())
.any(|x| x == mod_user.actor_id)
{
let community_moderator_form = CommunityModeratorForm {
community_id: community.id,
person_id: mod_user.id,
};
blocking(context.pool(), move |conn| {
CommunityModerator::join(conn, &community_moderator_form)
})
.await??;
}
}
Ok(())
}
pub(crate) async fn fetch_community_outbox(
context: &LemmyContext,
outbox: &Url,
recursion_counter: &mut i32,
) -> Result<(), LemmyError> {
let outbox = fetch_remote_object::<OrderedCollection>(
context.client(),
&context.settings(),
outbox,
recursion_counter,
)
.await?;
let outbox_activities = outbox.items().context(location_info!())?.clone();
let mut outbox_activities = outbox_activities.many().context(location_info!())?;
if outbox_activities.len() > 20 {
outbox_activities = outbox_activities[0..20].to_vec();
}
// We intentionally ignore errors here. This is because the outbox might contain posts from old
// Lemmy versions, or from other software which we cant parse. In that case, we simply skip the
// item and only parse the ones that work.
for activity in outbox_activities {
parse_outbox_item(activity, context, recursion_counter)
.await
.ok();
}
Ok(())
}
async fn parse_outbox_item(
announce: AnyBase,
context: &LemmyContext,
request_counter: &mut i32,
) -> Result<(), LemmyError> {
// TODO: instead of converting like this, we should create a struct CommunityOutbox with
// AnnounceActivity as inner type, but that gives me stackoverflow
let ser = serde_json::to_string(&announce)?;
let announce: AnnounceActivity = serde_json::from_str(&ser)?;
announce
.receive(&Data::new(context.clone()), request_counter)
.await?;
Ok(())
}
async fn fetch_community_mods(
context: &LemmyContext,
group: &Group,
recursion_counter: &mut i32,
) -> Result<Vec<Url>, LemmyError> {
if let Some(mods_url) = &group.moderators {
let mods = fetch_remote_object::<OrderedCollection>(
context.client(),
&context.settings(),
mods_url,
recursion_counter,
)
.await?;
let mods = mods
.items()
.map(|i| i.as_many())
.flatten()
.context(location_info!())?
.iter()
.filter_map(|i| i.as_xsd_any_uri())
.map(|u| u.to_owned())
.collect();
Ok(mods)
} else {
Ok(vec![])
}
}

View file

@ -1,49 +0,0 @@
use crate::check_is_apub_id_valid;
use anyhow::anyhow;
use lemmy_apub_lib::APUB_JSON_CONTENT_TYPE;
use lemmy_utils::{request::retry, settings::structs::Settings, LemmyError};
use log::info;
use reqwest::Client;
use serde::Deserialize;
use std::time::Duration;
use url::Url;
/// Maximum number of HTTP requests allowed to handle a single incoming activity (or a single object
/// fetch through the search).
///
/// A community fetch will load the outbox with up to 20 items, and fetch the creator for each item.
/// So we are looking at a maximum of 22 requests (rounded up just to be safe).
static MAX_REQUEST_NUMBER: i32 = 25;
/// Fetch any type of ActivityPub object, handling things like HTTP headers, deserialisation,
/// timeouts etc.
pub(in crate::fetcher) async fn fetch_remote_object<Response>(
client: &Client,
settings: &Settings,
url: &Url,
recursion_counter: &mut i32,
) -> Result<Response, LemmyError>
where
Response: for<'de> Deserialize<'de> + std::fmt::Debug,
{
*recursion_counter += 1;
if *recursion_counter > MAX_REQUEST_NUMBER {
return Err(anyhow!("Maximum recursion depth reached").into());
}
check_is_apub_id_valid(url, false, settings)?;
let timeout = Duration::from_secs(60);
let res = retry(|| {
client
.get(url.as_str())
.header("Accept", APUB_JSON_CONTENT_TYPE)
.timeout(timeout)
.send()
})
.await?;
let object = res.json().await?;
info!("Fetched remote object {}", url);
Ok(object)
}

View file

@ -1,5 +1,3 @@
pub mod community;
mod fetch;
pub mod object_id;
pub mod post_or_comment;
pub mod search;
@ -47,7 +45,7 @@ pub(crate) async fn get_or_fetch_and_upsert_actor(
///
/// TODO it won't pick up new avatars, summaries etc until a day after.
/// Actors need an "update" activity pushed to other servers to fix this.
fn should_refetch_actor(last_refreshed: NaiveDateTime) -> bool {
fn should_refetch_object(last_refreshed: NaiveDateTime) -> bool {
let update_interval = if cfg!(debug_assertions) {
// avoid infinite loop when fetching community outbox
chrono::Duration::seconds(ACTOR_REFETCH_INTERVAL_SECONDS_DEBUG)

View file

@ -1,14 +1,15 @@
use crate::fetcher::should_refetch_actor;
use crate::fetcher::should_refetch_object;
use anyhow::anyhow;
use diesel::NotFound;
use lemmy_apub_lib::{
traits::{ApubObject, FromApub},
APUB_JSON_CONTENT_TYPE,
};
use lemmy_apub_lib::{traits::ApubObject, APUB_JSON_CONTENT_TYPE};
use lemmy_db_schema::newtypes::DbUrl;
use lemmy_utils::{request::retry, settings::structs::Settings, LemmyError};
use lemmy_websocket::LemmyContext;
use reqwest::StatusCode;
use lemmy_utils::{
request::{build_user_agent, retry},
settings::structs::Settings,
LemmyError,
};
use log::info;
use reqwest::{Client, StatusCode};
use serde::{Deserialize, Serialize};
use std::{
fmt::{Debug, Display, Formatter},
@ -21,17 +22,25 @@ use url::Url;
/// fetch through the search). This should be configurable.
static REQUEST_LIMIT: i32 = 25;
// TODO: after moving this file to library, remove lazy_static dependency from apub crate
lazy_static! {
static ref CLIENT: Client = Client::builder()
.user_agent(build_user_agent(&Settings::get()))
.build()
.unwrap();
}
#[derive(Clone, PartialEq, Serialize, Deserialize, Debug)]
#[serde(transparent)]
pub struct ObjectId<Kind>(Url, #[serde(skip)] PhantomData<Kind>)
where
Kind: FromApub<DataType = LemmyContext> + ApubObject<DataType = LemmyContext> + Send + 'static,
for<'de2> <Kind as FromApub>::ApubType: serde::Deserialize<'de2>;
Kind: ApubObject + Send + 'static,
for<'de2> <Kind as ApubObject>::ApubType: serde::Deserialize<'de2>;
impl<Kind> ObjectId<Kind>
where
Kind: FromApub<DataType = LemmyContext> + ApubObject<DataType = LemmyContext> + Send + 'static,
for<'de> <Kind as FromApub>::ApubType: serde::Deserialize<'de>,
Kind: ApubObject + Send + 'static,
for<'de2> <Kind as ApubObject>::ApubType: serde::Deserialize<'de2>,
{
pub fn new<T>(url: T) -> Self
where
@ -47,10 +56,10 @@ where
/// Fetches an activitypub object, either from local database (if possible), or over http.
pub async fn dereference(
&self,
context: &LemmyContext,
data: &<Kind as ApubObject>::DataType,
request_counter: &mut i32,
) -> Result<Kind, LemmyError> {
let db_object = self.dereference_from_db(context).await?;
let db_object = self.dereference_from_db(data).await?;
// if its a local object, only fetch it from the database and not over http
if self.0.domain() == Some(&Settings::get().get_hostname_without_port()?) {
@ -60,44 +69,54 @@ where
};
}
// object found in database
if let Some(object) = db_object {
// object is old and should be refetched
if let Some(last_refreshed_at) = object.last_refreshed_at() {
// TODO: rename to should_refetch_object()
if should_refetch_actor(last_refreshed_at) {
if should_refetch_object(last_refreshed_at) {
return self
.dereference_from_http(context, request_counter, Some(object))
.dereference_from_http(data, request_counter, Some(object))
.await;
}
}
Ok(object)
} else {
}
// object not found, need to fetch over http
else {
self
.dereference_from_http(context, request_counter, None)
.dereference_from_http(data, request_counter, None)
.await
}
}
/// Fetch an object from the local db. Instead of falling back to http, this throws an error if
/// the object is not found in the database.
pub async fn dereference_local(&self, context: &LemmyContext) -> Result<Kind, LemmyError> {
let object = self.dereference_from_db(context).await?;
pub async fn dereference_local(
&self,
data: &<Kind as ApubObject>::DataType,
) -> Result<Kind, LemmyError> {
let object = self.dereference_from_db(data).await?;
object.ok_or_else(|| anyhow!("object not found in database {}", self).into())
}
/// returning none means the object was not found in local db
async fn dereference_from_db(&self, context: &LemmyContext) -> Result<Option<Kind>, LemmyError> {
async fn dereference_from_db(
&self,
data: &<Kind as ApubObject>::DataType,
) -> Result<Option<Kind>, LemmyError> {
let id = self.0.clone();
ApubObject::read_from_apub_id(id, context).await
ApubObject::read_from_apub_id(id, data).await
}
async fn dereference_from_http(
&self,
context: &LemmyContext,
data: &<Kind as ApubObject>::DataType,
request_counter: &mut i32,
db_object: Option<Kind>,
) -> Result<Kind, LemmyError> {
// dont fetch local objects this way
debug_assert!(self.0.domain() != Some(&Settings::get().hostname));
info!("Fetching remote object {}", self.to_string());
*request_counter += 1;
if *request_counter > REQUEST_LIMIT {
@ -105,8 +124,7 @@ where
}
let res = retry(|| {
context
.client()
CLIENT
.get(self.0.as_str())
.header("Accept", APUB_JSON_CONTENT_TYPE)
.timeout(Duration::from_secs(60))
@ -116,21 +134,21 @@ where
if res.status() == StatusCode::GONE {
if let Some(db_object) = db_object {
db_object.delete(context).await?;
db_object.delete(data).await?;
}
return Err(anyhow!("Fetched remote object {} which was deleted", self).into());
}
let res2: Kind::ApubType = res.json().await?;
Ok(Kind::from_apub(&res2, context, self.inner(), request_counter).await?)
Ok(Kind::from_apub(&res2, data, self.inner(), request_counter).await?)
}
}
impl<Kind> Display for ObjectId<Kind>
where
Kind: FromApub<DataType = LemmyContext> + ApubObject<DataType = LemmyContext> + Send + 'static,
for<'de> <Kind as FromApub>::ApubType: serde::Deserialize<'de>,
Kind: ApubObject + Send + 'static,
for<'de2> <Kind as ApubObject>::ApubType: serde::Deserialize<'de2>,
{
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", self.0.to_string())
@ -139,8 +157,8 @@ where
impl<Kind> From<ObjectId<Kind>> for Url
where
Kind: FromApub<DataType = LemmyContext> + ApubObject<DataType = LemmyContext> + Send + 'static,
for<'de> <Kind as FromApub>::ApubType: serde::Deserialize<'de>,
Kind: ApubObject + Send + 'static,
for<'de2> <Kind as ApubObject>::ApubType: serde::Deserialize<'de2>,
{
fn from(id: ObjectId<Kind>) -> Self {
id.0
@ -149,8 +167,8 @@ where
impl<Kind> From<ObjectId<Kind>> for DbUrl
where
Kind: FromApub<DataType = LemmyContext> + ApubObject<DataType = LemmyContext> + Send + 'static,
for<'de> <Kind as FromApub>::ApubType: serde::Deserialize<'de>,
Kind: ApubObject + Send + 'static,
for<'de2> <Kind as ApubObject>::ApubType: serde::Deserialize<'de2>,
{
fn from(id: ObjectId<Kind>) -> Self {
id.0.into()

View file

@ -3,7 +3,7 @@ use crate::objects::{
post::{ApubPost, Page},
};
use activitystreams::chrono::NaiveDateTime;
use lemmy_apub_lib::traits::{ApubObject, FromApub};
use lemmy_apub_lib::traits::ApubObject;
use lemmy_db_schema::source::{comment::CommentForm, post::PostForm};
use lemmy_utils::LemmyError;
use lemmy_websocket::LemmyContext;
@ -31,6 +31,8 @@ pub enum PageOrNote {
#[async_trait::async_trait(?Send)]
impl ApubObject for PostOrComment {
type DataType = LemmyContext;
type ApubType = PageOrNote;
type TombstoneType = ();
fn last_refreshed_at(&self) -> Option<NaiveDateTime> {
None
@ -59,12 +61,14 @@ impl ApubObject for PostOrComment {
PostOrComment::Comment(c) => c.delete(data).await,
}
}
}
#[async_trait::async_trait(?Send)]
impl FromApub for PostOrComment {
type ApubType = PageOrNote;
type DataType = LemmyContext;
async fn to_apub(&self, _data: &Self::DataType) -> Result<Self::ApubType, LemmyError> {
unimplemented!()
}
fn to_tombstone(&self) -> Result<Self::TombstoneType, LemmyError> {
unimplemented!()
}
async fn from_apub(
apub: &PageOrNote,

View file

@ -12,7 +12,7 @@ use anyhow::anyhow;
use itertools::Itertools;
use lemmy_api_common::blocking;
use lemmy_apub_lib::{
traits::{ApubObject, FromApub},
traits::ApubObject,
webfinger::{webfinger_resolve_actor, WebfingerType},
};
use lemmy_db_schema::{
@ -110,6 +110,8 @@ pub enum SearchableApubTypes {
#[async_trait::async_trait(?Send)]
impl ApubObject for SearchableObjects {
type DataType = LemmyContext;
type ApubType = SearchableApubTypes;
type TombstoneType = ();
fn last_refreshed_at(&self) -> Option<NaiveDateTime> {
match self {
@ -156,12 +158,14 @@ impl ApubObject for SearchableObjects {
SearchableObjects::Comment(c) => c.delete(data).await,
}
}
}
#[async_trait::async_trait(?Send)]
impl FromApub for SearchableObjects {
type ApubType = SearchableApubTypes;
type DataType = LemmyContext;
async fn to_apub(&self, _data: &Self::DataType) -> Result<Self::ApubType, LemmyError> {
unimplemented!()
}
fn to_tombstone(&self) -> Result<Self::TombstoneType, LemmyError> {
unimplemented!()
}
async fn from_apub(
apub: &Self::ApubType,

View file

@ -5,7 +5,7 @@ use crate::{
use actix_web::{body::Body, web, web::Path, HttpResponse};
use diesel::result::Error::NotFound;
use lemmy_api_common::blocking;
use lemmy_apub_lib::traits::ToApub;
use lemmy_apub_lib::traits::ApubObject;
use lemmy_db_schema::{newtypes::CommentId, source::comment::Comment, traits::Crud};
use lemmy_utils::LemmyError;
use lemmy_websocket::LemmyContext;
@ -30,9 +30,7 @@ pub(crate) async fn get_apub_comment(
}
if !comment.deleted {
Ok(create_apub_response(
&comment.to_apub(context.pool()).await?,
))
Ok(create_apub_response(&comment.to_apub(&**context).await?))
} else {
Ok(create_apub_tombstone_response(&comment.to_tombstone()?))
}

View file

@ -5,8 +5,13 @@ use crate::{
following::{follow::FollowCommunity, undo::UndoFollowCommunity},
report::Report,
},
collections::{
community_moderators::ApubCommunityModerators,
community_outbox::ApubCommunityOutbox,
CommunityContext,
},
context::lemmy_context,
generate_moderators_url,
fetcher::object_id::ObjectId,
generate_outbox_url,
http::{
create_apub_response,
@ -17,18 +22,14 @@ use crate::{
objects::community::ApubCommunity,
};
use activitystreams::{
base::{AnyBase, BaseExt},
collection::{CollectionExt, OrderedCollection, UnorderedCollection},
url::Url,
base::BaseExt,
collection::{CollectionExt, UnorderedCollection},
};
use actix_web::{body::Body, web, web::Payload, HttpRequest, HttpResponse};
use lemmy_api_common::blocking;
use lemmy_apub_lib::traits::{ActivityFields, ActivityHandler, ToApub};
use lemmy_db_schema::source::{activity::Activity, community::Community};
use lemmy_db_views_actor::{
community_follower_view::CommunityFollowerView,
community_moderator_view::CommunityModeratorView,
};
use lemmy_apub_lib::traits::{ActivityFields, ActivityHandler, ApubObject};
use lemmy_db_schema::source::community::Community;
use lemmy_db_views_actor::community_follower_view::CommunityFollowerView;
use lemmy_utils::LemmyError;
use lemmy_websocket::LemmyContext;
use log::trace;
@ -51,7 +52,7 @@ pub(crate) async fn get_apub_community_http(
.into();
if !community.deleted {
let apub = community.to_apub(context.pool()).await?;
let apub = community.to_apub(&**context).await?;
Ok(create_apub_response(&apub))
} else {
@ -133,41 +134,10 @@ pub(crate) async fn get_apub_community_outbox(
Community::read_from_name(conn, &info.community_name)
})
.await??;
let community_actor_id = community.actor_id.to_owned();
let activities = blocking(context.pool(), move |conn| {
Activity::read_community_outbox(conn, &community_actor_id)
})
.await??;
let activities = activities
.iter()
.map(AnyBase::from_arbitrary_json)
.collect::<Result<Vec<AnyBase>, serde_json::Error>>()?;
let len = activities.len();
let mut collection = OrderedCollection::new();
collection
.set_many_items(activities)
.set_many_contexts(lemmy_context())
.set_id(generate_outbox_url(&community.actor_id)?.into())
.set_total_items(len as u64);
Ok(create_apub_response(&collection))
}
pub(crate) async fn get_apub_community_inbox(
info: web::Path<CommunityQuery>,
context: web::Data<LemmyContext>,
) -> Result<HttpResponse<Body>, LemmyError> {
let community = blocking(context.pool(), move |conn| {
Community::read_from_name(conn, &info.community_name)
})
.await??;
let mut collection = OrderedCollection::new();
collection
.set_id(community.inbox_url.into())
.set_many_contexts(lemmy_context());
Ok(create_apub_response(&collection))
let id = ObjectId::new(generate_outbox_url(&community.actor_id)?.into_inner());
let outbox_data = CommunityContext(community.into(), context.get_ref().clone());
let outbox: ApubCommunityOutbox = id.dereference(&outbox_data, &mut 0).await?;
Ok(create_apub_response(&outbox.to_apub(&outbox_data).await?))
}
pub(crate) async fn get_apub_community_moderators(
@ -179,26 +149,10 @@ pub(crate) async fn get_apub_community_moderators(
})
.await??
.into();
// The attributed to, is an ordered vector with the creator actor_ids first,
// then the rest of the moderators
// TODO Technically the instance admins can mod the community, but lets
// ignore that for now
let cid = community.id;
let moderators = blocking(context.pool(), move |conn| {
CommunityModeratorView::for_community(conn, cid)
})
.await??;
let moderators: Vec<Url> = moderators
.into_iter()
.map(|m| m.moderator.actor_id.into())
.collect();
let mut collection = OrderedCollection::new();
collection
.set_id(generate_moderators_url(&community.actor_id)?.into())
.set_total_items(moderators.len() as u64)
.set_many_items(moderators)
.set_many_contexts(lemmy_context());
Ok(create_apub_response(&collection))
let id = ObjectId::new(generate_outbox_url(&community.actor_id)?.into_inner());
let outbox_data = CommunityContext(community, context.get_ref().clone());
let moderators: ApubCommunityModerators = id.dereference(&outbox_data, &mut 0).await?;
Ok(create_apub_response(
&moderators.to_apub(&outbox_data).await?,
))
}

View file

@ -24,7 +24,7 @@ use activitystreams::{
};
use actix_web::{body::Body, web, web::Payload, HttpRequest, HttpResponse};
use lemmy_api_common::blocking;
use lemmy_apub_lib::traits::{ActivityFields, ActivityHandler, ToApub};
use lemmy_apub_lib::traits::{ActivityFields, ActivityHandler, ApubObject};
use lemmy_db_schema::source::person::Person;
use lemmy_utils::LemmyError;
use lemmy_websocket::LemmyContext;
@ -51,7 +51,7 @@ pub(crate) async fn get_apub_person_http(
.into();
if !person.deleted {
let apub = person.to_apub(context.pool()).await?;
let apub = person.to_apub(&context).await?;
Ok(create_apub_response(&apub))
} else {
@ -109,19 +109,3 @@ pub(crate) async fn get_apub_person_outbox(
.set_total_items(0_u64);
Ok(create_apub_response(&collection))
}
pub(crate) async fn get_apub_person_inbox(
info: web::Path<PersonQuery>,
context: web::Data<LemmyContext>,
) -> Result<HttpResponse<Body>, LemmyError> {
let person = blocking(context.pool(), move |conn| {
Person::find_by_name(conn, &info.user_name)
})
.await??;
let mut collection = OrderedCollection::new();
collection
.set_id(person.inbox_url.into())
.set_many_contexts(lemmy_context());
Ok(create_apub_response(&collection))
}

View file

@ -5,7 +5,7 @@ use crate::{
use actix_web::{body::Body, web, HttpResponse};
use diesel::result::Error::NotFound;
use lemmy_api_common::blocking;
use lemmy_apub_lib::traits::ToApub;
use lemmy_apub_lib::traits::ApubObject;
use lemmy_db_schema::{newtypes::PostId, source::post::Post, traits::Crud};
use lemmy_utils::LemmyError;
use lemmy_websocket::LemmyContext;
@ -30,7 +30,7 @@ pub(crate) async fn get_apub_post(
}
if !post.deleted {
Ok(create_apub_response(&post.to_apub(context.pool()).await?))
Ok(create_apub_response(&post.to_apub(&context).await?))
} else {
Ok(create_apub_tombstone_response(&post.to_tombstone()?))
}

View file

@ -4,12 +4,11 @@ use crate::http::{
community_inbox,
get_apub_community_followers,
get_apub_community_http,
get_apub_community_inbox,
get_apub_community_moderators,
get_apub_community_outbox,
},
get_activity,
person::{get_apub_person_http, get_apub_person_inbox, get_apub_person_outbox, person_inbox},
person::{get_apub_person_http, get_apub_person_outbox, person_inbox},
post::get_apub_post,
shared_inbox,
};
@ -49,10 +48,6 @@ pub fn config(cfg: &mut web::ServiceConfig, settings: &Settings) {
"/c/{community_name}/outbox",
web::get().to(get_apub_community_outbox),
)
.route(
"/c/{community_name}/inbox",
web::get().to(get_apub_community_inbox),
)
.route(
"/c/{community_name}/moderators",
web::get().to(get_apub_community_moderators),
@ -62,7 +57,6 @@ pub fn config(cfg: &mut web::ServiceConfig, settings: &Settings) {
"/u/{user_name}/outbox",
web::get().to(get_apub_person_outbox),
)
.route("/u/{user_name}/inbox", web::get().to(get_apub_person_inbox))
.route("/post/{post_id}", web::get().to(get_apub_post))
.route("/comment/{comment_id}", web::get().to(get_apub_comment))
.route("/activities/{type_}/{id}", web::get().to(get_activity)),

View file

@ -1,10 +1,14 @@
pub mod activities;
pub(crate) mod collections;
mod context;
pub mod fetcher;
pub mod http;
pub mod migrations;
pub mod objects;
#[macro_use]
extern crate lazy_static;
use crate::fetcher::post_or_comment::PostOrComment;
use anyhow::{anyhow, Context};
use lemmy_api_common::blocking;

View file

@ -2,13 +2,13 @@ use crate::{
activities::{verify_is_public, verify_person_in_community},
context::lemmy_context,
fetcher::object_id::ObjectId,
objects::{create_tombstone, person::ApubPerson, post::ApubPost, Source},
objects::{person::ApubPerson, post::ApubPost, tombstone::Tombstone, Source},
PostOrComment,
};
use activitystreams::{
base::AnyBase,
chrono::NaiveDateTime,
object::{kind::NoteType, Tombstone},
object::kind::NoteType,
primitives::OneOrMany,
public,
unparsed::Unparsed,
@ -18,7 +18,7 @@ use chrono::{DateTime, FixedOffset};
use html2md::parse_html;
use lemmy_api_common::blocking;
use lemmy_apub_lib::{
traits::{ApubObject, FromApub, ToApub},
traits::ApubObject,
values::{MediaTypeHtml, MediaTypeMarkdown},
verify::verify_domains_match,
};
@ -31,7 +31,6 @@ use lemmy_db_schema::{
post::Post,
},
traits::Crud,
DbPool,
};
use lemmy_utils::{
utils::{convert_datetime, remove_slurs},
@ -159,6 +158,8 @@ impl From<Comment> for ApubComment {
#[async_trait::async_trait(?Send)]
impl ApubObject for ApubComment {
type DataType = LemmyContext;
type ApubType = Note;
type TombstoneType = Tombstone;
fn last_refreshed_at(&self) -> Option<NaiveDateTime> {
None
@ -184,23 +185,17 @@ impl ApubObject for ApubComment {
.await??;
Ok(())
}
}
#[async_trait::async_trait(?Send)]
impl ToApub for ApubComment {
type ApubType = Note;
type TombstoneType = Tombstone;
type DataType = DbPool;
async fn to_apub(&self, pool: &DbPool) -> Result<Note, LemmyError> {
async fn to_apub(&self, context: &LemmyContext) -> Result<Note, LemmyError> {
let creator_id = self.creator_id;
let creator = blocking(pool, move |conn| Person::read(conn, creator_id)).await??;
let creator = blocking(context.pool(), move |conn| Person::read(conn, creator_id)).await??;
let post_id = self.post_id;
let post = blocking(pool, move |conn| Post::read(conn, post_id)).await??;
let post = blocking(context.pool(), move |conn| Post::read(conn, post_id)).await??;
let in_reply_to = if let Some(comment_id) = self.parent_id {
let parent_comment = blocking(pool, move |conn| Comment::read(conn, comment_id)).await??;
let parent_comment =
blocking(context.pool(), move |conn| Comment::read(conn, comment_id)).await??;
ObjectId::<PostOrComment>::new(parent_comment.ap_id.into_inner())
} else {
ObjectId::<PostOrComment>::new(post.ap_id.into_inner())
@ -228,19 +223,11 @@ impl ToApub for ApubComment {
}
fn to_tombstone(&self) -> Result<Tombstone, LemmyError> {
create_tombstone(
self.deleted,
self.ap_id.to_owned().into(),
self.updated,
Ok(Tombstone::new(
NoteType::Note,
)
self.updated.unwrap_or(self.published),
))
}
}
#[async_trait::async_trait(?Send)]
impl FromApub for ApubComment {
type ApubType = Note;
type DataType = LemmyContext;
/// Converts a `Note` to `Comment`.
///
@ -324,6 +311,8 @@ mod tests {
#[actix_rt::test]
#[serial]
async fn test_parse_lemmy_comment() {
// TODO: changed ObjectId::dereference() so that it always fetches if
// last_refreshed_at() == None. But post doesnt store that and expects to never be refetched
let context = init_context();
let url = Url::parse("https://enterprise.lemmy.ml/comment/38741").unwrap();
let data = prepare_comment_test(&url, &context).await;
@ -339,7 +328,7 @@ mod tests {
assert!(!comment.local);
assert_eq!(request_counter, 0);
let to_apub = comment.to_apub(context.pool()).await.unwrap();
let to_apub = comment.to_apub(&context).await.unwrap();
assert_json_include!(actual: json, expected: to_apub);
Comment::delete(&*context.pool().get().unwrap(), comment.id).unwrap();

View file

@ -1,17 +1,22 @@
use crate::{
check_is_apub_id_valid,
collections::{
community_moderators::ApubCommunityModerators,
community_outbox::ApubCommunityOutbox,
CommunityContext,
},
context::lemmy_context,
fetcher::community::{fetch_community_outbox, update_community_mods},
fetcher::object_id::ObjectId,
generate_moderators_url,
generate_outbox_url,
objects::{create_tombstone, get_summary_from_string_or_source, ImageObject, Source},
objects::{get_summary_from_string_or_source, tombstone::Tombstone, ImageObject, Source},
CommunityType,
};
use activitystreams::{
actor::{kind::GroupType, Endpoints},
base::AnyBase,
chrono::NaiveDateTime,
object::{kind::ImageType, Tombstone},
object::kind::ImageType,
primitives::OneOrMany,
unparsed::Unparsed,
};
@ -20,7 +25,7 @@ use itertools::Itertools;
use lemmy_api_common::blocking;
use lemmy_apub_lib::{
signatures::PublicKey,
traits::{ActorType, ApubObject, FromApub, ToApub},
traits::{ActorType, ApubObject},
values::MediaTypeMarkdown,
verify::verify_domains_match,
};
@ -63,9 +68,9 @@ pub struct Group {
// lemmy extension
sensitive: Option<bool>,
// lemmy extension
pub(crate) moderators: Option<Url>,
pub(crate) moderators: Option<ObjectId<ApubCommunityModerators>>,
inbox: Url,
pub(crate) outbox: Url,
pub(crate) outbox: ObjectId<ApubCommunityOutbox>,
followers: Url,
endpoints: Endpoints<Url>,
public_key: PublicKey,
@ -138,6 +143,8 @@ impl From<Community> for ApubCommunity {
#[async_trait::async_trait(?Send)]
impl ApubObject for ApubCommunity {
type DataType = LemmyContext;
type ApubType = Group;
type TombstoneType = Tombstone;
fn last_refreshed_at(&self) -> Option<NaiveDateTime> {
Some(self.last_refreshed_at)
@ -163,6 +170,91 @@ impl ApubObject for ApubCommunity {
.await??;
Ok(())
}
async fn to_apub(&self, _context: &LemmyContext) -> Result<Group, LemmyError> {
let source = self.description.clone().map(|bio| Source {
content: bio,
media_type: MediaTypeMarkdown::Markdown,
});
let icon = self.icon.clone().map(|url| ImageObject {
kind: ImageType::Image,
url: url.into(),
});
let image = self.banner.clone().map(|url| ImageObject {
kind: ImageType::Image,
url: url.into(),
});
let group = Group {
context: lemmy_context(),
kind: GroupType::Group,
id: self.actor_id(),
preferred_username: self.name.clone(),
name: self.title.clone(),
summary: self.description.as_ref().map(|b| markdown_to_html(b)),
source,
icon,
image,
sensitive: Some(self.nsfw),
moderators: Some(ObjectId::<ApubCommunityModerators>::new(
generate_moderators_url(&self.actor_id)?.into_inner(),
)),
inbox: self.inbox_url.clone().into(),
outbox: ObjectId::new(generate_outbox_url(&self.actor_id)?),
followers: self.followers_url.clone().into(),
endpoints: Endpoints {
shared_inbox: self.shared_inbox_url.clone().map(|s| s.into()),
..Default::default()
},
public_key: self.get_public_key()?,
published: Some(convert_datetime(self.published)),
updated: self.updated.map(convert_datetime),
unparsed: Default::default(),
};
Ok(group)
}
fn to_tombstone(&self) -> Result<Tombstone, LemmyError> {
Ok(Tombstone::new(
GroupType::Group,
self.updated.unwrap_or(self.published),
))
}
/// Converts a `Group` to `Community`, inserts it into the database and updates moderators.
async fn from_apub(
group: &Group,
context: &LemmyContext,
expected_domain: &Url,
request_counter: &mut i32,
) -> Result<ApubCommunity, LemmyError> {
let form = Group::from_apub_to_form(group, expected_domain, &context.settings()).await?;
// Fetching mods and outbox is not necessary for Lemmy to work, so ignore errors. Besides,
// we need to ignore these errors so that tests can work entirely offline.
let community: ApubCommunity =
blocking(context.pool(), move |conn| Community::upsert(conn, &form))
.await??
.into();
let outbox_data = CommunityContext(community.clone(), context.clone());
group
.outbox
.dereference(&outbox_data, request_counter)
.await
.map_err(|e| debug!("{}", e))
.ok();
if let Some(moderators) = &group.moderators {
moderators
.dereference(&outbox_data, request_counter)
.await
.map_err(|e| debug!("{}", e))
.ok();
}
Ok(community)
}
}
impl ActorType for ApubCommunity {
@ -191,95 +283,6 @@ impl ActorType for ApubCommunity {
}
}
#[async_trait::async_trait(?Send)]
impl ToApub for ApubCommunity {
type ApubType = Group;
type TombstoneType = Tombstone;
type DataType = DbPool;
async fn to_apub(&self, _pool: &DbPool) -> Result<Group, LemmyError> {
let source = self.description.clone().map(|bio| Source {
content: bio,
media_type: MediaTypeMarkdown::Markdown,
});
let icon = self.icon.clone().map(|url| ImageObject {
kind: ImageType::Image,
url: url.into(),
});
let image = self.banner.clone().map(|url| ImageObject {
kind: ImageType::Image,
url: url.into(),
});
let group = Group {
context: lemmy_context(),
kind: GroupType::Group,
id: self.actor_id(),
preferred_username: self.name.clone(),
name: self.title.clone(),
summary: self.description.as_ref().map(|b| markdown_to_html(b)),
source,
icon,
image,
sensitive: Some(self.nsfw),
moderators: Some(generate_moderators_url(&self.actor_id)?.into()),
inbox: self.inbox_url.clone().into(),
outbox: generate_outbox_url(&self.actor_id)?.into(),
followers: self.followers_url.clone().into(),
endpoints: Endpoints {
shared_inbox: self.shared_inbox_url.clone().map(|s| s.into()),
..Default::default()
},
public_key: self.get_public_key()?,
published: Some(convert_datetime(self.published)),
updated: self.updated.map(convert_datetime),
unparsed: Default::default(),
};
Ok(group)
}
fn to_tombstone(&self) -> Result<Tombstone, LemmyError> {
create_tombstone(
self.deleted,
self.actor_id.to_owned().into(),
self.updated,
GroupType::Group,
)
}
}
#[async_trait::async_trait(?Send)]
impl FromApub for ApubCommunity {
type ApubType = Group;
type DataType = LemmyContext;
/// Converts a `Group` to `Community`, inserts it into the database and updates moderators.
async fn from_apub(
group: &Group,
context: &LemmyContext,
expected_domain: &Url,
request_counter: &mut i32,
) -> Result<ApubCommunity, LemmyError> {
let form = Group::from_apub_to_form(group, expected_domain, &context.settings()).await?;
// Fetching mods and outbox is not necessary for Lemmy to work, so ignore errors. Besides,
// we need to ignore these errors so that tests can work entirely offline.
let community = blocking(context.pool(), move |conn| Community::upsert(conn, &form)).await??;
update_community_mods(group, &community, context, request_counter)
.await
.map_err(|e| debug!("{}", e))
.ok();
// TODO: doing this unconditionally might cause infinite loop for some reason
fetch_community_outbox(context, &group.outbox, request_counter)
.await
.map_err(|e| debug!("{}", e))
.ok();
Ok(community.into())
}
}
#[async_trait::async_trait(?Send)]
impl CommunityType for Community {
fn followers_url(&self) -> Url {
@ -327,9 +330,11 @@ mod tests {
let mut json: Group = file_to_json_object("assets/lemmy-community.json");
let json_orig = json.clone();
// change these links so they dont fetch over the network
json.moderators =
Some(Url::parse("https://enterprise.lemmy.ml/c/tenforward/not_moderators").unwrap());
json.outbox = Url::parse("https://enterprise.lemmy.ml/c/tenforward/not_outbox").unwrap();
json.moderators = Some(ObjectId::new(
Url::parse("https://enterprise.lemmy.ml/c/tenforward/not_moderators").unwrap(),
));
json.outbox =
ObjectId::new(Url::parse("https://enterprise.lemmy.ml/c/tenforward/not_outbox").unwrap());
let url = Url::parse("https://enterprise.lemmy.ml/c/tenforward").unwrap();
let mut request_counter = 0;
@ -345,7 +350,7 @@ mod tests {
// this makes two requests to the (intentionally) broken outbox/moderators collections
assert_eq!(request_counter, 2);
let to_apub = community.to_apub(context.pool()).await.unwrap();
let to_apub = community.to_apub(&context).await.unwrap();
assert_json_include!(actual: json_orig, expected: to_apub);
Community::delete(&*context.pool().get().unwrap(), community.id).unwrap();

View file

@ -1,12 +1,6 @@
use activitystreams::{
base::BaseExt,
object::{kind::ImageType, Tombstone, TombstoneExt},
};
use anyhow::anyhow;
use chrono::NaiveDateTime;
use activitystreams::object::kind::ImageType;
use html2md::parse_html;
use lemmy_apub_lib::values::MediaTypeMarkdown;
use lemmy_utils::{utils::convert_datetime, LemmyError};
use serde::{Deserialize, Serialize};
use url::Url;
@ -15,6 +9,7 @@ pub mod community;
pub mod person;
pub mod post;
pub mod private_message;
pub mod tombstone;
#[derive(Clone, Debug, Deserialize, Serialize)]
#[serde(rename_all = "camelCase")]
@ -31,31 +26,6 @@ pub struct ImageObject {
url: Url,
}
/// Updated is actually the deletion time
fn create_tombstone<T>(
deleted: bool,
object_id: Url,
updated: Option<NaiveDateTime>,
former_type: T,
) -> Result<Tombstone, LemmyError>
where
T: ToString,
{
if deleted {
if let Some(updated) = updated {
let mut tombstone = Tombstone::new();
tombstone.set_id(object_id);
tombstone.set_former_type(former_type.to_string());
tombstone.set_deleted(convert_datetime(updated));
Ok(tombstone)
} else {
Err(anyhow!("Cant convert to tombstone because updated time was None.").into())
}
} else {
Err(anyhow!("Cant convert object to tombstone if it wasnt deleted").into())
}
}
fn get_summary_from_string_or_source(
raw: &Option<String>,
source: &Option<Source>,
@ -69,7 +39,6 @@ fn get_summary_from_string_or_source(
#[cfg(test)]
mod tests {
use super::*;
use actix::Actor;
use diesel::{
r2d2::{ConnectionManager, Pool},
@ -85,6 +54,7 @@ mod tests {
rate_limit::{rate_limiter::RateLimiter, RateLimit},
request::build_user_agent,
settings::structs::Settings,
LemmyError,
};
use lemmy_websocket::{chat_server::ChatServer, LemmyContext};
use reqwest::Client;

View file

@ -16,14 +16,13 @@ use chrono::{DateTime, FixedOffset};
use lemmy_api_common::blocking;
use lemmy_apub_lib::{
signatures::PublicKey,
traits::{ActorType, ApubObject, FromApub, ToApub},
traits::{ActorType, ApubObject},
values::MediaTypeMarkdown,
verify::verify_domains_match,
};
use lemmy_db_schema::{
naive_now,
source::person::{Person as DbPerson, PersonForm},
DbPool,
};
use lemmy_utils::{
utils::{check_slurs, check_slurs_opt, convert_datetime, markdown_to_html},
@ -80,7 +79,7 @@ impl Person {
}
}
#[derive(Clone, Debug)]
#[derive(Clone, Debug, PartialEq)]
pub struct ApubPerson(DbPerson);
impl Deref for ApubPerson {
@ -99,6 +98,8 @@ impl From<DbPerson> for ApubPerson {
#[async_trait::async_trait(?Send)]
impl ApubObject for ApubPerson {
type DataType = LemmyContext;
type ApubType = Person;
type TombstoneType = Tombstone;
fn last_refreshed_at(&self) -> Option<NaiveDateTime> {
Some(self.last_refreshed_at)
@ -124,43 +125,8 @@ impl ApubObject for ApubPerson {
.await??;
Ok(())
}
}
impl ActorType for ApubPerson {
fn is_local(&self) -> bool {
self.local
}
fn actor_id(&self) -> Url {
self.actor_id.to_owned().into_inner()
}
fn name(&self) -> String {
self.name.clone()
}
fn public_key(&self) -> Option<String> {
self.public_key.to_owned()
}
fn private_key(&self) -> Option<String> {
self.private_key.to_owned()
}
fn inbox_url(&self) -> Url {
self.inbox_url.clone().into()
}
fn shared_inbox_url(&self) -> Option<Url> {
self.shared_inbox_url.clone().map(|s| s.into_inner())
}
}
#[async_trait::async_trait(?Send)]
impl ToApub for ApubPerson {
type ApubType = Person;
type TombstoneType = Tombstone;
type DataType = DbPool;
async fn to_apub(&self, _pool: &DbPool) -> Result<Person, LemmyError> {
async fn to_apub(&self, _pool: &LemmyContext) -> Result<Person, LemmyError> {
let kind = if self.bot_account {
UserTypes::Service
} else {
@ -203,15 +169,10 @@ impl ToApub for ApubPerson {
};
Ok(person)
}
fn to_tombstone(&self) -> Result<Tombstone, LemmyError> {
unimplemented!()
}
}
#[async_trait::async_trait(?Send)]
impl FromApub for ApubPerson {
type ApubType = Person;
type DataType = LemmyContext;
async fn from_apub(
person: &Person,
@ -265,6 +226,34 @@ impl FromApub for ApubPerson {
}
}
impl ActorType for ApubPerson {
fn is_local(&self) -> bool {
self.local
}
fn actor_id(&self) -> Url {
self.actor_id.to_owned().into_inner()
}
fn name(&self) -> String {
self.name.clone()
}
fn public_key(&self) -> Option<String> {
self.public_key.to_owned()
}
fn private_key(&self) -> Option<String> {
self.private_key.to_owned()
}
fn inbox_url(&self) -> Url {
self.inbox_url.clone().into()
}
fn shared_inbox_url(&self) -> Option<Url> {
self.shared_inbox_url.clone().map(|s| s.into_inner())
}
}
#[cfg(test)]
mod tests {
use super::*;
@ -291,7 +280,7 @@ mod tests {
assert_eq!(person.bio.as_ref().unwrap().len(), 39);
assert_eq!(request_counter, 0);
let to_apub = person.to_apub(context.pool()).await.unwrap();
let to_apub = person.to_apub(&context).await.unwrap();
assert_json_include!(actual: json, expected: to_apub);
DbPerson::delete(&*context.pool().get().unwrap(), person.id).unwrap();

View file

@ -2,14 +2,11 @@ use crate::{
activities::{extract_community, verify_is_public, verify_person_in_community},
context::lemmy_context,
fetcher::object_id::ObjectId,
objects::{create_tombstone, person::ApubPerson, ImageObject, Source},
objects::{person::ApubPerson, tombstone::Tombstone, ImageObject, Source},
};
use activitystreams::{
base::AnyBase,
object::{
kind::{ImageType, PageType},
Tombstone,
},
object::kind::{ImageType, PageType},
primitives::OneOrMany,
public,
unparsed::Unparsed,
@ -17,7 +14,7 @@ use activitystreams::{
use chrono::{DateTime, FixedOffset, NaiveDateTime};
use lemmy_api_common::blocking;
use lemmy_apub_lib::{
traits::{ActorType, ApubObject, FromApub, ToApub},
traits::{ActorType, ApubObject},
values::{MediaTypeHtml, MediaTypeMarkdown},
verify::verify_domains_match,
};
@ -29,7 +26,6 @@ use lemmy_db_schema::{
post::{Post, PostForm},
},
traits::Crud,
DbPool,
};
use lemmy_utils::{
request::fetch_site_data,
@ -133,6 +129,8 @@ impl From<Post> for ApubPost {
#[async_trait::async_trait(?Send)]
impl ApubObject for ApubPost {
type DataType = LemmyContext;
type ApubType = Page;
type TombstoneType = Tombstone;
fn last_refreshed_at(&self) -> Option<NaiveDateTime> {
None
@ -158,20 +156,16 @@ impl ApubObject for ApubPost {
.await??;
Ok(())
}
}
#[async_trait::async_trait(?Send)]
impl ToApub for ApubPost {
type ApubType = Page;
type TombstoneType = Tombstone;
type DataType = DbPool;
// Turn a Lemmy post into an ActivityPub page that can be sent out over the network.
async fn to_apub(&self, pool: &DbPool) -> Result<Page, LemmyError> {
async fn to_apub(&self, context: &LemmyContext) -> Result<Page, LemmyError> {
let creator_id = self.creator_id;
let creator = blocking(pool, move |conn| Person::read(conn, creator_id)).await??;
let creator = blocking(context.pool(), move |conn| Person::read(conn, creator_id)).await??;
let community_id = self.community_id;
let community = blocking(pool, move |conn| Community::read(conn, community_id)).await??;
let community = blocking(context.pool(), move |conn| {
Community::read(conn, community_id)
})
.await??;
let source = self.body.clone().map(|body| Source {
content: body,
@ -205,19 +199,11 @@ impl ToApub for ApubPost {
}
fn to_tombstone(&self) -> Result<Tombstone, LemmyError> {
create_tombstone(
self.deleted,
self.ap_id.to_owned().into(),
self.updated,
Ok(Tombstone::new(
PageType::Page,
)
self.updated.unwrap_or(self.published),
))
}
}
#[async_trait::async_trait(?Send)]
impl FromApub for ApubPost {
type ApubType = Page;
type DataType = LemmyContext;
async fn from_apub(
page: &Page,
@ -315,7 +301,7 @@ mod tests {
assert!(post.stickied);
assert_eq!(request_counter, 0);
let to_apub = post.to_apub(context.pool()).await.unwrap();
let to_apub = post.to_apub(&context).await.unwrap();
assert_json_include!(actual: json, expected: to_apub);
Post::delete(&*context.pool().get().unwrap(), post.id).unwrap();

View file

@ -1,12 +1,12 @@
use crate::{
context::lemmy_context,
fetcher::object_id::ObjectId,
objects::{create_tombstone, person::ApubPerson, Source},
objects::{person::ApubPerson, Source},
};
use activitystreams::{
base::AnyBase,
chrono::NaiveDateTime,
object::{kind::NoteType, Tombstone},
object::Tombstone,
primitives::OneOrMany,
unparsed::Unparsed,
};
@ -15,7 +15,7 @@ use chrono::{DateTime, FixedOffset};
use html2md::parse_html;
use lemmy_api_common::blocking;
use lemmy_apub_lib::{
traits::{ApubObject, FromApub, ToApub},
traits::ApubObject,
values::{MediaTypeHtml, MediaTypeMarkdown},
verify::verify_domains_match,
};
@ -25,7 +25,6 @@ use lemmy_db_schema::{
private_message::{PrivateMessage, PrivateMessageForm},
},
traits::Crud,
DbPool,
};
use lemmy_utils::{utils::convert_datetime, LemmyError};
use lemmy_websocket::LemmyContext;
@ -104,6 +103,8 @@ impl From<PrivateMessage> for ApubPrivateMessage {
#[async_trait::async_trait(?Send)]
impl ApubObject for ApubPrivateMessage {
type DataType = LemmyContext;
type ApubType = Note;
type TombstoneType = Tombstone;
fn last_refreshed_at(&self) -> Option<NaiveDateTime> {
None
@ -126,20 +127,14 @@ impl ApubObject for ApubPrivateMessage {
// do nothing, because pm can't be fetched over http
unimplemented!()
}
}
#[async_trait::async_trait(?Send)]
impl ToApub for ApubPrivateMessage {
type ApubType = Note;
type TombstoneType = Tombstone;
type DataType = DbPool;
async fn to_apub(&self, pool: &DbPool) -> Result<Note, LemmyError> {
async fn to_apub(&self, context: &LemmyContext) -> Result<Note, LemmyError> {
let creator_id = self.creator_id;
let creator = blocking(pool, move |conn| Person::read(conn, creator_id)).await??;
let creator = blocking(context.pool(), move |conn| Person::read(conn, creator_id)).await??;
let recipient_id = self.recipient_id;
let recipient = blocking(pool, move |conn| Person::read(conn, recipient_id)).await??;
let recipient =
blocking(context.pool(), move |conn| Person::read(conn, recipient_id)).await??;
let note = Note {
context: lemmy_context(),
@ -161,19 +156,8 @@ impl ToApub for ApubPrivateMessage {
}
fn to_tombstone(&self) -> Result<Tombstone, LemmyError> {
create_tombstone(
self.deleted,
self.ap_id.to_owned().into(),
self.updated,
NoteType::Note,
)
unimplemented!()
}
}
#[async_trait::async_trait(?Send)]
impl FromApub for ApubPrivateMessage {
type ApubType = Note;
type DataType = LemmyContext;
async fn from_apub(
note: &Note,
@ -253,7 +237,7 @@ mod tests {
assert_eq!(pm.content.len(), 20);
assert_eq!(request_counter, 0);
let to_apub = pm.to_apub(context.pool()).await.unwrap();
let to_apub = pm.to_apub(&context).await.unwrap();
assert_json_include!(actual: json, expected: to_apub);
PrivateMessage::delete(&*context.pool().get().unwrap(), pm.id).unwrap();

View file

@ -0,0 +1,33 @@
use crate::context::lemmy_context;
use activitystreams::{
base::AnyBase,
chrono::{DateTime, FixedOffset, NaiveDateTime},
object::kind::TombstoneType,
primitives::OneOrMany,
};
use lemmy_utils::utils::convert_datetime;
use serde::{Deserialize, Serialize};
use serde_with::skip_serializing_none;
#[skip_serializing_none]
#[derive(Clone, Debug, Deserialize, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct Tombstone {
#[serde(rename = "@context")]
context: OneOrMany<AnyBase>,
#[serde(rename = "type")]
kind: TombstoneType,
former_type: String,
deleted: DateTime<FixedOffset>,
}
impl Tombstone {
pub fn new<T: ToString>(former_type: T, updated_time: NaiveDateTime) -> Tombstone {
Tombstone {
context: lemmy_context(),
kind: TombstoneType::Tombstone,
former_type: former_type.to_string(),
deleted: convert_datetime(updated_time),
}
}
}

View file

@ -30,6 +30,9 @@ pub trait ActivityHandler {
#[async_trait::async_trait(?Send)]
pub trait ApubObject {
type DataType;
type ApubType;
type TombstoneType;
/// If this object should be refetched after a certain interval, it should return the last refresh
/// time here. This is mainly used to update remote actors.
fn last_refreshed_at(&self) -> Option<NaiveDateTime>;
@ -42,6 +45,25 @@ pub trait ApubObject {
Self: Sized;
/// Marks the object as deleted in local db. Called when a tombstone is received.
async fn delete(self, data: &Self::DataType) -> Result<(), LemmyError>;
/// Trait for converting an object or actor into the respective ActivityPub type.
async fn to_apub(&self, data: &Self::DataType) -> Result<Self::ApubType, LemmyError>;
fn to_tombstone(&self) -> Result<Self::TombstoneType, LemmyError>;
/// Converts an object from ActivityPub type to Lemmy internal type.
///
/// * `apub` The object to read from
/// * `context` LemmyContext which holds DB pool, HTTP client etc
/// * `expected_domain` Domain where the object was received from. None in case of mod action.
/// * `mod_action_allowed` True if the object can be a mod activity, ignore `expected_domain` in this case
async fn from_apub(
apub: &Self::ApubType,
data: &Self::DataType,
expected_domain: &Url,
request_counter: &mut i32,
) -> Result<Self, LemmyError>
where
Self: Sized;
}
/// Common methods provided by ActivityPub actors (community and person). Not all methods are
@ -71,35 +93,3 @@ pub trait ActorType {
})
}
}
/// Trait for converting an object or actor into the respective ActivityPub type.
#[async_trait::async_trait(?Send)]
pub trait ToApub {
type ApubType;
type TombstoneType;
type DataType;
async fn to_apub(&self, data: &Self::DataType) -> Result<Self::ApubType, LemmyError>;
fn to_tombstone(&self) -> Result<Self::TombstoneType, LemmyError>;
}
#[async_trait::async_trait(?Send)]
pub trait FromApub {
type ApubType;
type DataType;
/// Converts an object from ActivityPub type to Lemmy internal type.
///
/// * `apub` The object to read from
/// * `context` LemmyContext which holds DB pool, HTTP client etc
/// * `expected_domain` Domain where the object was received from. None in case of mod action.
/// * `mod_action_allowed` True if the object can be a mod activity, ignore `expected_domain` in this case
async fn from_apub(
apub: &Self::ApubType,
data: &Self::DataType,
expected_domain: &Url,
request_counter: &mut i32,
) -> Result<Self, LemmyError>
where
Self: Sized;
}

View file

@ -126,16 +126,6 @@ impl Community {
community.select(actor_id).distinct().load::<String>(conn)
}
pub fn read_from_followers_url(
conn: &PgConnection,
followers_url_: &DbUrl,
) -> Result<Community, Error> {
use crate::schema::community::dsl::*;
community
.filter(followers_url.eq(followers_url_))
.first::<Self>(conn)
}
pub fn upsert(conn: &PgConnection, community_form: &CommunityForm) -> Result<Community, Error> {
use crate::schema::community::dsl::*;
insert_into(community)