Remove instance follows

This commit is contained in:
Felix 2020-04-17 16:39:03 +02:00
parent 86f172076b
commit 9c974fbe50
8 changed files with 5 additions and 97 deletions

View file

@ -24,7 +24,6 @@ services:
- LEMMY_JWT_SECRET=changeme - LEMMY_JWT_SECRET=changeme
- LEMMY_FRONT_END_DIR=/app/dist - LEMMY_FRONT_END_DIR=/app/dist
- LEMMY_FEDERATION__ENABLED=true - LEMMY_FEDERATION__ENABLED=true
- LEMMY_FEDERATION__FOLLOWED_INSTANCES=lemmy_beta:8550
- LEMMY_FEDERATION__TLS_ENABLED=false - LEMMY_FEDERATION__TLS_ENABLED=false
- LEMMY_PORT=8540 - LEMMY_PORT=8540
- LEMMY_SETUP__ADMIN_USERNAME=lemmy_alpha - LEMMY_SETUP__ADMIN_USERNAME=lemmy_alpha
@ -58,7 +57,6 @@ services:
- LEMMY_JWT_SECRET=changeme - LEMMY_JWT_SECRET=changeme
- LEMMY_FRONT_END_DIR=/app/dist - LEMMY_FRONT_END_DIR=/app/dist
- LEMMY_FEDERATION__ENABLED=true - LEMMY_FEDERATION__ENABLED=true
- LEMMY_FEDERATION__FOLLOWED_INSTANCES=lemmy_alpha:8540
- LEMMY_FEDERATION__TLS_ENABLED=false - LEMMY_FEDERATION__TLS_ENABLED=false
- LEMMY_PORT=8550 - LEMMY_PORT=8550
- LEMMY_SETUP__ADMIN_USERNAME=lemmy_beta - LEMMY_SETUP__ADMIN_USERNAME=lemmy_beta

View file

@ -54,8 +54,6 @@
federation: { federation: {
# whether to enable activitypub federation. this feature is in alpha, do not enable in production. # whether to enable activitypub federation. this feature is in alpha, do not enable in production.
enabled: false enabled: false
# comma seperated list of instances to follow
followed_instances: ""
# whether tls is required for activitypub. only disable this for debugging, never for producion. # whether tls is required for activitypub. only disable this for debugging, never for producion.
tls_enabled: true tls_enabled: true
} }

View file

@ -9,23 +9,22 @@ use crate::db::user_view::UserView;
use crate::db::{Crud, SearchType}; use crate::db::{Crud, SearchType};
use crate::routes::nodeinfo::{NodeInfo, NodeInfoWellKnown}; use crate::routes::nodeinfo::{NodeInfo, NodeInfoWellKnown};
use crate::settings::Settings; use crate::settings::Settings;
use activitystreams::collection::{OrderedCollection, UnorderedCollection}; use activitystreams::collection::OrderedCollection;
use activitystreams::object::Page; use activitystreams::object::Page;
use activitystreams::BaseBox; use activitystreams::BaseBox;
use diesel::result::Error::NotFound; use diesel::result::Error::NotFound;
use diesel::PgConnection; use diesel::PgConnection;
use failure::Error; use failure::Error;
use isahc::prelude::*; use isahc::prelude::*;
use log::warn;
use serde::Deserialize; use serde::Deserialize;
use std::time::Duration; use std::time::Duration;
use url::Url; use url::Url;
fn fetch_node_info(instance: &Instance) -> Result<NodeInfo, Error> { fn _fetch_node_info(domain: &str) -> Result<NodeInfo, Error> {
let well_known_uri = Url::parse(&format!( let well_known_uri = Url::parse(&format!(
"{}://{}/.well-known/nodeinfo", "{}://{}/.well-known/nodeinfo",
get_apub_protocol_string(), get_apub_protocol_string(),
instance.domain domain
))?; ))?;
let well_known = fetch_remote_object::<NodeInfoWellKnown>(&well_known_uri)?; let well_known = fetch_remote_object::<NodeInfoWellKnown>(&well_known_uri)?;
Ok(fetch_remote_object::<NodeInfo>(&well_known.links.href)?) Ok(fetch_remote_object::<NodeInfo>(&well_known.links.href)?)
@ -61,22 +60,6 @@ fn upsert_post(post_form: &PostForm, conn: &PgConnection) -> Result<Post, Error>
} }
} }
fn fetch_communities_from_instance(
community_list: &Url,
conn: &PgConnection,
) -> Result<Vec<Community>, Error> {
fetch_remote_object::<UnorderedCollection>(community_list)?
.collection_props
.get_many_items_base_boxes()
.unwrap()
.map(|b| -> Result<CommunityForm, Error> {
let group = b.to_owned().to_concrete::<GroupExt>()?;
Ok(CommunityForm::from_group(&group, conn)?)
})
.map(|cf| upsert_community(&cf?, conn))
.collect()
}
// TODO: add an optional param last_updated and only fetch if its too old // TODO: add an optional param last_updated and only fetch if its too old
pub fn fetch_remote_object<Response>(url: &Url) -> Result<Response, Error> pub fn fetch_remote_object<Response>(url: &Url) -> Result<Response, Error>
where where
@ -126,6 +109,7 @@ pub fn search_by_apub_id(query: &str, conn: &PgConnection) -> Result<SearchRespo
} }
SearchAcceptedObjects::Group(g) => { SearchAcceptedObjects::Group(g) => {
let c = upsert_community(&CommunityForm::from_group(&g, conn)?, conn)?; let c = upsert_community(&CommunityForm::from_group(&g, conn)?, conn)?;
fetch_community_outbox(&c, conn)?;
response.communities = vec![CommunityView::read(conn, c.id, None)?]; response.communities = vec![CommunityView::read(conn, c.id, None)?];
} }
SearchAcceptedObjects::Page(p) => { SearchAcceptedObjects::Page(p) => {
@ -133,14 +117,10 @@ pub fn search_by_apub_id(query: &str, conn: &PgConnection) -> Result<SearchRespo
response.posts = vec![PostView::read(conn, p.id, None)?]; response.posts = vec![PostView::read(conn, p.id, None)?];
} }
} }
dbg!(&response);
Ok(response) Ok(response)
} }
fn fetch_remote_community_posts( fn fetch_community_outbox(community: &Community, conn: &PgConnection) -> Result<Vec<Post>, Error> {
community: &Community,
conn: &PgConnection,
) -> Result<Vec<Post>, Error> {
let outbox_url = Url::parse(&community.get_outbox_url())?; let outbox_url = Url::parse(&community.get_outbox_url())?;
let outbox = fetch_remote_object::<OrderedCollection>(&outbox_url)?; let outbox = fetch_remote_object::<OrderedCollection>(&outbox_url)?;
let items = outbox.collection_props.get_many_items_base_boxes(); let items = outbox.collection_props.get_many_items_base_boxes();
@ -168,23 +148,3 @@ pub fn fetch_remote_community(apub_id: &Url, conn: &PgConnection) -> Result<Comm
let cf = CommunityForm::from_group(&group, conn)?; let cf = CommunityForm::from_group(&group, conn)?;
upsert_community(&cf, conn) upsert_community(&cf, conn)
} }
// TODO: in the future, this should only be done when an instance is followed for the first time
// after that, we should rely in the inbox, and fetch on demand when needed
pub fn fetch_all(conn: &PgConnection) -> Result<(), Error> {
for instance in &get_following_instances() {
let node_info = fetch_node_info(instance)?;
if let Some(community_list) = node_info.metadata.community_list_url {
let communities = fetch_communities_from_instance(&community_list, conn)?;
for c in communities {
fetch_remote_community_posts(&c, conn)?;
}
} else {
warn!(
"{} is not a Lemmy instance, federation is not supported",
instance.domain
);
}
}
Ok(())
}

View file

@ -27,10 +27,6 @@ pub enum EndpointType {
Comment, Comment,
} }
pub struct Instance {
domain: String,
}
fn create_apub_response<T>(json: &T) -> HttpResponse<Body> fn create_apub_response<T>(json: &T) -> HttpResponse<Body>
where where
T: serde::ser::Serialize, T: serde::ser::Serialize,
@ -92,14 +88,3 @@ pub fn gen_keypair_str() -> (String, String) {
fn vec_bytes_to_str(bytes: Vec<u8>) -> String { fn vec_bytes_to_str(bytes: Vec<u8>) -> String {
String::from_utf8_lossy(&bytes).into_owned() String::from_utf8_lossy(&bytes).into_owned()
} }
pub fn get_following_instances() -> Vec<Instance> {
Settings::get()
.federation
.followed_instances
.split(',')
.map(|i| Instance {
domain: i.to_string(),
})
.collect()
}

View file

@ -7,15 +7,10 @@ use actix_web::*;
use diesel::r2d2::{ConnectionManager, Pool}; use diesel::r2d2::{ConnectionManager, Pool};
use diesel::PgConnection; use diesel::PgConnection;
use failure::Error; use failure::Error;
use lemmy_server::apub::fetcher::fetch_all;
use lemmy_server::db::code_migrations::run_advanced_migrations; use lemmy_server::db::code_migrations::run_advanced_migrations;
use lemmy_server::routes::{api, federation, feeds, index, nodeinfo, webfinger, websocket}; use lemmy_server::routes::{api, federation, feeds, index, nodeinfo, webfinger, websocket};
use lemmy_server::settings::Settings; use lemmy_server::settings::Settings;
use lemmy_server::websocket::server::*; use lemmy_server::websocket::server::*;
use log::warn;
use std::thread;
use std::thread::sleep;
use std::time::Duration;
embed_migrations!(); embed_migrations!();
@ -39,16 +34,6 @@ async fn main() -> Result<(), Error> {
// Set up websocket server // Set up websocket server
let server = ChatServer::startup(pool.clone()).start(); let server = ChatServer::startup(pool.clone()).start();
thread::spawn(move || {
// some work here
sleep(Duration::from_secs(5));
println!("Fetching apub data");
match fetch_all(&conn) {
Ok(_) => {}
Err(e) => warn!("Error during apub fetch: {}", e),
}
});
println!( println!(
"Starting http server at {}:{}", "Starting http server at {}:{}",
settings.bind, settings.port settings.bind, settings.port

View file

@ -6,10 +6,6 @@ pub fn config(cfg: &mut web::ServiceConfig) {
if Settings::get().federation.enabled { if Settings::get().federation.enabled {
println!("federation enabled, host is {}", Settings::get().hostname); println!("federation enabled, host is {}", Settings::get().hostname);
cfg cfg
.route(
"/federation/communities",
web::get().to(apub::community::get_apub_community_list),
)
// TODO: check the user/community params for these // TODO: check the user/community params for these
.route( .route(
"/federation/c/{_}/inbox", "/federation/c/{_}/inbox",

View file

@ -61,13 +61,6 @@ async fn node_info(
local_comments: site_view.number_of_comments, local_comments: site_view.number_of_comments,
open_registrations: site_view.open_registration, open_registrations: site_view.open_registration,
}, },
metadata: NodeInfoMetadata {
community_list_url: Some(Url::parse(&format!(
"{}://{}/federation/communities",
get_apub_protocol_string(),
Settings::get().hostname
))?),
},
}) })
}) })
.await .await
@ -93,7 +86,6 @@ pub struct NodeInfo {
pub software: NodeInfoSoftware, pub software: NodeInfoSoftware,
pub protocols: Vec<String>, pub protocols: Vec<String>,
pub usage: NodeInfoUsage, pub usage: NodeInfoUsage,
pub metadata: NodeInfoMetadata,
} }
#[derive(Serialize, Deserialize, Debug)] #[derive(Serialize, Deserialize, Debug)]
@ -115,8 +107,3 @@ pub struct NodeInfoUsage {
pub struct NodeInfoUsers { pub struct NodeInfoUsers {
pub total: i64, pub total: i64,
} }
#[derive(Serialize, Deserialize, Debug)]
pub struct NodeInfoMetadata {
pub community_list_url: Option<Url>,
}

View file

@ -63,7 +63,6 @@ pub struct Database {
#[derive(Debug, Deserialize, Clone)] #[derive(Debug, Deserialize, Clone)]
pub struct Federation { pub struct Federation {
pub enabled: bool, pub enabled: bool,
pub followed_instances: String,
pub tls_enabled: bool, pub tls_enabled: bool,
} }