Adding instance software and version. Fixes #2222 (#2733)

* Adding instance software and version. Fixes #2222

* Fix clippy.

* Fix clippy 2

* Fixing some more issues.
This commit is contained in:
Dessalines 2023-02-18 09:36:12 -05:00 committed by GitHub
parent 47f4aa3550
commit 3735c6fabf
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
13 changed files with 201 additions and 92 deletions

View file

@ -71,7 +71,7 @@ tracing-error = "0.2.0"
tracing-log = "0.1.3" tracing-log = "0.1.3"
tracing-subscriber = { version = "0.3.15", features = ["env-filter"] } tracing-subscriber = { version = "0.3.15", features = ["env-filter"] }
url = { version = "2.3.1", features = ["serde"] } url = { version = "2.3.1", features = ["serde"] }
reqwest = { version = "0.11.12", features = ["json"] } reqwest = { version = "0.11.12", features = ["json", "blocking"] }
reqwest-middleware = "0.2.0" reqwest-middleware = "0.2.0"
reqwest-tracing = "0.4.0" reqwest-tracing = "0.4.0"
clokwerk = "0.3.5" clokwerk = "0.3.5"

View file

@ -1,7 +1,12 @@
use crate::sensitive::Sensitive; use crate::sensitive::Sensitive;
use lemmy_db_schema::{ use lemmy_db_schema::{
newtypes::{CommentId, CommunityId, LanguageId, PersonId, PostId}, newtypes::{CommentId, CommunityId, LanguageId, PersonId, PostId},
source::{language::Language, local_site::RegistrationMode, tagline::Tagline}, source::{
instance::Instance,
language::Language,
local_site::RegistrationMode,
tagline::Tagline,
},
ListingType, ListingType,
ModlogActionType, ModlogActionType,
SearchType, SearchType,
@ -239,9 +244,9 @@ pub struct LeaveAdmin {
#[derive(Debug, Serialize, Deserialize, Clone)] #[derive(Debug, Serialize, Deserialize, Clone)]
pub struct FederatedInstances { pub struct FederatedInstances {
pub linked: Vec<String>, pub linked: Vec<Instance>,
pub allowed: Option<Vec<String>>, pub allowed: Option<Vec<Instance>>,
pub blocked: Option<Vec<String>>, pub blocked: Option<Vec<Instance>>,
} }
#[derive(Debug, Serialize, Deserialize, Clone)] #[derive(Debug, Serialize, Deserialize, Clone)]

View file

@ -114,13 +114,13 @@ fn check_apub_id_valid(
} }
if let Some(blocked) = local_site_data.blocked_instances.as_ref() { if let Some(blocked) = local_site_data.blocked_instances.as_ref() {
if blocked.contains(&domain) { if blocked.iter().any(|i| domain.eq(&i.domain)) {
return Err("Domain is blocked"); return Err("Domain is blocked");
} }
} }
if let Some(allowed) = local_site_data.allowed_instances.as_ref() { if let Some(allowed) = local_site_data.allowed_instances.as_ref() {
if !allowed.contains(&domain) { if !allowed.iter().any(|i| domain.eq(&i.domain)) {
return Err("Domain is not in allowlist"); return Err("Domain is not in allowlist");
} }
} }
@ -131,8 +131,8 @@ fn check_apub_id_valid(
#[derive(Clone)] #[derive(Clone)]
pub(crate) struct LocalSiteData { pub(crate) struct LocalSiteData {
local_site: Option<LocalSite>, local_site: Option<LocalSite>,
allowed_instances: Option<Vec<String>>, allowed_instances: Option<Vec<Instance>>,
blocked_instances: Option<Vec<String>>, blocked_instances: Option<Vec<Instance>>,
} }
pub(crate) async fn fetch_local_site_data( pub(crate) async fn fetch_local_site_data(
@ -175,7 +175,10 @@ pub(crate) fn check_apub_id_valid_with_strictness(
if is_strict { if is_strict {
// need to allow this explicitly because apub receive might contain objects from our local // need to allow this explicitly because apub receive might contain objects from our local
// instance. // instance.
let mut allowed_and_local = allowed.clone(); let mut allowed_and_local = allowed
.iter()
.map(|i| i.domain.clone())
.collect::<Vec<String>>();
allowed_and_local.push(local_instance); allowed_and_local.push(local_instance);
if !allowed_and_local.contains(&domain) { if !allowed_and_local.contains(&domain) {

View file

@ -59,25 +59,24 @@ mod tests {
#[serial] #[serial]
async fn test_allowlist_insert_and_clear() { async fn test_allowlist_insert_and_clear() {
let pool = &build_db_pool_for_tests().await; let pool = &build_db_pool_for_tests().await;
let allowed = Some(vec![ let domains = vec![
"tld1.xyz".to_string(), "tld1.xyz".to_string(),
"tld2.xyz".to_string(), "tld2.xyz".to_string(),
"tld3.xyz".to_string(), "tld3.xyz".to_string(),
]); ];
let allowed = Some(domains.clone());
FederationAllowList::replace(pool, allowed).await.unwrap(); FederationAllowList::replace(pool, allowed).await.unwrap();
let allows = Instance::allowlist(pool).await.unwrap(); let allows = Instance::allowlist(pool).await.unwrap();
let allows_domains = allows
.iter()
.map(|i| i.domain.clone())
.collect::<Vec<String>>();
assert_eq!(3, allows.len()); assert_eq!(3, allows.len());
assert_eq!( assert_eq!(domains, allows_domains);
vec![
"tld1.xyz".to_string(),
"tld2.xyz".to_string(),
"tld3.xyz".to_string()
],
allows
);
// Now test clearing them via Some(empty vec) // Now test clearing them via Some(empty vec)
let clear_allows = Some(Vec::new()); let clear_allows = Some(Vec::new());

View file

@ -31,10 +31,10 @@ impl Instance {
Self::create(pool, domain).await Self::create(pool, domain).await
} }
pub async fn create_conn(conn: &mut AsyncPgConnection, domain: &str) -> Result<Self, Error> { pub async fn create_conn(conn: &mut AsyncPgConnection, domain: &str) -> Result<Self, Error> {
let form = InstanceForm { let form = InstanceForm::builder()
domain: domain.to_string(), .domain(domain.to_string())
updated: Some(naive_now()), .updated(Some(naive_now()))
}; .build();
Self::create_from_form_conn(conn, &form).await Self::create_from_form_conn(conn, &form).await
} }
pub async fn delete(pool: &DbPool, instance_id: InstanceId) -> Result<usize, Error> { pub async fn delete(pool: &DbPool, instance_id: InstanceId) -> Result<usize, Error> {
@ -47,31 +47,31 @@ impl Instance {
let conn = &mut get_conn(pool).await?; let conn = &mut get_conn(pool).await?;
diesel::delete(instance::table).execute(conn).await diesel::delete(instance::table).execute(conn).await
} }
pub async fn allowlist(pool: &DbPool) -> Result<Vec<String>, Error> { pub async fn allowlist(pool: &DbPool) -> Result<Vec<Self>, Error> {
let conn = &mut get_conn(pool).await?; let conn = &mut get_conn(pool).await?;
instance::table instance::table
.inner_join(federation_allowlist::table) .inner_join(federation_allowlist::table)
.select(instance::domain) .select(instance::all_columns)
.load::<String>(conn) .get_results(conn)
.await .await
} }
pub async fn blocklist(pool: &DbPool) -> Result<Vec<String>, Error> { pub async fn blocklist(pool: &DbPool) -> Result<Vec<Self>, Error> {
let conn = &mut get_conn(pool).await?; let conn = &mut get_conn(pool).await?;
instance::table instance::table
.inner_join(federation_blocklist::table) .inner_join(federation_blocklist::table)
.select(instance::domain) .select(instance::all_columns)
.load::<String>(conn) .get_results(conn)
.await .await
} }
pub async fn linked(pool: &DbPool) -> Result<Vec<String>, Error> { pub async fn linked(pool: &DbPool) -> Result<Vec<Self>, Error> {
let conn = &mut get_conn(pool).await?; let conn = &mut get_conn(pool).await?;
instance::table instance::table
.left_join(federation_blocklist::table) .left_join(federation_blocklist::table)
.filter(federation_blocklist::id.is_null()) .filter(federation_blocklist::id.is_null())
.select(instance::domain) .select(instance::all_columns)
.load::<String>(conn) .get_results(conn)
.await .await
} }
} }

View file

@ -648,6 +648,8 @@ table! {
instance(id) { instance(id) {
id -> Int4, id -> Int4,
domain -> Text, domain -> Text,
software -> Nullable<Text>,
version -> Nullable<Text>,
published -> Timestamp, published -> Timestamp,
updated -> Nullable<Timestamp>, updated -> Nullable<Timestamp>,
} }

View file

@ -1,21 +1,30 @@
use crate::newtypes::InstanceId; use crate::newtypes::InstanceId;
#[cfg(feature = "full")] #[cfg(feature = "full")]
use crate::schema::instance; use crate::schema::instance;
use serde::{Deserialize, Serialize};
use std::fmt::Debug; use std::fmt::Debug;
use typed_builder::TypedBuilder;
#[derive(PartialEq, Eq, Debug)] #[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize)]
#[cfg_attr(feature = "full", derive(Queryable, Identifiable))] #[cfg_attr(feature = "full", derive(Queryable, Identifiable))]
#[cfg_attr(feature = "full", diesel(table_name = instance))] #[cfg_attr(feature = "full", diesel(table_name = instance))]
pub struct Instance { pub struct Instance {
pub id: InstanceId, pub id: InstanceId,
pub domain: String, pub domain: String,
pub software: Option<String>,
pub version: Option<String>,
pub published: chrono::NaiveDateTime, pub published: chrono::NaiveDateTime,
pub updated: Option<chrono::NaiveDateTime>, pub updated: Option<chrono::NaiveDateTime>,
} }
#[derive(Clone, TypedBuilder)]
#[builder(field_defaults(default))]
#[cfg_attr(feature = "full", derive(Insertable, AsChangeset))] #[cfg_attr(feature = "full", derive(Insertable, AsChangeset))]
#[cfg_attr(feature = "full", diesel(table_name = instance))] #[cfg_attr(feature = "full", diesel(table_name = instance))]
pub struct InstanceForm { pub struct InstanceForm {
#[builder(!default)]
pub domain: String, pub domain: String,
pub software: Option<String>,
pub version: Option<String>,
pub updated: Option<chrono::NaiveDateTime>, pub updated: Option<chrono::NaiveDateTime>,
} }

View file

@ -34,27 +34,27 @@ async fn node_info(context: web::Data<LemmyContext>) -> Result<HttpResponse, Err
.map_err(|_| ErrorBadRequest(LemmyError::from(anyhow!("not_found"))))?; .map_err(|_| ErrorBadRequest(LemmyError::from(anyhow!("not_found"))))?;
let protocols = if site_view.local_site.federation_enabled { let protocols = if site_view.local_site.federation_enabled {
vec!["activitypub".to_string()] Some(vec!["activitypub".to_string()])
} else { } else {
vec![] None
}; };
let open_registrations = site_view.local_site.registration_mode == RegistrationMode::Open; let open_registrations = Some(site_view.local_site.registration_mode == RegistrationMode::Open);
let json = NodeInfo { let json = NodeInfo {
version: "2.0".to_string(), version: Some("2.0".to_string()),
software: NodeInfoSoftware { software: Some(NodeInfoSoftware {
name: "lemmy".to_string(), name: Some("lemmy".to_string()),
version: version::VERSION.to_string(), version: Some(version::VERSION.to_string()),
}, }),
protocols, protocols,
usage: NodeInfoUsage { usage: Some(NodeInfoUsage {
users: NodeInfoUsers { users: Some(NodeInfoUsers {
total: site_view.counts.users, total: Some(site_view.counts.users),
active_halfyear: site_view.counts.users_active_half_year, active_halfyear: Some(site_view.counts.users_active_half_year),
active_month: site_view.counts.users_active_month, active_month: Some(site_view.counts.users_active_month),
}, }),
local_posts: site_view.counts.posts, local_posts: Some(site_view.counts.posts),
local_comments: site_view.counts.comments, local_comments: Some(site_view.counts.comments),
}, }),
open_registrations, open_registrations,
}; };
@ -72,34 +72,35 @@ struct NodeInfoWellKnownLinks {
pub href: Url, pub href: Url,
} }
#[derive(Serialize, Deserialize, Debug)] #[derive(Serialize, Deserialize, Debug, Default)]
#[serde(rename_all = "camelCase")] #[serde(rename_all = "camelCase", default)]
struct NodeInfo { pub struct NodeInfo {
pub version: String, pub version: Option<String>,
pub software: NodeInfoSoftware, pub software: Option<NodeInfoSoftware>,
pub protocols: Vec<String>, pub protocols: Option<Vec<String>>,
pub usage: NodeInfoUsage, pub usage: Option<NodeInfoUsage>,
pub open_registrations: bool, pub open_registrations: Option<bool>,
} }
#[derive(Serialize, Deserialize, Debug)] #[derive(Serialize, Deserialize, Debug, Default)]
struct NodeInfoSoftware { #[serde(default)]
pub name: String, pub struct NodeInfoSoftware {
pub version: String, pub name: Option<String>,
pub version: Option<String>,
} }
#[derive(Serialize, Deserialize, Debug)] #[derive(Serialize, Deserialize, Debug, Default)]
#[serde(rename_all = "camelCase")] #[serde(rename_all = "camelCase", default)]
struct NodeInfoUsage { pub struct NodeInfoUsage {
pub users: NodeInfoUsers, pub users: Option<NodeInfoUsers>,
pub local_posts: i64, pub local_posts: Option<i64>,
pub local_comments: i64, pub local_comments: Option<i64>,
} }
#[derive(Serialize, Deserialize, Debug)] #[derive(Serialize, Deserialize, Debug, Default)]
#[serde(rename_all = "camelCase")] #[serde(rename_all = "camelCase", default)]
struct NodeInfoUsers { pub struct NodeInfoUsers {
pub total: i64, pub total: Option<i64>,
pub active_halfyear: i64, pub active_halfyear: Option<i64>,
pub active_month: i64, pub active_month: Option<i64>,
} }

View file

@ -0,0 +1,2 @@
alter table instance drop column software;
alter table instance drop column version;

View file

@ -0,0 +1,4 @@
-- Add Software and Version columns from nodeinfo to the instance table
alter table instance add column software varchar(255);
alter table instance add column version varchar(255);

View file

@ -1,6 +1,10 @@
#!/bin/bash #!/bin/bash
set -e set -e
CWD="$(cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
cd $CWD/../
cargo clippy --workspace --fix --allow-staged --allow-dirty --tests --all-targets --all-features -- \ cargo clippy --workspace --fix --allow-staged --allow-dirty --tests --all-targets --all-features -- \
-D warnings -D deprecated -D clippy::perf -D clippy::complexity \ -D warnings -D deprecated -D clippy::perf -D clippy::complexity \
-D clippy::style -D clippy::correctness -D clippy::suspicious \ -D clippy::style -D clippy::correctness -D clippy::suspicious \
@ -10,3 +14,5 @@ cargo clippy --workspace --fix --allow-staged --allow-dirty --tests --all-target
-D clippy::manual_string_new -D clippy::redundant_closure_for_method_calls \ -D clippy::manual_string_new -D clippy::redundant_closure_for_method_calls \
-D clippy::unused_self \ -D clippy::unused_self \
-A clippy::uninlined_format_args -A clippy::uninlined_format_args
cargo +nightly fmt

View file

@ -42,7 +42,7 @@ use tracing_subscriber::{filter::Targets, layer::SubscriberExt, Layer, Registry}
use url::Url; use url::Url;
/// Max timeout for http requests /// Max timeout for http requests
const REQWEST_TIMEOUT: Duration = Duration::from_secs(10); pub(crate) const REQWEST_TIMEOUT: Duration = Duration::from_secs(10);
/// Placing the main function in lib.rs allows other crates to import it and embed Lemmy /// Placing the main function in lib.rs allows other crates to import it and embed Lemmy
pub async fn start_lemmy_server() -> Result<(), LemmyError> { pub async fn start_lemmy_server() -> Result<(), LemmyError> {
@ -73,11 +73,6 @@ pub async fn start_lemmy_server() -> Result<(), LemmyError> {
let pool = build_db_pool(&settings).await?; let pool = build_db_pool(&settings).await?;
run_advanced_migrations(&pool, &settings).await?; run_advanced_migrations(&pool, &settings).await?;
// Schedules various cleanup tasks for the DB
thread::spawn(move || {
scheduled_tasks::setup(db_url).expect("Couldn't set up scheduled_tasks");
});
// Initialize the secrets // Initialize the secrets
let secret = Secret::init(&pool) let secret = Secret::init(&pool)
.await .await
@ -106,8 +101,9 @@ pub async fn start_lemmy_server() -> Result<(), LemmyError> {
settings.bind, settings.port settings.bind, settings.port
); );
let user_agent = build_user_agent(&settings);
let reqwest_client = Client::builder() let reqwest_client = Client::builder()
.user_agent(build_user_agent(&settings)) .user_agent(user_agent.clone())
.timeout(REQWEST_TIMEOUT) .timeout(REQWEST_TIMEOUT)
.build()?; .build()?;
@ -128,6 +124,11 @@ pub async fn start_lemmy_server() -> Result<(), LemmyError> {
.with(TracingMiddleware::default()) .with(TracingMiddleware::default())
.build(); .build();
// Schedules various cleanup tasks for the DB
thread::spawn(move || {
scheduled_tasks::setup(db_url, user_agent).expect("Couldn't set up scheduled_tasks");
});
let chat_server = Arc::new(ChatServer::startup()); let chat_server = Arc::new(ChatServer::startup());
// Create Http server with websocket support // Create Http server with websocket support

View file

@ -2,17 +2,25 @@ use clokwerk::{Scheduler, TimeUnits};
// Import week days and WeekDay // Import week days and WeekDay
use diesel::{sql_query, PgConnection, RunQueryDsl}; use diesel::{sql_query, PgConnection, RunQueryDsl};
use diesel::{Connection, ExpressionMethods, QueryDsl}; use diesel::{Connection, ExpressionMethods, QueryDsl};
use lemmy_utils::error::LemmyError; use lemmy_db_schema::{
source::instance::{Instance, InstanceForm},
utils::naive_now,
};
use lemmy_routes::nodeinfo::NodeInfo;
use lemmy_utils::{error::LemmyError, REQWEST_TIMEOUT};
use reqwest::blocking::Client;
use std::{thread, time::Duration}; use std::{thread, time::Duration};
use tracing::info; use tracing::info;
/// Schedules various cleanup tasks for lemmy in a background thread /// Schedules various cleanup tasks for lemmy in a background thread
pub fn setup(db_url: String) -> Result<(), LemmyError> { pub fn setup(db_url: String, user_agent: String) -> Result<(), LemmyError> {
// Setup the connections // Setup the connections
let mut scheduler = Scheduler::new(); let mut scheduler = Scheduler::new();
let mut conn = PgConnection::establish(&db_url).expect("could not establish connection"); let mut conn = PgConnection::establish(&db_url).expect("could not establish connection");
let mut conn_2 = PgConnection::establish(&db_url).expect("could not establish connection");
active_counts(&mut conn); active_counts(&mut conn);
update_banned_when_expired(&mut conn); update_banned_when_expired(&mut conn);
@ -33,6 +41,11 @@ pub fn setup(db_url: String) -> Result<(), LemmyError> {
clear_old_activities(&mut conn); clear_old_activities(&mut conn);
}); });
update_instance_software(&mut conn_2, &user_agent);
scheduler.every(1.days()).run(move || {
update_instance_software(&mut conn_2, &user_agent);
});
// Manually run the scheduler in an event loop // Manually run the scheduler in an event loop
loop { loop {
scheduler.run_pending(); scheduler.run_pending();
@ -120,3 +133,67 @@ fn drop_ccnew_indexes(conn: &mut PgConnection) {
.execute(conn) .execute(conn)
.expect("drop ccnew indexes"); .expect("drop ccnew indexes");
} }
/// Updates the instance software and version
fn update_instance_software(conn: &mut PgConnection, user_agent: &str) {
use lemmy_db_schema::schema::instance;
info!("Updating instances software and versions...");
let client = Client::builder()
.user_agent(user_agent)
.timeout(REQWEST_TIMEOUT)
.build()
.expect("couldnt build reqwest client");
let instances = instance::table
.get_results::<Instance>(conn)
.expect("no instances found");
for instance in instances {
let node_info_url = format!("https://{}/nodeinfo/2.0.json", instance.domain);
// Skip it if it can't connect
let res = client
.get(&node_info_url)
.send()
.ok()
.and_then(|t| t.json::<NodeInfo>().ok());
if let Some(node_info) = res {
let software = node_info.software.as_ref();
let form = InstanceForm::builder()
.domain(instance.domain)
.software(software.and_then(|s| s.name.clone()))
.version(software.and_then(|s| s.version.clone()))
.updated(Some(naive_now()))
.build();
diesel::update(instance::table.find(instance.id))
.set(form)
.execute(conn)
.expect("update site instance software");
}
}
info!("Done.");
}
#[cfg(test)]
mod tests {
use lemmy_routes::nodeinfo::NodeInfo;
use reqwest::Client;
#[tokio::test]
async fn test_nodeinfo() {
let client = Client::builder().build().unwrap();
let lemmy_ml_nodeinfo = client
.get("https://lemmy.ml/nodeinfo/2.0.json")
.send()
.await
.unwrap()
.json::<NodeInfo>()
.await
.unwrap();
assert_eq!(lemmy_ml_nodeinfo.software.unwrap().name.unwrap(), "lemmy");
}
}