From 53a70d47c5cb4b6dd94d077f93bd9fa0196973ae Mon Sep 17 00:00:00 2001 From: Dessalines Date: Mon, 16 Aug 2021 23:59:27 -0400 Subject: [PATCH] Changing PostLink to SiteMetadata, adding it to the API. --- ansible/templates/nginx.conf | 2 +- crates/api/src/lib.rs | 3 ++ crates/api/src/post.rs | 19 +++++++++- crates/api_common/src/post.rs | 11 ++++++ crates/api_crud/src/post/create.rs | 8 ++--- crates/api_crud/src/post/update.rs | 8 ++--- crates/apub/src/objects/post.rs | 8 ++--- crates/routes/src/lib.rs | 1 - crates/routes/src/post_link_tags.rs | 29 --------------- crates/utils/src/request.rs | 56 +++++++++++++++-------------- crates/websocket/src/lib.rs | 1 + src/api_routes.rs | 6 +++- src/main.rs | 13 +++---- 13 files changed, 84 insertions(+), 81 deletions(-) delete mode 100644 crates/routes/src/post_link_tags.rs diff --git a/ansible/templates/nginx.conf b/ansible/templates/nginx.conf index de0c197c3..132821c76 100644 --- a/ansible/templates/nginx.conf +++ b/ansible/templates/nginx.conf @@ -80,7 +80,7 @@ server { } # backend - location ~ ^/(api|pictrs|feeds|nodeinfo|.well-known|post_link_tags) { + location ~ ^/(api|pictrs|feeds|nodeinfo|.well-known) { proxy_pass http://0.0.0.0:{{ lemmy_port }}; proxy_http_version 1.1; proxy_set_header Upgrade $http_upgrade; diff --git a/crates/api/src/lib.rs b/crates/api/src/lib.rs index 6234e7565..1c194b311 100644 --- a/crates/api/src/lib.rs +++ b/crates/api/src/lib.rs @@ -121,6 +121,9 @@ pub async fn match_websocket_operation( UserOperation::ResolvePostReport => { do_websocket_operation::(context, id, op, data).await } + UserOperation::GetSiteMetadata => { + do_websocket_operation::(context, id, op, data).await + } // Comment ops UserOperation::MarkCommentAsRead => { diff --git a/crates/api/src/post.rs b/crates/api/src/post.rs index 3de027313..1cb6c587e 100644 --- a/crates/api/src/post.rs +++ b/crates/api/src/post.rs @@ -23,7 +23,7 @@ use lemmy_apub::{ use lemmy_db_queries::{source::post::Post_, Crud, Likeable, Saveable}; use lemmy_db_schema::source::{moderator::*, post::*}; use lemmy_db_views::post_view::PostView; -use lemmy_utils::{ApiError, ConnectionId, LemmyError}; +use lemmy_utils::{request::fetch_site_metadata, ApiError, ConnectionId, LemmyError}; use lemmy_websocket::{send::send_post_ws_message, LemmyContext, UserOperation}; use std::convert::TryInto; @@ -285,3 +285,20 @@ impl Perform for SavePost { Ok(PostResponse { post_view }) } } + +#[async_trait::async_trait(?Send)] +impl Perform for GetSiteMetadata { + type Response = GetSiteMetadataResponse; + + async fn perform( + &self, + context: &Data, + _websocket_id: Option, + ) -> Result { + let data: &Self = self; + + let metadata = fetch_site_metadata(context.client(), &data.url).await?; + + Ok(GetSiteMetadataResponse { metadata }) + } +} diff --git a/crates/api_common/src/post.rs b/crates/api_common/src/post.rs index bb6b36249..5a83a0c50 100644 --- a/crates/api_common/src/post.rs +++ b/crates/api_common/src/post.rs @@ -8,6 +8,7 @@ use lemmy_db_views_actor::{ community_moderator_view::CommunityModeratorView, community_view::CommunityView, }; +use lemmy_utils::request::SiteMetadata; use serde::{Deserialize, Serialize}; use url::Url; @@ -148,3 +149,13 @@ pub struct ListPostReports { pub struct ListPostReportsResponse { pub posts: Vec, } + +#[derive(Deserialize, Debug)] +pub struct GetSiteMetadata { + pub url: Url, +} + +#[derive(Serialize, Clone, Debug)] +pub struct GetSiteMetadataResponse { + pub metadata: SiteMetadata, +} diff --git a/crates/api_crud/src/post/create.rs b/crates/api_crud/src/post/create.rs index 54b43718e..4e519ce0d 100644 --- a/crates/api_crud/src/post/create.rs +++ b/crates/api_crud/src/post/create.rs @@ -20,7 +20,7 @@ use lemmy_apub::{ use lemmy_db_queries::{source::post::Post_, Crud, Likeable}; use lemmy_db_schema::source::post::*; use lemmy_utils::{ - request::fetch_post_links_and_pictrs_data, + request::fetch_site_metadata_and_pictrs_data, utils::{check_slurs, check_slurs_opt, clean_url_params, is_valid_post_title}, ApiError, ConnectionId, @@ -51,9 +51,9 @@ impl PerformCrud for CreatePost { // Fetch post links and pictrs cached image let data_url = data.url.as_ref(); - let (post_links_res, pictrs_thumbnail) = - fetch_post_links_and_pictrs_data(context.client(), data_url).await?; - let (embed_title, embed_description, embed_html) = post_links_res + let (metadata_res, pictrs_thumbnail) = + fetch_site_metadata_and_pictrs_data(context.client(), data_url).await?; + let (embed_title, embed_description, embed_html) = metadata_res .map(|u| (u.title, u.description, u.html)) .unwrap_or((None, None, None)); diff --git a/crates/api_crud/src/post/update.rs b/crates/api_crud/src/post/update.rs index f65602af9..b31feaee3 100644 --- a/crates/api_crud/src/post/update.rs +++ b/crates/api_crud/src/post/update.rs @@ -5,7 +5,7 @@ use lemmy_apub::activities::{post::create_or_update::CreateOrUpdatePost, CreateO use lemmy_db_queries::{source::post::Post_, Crud}; use lemmy_db_schema::{naive_now, source::post::*}; use lemmy_utils::{ - request::fetch_post_links_and_pictrs_data, + request::fetch_site_metadata_and_pictrs_data, utils::{check_slurs_opt, clean_url_params, is_valid_post_title}, ApiError, ConnectionId, @@ -51,9 +51,9 @@ impl PerformCrud for EditPost { // Fetch post links and Pictrs cached image let data_url = data.url.as_ref(); - let (post_links_res, pictrs_thumbnail) = - fetch_post_links_and_pictrs_data(context.client(), data_url).await?; - let (embed_title, embed_description, embed_html) = post_links_res + let (metadata_res, pictrs_thumbnail) = + fetch_site_metadata_and_pictrs_data(context.client(), data_url).await?; + let (embed_title, embed_description, embed_html) = metadata_res .map(|u| (u.title, u.description, u.html)) .unwrap_or((None, None, None)); diff --git a/crates/apub/src/objects/post.rs b/crates/apub/src/objects/post.rs index 918c336c4..024ab6f1a 100644 --- a/crates/apub/src/objects/post.rs +++ b/crates/apub/src/objects/post.rs @@ -31,7 +31,7 @@ use lemmy_db_schema::{ }, }; use lemmy_utils::{ - request::fetch_post_links_and_pictrs_data, + request::fetch_site_metadata_and_pictrs_data, utils::{check_slurs, convert_datetime, markdown_to_html, remove_slurs}, LemmyError, }; @@ -188,12 +188,12 @@ impl FromApub for Post { let community = extract_community(&page.to, context, request_counter).await?; let thumbnail_url: Option = page.image.clone().map(|i| i.url); - let (post_links_res, pictrs_thumbnail) = if let Some(url) = &page.url { - fetch_post_links_and_pictrs_data(context.client(), Some(url)).await? + let (metadata_res, pictrs_thumbnail) = if let Some(url) = &page.url { + fetch_site_metadata_and_pictrs_data(context.client(), Some(url)).await? } else { (None, thumbnail_url) }; - let (embed_title, embed_description, embed_html) = post_links_res + let (embed_title, embed_description, embed_html) = metadata_res .map(|u| (u.title, u.description, u.html)) .unwrap_or((None, None, None)); diff --git a/crates/routes/src/lib.rs b/crates/routes/src/lib.rs index 8b9ae013c..b966b766a 100644 --- a/crates/routes/src/lib.rs +++ b/crates/routes/src/lib.rs @@ -4,5 +4,4 @@ extern crate lazy_static; pub mod feeds; pub mod images; pub mod nodeinfo; -pub mod post_link_tags; pub mod webfinger; diff --git a/crates/routes/src/post_link_tags.rs b/crates/routes/src/post_link_tags.rs deleted file mode 100644 index f5868e57c..000000000 --- a/crates/routes/src/post_link_tags.rs +++ /dev/null @@ -1,29 +0,0 @@ -use actix_web::{error::ErrorBadRequest, web::Query, *}; -use anyhow::anyhow; -use lemmy_utils::{request::fetch_post_link_tags, LemmyError}; -use lemmy_websocket::LemmyContext; -use serde::Deserialize; -use url::Url; - -#[derive(Deserialize)] -struct Params { - url: String, -} - -pub fn config(cfg: &mut web::ServiceConfig) { - cfg.route("/post_link_tags", web::get().to(get_post_links_response)); -} - -async fn get_post_links_response( - info: Query, - context: web::Data, -) -> Result { - let url = - Url::parse(&info.url).map_err(|_| ErrorBadRequest(LemmyError::from(anyhow!("not_found"))))?; - - let json = fetch_post_link_tags(context.client(), &url) - .await - .map_err(|_| ErrorBadRequest(LemmyError::from(anyhow!("not_found"))))?; - - Ok(HttpResponse::Ok().json(json)) -} diff --git a/crates/utils/src/request.rs b/crates/utils/src/request.rs index 0190021ba..1882e538c 100644 --- a/crates/utils/src/request.rs +++ b/crates/utils/src/request.rs @@ -48,16 +48,16 @@ where response.expect("retry http request") } -#[derive(Deserialize, Serialize, Debug, PartialEq)] -pub struct PostLinkTags { +#[derive(Deserialize, Serialize, Debug, PartialEq, Clone)] +pub struct SiteMetadata { pub title: Option, pub description: Option, - thumbnail_url: Option, + image: Option, pub html: Option, } -/// Fetches the post link html tags (like title, description, thumbnail, etc) -pub async fn fetch_post_link_tags(client: &Client, url: &Url) -> Result { +/// Fetches the post link html tags (like title, description, image, etc) +pub async fn fetch_site_metadata(client: &Client, url: &Url) -> Result { let response = retry(|| client.get(url.as_str()).send()).await?; let html = response @@ -65,12 +65,12 @@ pub async fn fetch_post_link_tags(client: &Client, url: &Url) -> Result Result { +fn html_to_site_metadata(html: &str) -> Result { let page = HTML::from_string(html.to_string(), None)?; let page_title = page.title; @@ -95,12 +95,12 @@ fn html_to_post_link_tags(html: &str) -> Result { let title = og_title.or(page_title); let description = og_description.or(page_description); - let thumbnail_url = og_image; + let image = og_image; - Ok(PostLinkTags { + Ok(SiteMetadata { title, description, - thumbnail_url, + image, html: None, }) } @@ -148,19 +148,21 @@ pub(crate) async fn fetch_pictrs( } /// Both are options, since the URL might be either an html page, or an image -pub async fn fetch_post_links_and_pictrs_data( +pub async fn fetch_site_metadata_and_pictrs_data( client: &Client, url: Option<&Url>, -) -> Result<(Option, Option), LemmyError> { +) -> Result<(Option, Option), LemmyError> { match &url { Some(url) => { - // Fetch post-links data - let post_links_res_option = fetch_post_link_tags(client, url).await.ok(); + // Fetch metadata + // Ignore errors, since it may be an image, or not have the data. + // Warning, this may ignore SSL errors + let metadata_option = fetch_site_metadata(client, url).await.ok(); // Fetch pictrs thumbnail - let pictrs_hash = match &post_links_res_option { - Some(post_link_res) => match &post_link_res.thumbnail_url { - Some(post_links_thumbnail_url) => fetch_pictrs(client, post_links_thumbnail_url) + let pictrs_hash = match &metadata_option { + Some(metadata_res) => match &metadata_res.image { + Some(metadata_image) => fetch_pictrs(client, metadata_image) .await? .map(|r| r.files[0].file.to_owned()), // Try to generate a small thumbnail if there's a full sized one from post-links @@ -185,7 +187,7 @@ pub async fn fetch_post_links_and_pictrs_data( }) .flatten(); - Ok((post_links_res_option, pictrs_thumbnail)) + Ok((metadata_option, pictrs_thumbnail)) } None => Ok((None, None)), } @@ -208,32 +210,32 @@ async fn is_image_content_type(client: &Client, test: &Url) -> Result<(), LemmyE #[cfg(test)] mod tests { - use crate::request::fetch_post_link_tags; + use crate::request::fetch_site_metadata; use url::Url; - use super::PostLinkTags; + use super::SiteMetadata; // These helped with testing #[actix_rt::test] - async fn test_post_links() { + async fn test_site_metadata() { let client = reqwest::Client::default(); let sample_url = Url::parse("https://www.redspark.nu/en/peoples-war/district-leader-of-chand-led-cpn-arrested-in-bhojpur/").unwrap(); - let sample_res = fetch_post_link_tags(&client, &sample_url).await.unwrap(); + let sample_res = fetch_site_metadata(&client, &sample_url).await.unwrap(); assert_eq!( - PostLinkTags { + SiteMetadata { title: Some("District Leader Of Chand Led CPN Arrested In Bhojpur - Redspark".to_string()), description: Some("BHOJPUR: A district leader of the outlawed Netra Bikram Chand alias Biplav-led outfit has been arrested. According to District Police".to_string()), - thumbnail_url: Some(Url::parse("https://www.redspark.nu/wp-content/uploads/2020/03/netra-bikram-chand-attends-program-1272019033653-1000x0-845x653-1.jpg").unwrap()), + image: Some(Url::parse("https://www.redspark.nu/wp-content/uploads/2020/03/netra-bikram-chand-attends-program-1272019033653-1000x0-845x653-1.jpg").unwrap()), html: None, }, sample_res); let youtube_url = Url::parse("https://www.youtube.com/watch?v=IquO_TcMZIQ").unwrap(); - let youtube_res = fetch_post_link_tags(&client, &youtube_url).await.unwrap(); + let youtube_res = fetch_site_metadata(&client, &youtube_url).await.unwrap(); assert_eq!( - PostLinkTags { + SiteMetadata { title: Some("A Hard Look at Rent and Rent Seeking with Michael Hudson & Pepe Escobar".to_string()), description: Some("An interactive discussion on wealth inequality and the “Great Game” on the control of natural resources.In this webinar organized jointly by the Henry George...".to_string()), - thumbnail_url: Some(Url::parse("https://i.ytimg.com/vi/IquO_TcMZIQ/maxresdefault.jpg").unwrap()), + image: Some(Url::parse("https://i.ytimg.com/vi/IquO_TcMZIQ/maxresdefault.jpg").unwrap()), html: None, }, youtube_res); } diff --git a/crates/websocket/src/lib.rs b/crates/websocket/src/lib.rs index b866a2a21..e01514030 100644 --- a/crates/websocket/src/lib.rs +++ b/crates/websocket/src/lib.rs @@ -125,6 +125,7 @@ pub enum UserOperation { CommunityJoin, ModJoin, ChangePassword, + GetSiteMetadata, } #[derive(EnumString, ToString, Debug, Clone)] diff --git a/src/api_routes.rs b/src/api_routes.rs index f8e8f7e18..a3bf39fe1 100644 --- a/src/api_routes.rs +++ b/src/api_routes.rs @@ -88,7 +88,11 @@ pub fn config(cfg: &mut web::ServiceConfig, rate_limit: &RateLimit) { "/report/resolve", web::put().to(route_post::), ) - .route("/report/list", web::get().to(route_get::)), + .route("/report/list", web::get().to(route_get::)) + .route( + "/site_metadata", + web::get().to(route_get::), + ), ) // Comment .service( diff --git a/src/main.rs b/src/main.rs index b1a076ddb..504ccc80e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -12,7 +12,7 @@ use lemmy_api_common::blocking; use lemmy_api_crud::match_websocket_operation_crud; use lemmy_apub::activity_queue::create_activity_queue; use lemmy_db_queries::get_database_url_from_env; -use lemmy_routes::{feeds, images, nodeinfo, post_link_tags, webfinger}; +use lemmy_routes::{feeds, images, nodeinfo, webfinger}; use lemmy_server::{api_routes, code_migrations::run_advanced_migrations, scheduled_tasks}; use lemmy_utils::{ rate_limit::{rate_limiter::RateLimiter, RateLimit}, @@ -20,6 +20,7 @@ use lemmy_utils::{ LemmyError, }; use lemmy_websocket::{chat_server::ChatServer, LemmyContext}; +use reqwest::Client; use std::{sync::Arc, thread}; use tokio::sync::Mutex; @@ -66,17 +67,12 @@ async fn main() -> Result<(), LemmyError> { let activity_queue = create_activity_queue(); - // Required because docker installs try to use TLS, which is disabled as a reqwest feature flag - let client = reqwest::ClientBuilder::new() - .danger_accept_invalid_certs(true) - .build()?; - let chat_server = ChatServer::startup( pool.clone(), rate_limiter.clone(), |c, i, o, d| Box::pin(match_websocket_operation(c, i, o, d)), |c, i, o, d| Box::pin(match_websocket_operation_crud(c, i, o, d)), - client.clone(), + Client::default(), activity_queue.clone(), ) .start(); @@ -86,7 +82,7 @@ async fn main() -> Result<(), LemmyError> { let context = LemmyContext::create( pool.clone(), chat_server.to_owned(), - client.to_owned(), + Client::default(), activity_queue.to_owned(), ); let rate_limiter = rate_limiter.clone(); @@ -100,7 +96,6 @@ async fn main() -> Result<(), LemmyError> { .configure(|cfg| images::config(cfg, &rate_limiter)) .configure(nodeinfo::config) .configure(webfinger::config) - .configure(post_link_tags::config) }) .bind((settings.bind, settings.port))? .run()