Add logging for pictrs uploads (#3927)

* Add logging for pictrs uploads

* cleanup
This commit is contained in:
Anon 2023-09-06 08:13:30 -05:00 committed by GitHub
parent 797d26fdf4
commit fe3ebea95a
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
12 changed files with 184 additions and 19 deletions

View file

@ -1,17 +1,19 @@
use actix_web::web::{Data, Json};
use lemmy_api_common::{
context::LemmyContext,
request::purge_image_from_pictrs,
request::delete_image_from_pictrs,
site::{PurgeItemResponse, PurgePerson},
utils::{is_admin, local_user_view_from_jwt, purge_image_posts_for_person, sanitize_html_opt},
utils::{is_admin, local_user_view_from_jwt, sanitize_html_opt},
};
use lemmy_db_schema::{
source::{
image_upload::ImageUpload,
moderator::{AdminPurgePerson, AdminPurgePersonForm},
person::Person,
},
traits::Crud,
};
use lemmy_db_views::structs::LocalUserView;
use lemmy_utils::error::LemmyError;
#[tracing::instrument(skip(context))]
@ -26,18 +28,17 @@ pub async fn purge_person(
// Read the person to get their images
let person_id = data.person_id;
let person = Person::read(&mut context.pool(), person_id).await?;
if let Some(banner) = person.banner {
purge_image_from_pictrs(&banner, &context).await.ok();
let local_user = LocalUserView::read_person(&mut context.pool(), person_id).await?;
let pictrs_uploads =
ImageUpload::get_all_by_local_user_id(&mut context.pool(), &local_user.local_user.id).await?;
for upload in pictrs_uploads {
delete_image_from_pictrs(&upload.pictrs_alias, &upload.pictrs_delete_token, &context)
.await
.ok();
}
if let Some(avatar) = person.avatar {
purge_image_from_pictrs(&avatar, &context).await.ok();
}
purge_image_posts_for_person(person_id, &context).await?;
Person::delete(&mut context.pool(), person_id).await?;
// Mod tables

View file

@ -158,7 +158,6 @@ pub async fn purge_image_from_pictrs(
image_url: &Url,
context: &LemmyContext,
) -> Result<(), LemmyError> {
let pictrs_config = context.settings().pictrs_config()?;
is_image_content_type(context.client(), image_url).await?;
let alias = image_url
@ -167,7 +166,15 @@ pub async fn purge_image_from_pictrs(
.next_back()
.ok_or(LemmyErrorType::ImageUrlMissingLastPathSegment)?;
let purge_url = format!("{}/internal/purge?alias={}", pictrs_config.url, alias);
purge_image_from_pictrs_by_alias(alias, context).await
}
pub async fn purge_image_from_pictrs_by_alias(
alias: &str,
context: &LemmyContext,
) -> Result<(), LemmyError> {
let pictrs_config = context.settings().pictrs_config()?;
let purge_url = format!("{}internal/purge?alias={}", pictrs_config.url, alias);
let pictrs_api_key = pictrs_config
.api_key
@ -189,6 +196,26 @@ pub async fn purge_image_from_pictrs(
}
}
pub async fn delete_image_from_pictrs(
alias: &str,
delete_token: &str,
context: &LemmyContext,
) -> Result<(), LemmyError> {
let pictrs_config = context.settings().pictrs_config()?;
let url = format!(
"{}image/delete/{}/{}",
pictrs_config.url, &delete_token, &alias
);
context
.client()
.delete(&url)
.timeout(REQWEST_TIMEOUT)
.send()
.await
.map_err(LemmyError::from)?;
Ok(())
}
/// Both are options, since the URL might be either an html page, or an image
/// Returns the SiteMetadata, and an image URL, if there is a picture associated
#[tracing::instrument(skip_all)]

View file

@ -0,0 +1,47 @@
use crate::{
newtypes::{ImageUploadId, LocalUserId},
schema::image_upload::dsl::{image_upload, local_user_id, pictrs_alias},
source::image_upload::{ImageUpload, ImageUploadForm},
utils::{get_conn, DbPool},
};
use diesel::{insert_into, result::Error, ExpressionMethods, QueryDsl, Table};
use diesel_async::RunQueryDsl;
impl ImageUpload {
pub async fn create(pool: &mut DbPool<'_>, form: &ImageUploadForm) -> Result<Self, Error> {
let conn = &mut get_conn(pool).await?;
insert_into(image_upload)
.values(form)
.get_result::<Self>(conn)
.await
}
pub async fn get_all_by_local_user_id(
pool: &mut DbPool<'_>,
user_id: &LocalUserId,
) -> Result<Vec<Self>, Error> {
let conn = &mut get_conn(pool).await?;
image_upload
.filter(local_user_id.eq(user_id))
.select(image_upload::all_columns())
.load::<ImageUpload>(conn)
.await
}
pub async fn delete(
pool: &mut DbPool<'_>,
image_upload_id: ImageUploadId,
) -> Result<usize, Error> {
let conn = &mut get_conn(pool).await?;
diesel::delete(image_upload.find(image_upload_id))
.execute(conn)
.await
}
pub async fn delete_by_alias(pool: &mut DbPool<'_>, alias: &str) -> Result<usize, Error> {
let conn = &mut get_conn(pool).await?;
diesel::delete(image_upload.filter(pictrs_alias.eq(alias)))
.execute(conn)
.await
}
}

View file

@ -10,6 +10,7 @@ pub mod custom_emoji;
pub mod email_verification;
pub mod federation_allowlist;
pub mod federation_blocklist;
pub mod image_upload;
pub mod instance;
pub mod language;
pub mod local_site;

View file

@ -137,6 +137,12 @@ pub struct CommunityLanguageId(pub i32);
/// The comment reply id.
pub struct CommentReplyId(i32);
#[derive(Debug, Copy, Clone, Hash, Eq, PartialEq, Serialize, Deserialize, Default)]
#[cfg_attr(feature = "full", derive(DieselNewType, TS))]
#[cfg_attr(feature = "full", ts(export))]
/// The Image Upload id.
pub struct ImageUploadId(i32);
#[derive(Debug, Copy, Clone, Hash, Eq, PartialEq, Serialize, Deserialize, Default)]
#[cfg_attr(feature = "full", derive(DieselNewType, TS))]
#[cfg_attr(feature = "full", ts(export))]

View file

@ -299,6 +299,16 @@ diesel::table! {
}
}
diesel::table! {
image_upload (id) {
id -> Int4,
local_user_id -> Int4,
pictrs_alias -> Text,
pictrs_delete_token -> Text,
published -> Timestamptz,
}
}
diesel::table! {
instance (id) {
id -> Int4,
@ -405,9 +415,9 @@ diesel::table! {
totp_2fa_secret -> Nullable<Text>,
totp_2fa_url -> Nullable<Text>,
open_links_in_new_tab -> Bool,
infinite_scroll_enabled -> Bool,
blur_nsfw -> Bool,
auto_expand -> Bool,
infinite_scroll_enabled -> Bool,
admin -> Bool,
post_listing_mode -> PostListingModeEnum,
}
@ -893,6 +903,7 @@ diesel::joinable!(custom_emoji_keyword -> custom_emoji (custom_emoji_id));
diesel::joinable!(email_verification -> local_user (local_user_id));
diesel::joinable!(federation_allowlist -> instance (instance_id));
diesel::joinable!(federation_blocklist -> instance (instance_id));
diesel::joinable!(image_upload -> local_user (local_user_id));
diesel::joinable!(local_site -> site (site_id));
diesel::joinable!(local_site_rate_limit -> local_site (local_site_id));
diesel::joinable!(local_user -> person (person_id));
@ -967,6 +978,7 @@ diesel::allow_tables_to_appear_in_same_query!(
email_verification,
federation_allowlist,
federation_blocklist,
image_upload,
instance,
language,
local_site,

View file

@ -0,0 +1,36 @@
use crate::newtypes::{ImageUploadId, LocalUserId};
#[cfg(feature = "full")]
use crate::schema::image_upload;
use chrono::{DateTime, Utc};
use serde::{Deserialize, Serialize};
use serde_with::skip_serializing_none;
use std::fmt::Debug;
#[cfg(feature = "full")]
use ts_rs::TS;
use typed_builder::TypedBuilder;
#[skip_serializing_none]
#[derive(PartialEq, Eq, Debug, Clone, Serialize, Deserialize)]
#[cfg_attr(feature = "full", derive(Queryable, Associations, Identifiable, TS))]
#[cfg_attr(feature = "full", diesel(table_name = image_upload))]
#[cfg_attr(feature = "full", ts(export))]
#[cfg_attr(
feature = "full",
diesel(belongs_to(crate::source::local_user::LocalUser))
)]
pub struct ImageUpload {
pub id: ImageUploadId,
pub local_user_id: LocalUserId,
pub pictrs_alias: String,
pub pictrs_delete_token: String,
pub published: DateTime<Utc>,
}
#[derive(Debug, Clone, TypedBuilder)]
#[cfg_attr(feature = "full", derive(Insertable, AsChangeset))]
#[cfg_attr(feature = "full", diesel(table_name = image_upload))]
pub struct ImageUploadForm {
pub local_user_id: LocalUserId,
pub pictrs_alias: String,
pub pictrs_delete_token: String,
}

View file

@ -15,6 +15,7 @@ pub mod custom_emoji_keyword;
pub mod email_verification;
pub mod federation_allowlist;
pub mod federation_blocklist;
pub mod image_upload;
pub mod instance;
pub mod language;
pub mod local_site;

View file

@ -12,7 +12,13 @@ use actix_web::{
};
use futures::stream::{Stream, StreamExt};
use lemmy_api_common::{context::LemmyContext, utils::local_user_view_from_jwt};
use lemmy_db_schema::source::local_site::LocalSite;
use lemmy_db_schema::{
newtypes::LocalUserId,
source::{
image_upload::{ImageUpload, ImageUploadForm},
local_site::LocalSite,
},
};
use lemmy_utils::{claims::Claims, rate_limit::RateLimitCell, REQWEST_TIMEOUT};
use reqwest::Body;
use reqwest_middleware::{ClientWithMiddleware, RequestBuilder};
@ -95,8 +101,8 @@ async fn upload(
let jwt = req.cookie("jwt").ok_or(error::ErrorUnauthorized(
"No auth header for picture upload",
))?;
if Claims::decode(jwt.value(), &context.secret().jwt_secret).is_err() {
let claims = Claims::decode(jwt.value(), &context.secret().jwt_secret);
if claims.is_err() {
return Ok(HttpResponse::Unauthorized().finish());
};
@ -108,7 +114,6 @@ async fn upload(
if let Some(addr) = req.head().peer_addr {
client_req = client_req.header("X-Forwarded-For", addr.to_string())
};
let res = client_req
.body(Body::wrap_stream(make_send(body)))
.send()
@ -117,6 +122,19 @@ async fn upload(
let status = res.status();
let images = res.json::<Images>().await.map_err(error::ErrorBadRequest)?;
if let Some(images) = &images.files {
let local_user_id = LocalUserId(claims?.claims.sub);
for uploaded_image in images {
let form = ImageUploadForm {
local_user_id,
pictrs_alias: uploaded_image.file.to_string(),
pictrs_delete_token: uploaded_image.delete_token.to_string(),
};
ImageUpload::create(&mut context.pool(), &form)
.await
.map_err(error::ErrorBadRequest)?;
}
}
Ok(HttpResponse::build(status).json(images))
}
@ -215,6 +233,10 @@ async fn delete(
let res = client_req.send().await.map_err(error::ErrorBadRequest)?;
ImageUpload::delete_by_alias(&mut context.pool(), &file)
.await
.map_err(error::ErrorBadRequest)?;
Ok(HttpResponse::build(res.status()).body(BodyStream::new(res.bytes_stream())))
}

View file

@ -0,0 +1,2 @@
DROP TABLE image_upload;

View file

@ -0,0 +1,10 @@
CREATE TABLE image_upload (
id serial PRIMARY KEY,
local_user_id int REFERENCES local_user ON UPDATE CASCADE ON DELETE CASCADE NOT NULL,
pictrs_alias text NOT NULL UNIQUE,
pictrs_delete_token text NOT NULL,
published timestamptz DEFAULT now() NOT NULL
);
CREATE INDEX idx_image_upload_local_user_id ON image_upload (local_user_id);

View file

@ -11,5 +11,5 @@ find migrations -type f -name "*.sql" -print0 | while read -d $'\0' FILE
do
TMP_FILE="/tmp/tmp_pg_format.sql"
pg_format $FILE > $TMP_FILE
diff $FILE $TMP_FILE
diff -u $FILE $TMP_FILE
done