From c34fffc2c48b82e7c7aa2f139850c2917fc36438 Mon Sep 17 00:00:00 2001 From: nutomic Date: Wed, 5 Aug 2020 16:00:00 +0000 Subject: [PATCH] Proxy pictrs requests through Lemmy (fixes #371) (#77) fix check_only value for image rate limit Fix image rate limit Add rate limit for image uploads Proxy pictrs requests through Lemmy (fixes #371) Co-authored-by: Felix Ableitner Reviewed-on: https://yerbamate.dev/LemmyNet/lemmy/pulls/77 --- ansible/templates/nginx.conf | 12 --- docker/dev/docker-compose.yml | 3 +- docker/federation/nginx.conf | 36 ------- docker/lemmy.hjson | 9 ++ server/config/defaults.hjson | 6 ++ server/lemmy_utils/src/settings.rs | 3 + server/src/main.rs | 5 +- server/src/rate_limit/mod.rs | 13 +++ server/src/rate_limit/rate_limiter.rs | 1 + server/src/routes/images.rs | 146 ++++++++++++++++++++++++++ server/src/routes/mod.rs | 1 + 11 files changed, 184 insertions(+), 51 deletions(-) create mode 100644 server/src/routes/images.rs diff --git a/ansible/templates/nginx.conf b/ansible/templates/nginx.conf index 4f66292c3..092f85520 100644 --- a/ansible/templates/nginx.conf +++ b/ansible/templates/nginx.conf @@ -74,18 +74,6 @@ server { return 301 /pictrs/image/$1; } - # pict-rs images - location /pictrs { - location /pictrs/image { - proxy_pass http://0.0.0.0:8537/image; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - } - # Block the import - return 403; - } - location /iframely/ { proxy_pass http://0.0.0.0:8061/; proxy_set_header X-Real-IP $remote_addr; diff --git a/docker/dev/docker-compose.yml b/docker/dev/docker-compose.yml index 51a3ecdab..257ad6c63 100644 --- a/docker/dev/docker-compose.yml +++ b/docker/dev/docker-compose.yml @@ -21,7 +21,8 @@ services: postgres: image: postgres:12-alpine ports: - - "127.0.0.1:5432:5432" + # use a different port so it doesnt conflict with postgres running on the host + - "127.0.0.1:5433:5432" environment: - POSTGRES_USER=lemmy - POSTGRES_PASSWORD=password diff --git a/docker/federation/nginx.conf b/docker/federation/nginx.conf index 573067981..b7901c19c 100644 --- a/docker/federation/nginx.conf +++ b/docker/federation/nginx.conf @@ -26,18 +26,6 @@ http { proxy_set_header Connection "upgrade"; } - # pict-rs images - location /pictrs { - location /pictrs/image { - proxy_pass http://pictrs:8080/image; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - } - # Block the import - return 403; - } - location /iframely/ { proxy_pass http://iframely:80/; proxy_set_header X-Real-IP $remote_addr; @@ -69,18 +57,6 @@ http { proxy_set_header Connection "upgrade"; } - # pict-rs images - location /pictrs { - location /pictrs/image { - proxy_pass http://pictrs:8080/image; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - } - # Block the import - return 403; - } - location /iframely/ { proxy_pass http://iframely:80/; proxy_set_header X-Real-IP $remote_addr; @@ -112,18 +88,6 @@ http { proxy_set_header Connection "upgrade"; } - # pict-rs images - location /pictrs { - location /pictrs/image { - proxy_pass http://pictrs:8080/image; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - } - # Block the import - return 403; - } - location /iframely/ { proxy_pass http://iframely:80/; proxy_set_header X-Real-IP $remote_addr; diff --git a/docker/lemmy.hjson b/docker/lemmy.hjson index 89da46891..d17394767 100644 --- a/docker/lemmy.hjson +++ b/docker/lemmy.hjson @@ -2,6 +2,15 @@ # for more info about the config, check out the documentation # https://dev.lemmy.ml/docs/administration_configuration.html + setup: { + # username for the admin user + admin_username: "lemmy" + # password for the admin user + admin_password: "lemmy" + # name of the site (can be changed later) + site_name: "lemmy-test" + } + # the domain name of your instance (eg "dev.lemmy.ml") hostname: "my_domain" # address where lemmy should listen for incoming requests diff --git a/server/config/defaults.hjson b/server/config/defaults.hjson index 5238455a7..9e9fc9988 100644 --- a/server/config/defaults.hjson +++ b/server/config/defaults.hjson @@ -35,6 +35,8 @@ jwt_secret: "changeme" # The location of the frontend front_end_dir: "../ui/dist" + # address where pictrs is available + pictrs_url: "http://pictrs:8080" # rate limits for various user actions, by user ip rate_limit: { # maximum number of messages created in interval @@ -49,6 +51,10 @@ register: 3 # interval length for registration limit register_per_second: 3600 + # maximum number of image uploads in interval + image: 6 + # interval length for image uploads + image_per_second: 3600 } # settings related to activitypub federation federation: { diff --git a/server/lemmy_utils/src/settings.rs b/server/lemmy_utils/src/settings.rs index b7cc2c45f..6a566de7e 100644 --- a/server/lemmy_utils/src/settings.rs +++ b/server/lemmy_utils/src/settings.rs @@ -14,6 +14,7 @@ pub struct Settings { pub port: u16, pub jwt_secret: String, pub front_end_dir: String, + pub pictrs_url: String, pub rate_limit: RateLimitConfig, pub email: Option, pub federation: Federation, @@ -36,6 +37,8 @@ pub struct RateLimitConfig { pub post_per_second: i32, pub register: i32, pub register_per_second: i32, + pub image: i32, + pub image_per_second: i32, } #[derive(Debug, Deserialize, Clone)] diff --git a/server/src/main.rs b/server/src/main.rs index 7689d7ad1..b27ddb9cb 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -27,7 +27,7 @@ use lemmy_server::{ blocking, code_migrations::run_advanced_migrations, rate_limit::{rate_limiter::RateLimiter, RateLimit}, - routes::{api, federation, feeds, index, nodeinfo, webfinger}, + routes::*, websocket::server::*, LemmyError, }; @@ -91,9 +91,10 @@ async fn main() -> Result<(), LemmyError> { .data(server.clone()) .data(Client::default()) // The routes - .configure(move |cfg| api::config(cfg, &rate_limiter)) + .configure(|cfg| api::config(cfg, &rate_limiter)) .configure(federation::config) .configure(feeds::config) + .configure(|cfg| images::config(cfg, &rate_limiter)) .configure(index::config) .configure(nodeinfo::config) .configure(webfinger::config) diff --git a/server/src/rate_limit/mod.rs b/server/src/rate_limit/mod.rs index 513c923c6..39df72650 100644 --- a/server/src/rate_limit/mod.rs +++ b/server/src/rate_limit/mod.rs @@ -45,6 +45,10 @@ impl RateLimit { self.kind(RateLimitType::Register) } + pub fn image(&self) -> RateLimited { + self.kind(RateLimitType::Image) + } + fn kind(&self, type_: RateLimitType) -> RateLimited { RateLimited { rate_limiter: self.rate_limiter.clone(), @@ -101,6 +105,15 @@ impl RateLimited { true, )?; } + RateLimitType::Image => { + limiter.check_rate_limit_full( + self.type_, + &ip_addr, + rate_limit.image, + rate_limit.image_per_second, + false, + )?; + } }; } diff --git a/server/src/rate_limit/rate_limiter.rs b/server/src/rate_limit/rate_limiter.rs index 20a617c2f..f1a388412 100644 --- a/server/src/rate_limit/rate_limiter.rs +++ b/server/src/rate_limit/rate_limiter.rs @@ -15,6 +15,7 @@ pub enum RateLimitType { Message, Register, Post, + Image, } /// Rate limiting based on rate type and IP addr diff --git a/server/src/routes/images.rs b/server/src/routes/images.rs new file mode 100644 index 000000000..8c94535a2 --- /dev/null +++ b/server/src/routes/images.rs @@ -0,0 +1,146 @@ +use crate::rate_limit::RateLimit; +use actix::clock::Duration; +use actix_web::{body::BodyStream, http::StatusCode, *}; +use awc::Client; +use lemmy_utils::settings::Settings; +use serde::{Deserialize, Serialize}; + +const THUMBNAIL_SIZES: &[u64] = &[256]; + +pub fn config(cfg: &mut web::ServiceConfig, rate_limit: &RateLimit) { + let client = Client::build() + .header("User-Agent", "pict-rs-frontend, v0.1.0") + .timeout(Duration::from_secs(30)) + .finish(); + + cfg + .data(client) + .service( + web::resource("/pictrs/image") + .wrap(rate_limit.image()) + .route(web::post().to(upload)), + ) + .service(web::resource("/pictrs/image/{filename}").route(web::get().to(full_res))) + .service( + web::resource("/pictrs/image/thumbnail{size}/{filename}").route(web::get().to(thumbnail)), + ) + .service(web::resource("/pictrs/image/delete/{token}/{filename}").route(web::get().to(delete))); +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct Image { + file: String, + delete_token: String, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct Images { + msg: String, + files: Option>, +} + +async fn upload( + req: HttpRequest, + body: web::Payload, + client: web::Data, +) -> Result { + // TODO: check auth and rate limit here + + let mut res = client + .request_from(format!("{}/image", Settings::get().pictrs_url), req.head()) + .if_some(req.head().peer_addr, |addr, req| { + req.header("X-Forwarded-For", addr.to_string()) + }) + .send_stream(body) + .await?; + + let images = res.json::().await?; + + Ok(HttpResponse::build(res.status()).json(images)) +} + +async fn full_res( + filename: web::Path, + req: HttpRequest, + client: web::Data, +) -> Result { + let url = format!( + "{}/image/{}", + Settings::get().pictrs_url, + &filename.into_inner() + ); + image(url, req, client).await +} + +async fn thumbnail( + parts: web::Path<(u64, String)>, + req: HttpRequest, + client: web::Data, +) -> Result { + let (size, file) = parts.into_inner(); + + if THUMBNAIL_SIZES.contains(&size) { + let url = format!( + "{}/image/thumbnail{}/{}", + Settings::get().pictrs_url, + size, + &file + ); + + return image(url, req, client).await; + } + + Ok(HttpResponse::NotFound().finish()) +} + +async fn image( + url: String, + req: HttpRequest, + client: web::Data, +) -> Result { + let res = client + .request_from(url, req.head()) + .if_some(req.head().peer_addr, |addr, req| { + req.header("X-Forwarded-For", addr.to_string()) + }) + .no_decompress() + .send() + .await?; + + if res.status() == StatusCode::NOT_FOUND { + return Ok(HttpResponse::NotFound().finish()); + } + + let mut client_res = HttpResponse::build(res.status()); + + for (name, value) in res.headers().iter().filter(|(h, _)| *h != "connection") { + client_res.header(name.clone(), value.clone()); + } + + Ok(client_res.body(BodyStream::new(res))) +} + +async fn delete( + components: web::Path<(String, String)>, + req: HttpRequest, + client: web::Data, +) -> Result { + let (token, file) = components.into_inner(); + + let url = format!( + "{}/image/delete/{}/{}", + Settings::get().pictrs_url, + &token, + &file + ); + let res = client + .request_from(url, req.head()) + .if_some(req.head().peer_addr, |addr, req| { + req.header("X-Forwarded-For", addr.to_string()) + }) + .no_decompress() + .send() + .await?; + + Ok(HttpResponse::build(res.status()).body(BodyStream::new(res))) +} diff --git a/server/src/routes/mod.rs b/server/src/routes/mod.rs index bcb7e45fa..4a7d30993 100644 --- a/server/src/routes/mod.rs +++ b/server/src/routes/mod.rs @@ -1,6 +1,7 @@ pub mod api; pub mod federation; pub mod feeds; +pub mod images; pub mod index; pub mod nodeinfo; pub mod webfinger;