Proxy pictrs requests through Lemmy (fixes #371) (#77)

fix check_only value for image rate limit

Fix image rate limit

Add rate limit for image uploads

Proxy pictrs requests through Lemmy (fixes #371)

Co-authored-by: Felix Ableitner <me@nutomic.com>
Reviewed-on: https://yerbamate.dev/LemmyNet/lemmy/pulls/77
This commit is contained in:
nutomic 2020-08-05 16:00:00 +00:00
parent 37b438a77f
commit c34fffc2c4
11 changed files with 184 additions and 51 deletions

View file

@ -74,18 +74,6 @@ server {
return 301 /pictrs/image/$1; return 301 /pictrs/image/$1;
} }
# pict-rs images
location /pictrs {
location /pictrs/image {
proxy_pass http://0.0.0.0:8537/image;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header Host $host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}
# Block the import
return 403;
}
location /iframely/ { location /iframely/ {
proxy_pass http://0.0.0.0:8061/; proxy_pass http://0.0.0.0:8061/;
proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Real-IP $remote_addr;

View file

@ -21,7 +21,8 @@ services:
postgres: postgres:
image: postgres:12-alpine image: postgres:12-alpine
ports: ports:
- "127.0.0.1:5432:5432" # use a different port so it doesnt conflict with postgres running on the host
- "127.0.0.1:5433:5432"
environment: environment:
- POSTGRES_USER=lemmy - POSTGRES_USER=lemmy
- POSTGRES_PASSWORD=password - POSTGRES_PASSWORD=password

View file

@ -26,18 +26,6 @@ http {
proxy_set_header Connection "upgrade"; proxy_set_header Connection "upgrade";
} }
# pict-rs images
location /pictrs {
location /pictrs/image {
proxy_pass http://pictrs:8080/image;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header Host $host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}
# Block the import
return 403;
}
location /iframely/ { location /iframely/ {
proxy_pass http://iframely:80/; proxy_pass http://iframely:80/;
proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Real-IP $remote_addr;
@ -69,18 +57,6 @@ http {
proxy_set_header Connection "upgrade"; proxy_set_header Connection "upgrade";
} }
# pict-rs images
location /pictrs {
location /pictrs/image {
proxy_pass http://pictrs:8080/image;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header Host $host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}
# Block the import
return 403;
}
location /iframely/ { location /iframely/ {
proxy_pass http://iframely:80/; proxy_pass http://iframely:80/;
proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Real-IP $remote_addr;
@ -112,18 +88,6 @@ http {
proxy_set_header Connection "upgrade"; proxy_set_header Connection "upgrade";
} }
# pict-rs images
location /pictrs {
location /pictrs/image {
proxy_pass http://pictrs:8080/image;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header Host $host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}
# Block the import
return 403;
}
location /iframely/ { location /iframely/ {
proxy_pass http://iframely:80/; proxy_pass http://iframely:80/;
proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Real-IP $remote_addr;

9
docker/lemmy.hjson vendored
View file

@ -2,6 +2,15 @@
# for more info about the config, check out the documentation # for more info about the config, check out the documentation
# https://dev.lemmy.ml/docs/administration_configuration.html # https://dev.lemmy.ml/docs/administration_configuration.html
setup: {
# username for the admin user
admin_username: "lemmy"
# password for the admin user
admin_password: "lemmy"
# name of the site (can be changed later)
site_name: "lemmy-test"
}
# the domain name of your instance (eg "dev.lemmy.ml") # the domain name of your instance (eg "dev.lemmy.ml")
hostname: "my_domain" hostname: "my_domain"
# address where lemmy should listen for incoming requests # address where lemmy should listen for incoming requests

View file

@ -35,6 +35,8 @@
jwt_secret: "changeme" jwt_secret: "changeme"
# The location of the frontend # The location of the frontend
front_end_dir: "../ui/dist" front_end_dir: "../ui/dist"
# address where pictrs is available
pictrs_url: "http://pictrs:8080"
# rate limits for various user actions, by user ip # rate limits for various user actions, by user ip
rate_limit: { rate_limit: {
# maximum number of messages created in interval # maximum number of messages created in interval
@ -49,6 +51,10 @@
register: 3 register: 3
# interval length for registration limit # interval length for registration limit
register_per_second: 3600 register_per_second: 3600
# maximum number of image uploads in interval
image: 6
# interval length for image uploads
image_per_second: 3600
} }
# settings related to activitypub federation # settings related to activitypub federation
federation: { federation: {

View file

@ -14,6 +14,7 @@ pub struct Settings {
pub port: u16, pub port: u16,
pub jwt_secret: String, pub jwt_secret: String,
pub front_end_dir: String, pub front_end_dir: String,
pub pictrs_url: String,
pub rate_limit: RateLimitConfig, pub rate_limit: RateLimitConfig,
pub email: Option<EmailConfig>, pub email: Option<EmailConfig>,
pub federation: Federation, pub federation: Federation,
@ -36,6 +37,8 @@ pub struct RateLimitConfig {
pub post_per_second: i32, pub post_per_second: i32,
pub register: i32, pub register: i32,
pub register_per_second: i32, pub register_per_second: i32,
pub image: i32,
pub image_per_second: i32,
} }
#[derive(Debug, Deserialize, Clone)] #[derive(Debug, Deserialize, Clone)]

View file

@ -27,7 +27,7 @@ use lemmy_server::{
blocking, blocking,
code_migrations::run_advanced_migrations, code_migrations::run_advanced_migrations,
rate_limit::{rate_limiter::RateLimiter, RateLimit}, rate_limit::{rate_limiter::RateLimiter, RateLimit},
routes::{api, federation, feeds, index, nodeinfo, webfinger}, routes::*,
websocket::server::*, websocket::server::*,
LemmyError, LemmyError,
}; };
@ -91,9 +91,10 @@ async fn main() -> Result<(), LemmyError> {
.data(server.clone()) .data(server.clone())
.data(Client::default()) .data(Client::default())
// The routes // The routes
.configure(move |cfg| api::config(cfg, &rate_limiter)) .configure(|cfg| api::config(cfg, &rate_limiter))
.configure(federation::config) .configure(federation::config)
.configure(feeds::config) .configure(feeds::config)
.configure(|cfg| images::config(cfg, &rate_limiter))
.configure(index::config) .configure(index::config)
.configure(nodeinfo::config) .configure(nodeinfo::config)
.configure(webfinger::config) .configure(webfinger::config)

View file

@ -45,6 +45,10 @@ impl RateLimit {
self.kind(RateLimitType::Register) self.kind(RateLimitType::Register)
} }
pub fn image(&self) -> RateLimited {
self.kind(RateLimitType::Image)
}
fn kind(&self, type_: RateLimitType) -> RateLimited { fn kind(&self, type_: RateLimitType) -> RateLimited {
RateLimited { RateLimited {
rate_limiter: self.rate_limiter.clone(), rate_limiter: self.rate_limiter.clone(),
@ -101,6 +105,15 @@ impl RateLimited {
true, true,
)?; )?;
} }
RateLimitType::Image => {
limiter.check_rate_limit_full(
self.type_,
&ip_addr,
rate_limit.image,
rate_limit.image_per_second,
false,
)?;
}
}; };
} }

View file

@ -15,6 +15,7 @@ pub enum RateLimitType {
Message, Message,
Register, Register,
Post, Post,
Image,
} }
/// Rate limiting based on rate type and IP addr /// Rate limiting based on rate type and IP addr

146
server/src/routes/images.rs Normal file
View file

@ -0,0 +1,146 @@
use crate::rate_limit::RateLimit;
use actix::clock::Duration;
use actix_web::{body::BodyStream, http::StatusCode, *};
use awc::Client;
use lemmy_utils::settings::Settings;
use serde::{Deserialize, Serialize};
const THUMBNAIL_SIZES: &[u64] = &[256];
pub fn config(cfg: &mut web::ServiceConfig, rate_limit: &RateLimit) {
let client = Client::build()
.header("User-Agent", "pict-rs-frontend, v0.1.0")
.timeout(Duration::from_secs(30))
.finish();
cfg
.data(client)
.service(
web::resource("/pictrs/image")
.wrap(rate_limit.image())
.route(web::post().to(upload)),
)
.service(web::resource("/pictrs/image/{filename}").route(web::get().to(full_res)))
.service(
web::resource("/pictrs/image/thumbnail{size}/{filename}").route(web::get().to(thumbnail)),
)
.service(web::resource("/pictrs/image/delete/{token}/{filename}").route(web::get().to(delete)));
}
#[derive(Debug, Serialize, Deserialize)]
pub struct Image {
file: String,
delete_token: String,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct Images {
msg: String,
files: Option<Vec<Image>>,
}
async fn upload(
req: HttpRequest,
body: web::Payload,
client: web::Data<Client>,
) -> Result<HttpResponse, Error> {
// TODO: check auth and rate limit here
let mut res = client
.request_from(format!("{}/image", Settings::get().pictrs_url), req.head())
.if_some(req.head().peer_addr, |addr, req| {
req.header("X-Forwarded-For", addr.to_string())
})
.send_stream(body)
.await?;
let images = res.json::<Images>().await?;
Ok(HttpResponse::build(res.status()).json(images))
}
async fn full_res(
filename: web::Path<String>,
req: HttpRequest,
client: web::Data<Client>,
) -> Result<HttpResponse, Error> {
let url = format!(
"{}/image/{}",
Settings::get().pictrs_url,
&filename.into_inner()
);
image(url, req, client).await
}
async fn thumbnail(
parts: web::Path<(u64, String)>,
req: HttpRequest,
client: web::Data<Client>,
) -> Result<HttpResponse, Error> {
let (size, file) = parts.into_inner();
if THUMBNAIL_SIZES.contains(&size) {
let url = format!(
"{}/image/thumbnail{}/{}",
Settings::get().pictrs_url,
size,
&file
);
return image(url, req, client).await;
}
Ok(HttpResponse::NotFound().finish())
}
async fn image(
url: String,
req: HttpRequest,
client: web::Data<Client>,
) -> Result<HttpResponse, Error> {
let res = client
.request_from(url, req.head())
.if_some(req.head().peer_addr, |addr, req| {
req.header("X-Forwarded-For", addr.to_string())
})
.no_decompress()
.send()
.await?;
if res.status() == StatusCode::NOT_FOUND {
return Ok(HttpResponse::NotFound().finish());
}
let mut client_res = HttpResponse::build(res.status());
for (name, value) in res.headers().iter().filter(|(h, _)| *h != "connection") {
client_res.header(name.clone(), value.clone());
}
Ok(client_res.body(BodyStream::new(res)))
}
async fn delete(
components: web::Path<(String, String)>,
req: HttpRequest,
client: web::Data<Client>,
) -> Result<HttpResponse, Error> {
let (token, file) = components.into_inner();
let url = format!(
"{}/image/delete/{}/{}",
Settings::get().pictrs_url,
&token,
&file
);
let res = client
.request_from(url, req.head())
.if_some(req.head().peer_addr, |addr, req| {
req.header("X-Forwarded-For", addr.to_string())
})
.no_decompress()
.send()
.await?;
Ok(HttpResponse::build(res.status()).body(BodyStream::new(res)))
}

View file

@ -1,6 +1,7 @@
pub mod api; pub mod api;
pub mod federation; pub mod federation;
pub mod feeds; pub mod feeds;
pub mod images;
pub mod index; pub mod index;
pub mod nodeinfo; pub mod nodeinfo;
pub mod webfinger; pub mod webfinger;