Dont pass accept-encoding header to pictrs (ref #1734) (#1738)

* Allow image upload in docker/dev/ (replaces #1765)

* Dont pass accept-encoding header to pictrs (ref #1734)

* Fix clippy errors
This commit is contained in:
Nutomic 2021-09-17 15:44:20 +00:00 committed by GitHub
parent 007a43c012
commit e84e1193fc
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 73 additions and 83 deletions

66
Cargo.lock generated
View file

@ -70,12 +70,10 @@ dependencies = [
"ahash", "ahash",
"base64 0.13.0", "base64 0.13.0",
"bitflags", "bitflags",
"brotli2",
"bytes", "bytes",
"bytestring", "bytestring",
"derive_more", "derive_more",
"encoding_rs", "encoding_rs",
"flate2",
"futures-core", "futures-core",
"futures-util", "futures-util",
"h2", "h2",
@ -97,7 +95,6 @@ dependencies = [
"smallvec", "smallvec",
"time 0.2.27", "time 0.2.27",
"tokio", "tokio",
"zstd",
] ]
[[package]] [[package]]
@ -378,7 +375,6 @@ dependencies = [
"base64 0.13.0", "base64 0.13.0",
"bytes", "bytes",
"cfg-if", "cfg-if",
"cookie",
"derive_more", "derive_more",
"futures-core", "futures-core",
"itoa", "itoa",
@ -530,26 +526,6 @@ dependencies = [
"opaque-debug 0.3.0", "opaque-debug 0.3.0",
] ]
[[package]]
name = "brotli-sys"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4445dea95f4c2b41cde57cc9fee236ae4dbae88d8fcbdb4750fc1bb5d86aaecd"
dependencies = [
"cc",
"libc",
]
[[package]]
name = "brotli2"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0cb036c3eade309815c15ddbacec5b22c4d1f3983a774ab2eac2e3e9ea85568e"
dependencies = [
"brotli-sys",
"libc",
]
[[package]] [[package]]
name = "bumpalo" name = "bumpalo"
version = "3.7.0" version = "3.7.0"
@ -614,9 +590,6 @@ name = "cc"
version = "1.0.69" version = "1.0.69"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e70cc2f62c6ce1868963827bd677764c62d07c3d9a3e1fb1177ee1a9ab199eb2" checksum = "e70cc2f62c6ce1868963827bd677764c62d07c3d9a3e1fb1177ee1a9ab199eb2"
dependencies = [
"jobserver",
]
[[package]] [[package]]
name = "cfg-if" name = "cfg-if"
@ -1559,15 +1532,6 @@ version = "0.4.8"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4" checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4"
[[package]]
name = "jobserver"
version = "0.1.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "af25a77299a7f711a01975c35a6a424eb6862092cc2d6c72c4ed6cbc56dfc1fa"
dependencies = [
"libc",
]
[[package]] [[package]]
name = "jpeg-decoder" name = "jpeg-decoder"
version = "0.1.22" version = "0.1.22"
@ -1867,6 +1831,7 @@ name = "lemmy_routes"
version = "0.12.2-rc.1" version = "0.12.2-rc.1"
dependencies = [ dependencies = [
"actix", "actix",
"actix-http",
"actix-web", "actix-web",
"actix-web-actors", "actix-web-actors",
"anyhow", "anyhow",
@ -4001,32 +3966,3 @@ name = "xdg"
version = "2.2.0" version = "2.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d089681aa106a86fade1b0128fb5daf07d5867a509ab036d99988dec80429a57" checksum = "d089681aa106a86fade1b0128fb5daf07d5867a509ab036d99988dec80429a57"
[[package]]
name = "zstd"
version = "0.7.0+zstd.1.4.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9428752481d8372e15b1bf779ea518a179ad6c771cca2d2c60e4fbff3cc2cd52"
dependencies = [
"zstd-safe",
]
[[package]]
name = "zstd-safe"
version = "3.1.0+zstd.1.4.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5aa1926623ad7fe406e090555387daf73db555b948134b4d73eac5eb08fb666d"
dependencies = [
"libc",
"zstd-sys",
]
[[package]]
name = "zstd-sys"
version = "1.5.0+zstd.1.4.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4e6c094340240369025fc6b731b054ee2a834328fa584310ac96aa4baebdc465"
dependencies = [
"cc",
"libc",
]

View file

@ -21,15 +21,14 @@ diesel = "1.4.7"
actix = "0.12.0" actix = "0.12.0"
actix-web = { version = "4.0.0-beta.8", default-features = false, features = ["rustls"] } actix-web = { version = "4.0.0-beta.8", default-features = false, features = ["rustls"] }
actix-web-actors = { version = "4.0.0-beta.6", default-features = false } actix-web-actors = { version = "4.0.0-beta.6", default-features = false }
actix-http = "3.0.0-beta.9"
sha2 = "0.9.5" sha2 = "0.9.5"
log = "0.4.14" log = "0.4.14"
anyhow = "1.0.43" anyhow = "1.0.43"
chrono = { version = "0.4.19", features = ["serde"] } chrono = { version = "0.4.19", features = ["serde"] }
rss = "1.10.0" rss = "1.10.0"
serde = { version = "1.0.129", features = ["derive"] } serde = { version = "1.0.129", features = ["derive"] }
# TODO awc should use default-features = false, because gzip is a heavy dependency. awc = { version = "3.0.0-beta.7", default-features = false }
# This must wait for pictrs to have a configurable disabled gzip
awc = "3.0.0-beta.7"
url = { version = "2.2.2", features = ["serde"] } url = { version = "2.2.2", features = ["serde"] }
strum = "0.21.0" strum = "0.21.0"
lazy_static = "1.4.0" lazy_static = "1.4.0"

View file

@ -1,3 +1,4 @@
use actix_http::http::header::ACCEPT_ENCODING;
use actix_web::{body::BodyStream, http::StatusCode, web::Data, *}; use actix_web::{body::BodyStream, http::StatusCode, web::Data, *};
use anyhow::anyhow; use anyhow::anyhow;
use awc::Client; use awc::Client;
@ -56,6 +57,8 @@ async fn upload(
}; };
let mut client_req = client.request_from(format!("{}/image", pictrs_url()?), req.head()); let mut client_req = client.request_from(format!("{}/image", pictrs_url()?), req.head());
// remove content-encoding header so that pictrs doesnt send gzipped response
client_req.headers_mut().remove(ACCEPT_ENCODING);
if let Some(addr) = req.head().peer_addr { if let Some(addr) = req.head().peer_addr {
client_req = client_req.insert_header(("X-Forwarded-For", addr.to_string())) client_req = client_req.insert_header(("X-Forwarded-For", addr.to_string()))
@ -103,6 +106,7 @@ async fn image(
client: web::Data<Client>, client: web::Data<Client>,
) -> Result<HttpResponse, Error> { ) -> Result<HttpResponse, Error> {
let mut client_req = client.request_from(url, req.head()); let mut client_req = client.request_from(url, req.head());
client_req.headers_mut().remove(ACCEPT_ENCODING);
if let Some(addr) = req.head().peer_addr { if let Some(addr) = req.head().peer_addr {
client_req = client_req.insert_header(("X-Forwarded-For", addr.to_string())) client_req = client_req.insert_header(("X-Forwarded-For", addr.to_string()))
@ -137,6 +141,7 @@ async fn delete(
let url = format!("{}/image/delete/{}/{}", pictrs_url()?, &token, &file); let url = format!("{}/image/delete/{}/{}", pictrs_url()?, &token, &file);
let mut client_req = client.request_from(url, req.head()); let mut client_req = client.request_from(url, req.head());
client_req.headers_mut().remove(ACCEPT_ENCODING);
if let Some(addr) = req.head().peer_addr { if let Some(addr) = req.head().peer_addr {
client_req = client_req.insert_header(("X-Forwarded-For", addr.to_string())) client_req = client_req.insert_header(("X-Forwarded-For", addr.to_string()))

View file

@ -18,19 +18,11 @@ pub(crate) enum RateLimitType {
} }
/// Rate limiting based on rate type and IP addr /// Rate limiting based on rate type and IP addr
#[derive(Debug, Clone)] #[derive(Debug, Clone, Default)]
pub struct RateLimiter { pub struct RateLimiter {
buckets: HashMap<RateLimitType, HashMap<IpAddr, RateLimitBucket>>, buckets: HashMap<RateLimitType, HashMap<IpAddr, RateLimitBucket>>,
} }
impl Default for RateLimiter {
fn default() -> Self {
Self {
buckets: HashMap::<RateLimitType, HashMap<IpAddr, RateLimitBucket>>::new(),
}
}
}
impl RateLimiter { impl RateLimiter {
fn insert_ip(&mut self, ip: &IpAddr) { fn insert_ip(&mut self, ip: &IpAddr) {
for rate_limit_type in RateLimitType::iter() { for rate_limit_type in RateLimitType::iter() {

View file

@ -114,6 +114,7 @@ pub(crate) struct PictrsResponse {
#[derive(Deserialize, Debug, Clone)] #[derive(Deserialize, Debug, Clone)]
pub(crate) struct PictrsFile { pub(crate) struct PictrsFile {
file: String, file: String,
#[allow(dead_code)]
delete_token: String, delete_token: String,
} }

View file

@ -1,6 +1,16 @@
version: '3.3' version: '3.3'
services: services:
nginx:
image: nginx:1-alpine
ports:
- "1236:1236"
volumes:
- ./nginx.conf:/etc/nginx/nginx.conf
restart: always
depends_on:
- pictrs
- lemmy-ui
lemmy: lemmy:
image: lemmy-dev:latest image: lemmy-dev:latest
@ -17,12 +27,10 @@ services:
lemmy-ui: lemmy-ui:
image: dessalines/lemmy-ui:dev image: dessalines/lemmy-ui:dev
ports:
- "1235:1234"
restart: always restart: always
environment: environment:
- LEMMY_INTERNAL_HOST=lemmy:8536 - LEMMY_INTERNAL_HOST=lemmy:8536
- LEMMY_EXTERNAL_HOST=localhost:8536 - LEMMY_EXTERNAL_HOST=localhost:1234
- LEMMY_HTTPS=false - LEMMY_HTTPS=false
depends_on: depends_on:
- lemmy - lemmy
@ -42,8 +50,6 @@ services:
pictrs: pictrs:
image: asonix/pictrs:v0.2.6-r2 image: asonix/pictrs:v0.2.6-r2
ports:
- "8537:8080"
user: 991:991 user: 991:991
volumes: volumes:
- ./volumes/pictrs:/mnt - ./volumes/pictrs:/mnt

51
docker/dev/nginx.conf Normal file
View file

@ -0,0 +1,51 @@
worker_processes 1;
events {
worker_connections 1024;
}
http {
upstream lemmy {
server "lemmy:8536";
}
upstream lemmy-ui {
server "lemmy-ui:1234";
}
server {
listen 1236;
server_name localhost;
# frontend
location / {
set $proxpass "http://lemmy-ui";
if ($http_accept = "application/activity+json") {
set $proxpass "http://lemmy";
}
if ($http_accept = "application/ld+json; profile=\"https://www.w3.org/ns/activitystreams\"") {
set $proxpass "http://lemmy";
}
if ($request_method = POST) {
set $proxpass "http://lemmy";
}
proxy_pass $proxpass;
rewrite ^(.+)/+$ $1 permanent;
# Send actual client IP upstream
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header Host $host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}
# backend
location ~ ^/(api|pictrs|feeds|nodeinfo|.well-known) {
proxy_pass "http://lemmy";
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
# Add IP forwarding headers
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header Host $host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
}
}
}

View file

@ -11,7 +11,7 @@ services:
- "8580:8580" - "8580:8580"
volumes: volumes:
- ./nginx.conf:/etc/nginx/nginx.conf - ./nginx.conf:/etc/nginx/nginx.conf
restart: on-failure restart: always
depends_on: depends_on:
- pictrs - pictrs
- lemmy-alpha-ui - lemmy-alpha-ui