diff --git a/Cargo.lock b/Cargo.lock index a44bb75..601b5e3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -21,9 +21,9 @@ dependencies = [ [[package]] name = "actix-form-data" -version = "0.7.0-beta.6" +version = "0.7.0-beta.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0588d156cb871d8c237d55ce398e2a65727370fb98352ba5b65c15a2f834b0f" +checksum = "6c2355a841a5d9a6c616d6a4f31336064116d206e6c1830de22730f983613a05" dependencies = [ "actix-multipart", "actix-web", @@ -340,18 +340,18 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.53", + "syn 2.0.55", ] [[package]] name = "async-trait" -version = "0.1.78" +version = "0.1.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "461abc97219de0eaaf81fe3ef974a540158f3d079c2ab200f891f1a2ef201e85" +checksum = "a507401cad91ec6a857ed5513a2073c82a9b9048762b885bb98655b306964681" dependencies = [ "proc-macro2", "quote", - "syn 2.0.53", + "syn 2.0.55", ] [[package]] @@ -362,9 +362,9 @@ checksum = "c59bdb34bc650a32731b31bd8f0829cc15d24a708ee31559e0bb34f2bc320cba" [[package]] name = "autocfg" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +checksum = "f1fdabc7756949593fe60f30ec81974b613357de856987752631dea1e3394c80" [[package]] name = "axum" @@ -538,9 +538,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "chrono" -version = "0.4.35" +version = "0.4.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8eaf5903dcbc0a39312feb77df2ff4c76387d591b9fc7b04a238dcf8bb62639a" +checksum = "8a0d04d43504c61aa6c7531f1871dd0d418d91130162063b789da00fd7057a5e" dependencies = [ "android-tzdata", "iana-time-zone", @@ -550,9 +550,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.3" +version = "4.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "949626d00e063efc93b6dca932419ceb5432f99769911c0b995f7e884c778813" +checksum = "90bc066a67923782aa8515dbaea16946c5bcc5addbd668bb80af688e53e548a0" dependencies = [ "clap_builder", "clap_derive", @@ -572,14 +572,14 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.3" +version = "4.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90239a040c80f5e14809ca132ddc4176ab33d5e17e49691793296e3fcb34d72f" +checksum = "528131438037fd55894f62d6e9f068b8f45ac57ffa77517819645d10aed04f64" dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.53", + "syn 2.0.55", ] [[package]] @@ -839,7 +839,7 @@ dependencies = [ "heck 0.4.1", "proc-macro2", "quote", - "syn 2.0.53", + "syn 2.0.55", ] [[package]] @@ -851,7 +851,7 @@ dependencies = [ "diesel_table_macro_syntax", "proc-macro2", "quote", - "syn 2.0.53", + "syn 2.0.55", ] [[package]] @@ -860,7 +860,7 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc5557efc453706fed5e4fa85006fe9817c224c3f480a34c7e5959fd700921c5" dependencies = [ - "syn 2.0.53", + "syn 2.0.55", ] [[package]] @@ -1019,7 +1019,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.53", + "syn 2.0.55", ] [[package]] @@ -1367,9 +1367,9 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.10" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" +checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" [[package]] name = "js-sys" @@ -1464,9 +1464,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.7.1" +version = "2.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149" +checksum = "6c8640c5d730cb13ebd907d8d04b52f55ac9a2eec55b440c8892f40d56c76c1d" [[package]] name = "metrics" @@ -1852,7 +1852,7 @@ dependencies = [ "reqwest", "reqwest-middleware", "reqwest-tracing", - "rustls 0.22.2", + "rustls 0.22.3", "rustls-channel-resolver", "rustls-pemfile 2.1.1", "rusty-s3", @@ -1900,7 +1900,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", - "syn 2.0.53", + "syn 2.0.55", ] [[package]] @@ -2009,7 +2009,7 @@ dependencies = [ "itertools", "proc-macro2", "quote", - "syn 2.0.53", + "syn 2.0.55", ] [[package]] @@ -2154,7 +2154,7 @@ dependencies = [ "quote", "refinery-core", "regex", - "syn 2.0.53", + "syn 2.0.55", ] [[package]] @@ -2166,7 +2166,7 @@ dependencies = [ "aho-corasick", "memchr", "regex-automata 0.4.6", - "regex-syntax 0.8.2", + "regex-syntax 0.8.3", ] [[package]] @@ -2186,7 +2186,7 @@ checksum = "86b83b8b9847f9bf95ef68afb0b8e6cdb80f498442f5179a29fad448fcc1eaea" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.8.2", + "regex-syntax 0.8.3", ] [[package]] @@ -2197,9 +2197,9 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "regex-syntax" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" +checksum = "adad44e29e4c806119491a7f06f03de4d1af22c3a680dd47f1e6e179439d1f56" [[package]] name = "reqwest" @@ -2341,9 +2341,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.22.2" +version = "0.22.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e87c9956bd9807afa1f77e0f7594af32566e830e088a5576d27c5b6f30f49d41" +checksum = "99008d7ad0bbbea527ec27bddbc0e432c5b87d8175178cee68d2eec9c4a1813c" dependencies = [ "log", "ring", @@ -2360,7 +2360,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ffbd1941204442f051576a9a7ea8e8db074ad7fd43db1eb3378c3633f9f9e166" dependencies = [ "nanorand", - "rustls 0.22.2", + "rustls 0.22.3", ] [[package]] @@ -2384,9 +2384,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.4.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "868e20fada228fefaf6b652e00cc73623d54f8171e7352c18bb281571f2d92da" +checksum = "ecd36cc4259e3e4514335c4a138c6b43171a8d61d8f5c9348f9fc7529416f247" [[package]] name = "rustls-webpki" @@ -2513,14 +2513,14 @@ checksum = "7eb0b34b42edc17f6b7cac84a52a1c5f0e1bb2227e997ca9011ea3dd34e8610b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.53", + "syn 2.0.55", ] [[package]] name = "serde_json" -version = "1.0.114" +version = "1.0.115" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5f09b1bd632ef549eaa9f60a1f8de742bdbc698e6cee2095fc84dde5f549ae0" +checksum = "12dc5c46daa8e9fdf4f5e71b6cf9a53f2487da0e86e55808e2d35539666497dd" dependencies = [ "itoa", "ryu", @@ -2741,9 +2741,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.53" +version = "2.0.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7383cd0e49fff4b6b90ca5670bfd3e9d6a733b3f90c686605aa7eec8c4996032" +checksum = "002a1b3dbf967edfafc32655d0f377ab0bb7b994aa1d32c8cc7e9b8bf3ebb8f0" dependencies = [ "proc-macro2", "quote", @@ -2803,7 +2803,7 @@ checksum = "c61f3ba182994efc43764a46c018c347bc492c79f024e705f46567b418f6d4f7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.53", + "syn 2.0.55", ] [[package]] @@ -2900,7 +2900,7 @@ checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.53", + "syn 2.0.55", ] [[package]] @@ -2937,7 +2937,7 @@ checksum = "0ea13f22eda7127c827983bdaf0d7fff9df21c8817bab02815ac277a21143677" dependencies = [ "futures", "ring", - "rustls 0.22.2", + "rustls 0.22.3", "tokio", "tokio-postgres", "tokio-rustls 0.25.0", @@ -2960,7 +2960,7 @@ version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "775e0c0f0adb3a2f22a00c4745d728b479985fc15ee7ca6a2608388c5569860f" dependencies = [ - "rustls 0.22.2", + "rustls 0.22.3", "rustls-pki-types", "tokio", ] @@ -3160,7 +3160,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.53", + "syn 2.0.55", ] [[package]] @@ -3390,7 +3390,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.53", + "syn 2.0.55", "wasm-bindgen-shared", ] @@ -3424,7 +3424,7 @@ checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.53", + "syn 2.0.55", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -3730,7 +3730,7 @@ checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.53", + "syn 2.0.55", ] [[package]] @@ -3750,5 +3750,5 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.53", + "syn 2.0.55", ] diff --git a/Cargo.toml b/Cargo.toml index ea61a6e..61869ce 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,7 +19,7 @@ poll-timer-warnings = [] random-errors = ["dep:nanorand"] [dependencies] -actix-form-data = "0.7.0-beta.6" +actix-form-data = "0.7.0-beta.7" actix-web = { version = "4.0.0", default-features = false, features = ["rustls-0_22"] } async-trait = "0.1.51" barrel = { version = "0.7.0", features = ["pg"] } @@ -58,8 +58,8 @@ rustls-channel-resolver = "0.2.0" rustls-pemfile = "2.0.0" rusty-s3 = "0.5.0" serde = { version = "1.0", features = ["derive"] } -serde-tuple-vec-map = "1.0.1" serde_json = "1.0" +serde-tuple-vec-map = "1.0.1" serde_urlencoded = "0.7.1" sha2 = "0.10.0" sled = { version = "0.34.7" } diff --git a/flake.nix b/flake.nix index 9224cf2..f475255 100644 --- a/flake.nix +++ b/flake.nix @@ -33,11 +33,13 @@ cargo-outdated certstrap clippy + curl diesel-cli exiftool ffmpeg_6-full garage imagemagick + jq minio-client rust-analyzer rustc diff --git a/src/error.rs b/src/error.rs index 5529499..d5731c9 100644 --- a/src/error.rs +++ b/src/error.rs @@ -82,7 +82,7 @@ pub(crate) enum UploadError { Io(#[from] std::io::Error), #[error("Error validating upload")] - Validation(#[from] crate::validate::ValidationError), + Validation(#[from] crate::ingest::ValidationError), #[error("Error in store")] Store(#[source] crate::store::StoreError), @@ -111,6 +111,12 @@ pub(crate) enum UploadError { #[error("Invalid job popped from job queue: {1}")] InvalidJob(#[source] serde_json::Error, String), + #[error("Invalid query supplied")] + InvalidQuery(#[source] actix_web::error::QueryPayloadError), + + #[error("Invalid json supplied")] + InvalidJson(#[source] actix_web::error::JsonPayloadError), + #[error("pict-rs is in read-only mode")] ReadOnly, @@ -209,6 +215,8 @@ impl UploadError { Self::ProcessTimeout => ErrorCode::COMMAND_TIMEOUT, Self::FailedExternalValidation => ErrorCode::FAILED_EXTERNAL_VALIDATION, Self::InvalidJob(_, _) => ErrorCode::INVALID_JOB, + Self::InvalidQuery(_) => ErrorCode::INVALID_QUERY, + Self::InvalidJson(_) => ErrorCode::INVALID_JSON, #[cfg(feature = "random-errors")] Self::RandomError => ErrorCode::RANDOM_ERROR, } @@ -248,7 +256,7 @@ impl ResponseError for Error { fn status_code(&self) -> StatusCode { match self.kind() { Some(UploadError::Upload(actix_form_data::Error::FileSize)) - | Some(UploadError::Validation(crate::validate::ValidationError::Filesize)) => { + | Some(UploadError::Validation(crate::ingest::ValidationError::Filesize)) => { StatusCode::PAYLOAD_TOO_LARGE } Some( @@ -261,6 +269,8 @@ impl ResponseError for Error { )) | UploadError::Repo(crate::repo::RepoError::AlreadyClaimed) | UploadError::Validation(_) + | UploadError::InvalidQuery(_) + | UploadError::InvalidJson(_) | UploadError::UnsupportedProcessExtension | UploadError::ReadOnly | UploadError::FailedExternalValidation diff --git a/src/error_code.rs b/src/error_code.rs index 9e9b936..5e3f2ef 100644 --- a/src/error_code.rs +++ b/src/error_code.rs @@ -100,6 +100,9 @@ impl ErrorCode { pub(crate) const VIDEO_DISABLED: ErrorCode = ErrorCode { code: "video-disabled", }; + pub(crate) const MEDIA_DISALLOWED: ErrorCode = ErrorCode { + code: "media-disallowed", + }; pub(crate) const HTTP_CLIENT_ERROR: ErrorCode = ErrorCode { code: "http-client-error", }; @@ -147,6 +150,12 @@ impl ErrorCode { pub(crate) const INVALID_JOB: ErrorCode = ErrorCode { code: "invalid-job", }; + pub(crate) const INVALID_QUERY: ErrorCode = ErrorCode { + code: "invalid-query", + }; + pub(crate) const INVALID_JSON: ErrorCode = ErrorCode { + code: "invalid-json", + }; #[cfg(feature = "random-errors")] pub(crate) const RANDOM_ERROR: ErrorCode = ErrorCode { code: "random-error", diff --git a/src/ingest.rs b/src/ingest.rs index 3d5004b..d8fb592 100644 --- a/src/ingest.rs +++ b/src/ingest.rs @@ -1,3 +1,6 @@ +mod hasher; +mod validate; + use std::{cell::RefCell, rc::Rc, sync::Arc, time::Duration}; use crate::{ @@ -9,16 +12,17 @@ use crate::{ repo::{Alias, ArcRepo, DeleteToken, Hash}, state::State, store::Store, + UploadQuery, }; use actix_web::web::Bytes; use futures_core::Stream; use reqwest::Body; - use tracing::{Instrument, Span}; -mod hasher; use hasher::Hasher; +pub(crate) use validate::ValidationError; + #[derive(Debug)] pub(crate) struct Session { repo: ArcRepo, @@ -31,6 +35,7 @@ pub(crate) struct Session { async fn process_ingest( state: &State, stream: impl Stream>, + upload_query: &UploadQuery, ) -> Result< ( InternalFormat, @@ -54,11 +59,18 @@ where let permit = crate::process_semaphore().acquire().await?; tracing::trace!("Validating bytes"); - let (input_type, process_read) = crate::validate::validate_bytes_stream(state, bytes) - .with_poll_timer("validate-bytes-stream") - .await?; + let (input_type, process_read) = + validate::validate_bytes_stream(state, bytes, &upload_query.limits) + .with_poll_timer("validate-bytes-stream") + .await?; - let process_read = if let Some(operations) = state.config.media.preprocess_steps() { + let operations = if upload_query.operations.is_empty() { + state.config.media.preprocess_steps() + } else { + Some(upload_query.operations.as_ref()) + }; + + let process_read = if let Some(operations) = operations { if let Some(format) = input_type.processable_format() { let (_, magick_args) = crate::processor::build_chain(operations, format.file_extension())?; @@ -159,6 +171,7 @@ pub(crate) async fn ingest( state: &State, stream: impl Stream>, declared_alias: Option, + upload_query: &UploadQuery, ) -> Result where S: Store, @@ -166,7 +179,7 @@ where let (input_type, identifier, details, hash_state) = if state.config.server.danger_dummy_mode { dummy_ingest(state, stream).await? } else { - process_ingest(state, stream) + process_ingest(state, stream, upload_query) .with_poll_timer("ingest-future") .await? }; diff --git a/src/validate.rs b/src/ingest/validate.rs similarity index 79% rename from src/validate.rs rename to src/ingest/validate.rs index 5460553..10fc8d2 100644 --- a/src/validate.rs +++ b/src/ingest/validate.rs @@ -14,6 +14,7 @@ use crate::{ future::WithPollTimer, process::{Process, ProcessRead}, state::State, + UploadLimits, }; #[derive(Debug, thiserror::Error)] @@ -38,6 +39,9 @@ pub(crate) enum ValidationError { #[error("Video is disabled")] VideoDisabled, + + #[error("Media type wasn't allowed for this upload")] + MediaDisallowed, } impl ValidationError { @@ -50,6 +54,7 @@ impl ValidationError { Self::Empty => ErrorCode::VALIDATE_FILE_EMPTY, Self::Filesize => ErrorCode::VALIDATE_FILE_SIZE, Self::VideoDisabled => ErrorCode::VIDEO_DISABLED, + Self::MediaDisallowed => ErrorCode::MEDIA_DISALLOWED, } } } @@ -60,6 +65,7 @@ const MEGABYTES: usize = 1024 * 1024; pub(crate) async fn validate_bytes_stream( state: &State, bytes: BytesStream, + upload_limits: &UploadLimits, ) -> Result<(InternalFormat, ProcessRead), Error> { if bytes.is_empty() { return Err(ValidationError::Empty.into()); @@ -74,14 +80,16 @@ pub(crate) async fn validate_bytes_stream( .with_poll_timer("discover-bytes-stream") .await?; + validate_upload(bytes.len(), width, height, frames, upload_limits)?; + match &input { - InputFile::Image(input) => { + InputFile::Image(input) if *upload_limits.allow_image => { let (format, process) = process_image_command(state, *input, bytes.len(), width, height).await?; Ok((format, process.drive_with_stream(bytes.into_io_stream()))) } - InputFile::Animation(input) => { + InputFile::Animation(input) if *upload_limits.allow_animation => { let (format, process) = process_animation_command( state, *input, @@ -94,20 +102,67 @@ pub(crate) async fn validate_bytes_stream( Ok((format, process.drive_with_stream(bytes.into_io_stream()))) } - InputFile::Video(input) => { + InputFile::Video(input) if *upload_limits.allow_video => { let (format, process_read) = process_video(state, bytes, *input, width, height, frames.unwrap_or(1)).await?; Ok((format, process_read)) } + _ => Err(ValidationError::MediaDisallowed.into()), } } +fn validate_upload( + size: usize, + width: u16, + height: u16, + frames: Option, + upload_limits: &UploadLimits, +) -> Result<(), ValidationError> { + if upload_limits + .max_width + .is_some_and(|max_width| width > *max_width) + { + return Err(ValidationError::Width); + } + + if upload_limits + .max_height + .is_some_and(|max_height| height > *max_height) + { + return Err(ValidationError::Height); + } + + if upload_limits + .max_frame_count + .zip(frames) + .is_some_and(|(max_frame_count, frames)| frames > *max_frame_count) + { + return Err(ValidationError::Frames); + } + + if upload_limits + .max_area + .is_some_and(|max_area| u32::from(width) * u32::from(height) > *max_area) + { + return Err(ValidationError::Area); + } + + if upload_limits + .max_file_size + .is_some_and(|max_file_size| size > *max_file_size * MEGABYTES) + { + return Err(ValidationError::Filesize); + } + + Ok(()) +} + #[tracing::instrument(skip(state))] async fn process_image_command( state: &State, input: ImageInput, - length: usize, + size: usize, width: u16, height: u16, ) -> Result<(InternalFormat, Process), Error> { @@ -122,7 +177,7 @@ async fn process_image_command( if u32::from(width) * u32::from(height) > validations.max_area { return Err(ValidationError::Area.into()); } - if length > validations.max_file_size * MEGABYTES { + if size > validations.max_file_size * MEGABYTES { return Err(ValidationError::Filesize.into()); } @@ -172,14 +227,14 @@ fn validate_animation( async fn process_animation_command( state: &State, input: AnimationFormat, - length: usize, + size: usize, width: u16, height: u16, frames: u32, ) -> Result<(InternalFormat, Process), Error> { let validations = &state.config.media.animation; - validate_animation(length, width, height, frames, validations)?; + validate_animation(size, width, height, frames, validations)?; let AnimationOutput { format, diff --git a/src/validate/exiftool.rs b/src/ingest/validate/exiftool.rs similarity index 100% rename from src/validate/exiftool.rs rename to src/ingest/validate/exiftool.rs diff --git a/src/validate/ffmpeg.rs b/src/ingest/validate/ffmpeg.rs similarity index 100% rename from src/validate/ffmpeg.rs rename to src/ingest/validate/ffmpeg.rs diff --git a/src/validate/magick.rs b/src/ingest/validate/magick.rs similarity index 100% rename from src/validate/magick.rs rename to src/ingest/validate/magick.rs diff --git a/src/lib.rs b/src/lib.rs index bc19baa..89da947 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -35,7 +35,6 @@ mod stream; mod sync; mod tls; mod tmp_file; -mod validate; use actix_form_data::{Field, Form, FormData, Multipart, Value}; use actix_web::{ @@ -59,6 +58,7 @@ use std::{ marker::PhantomData, path::Path, path::PathBuf, + rc::Rc, sync::{Arc, OnceLock}, time::{Duration, SystemTime}, }; @@ -147,22 +147,64 @@ async fn ensure_details_identifier( } } +#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)] +#[serde(default)] +struct UploadLimits { + max_width: Option>, + max_height: Option>, + max_area: Option>, + max_frame_count: Option>, + max_file_size: Option>, + allow_image: Serde, + allow_animation: Serde, + allow_video: Serde, +} + +impl Default for UploadLimits { + fn default() -> Self { + Self { + max_width: None, + max_height: None, + max_area: None, + max_frame_count: None, + max_file_size: None, + allow_image: Serde::new(true), + allow_animation: Serde::new(true), + allow_video: Serde::new(true), + } + } +} + +#[derive(Clone, Default, Debug, serde::Deserialize, serde::Serialize)] +struct UploadQuery { + #[serde(flatten)] + limits: UploadLimits, + + #[serde(with = "tuple_vec_map", flatten)] + operations: Vec<(String, String)>, +} + struct Upload(Value, PhantomData); impl FormData for Upload { type Item = Session; type Error = Error; - fn form(req: &HttpRequest) -> Form { + fn form(req: &HttpRequest) -> Result, Self::Error> { let state = req .app_data::>>() .expect("No state in request") .clone(); + let web::Query(upload_query) = web::Query::::from_query(req.query_string()) + .map_err(UploadError::InvalidQuery)?; + + let upload_query = Rc::new(upload_query); + // Create a new Multipart Form validator // // This form is expecting a single array field, 'images' with at most 10 files in it - Form::new() + Ok(Form::new() .max_files(state.config.server.max_file_count) .max_file_size(state.config.media.max_file_size * MEGABYTES) .transform_error(transform_error) @@ -170,6 +212,7 @@ impl FormData for Upload { "images", Field::array(Field::file(move |filename, _, stream| { let state = state.clone(); + let upload_query = upload_query.clone(); metrics::counter!(crate::init_metrics::FILES, "upload" => "inline") .increment(1); @@ -184,13 +227,13 @@ impl FormData for Upload { let stream = crate::stream::from_err(stream); - ingest::ingest(&state, stream, None).await + ingest::ingest(&state, stream, None, &upload_query).await } .with_poll_timer("file-upload") .instrument(span), ) })), - ) + )) } fn extract(value: Value) -> Result { @@ -204,16 +247,21 @@ impl FormData for Import { type Item = Session; type Error = Error; - fn form(req: &actix_web::HttpRequest) -> Form { + fn form(req: &actix_web::HttpRequest) -> Result, Self::Error> { let state = req .app_data::>>() .expect("No state in request") .clone(); + let web::Query(upload_query) = web::Query::::from_query(req.query_string()) + .map_err(UploadError::InvalidQuery)?; + + let upload_query = Rc::new(upload_query); + // Create a new Multipart Form validator for internal imports // // This form is expecting a single array field, 'images' with at most 10 files in it - Form::new() + Ok(Form::new() .max_files(state.config.server.max_file_count) .max_file_size(state.config.media.max_file_size * MEGABYTES) .transform_error(transform_error) @@ -221,6 +269,7 @@ impl FormData for Import { "images", Field::array(Field::file(move |filename, _, stream| { let state = state.clone(); + let upload_query = upload_query.clone(); metrics::counter!(crate::init_metrics::FILES, "import" => "inline") .increment(1); @@ -235,14 +284,19 @@ impl FormData for Import { let stream = crate::stream::from_err(stream); - ingest::ingest(&state, stream, Some(Alias::from_existing(&filename))) - .await + ingest::ingest( + &state, + stream, + Some(Alias::from_existing(&filename)), + &upload_query, + ) + .await } .with_poll_timer("file-import") .instrument(span), ) })), - ) + )) } fn extract(value: Value) -> Result @@ -320,16 +374,16 @@ impl FormData for BackgroundedUpload { type Item = Backgrounded; type Error = Error; - fn form(req: &actix_web::HttpRequest) -> Form { - // Create a new Multipart Form validator for backgrounded uploads - // - // This form is expecting a single array field, 'images' with at most 10 files in it + fn form(req: &actix_web::HttpRequest) -> Result, Self::Error> { let state = req .app_data::>>() .expect("No state in request") .clone(); - Form::new() + // Create a new Multipart Form validator for backgrounded uploads + // + // This form is expecting a single array field, 'images' with at most 10 files in it + Ok(Form::new() .max_files(state.config.server.max_file_count) .max_file_size(state.config.media.max_file_size * MEGABYTES) .transform_error(transform_error) @@ -357,7 +411,7 @@ impl FormData for BackgroundedUpload { .instrument(span), ) })), - ) + )) } fn extract(value: Value) -> Result @@ -372,7 +426,10 @@ impl FormData for BackgroundedUpload { async fn upload_backgrounded( Multipart(BackgroundedUpload(value, _)): Multipart>, state: web::Data>, + upload_query: web::Query, ) -> Result { + let upload_query = upload_query.into_inner(); + let images = value .map() .and_then(|mut m| m.remove("images")) @@ -389,7 +446,14 @@ async fn upload_backgrounded( let upload_id = image.result.upload_id().expect("Upload ID exists"); let identifier = image.result.identifier().expect("Identifier exists"); - queue::queue_ingest(&state.repo, identifier, upload_id, None).await?; + queue::queue_ingest( + &state.repo, + identifier, + upload_id, + None, + upload_query.clone(), + ) + .await?; files.push(serde_json::json!({ "upload_id": upload_id.to_string(), @@ -462,11 +526,21 @@ struct UrlQuery { backgrounded: bool, } +#[derive(Debug, serde::Deserialize)] +struct DownloadQuery { + #[serde(flatten)] + url_query: UrlQuery, + + #[serde(flatten)] + upload_query: UploadQuery, +} + async fn ingest_inline( stream: impl Stream>, state: &State, + upload_query: &UploadQuery, ) -> Result<(Alias, DeleteToken, Details), Error> { - let session = ingest::ingest(state, stream, None).await?; + let session = ingest::ingest(state, stream, None, upload_query).await?; let alias = session.alias().expect("alias should exist").to_owned(); @@ -480,15 +554,20 @@ async fn ingest_inline( /// download an image from a URL #[tracing::instrument(name = "Downloading file", skip(state))] async fn download( - query: web::Query, + download_query: web::Query, state: web::Data>, ) -> Result { - let stream = download_stream(&query.url, &state).await?; + let DownloadQuery { + url_query, + upload_query, + } = download_query.into_inner(); - if query.backgrounded { - do_download_backgrounded(stream, state).await + let stream = download_stream(&url_query.url, &state).await?; + + if url_query.backgrounded { + do_download_backgrounded(stream, state, upload_query).await } else { - do_download_inline(stream, &state).await + do_download_inline(stream, &state, &upload_query).await } } @@ -518,10 +597,11 @@ async fn download_stream( async fn do_download_inline( stream: impl Stream>, state: &State, + upload_query: &UploadQuery, ) -> Result { metrics::counter!(crate::init_metrics::FILES, "download" => "inline").increment(1); - let (alias, delete_token, details) = ingest_inline(stream, state).await?; + let (alias, delete_token, details) = ingest_inline(stream, state, upload_query).await?; Ok(HttpResponse::Created().json(&serde_json::json!({ "msg": "ok", @@ -537,6 +617,7 @@ async fn do_download_inline( async fn do_download_backgrounded( stream: impl Stream>, state: web::Data>, + upload_query: UploadQuery, ) -> Result { metrics::counter!(crate::init_metrics::FILES, "download" => "background").increment(1); @@ -545,7 +626,7 @@ async fn do_download_backgrounded( let upload_id = backgrounded.upload_id().expect("Upload ID exists"); let identifier = backgrounded.identifier().expect("Identifier exists"); - queue::queue_ingest(&state.repo, identifier, upload_id, None).await?; + queue::queue_ingest(&state.repo, identifier, upload_id, None, upload_query).await?; backgrounded.disarm(); @@ -1212,7 +1293,7 @@ async fn proxy_alias_from_query( } else if !state.config.server.read_only { let stream = download_stream(proxy.as_str(), state).await?; - let (alias, _, _) = ingest_inline(stream, state).await?; + let (alias, _, _) = ingest_inline(stream, state, &Default::default()).await?; state.repo.relate_url(proxy, alias.clone()).await?; @@ -1497,6 +1578,16 @@ fn build_client() -> Result { .build()) } +fn query_config() -> web::QueryConfig { + web::QueryConfig::default() + .error_handler(|err, _| Error::from(UploadError::InvalidQuery(err)).into()) +} + +fn json_config() -> web::JsonConfig { + web::JsonConfig::default() + .error_handler(|err, _| Error::from(UploadError::InvalidJson(err)).into()) +} + fn configure_endpoints( config: &mut web::ServiceConfig, state: State, @@ -1504,6 +1595,8 @@ fn configure_endpoints( extra_config: F, ) { config + .app_data(query_config()) + .app_data(json_config()) .app_data(web::Data::new(state.clone())) .app_data(web::Data::new(process_map.clone())) .route("/healthz", web::get().to(healthz::)) diff --git a/src/queue.rs b/src/queue.rs index 42da976..96cebf8 100644 --- a/src/queue.rs +++ b/src/queue.rs @@ -7,6 +7,7 @@ use crate::{ serde_str::Serde, state::State, store::Store, + UploadQuery, }; use std::{ @@ -56,6 +57,8 @@ enum Process { identifier: String, upload_id: Serde, declared_alias: Option>, + #[serde(default)] + upload_query: UploadQuery, }, Generate { target_format: InputProcessableFormat, @@ -158,11 +161,13 @@ pub(crate) async fn queue_ingest( identifier: &Arc, upload_id: UploadId, declared_alias: Option, + upload_query: UploadQuery, ) -> Result<(), Error> { let job = serde_json::to_value(Process::Ingest { identifier: identifier.to_string(), declared_alias: declared_alias.map(Serde::new), upload_id: Serde::new(upload_id), + upload_query, }) .map_err(UploadError::PushJob)?; repo.push(PROCESS_QUEUE, job, None).await?; diff --git a/src/queue/process.rs b/src/queue/process.rs index 653ca26..76ff626 100644 --- a/src/queue/process.rs +++ b/src/queue/process.rs @@ -12,6 +12,7 @@ use crate::{ serde_str::Serde, state::State, store::Store, + UploadQuery, }; use std::{path::PathBuf, sync::Arc}; @@ -37,12 +38,14 @@ where identifier, upload_id, declared_alias, + upload_query, } => { process_ingest( state, Arc::from(identifier), Serde::into_inner(upload_id), declared_alias.map(Serde::into_inner), + upload_query, ) .with_poll_timer("process-ingest") .await? @@ -110,6 +113,7 @@ async fn process_ingest( unprocessed_identifier: Arc, upload_id: UploadId, declared_alias: Option, + upload_query: UploadQuery, ) -> JobResult where S: Store + 'static, @@ -129,7 +133,8 @@ where let stream = crate::stream::from_err(state2.store.to_stream(&ident, None, None).await?); - let session = crate::ingest::ingest(&state2, stream, declared_alias).await?; + let session = + crate::ingest::ingest(&state2, stream, declared_alias, &upload_query).await?; Ok(session) as Result } diff --git a/src/serde_str.rs b/src/serde_str.rs index 6b7962f..21e3344 100644 --- a/src/serde_str.rs +++ b/src/serde_str.rs @@ -3,7 +3,7 @@ use std::{ str::FromStr, }; -#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] pub(crate) struct Serde { inner: T, } @@ -44,6 +44,17 @@ impl DerefMut for Serde { } } +impl Default for Serde +where + T: Default, +{ + fn default() -> Self { + Serde { + inner: T::default(), + } + } +} + impl FromStr for Serde where T: FromStr,