Add tests with system deps

These new tests each spin up their own instance of pict-rs. This allows
for configuring pict-rs differently depending on the test that needs to
be run. It also opens the possibility of writing tests with object
storage and postgres requirements in the future.

So far there's pretty minimal test coverage. Just uploading,
downloading, deleting, and checking validation errors for images and
animations. In the future, tests need to be added for videos, many
variations of the process endpoint, and the admin endpoints.

The new tests are locked behind a configuration option (not a feature)
called `system_deps`. They can be enabled by passing

`RUSTFLAGS='--cfg system_deps'`

This is to prevent tests from running and failing in environments where
the exiftool, imagemagick, and ffmpeg binaries are not present.

This builds imagemagick from source for a few reasons
1. binaries for imagemagick 7 are not available for debian 12
2. the imagemagick appimage does not support avif files
3. the imagemagick appimage is limited to x86_64
This commit is contained in:
asonix 2025-03-27 18:42:13 -05:00
parent 742677fff6
commit a71812930e
12 changed files with 879 additions and 7 deletions

View file

@ -52,8 +52,45 @@ jobs:
name: Cargo Cache
uses: https://git.asonix.dog/asonix/actions/cache-rust-dependencies@main
-
name: Test
run: cargo test
name: Install apt dependencies
run: |
set -x
apt-get update
apt-get -y install ffmpeg exiftool
-
name: Install imagemagick
run: |
set -x
apt-get update
apt-get -y install \
build-essential \
libgif-dev \
libheif-dev \
libjpeg-dev \
libjxl-dev \
liblcms2-dev \
libltdl-dev \
libpng-dev \
libtiff-dev \
libwebp-dev \
libxml2-dev
git clone --depth 1 \
--branch 7.1.1-46 \
https://github.com/ImageMagick/ImageMagick.git \
ImageMagick-7.1.1
cd ImageMagick-7.1.1
./configure
make -j $(nproc)
make install
cd ..
-
name: Run integration tests
run: |
cargo test
env:
RUSTFLAGS: --cfg tokio_unstable --cfg system_deps
LD_LIBRARY_PATH: "/usr/local/lib"
check:
strategy:

17
Cargo.lock generated
View file

@ -1934,6 +1934,16 @@ version = "0.3.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a"
[[package]]
name = "mime_guess"
version = "2.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e"
dependencies = [
"mime",
"unicase",
]
[[package]]
name = "minimal-lexical"
version = "0.2.1"
@ -2782,6 +2792,7 @@ dependencies = [
"js-sys",
"log",
"mime",
"mime_guess",
"once_cell",
"percent-encoding",
"pin-project-lite",
@ -3788,6 +3799,12 @@ version = "1.18.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1dccffe3ce07af9386bfd29e80c0ab1a8205a2fc34e4bcd40364df902cfa8f3f"
[[package]]
name = "unicase"
version = "2.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "75b844d17643ee918803943289730bec8aac480150456169e647ed0b576ba539"
[[package]]
name = "unicode-bidi"
version = "0.3.18"

View file

@ -9,8 +9,12 @@ repository = "https://git.asonix.dog/asonix/pict-rs"
edition = "2021"
rust-version = "1.82"
[lints.rust]
unexpected_cfgs = { level = "warn", check-cfg = ['cfg(tokio_unstable)'] }
[lints.rust.unexpected_cfgs]
level = "warn"
check-cfg = [
'cfg(tokio_unstable)',
'cfg(system_deps)'
]
[profile.release]
strip = true
@ -51,7 +55,7 @@ opentelemetry = "0.28.0"
opentelemetry-otlp = { version = "0.28.0", features = ["grpc-tonic"] }
pin-project-lite = "0.2.14"
refinery = { version = "0.8.14", features = ["tokio-postgres", "postgres"] }
reqwest = { version = "0.12.5", default-features = false, features = ["json", "rustls-tls-no-provider", "stream"] }
reqwest = { version = "0.12.15", default-features = false, features = ["json", "rustls-tls-no-provider", "stream"] }
reqwest-middleware = "0.4.0"
reqwest-tracing = "0.5.0"
# pinned to tokio-postgres-generic-rustls
@ -102,3 +106,6 @@ image = { version = "0.25.5", default-features = false, features = ["gif", "jpeg
version = "0.7.16"
default-features = false
features = ["opentelemetry_0_28"]
[dev-dependencies]
reqwest = { version = "0.12.15", default-features = false, features = ["json", "rustls-tls-no-provider", "stream", "multipart"] }

BIN
client-examples/awoo.webp Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.1 MiB

BIN
client-examples/earth.avif Normal file

Binary file not shown.

View file

@ -109,8 +109,6 @@ confidence-threshold = 0.6
# Allow 1 or more licenses on a per-crate basis, so that particular licenses
# aren't accepted for every possible crate as with the normal allow list
exceptions = [
# OpenSSL license is unavoidable for BoringSSL derivatives
{ allow = ["OpenSSL"], crate = "ring" },
# Each entry is the crate and version constraint, and its specific allow
# list
#{ allow = ["Zlib"], crate = "adler32" },
@ -221,6 +219,7 @@ skip = [
# non-direct dependencies
"base64",
"bitflags",
"derive_more",
"h2",
"hashbrown",
"heck",
@ -237,6 +236,15 @@ skip = [
# Ignore duplicates for systems we don't target
"redox_syscall",
"windows-sys",
"windows_aarch64_gnullvm",
"windows_aarch64_msvc",
"windows_i686_gnu",
"windows_i686_gnullvm",
"windows_i686_msvc",
"windows-targets",
"windows_x86_64_gnu",
"windows_x86_64_gnullvm",
"windows_x86_64_msvc",
#"ansi_term@0.11.0",
#{ crate = "ansi_term@0.11.0", reason = "you can specify a reason why it can't be updated/removed" },
]

View file

@ -66,6 +66,8 @@
tokio-console
];
RUSTFLAGS = "--cfg tokio_unstable --cfg system_deps";
RUST_SRC_PATH = "${pkgs.rust.packages.stable.rustPlatform.rustLibSrc}";
};
});

185
tests/animation.rs Normal file
View file

@ -0,0 +1,185 @@
#![cfg(system_deps)]
use common::{pict_rs_test_config, upload_form, with_pict_rs, PictRsResult, UploadResponse};
mod common;
#[test]
fn cannot_upload_too_wide_animation() {
let address = "127.0.0.1:9100";
let mut config = pict_rs_test_config(address);
config["media"]["animation"]["max_width"] = 100.into();
with_pict_rs(config, || async {
let client = reqwest::Client::new();
let form = upload_form(["./client-examples/earth.gif"]).await;
let response = client
.post(format!("http://{address}/image"))
.multipart(form)
.send()
.await
.expect("send request");
let upload = response
.json::<PictRsResult<UploadResponse>>()
.await
.expect("valid response");
match upload {
PictRsResult::Ok(_) => panic!("request should have errored"),
PictRsResult::Err { msg, code } => {
assert_eq!(msg, "Too wide");
assert_eq!(code, "validate-width");
}
}
})
.unwrap();
}
#[test]
fn cannot_upload_too_tall_animation() {
let address = "127.0.0.1:9101";
let mut config = pict_rs_test_config(address);
config["media"]["animation"]["max_height"] = 100.into();
with_pict_rs(config, || async {
let client = reqwest::Client::new();
let form = upload_form(["./client-examples/earth.gif"]).await;
let response = client
.post(format!("http://{address}/image"))
.multipart(form)
.send()
.await
.expect("send request");
let upload = response
.json::<PictRsResult<UploadResponse>>()
.await
.expect("valid response");
match upload {
PictRsResult::Ok(_) => panic!("request should have errored"),
PictRsResult::Err { msg, code } => {
assert_eq!(msg, "Too tall");
assert_eq!(code, "validate-height");
}
}
})
.unwrap();
}
#[test]
fn cannot_upload_too_much_area_animation() {
let address = "127.0.0.1:9102";
let mut config = pict_rs_test_config(address);
config["media"]["animation"]["max_area"] = 100.into();
with_pict_rs(config, || async {
let client = reqwest::Client::new();
let form = upload_form(["./client-examples/earth.gif"]).await;
let response = client
.post(format!("http://{address}/image"))
.multipart(form)
.send()
.await
.expect("send request");
let upload = response
.json::<PictRsResult<UploadResponse>>()
.await
.expect("valid response");
match upload {
PictRsResult::Ok(_) => panic!("request should have errored"),
PictRsResult::Err { msg, code } => {
assert_eq!(msg, "Too many pixels");
assert_eq!(code, "validate-area");
}
}
})
.unwrap();
}
#[test]
fn cannot_upload_too_many_frames_animation() {
let address = "127.0.0.1:9103";
let mut config = pict_rs_test_config(address);
config["media"]["animation"]["max_frame_count"] = 3.into();
with_pict_rs(config, || async {
let client = reqwest::Client::new();
let form = upload_form(["./client-examples/earth.gif"]).await;
let response = client
.post(format!("http://{address}/image"))
.multipart(form)
.send()
.await
.expect("send request");
let upload = response
.json::<PictRsResult<UploadResponse>>()
.await
.expect("valid response");
match upload {
PictRsResult::Ok(_) => panic!("request should have errored"),
PictRsResult::Err { msg, code } => {
assert_eq!(msg, "Too many frames");
assert_eq!(code, "validate-frames");
}
}
})
.unwrap();
}
#[test]
fn cannot_upload_too_large_animation() {
let address = "127.0.0.1:9104";
let mut config = pict_rs_test_config(address);
config["media"]["animation"]["max_file_size"] = 1.into();
with_pict_rs(config, || async {
let client = reqwest::Client::new();
let form = upload_form(["./client-examples/earth.avif"]).await;
let response = client
.post(format!("http://{address}/image"))
.multipart(form)
.send()
.await
.expect("send request");
let upload = response
.json::<PictRsResult<UploadResponse>>()
.await
.expect("valid response");
match upload {
PictRsResult::Ok(_) => panic!("request should have errored"),
PictRsResult::Err { msg, code } => {
assert_eq!(msg, "Filesize too large");
assert_eq!(code, "validate-file-size");
}
}
})
.unwrap();
}

89
tests/background.rs Normal file
View file

@ -0,0 +1,89 @@
#![cfg(system_deps)]
use common::{
pict_rs_test_config, upload_form, with_pict_rs, OkString, PictRsResult, UploadResponse,
};
mod common;
#[derive(serde::Deserialize)]
struct BackgroundResponse {
#[allow(unused)]
msg: OkString,
uploads: Vec<Upload>,
}
#[derive(serde::Deserialize)]
struct Upload {
upload_id: String,
}
#[test]
fn can_upload_and_download_file() {
let address = "127.0.0.1:8090";
let config = pict_rs_test_config(address);
with_pict_rs(config, || async {
let client = reqwest::Client::new();
let form = upload_form(["./client-examples/earth.gif"]).await;
let response = client
.post(format!("http://{address}/image/backgrounded"))
.multipart(form)
.send()
.await
.expect("send request");
let backgrounded = response
.json::<PictRsResult<BackgroundResponse>>()
.await
.expect("valid response")
.unwrap();
assert_eq!(backgrounded.uploads.len(), 1);
let upload_id = &backgrounded.uploads[0].upload_id;
let response = loop {
let response = client
.get(format!(
"http://{address}/image/backgrounded/claim?upload_id={upload_id}"
))
.send()
.await
.expect("send request");
if response.status() == 200 {
break response;
}
assert!(response.status().is_success());
};
let upload = response
.json::<PictRsResult<UploadResponse>>()
.await
.expect("valid response")
.unwrap();
assert_eq!(upload.files.len(), 1);
let alias = &upload.files[0].file;
let response = client
.get(format!("http://{address}/image/original/{alias}"))
.send()
.await
.expect("download file");
assert!(response.status().is_success(), "download failed");
let length = response.bytes().await.expect("downlaod bytes").len();
assert!(length > 0);
})
.unwrap();
}

243
tests/basic.rs Normal file
View file

@ -0,0 +1,243 @@
#![cfg(system_deps)]
use common::{pict_rs_test_config, upload_form, with_pict_rs, PictRsResult, UploadResponse};
mod common;
#[test]
fn can_upload_and_download_file() {
let address = "127.0.0.1:8090";
let config = pict_rs_test_config(address);
with_pict_rs(config, || async {
let client = reqwest::Client::new();
let form = upload_form(["./client-examples/earth.gif"]).await;
let response = client
.post(format!("http://{address}/image"))
.multipart(form)
.send()
.await
.expect("send request");
let upload = response
.json::<PictRsResult<UploadResponse>>()
.await
.expect("valid response")
.unwrap();
assert_eq!(upload.files.len(), 1);
let alias = &upload.files[0].file;
let response = client
.get(format!("http://{address}/image/original/{alias}"))
.send()
.await
.expect("download file");
assert!(response.status().is_success(), "download failed");
let length = response.bytes().await.expect("downlaod bytes").len();
assert!(length > 0);
})
.unwrap();
}
#[test]
fn can_upload_and_download_multiple_files() {
let address = "127.0.0.1:8091";
let mut config = pict_rs_test_config(address);
config["server"]["max_file_count"] = 4.into();
with_pict_rs(config, || async {
let client = reqwest::Client::new();
let files = [
"./client-examples/earth.gif",
"./client-examples/cat.jpg",
"./client-examples/scene.webp",
"./client-examples/test.png",
];
let form = upload_form(files).await;
let response = client
.post(format!("http://{address}/image"))
.multipart(form)
.send()
.await
.expect("send request");
let upload = response
.json::<PictRsResult<UploadResponse>>()
.await
.expect("valid response")
.unwrap();
assert_eq!(upload.files.len(), files.len());
for file in upload.files {
let alias = file.file;
let response = client
.get(format!("http://{address}/image/original/{alias}"))
.send()
.await
.expect("download file");
assert!(response.status().is_success(), "download failed");
let length = response.bytes().await.expect("downlaod bytes").len();
assert!(length > 0);
}
})
.unwrap();
}
#[test]
fn can_delete_uploaded_file() {
let address = "127.0.0.1:8092";
let config = pict_rs_test_config(address);
with_pict_rs(config, || async {
let client = reqwest::Client::new();
let form = upload_form(["./client-examples/earth.gif"]).await;
let response = client
.post(format!("http://{address}/image"))
.multipart(form)
.send()
.await
.expect("send request");
let upload = response
.json::<PictRsResult<UploadResponse>>()
.await
.expect("valid response")
.unwrap();
assert_eq!(upload.files.len(), 1);
let alias = &upload.files[0].file;
let response = client
.get(format!("http://{address}/image/original/{alias}"))
.send()
.await
.expect("download file");
assert!(response.status().is_success(), "download failed");
let length = response.bytes().await.expect("downlaod bytes").len();
assert!(length > 0);
let delete_token = &upload.files[0].delete_token;
let response = client
.delete(format!(
"http://{address}/image/delete/{delete_token}/{alias}"
))
.send()
.await
.expect("send request");
assert!(response.status().is_success());
let response = client
.get(format!("http://{address}/image/original/{alias}"))
.send()
.await
.expect("download file");
assert!(response.status().is_client_error());
})
.unwrap();
}
#[test]
fn cannot_upload_too_many_files() {
let address = "127.0.0.1:8093";
let mut config = pict_rs_test_config(address);
config["server"]["max_file_count"] = 1.into();
with_pict_rs(config, || async {
let client = reqwest::Client::new();
let form = upload_form([
"./client-examples/earth.gif",
"./client-examples/cat.jpg",
"./client-examples/scene.webp",
"./client-examples/test.png",
])
.await;
let response = client
.post(format!("http://{address}/image"))
.multipart(form)
.send()
.await
.expect("send request");
let upload = response
.json::<PictRsResult<UploadResponse>>()
.await
.expect("valid response");
match upload {
PictRsResult::Ok(_) => panic!("request should have errored"),
PictRsResult::Err { msg, code } => {
assert_eq!(msg, "Too many files in request");
assert_eq!(code, "file-upload-error");
}
}
})
.unwrap();
}
#[test]
fn cannot_upload_too_big_file() {
let address = "127.0.0.1:8094";
let mut config = pict_rs_test_config(address);
config["media"]["max_file_size"] = 1.into();
with_pict_rs(config, || async {
let client = reqwest::Client::new();
let form = upload_form(["./client-examples/awoo.webp"]).await;
let response = client
.post(format!("http://{address}/image"))
.multipart(form)
.send()
.await
.expect("send request");
let upload = response
.json::<PictRsResult<UploadResponse>>()
.await
.expect("valid response");
match upload {
PictRsResult::Ok(_) => panic!("request should have errored"),
PictRsResult::Err { msg, code } => {
assert_eq!(msg, "File too large");
assert_eq!(code, "validate-file-size");
}
}
})
.unwrap();
}

135
tests/common/mod.rs Normal file
View file

@ -0,0 +1,135 @@
#![allow(dead_code)]
use std::future::Future;
pub fn with_pict_rs<F, Fut>(config: serde_json::Value, callback: F) -> color_eyre::Result<()>
where
F: Fn() -> Fut,
Fut: Future<Output = ()>,
{
tokio::runtime::Builder::new_multi_thread()
.enable_all()
.build()?
.block_on(async move {
tokio::task::LocalSet::new()
.run_until(async move {
let mut pict_rs_handle = spawn_pict_rs(config);
// give time to spin up
tokio::time::sleep(std::time::Duration::from_millis(500)).await;
tokio::select! {
_ = (callback)() => {
pict_rs_handle.abort();
let _ = pict_rs_handle.await;
}
res = &mut pict_rs_handle => {
res.unwrap();
}
}
Ok(())
})
.await
})
}
pub async fn upload_form<I>(files: I) -> reqwest::multipart::Form
where
I: IntoIterator<Item = &'static str>,
{
let mut form = reqwest::multipart::Form::new();
for file in files {
form = form.file("images[]", file).await.expect("read file");
}
form
}
pub fn pict_rs_test_config(address: &str) -> serde_json::Value {
let directory = format!("/tmp/pict-rs-test/{}", uuid::Uuid::now_v7());
serde_json::json!({
"server": {
"address": address,
"temporary_directory": format!("{directory}/tmp")
},
"repo": {
"type": "sled",
"path": format!("{directory}/sled-repo")
},
"store": {
"type": "filesystem",
"path": format!("{directory}/files")
}
})
}
fn spawn_pict_rs(config: serde_json::Value) -> tokio::task::JoinHandle<()> {
tokio::task::spawn_local(async move {
pict_rs::ConfigSource::memory(config)
.init::<String>(None)
.expect("init pict-rs config")
.install_crypto_provider()
.run()
.await
.expect("run pict-rs")
})
}
#[derive(serde::Deserialize)]
#[serde(untagged)]
pub enum PictRsResult<T> {
Ok(T),
Err { msg: String, code: String },
}
impl<T> PictRsResult<T> {
pub fn unwrap(self) -> T {
match self {
Self::Ok(t) => t,
Self::Err { msg, code } => panic!("{code}: {msg}"),
}
}
}
#[derive(serde::Deserialize)]
pub enum OkString {
#[serde(rename = "ok")]
Ok,
}
#[derive(serde::Deserialize)]
pub struct UploadResponse {
pub msg: OkString,
pub files: Vec<File>,
}
#[derive(serde::Deserialize)]
pub struct File {
pub delete_token: String,
pub file: String,
pub details: Details,
}
#[derive(serde::Deserialize)]
pub struct DetailsResponse {
pub msg: OkString,
#[serde(flatten)]
pub details: Details,
}
#[derive(serde::Deserialize)]
pub struct Details {
pub width: u16,
pub height: u16,
pub frames: Option<u32>,
pub blurhash: String,
pub content_type: String,
#[serde(with = "time::serde::rfc3339")]
pub created_at: time::OffsetDateTime,
}

149
tests/image.rs Normal file
View file

@ -0,0 +1,149 @@
#![cfg(system_deps)]
use common::{pict_rs_test_config, upload_form, with_pict_rs, PictRsResult, UploadResponse};
mod common;
#[test]
fn cannot_upload_too_wide_image() {
let address = "127.0.0.1:9000";
let mut config = pict_rs_test_config(address);
config["media"]["image"]["max_width"] = 100.into();
with_pict_rs(config, || async {
let client = reqwest::Client::new();
let form = upload_form(["./client-examples/cat.jpg"]).await;
let response = client
.post(format!("http://{address}/image"))
.multipart(form)
.send()
.await
.expect("send request");
let upload = response
.json::<PictRsResult<UploadResponse>>()
.await
.expect("valid response");
match upload {
PictRsResult::Ok(_) => panic!("request should have errored"),
PictRsResult::Err { msg, code } => {
assert_eq!(msg, "Too wide");
assert_eq!(code, "validate-width");
}
}
})
.unwrap();
}
#[test]
fn cannot_upload_too_tall_image() {
let address = "127.0.0.1:9001";
let mut config = pict_rs_test_config(address);
config["media"]["image"]["max_height"] = 100.into();
with_pict_rs(config, || async {
let client = reqwest::Client::new();
let form = upload_form(["./client-examples/cat.jpg"]).await;
let response = client
.post(format!("http://{address}/image"))
.multipart(form)
.send()
.await
.expect("send request");
let upload = response
.json::<PictRsResult<UploadResponse>>()
.await
.expect("valid response");
match upload {
PictRsResult::Ok(_) => panic!("request should have errored"),
PictRsResult::Err { msg, code } => {
assert_eq!(msg, "Too tall");
assert_eq!(code, "validate-height");
}
}
})
.unwrap();
}
#[test]
fn cannot_upload_too_much_area_image() {
let address = "127.0.0.1:9002";
let mut config = pict_rs_test_config(address);
config["media"]["image"]["max_area"] = 100.into();
with_pict_rs(config, || async {
let client = reqwest::Client::new();
let form = upload_form(["./client-examples/cat.jpg"]).await;
let response = client
.post(format!("http://{address}/image"))
.multipart(form)
.send()
.await
.expect("send request");
let upload = response
.json::<PictRsResult<UploadResponse>>()
.await
.expect("valid response");
match upload {
PictRsResult::Ok(_) => panic!("request should have errored"),
PictRsResult::Err { msg, code } => {
assert_eq!(msg, "Too many pixels");
assert_eq!(code, "validate-area");
}
}
})
.unwrap();
}
#[test]
fn cannot_upload_too_large_image() {
let address = "127.0.0.1:9003";
let mut config = pict_rs_test_config(address);
config["media"]["image"]["max_file_size"] = 1.into();
with_pict_rs(config, || async {
let client = reqwest::Client::new();
let form = upload_form(["./client-examples/awoo.webp"]).await;
let response = client
.post(format!("http://{address}/image"))
.multipart(form)
.send()
.await
.expect("send request");
let upload = response
.json::<PictRsResult<UploadResponse>>()
.await
.expect("valid response");
match upload {
PictRsResult::Ok(_) => panic!("request should have errored"),
PictRsResult::Err { msg, code } => {
assert_eq!(msg, "Filesize too large");
assert_eq!(code, "validate-file-size");
}
}
})
.unwrap();
}