mirror of
https://git.asonix.dog/asonix/pict-rs.git
synced 2024-06-16 12:10:43 +00:00
Pass a State value around rather than a bunch of arguments
This commit is contained in:
parent
c3e322f2c9
commit
cf7c753e65
120
src/generate.rs
120
src/generate.rs
|
@ -8,7 +8,8 @@ use crate::{
|
||||||
formats::{ImageFormat, InputProcessableFormat, InternalVideoFormat, ProcessableFormat},
|
formats::{ImageFormat, InputProcessableFormat, InternalVideoFormat, ProcessableFormat},
|
||||||
future::{WithMetrics, WithTimeout},
|
future::{WithMetrics, WithTimeout},
|
||||||
magick::PolicyDir,
|
magick::PolicyDir,
|
||||||
repo::{ArcRepo, Hash, VariantAlreadyExists},
|
repo::{Hash, VariantAlreadyExists},
|
||||||
|
state::State,
|
||||||
store::Store,
|
store::Store,
|
||||||
tmp_file::TmpDir,
|
tmp_file::TmpDir,
|
||||||
};
|
};
|
||||||
|
@ -49,47 +50,43 @@ impl Drop for MetricsGuard {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(clippy::too_many_arguments)]
|
#[tracing::instrument(skip(state, process_map, hash))]
|
||||||
#[tracing::instrument(skip(tmp_dir, policy_dir, repo, store, hash, process_map, config))]
|
|
||||||
pub(crate) async fn generate<S: Store + 'static>(
|
pub(crate) async fn generate<S: Store + 'static>(
|
||||||
tmp_dir: &TmpDir,
|
state: &State<S>,
|
||||||
policy_dir: &PolicyDir,
|
|
||||||
repo: &ArcRepo,
|
|
||||||
store: &S,
|
|
||||||
process_map: &ProcessMap,
|
process_map: &ProcessMap,
|
||||||
format: InputProcessableFormat,
|
format: InputProcessableFormat,
|
||||||
thumbnail_path: PathBuf,
|
thumbnail_path: PathBuf,
|
||||||
thumbnail_args: Vec<String>,
|
thumbnail_args: Vec<String>,
|
||||||
original_details: &Details,
|
original_details: &Details,
|
||||||
config: &crate::config::Configuration,
|
|
||||||
hash: Hash,
|
hash: Hash,
|
||||||
) -> Result<(Details, Bytes), Error> {
|
) -> Result<(Details, Bytes), Error> {
|
||||||
if config.server.danger_dummy_mode {
|
if state.config.server.danger_dummy_mode {
|
||||||
let identifier = repo
|
let identifier = state
|
||||||
|
.repo
|
||||||
.identifier(hash)
|
.identifier(hash)
|
||||||
.await?
|
.await?
|
||||||
.ok_or(UploadError::MissingIdentifier)?;
|
.ok_or(UploadError::MissingIdentifier)?;
|
||||||
|
|
||||||
let bytes = store.to_bytes(&identifier, None, None).await?.into_bytes();
|
let bytes = state
|
||||||
|
.store
|
||||||
|
.to_bytes(&identifier, None, None)
|
||||||
|
.await?
|
||||||
|
.into_bytes();
|
||||||
|
|
||||||
Ok((original_details.clone(), bytes))
|
Ok((original_details.clone(), bytes))
|
||||||
} else {
|
} else {
|
||||||
let process_fut = process(
|
let process_fut = process(
|
||||||
tmp_dir,
|
state,
|
||||||
policy_dir,
|
|
||||||
repo,
|
|
||||||
store,
|
|
||||||
format,
|
format,
|
||||||
thumbnail_path.clone(),
|
thumbnail_path.clone(),
|
||||||
thumbnail_args,
|
thumbnail_args,
|
||||||
original_details,
|
original_details,
|
||||||
config,
|
|
||||||
hash.clone(),
|
hash.clone(),
|
||||||
);
|
);
|
||||||
|
|
||||||
let (details, bytes) = process_map
|
let (details, bytes) = process_map
|
||||||
.process(hash, thumbnail_path, process_fut)
|
.process(hash, thumbnail_path, process_fut)
|
||||||
.with_timeout(Duration::from_secs(config.media.process_timeout * 4))
|
.with_timeout(Duration::from_secs(state.config.media.process_timeout * 4))
|
||||||
.with_metrics("pict-rs.generate.process")
|
.with_metrics("pict-rs.generate.process")
|
||||||
.await
|
.await
|
||||||
.map_err(|_| UploadError::ProcessTimeout)??;
|
.map_err(|_| UploadError::ProcessTimeout)??;
|
||||||
|
@ -99,37 +96,21 @@ pub(crate) async fn generate<S: Store + 'static>(
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(clippy::too_many_arguments)]
|
#[allow(clippy::too_many_arguments)]
|
||||||
#[tracing::instrument(skip(tmp_dir, policy_dir, repo, store, hash, config))]
|
#[tracing::instrument(skip(state, hash))]
|
||||||
async fn process<S: Store + 'static>(
|
async fn process<S: Store + 'static>(
|
||||||
tmp_dir: &TmpDir,
|
state: &State<S>,
|
||||||
policy_dir: &PolicyDir,
|
|
||||||
repo: &ArcRepo,
|
|
||||||
store: &S,
|
|
||||||
output_format: InputProcessableFormat,
|
output_format: InputProcessableFormat,
|
||||||
thumbnail_path: PathBuf,
|
thumbnail_path: PathBuf,
|
||||||
thumbnail_args: Vec<String>,
|
thumbnail_args: Vec<String>,
|
||||||
original_details: &Details,
|
original_details: &Details,
|
||||||
config: &crate::config::Configuration,
|
|
||||||
hash: Hash,
|
hash: Hash,
|
||||||
) -> Result<(Details, Bytes), Error> {
|
) -> Result<(Details, Bytes), Error> {
|
||||||
let guard = MetricsGuard::guard();
|
let guard = MetricsGuard::guard();
|
||||||
let permit = crate::process_semaphore().acquire().await?;
|
let permit = crate::process_semaphore().acquire().await?;
|
||||||
|
|
||||||
let identifier = input_identifier(
|
let identifier = input_identifier(state, output_format, hash.clone(), original_details).await?;
|
||||||
tmp_dir,
|
|
||||||
policy_dir,
|
|
||||||
repo,
|
|
||||||
store,
|
|
||||||
output_format,
|
|
||||||
hash.clone(),
|
|
||||||
original_details,
|
|
||||||
&config.media,
|
|
||||||
)
|
|
||||||
.await?;
|
|
||||||
|
|
||||||
let input_details =
|
let input_details = crate::ensure_details_identifier(state, &identifier).await?;
|
||||||
crate::ensure_details_identifier(tmp_dir, policy_dir, repo, store, config, &identifier)
|
|
||||||
.await?;
|
|
||||||
|
|
||||||
let input_format = input_details
|
let input_format = input_details
|
||||||
.internal_format()
|
.internal_format()
|
||||||
|
@ -139,21 +120,19 @@ async fn process<S: Store + 'static>(
|
||||||
let format = input_format.process_to(output_format);
|
let format = input_format.process_to(output_format);
|
||||||
|
|
||||||
let quality = match format {
|
let quality = match format {
|
||||||
ProcessableFormat::Image(format) => config.media.image.quality_for(format),
|
ProcessableFormat::Image(format) => state.config.media.image.quality_for(format),
|
||||||
ProcessableFormat::Animation(format) => config.media.animation.quality_for(format),
|
ProcessableFormat::Animation(format) => state.config.media.animation.quality_for(format),
|
||||||
};
|
};
|
||||||
|
|
||||||
let stream = store.to_stream(&identifier, None, None).await?;
|
let stream = state.store.to_stream(&identifier, None, None).await?;
|
||||||
|
|
||||||
let vec = crate::magick::process_image_stream_read(
|
let vec = crate::magick::process_image_stream_read(
|
||||||
tmp_dir,
|
state,
|
||||||
policy_dir,
|
|
||||||
stream,
|
stream,
|
||||||
thumbnail_args,
|
thumbnail_args,
|
||||||
input_format,
|
input_format,
|
||||||
format,
|
format,
|
||||||
quality,
|
quality,
|
||||||
config.media.process_timeout,
|
|
||||||
)
|
)
|
||||||
.await?
|
.await?
|
||||||
.into_vec()
|
.into_vec()
|
||||||
|
@ -165,18 +144,20 @@ async fn process<S: Store + 'static>(
|
||||||
drop(permit);
|
drop(permit);
|
||||||
|
|
||||||
let details = Details::from_bytes(
|
let details = Details::from_bytes(
|
||||||
tmp_dir,
|
&state.tmp_dir,
|
||||||
policy_dir,
|
&state.policy_dir,
|
||||||
config.media.process_timeout,
|
&state.config.media.process_timeout,
|
||||||
bytes.clone(),
|
bytes.clone(),
|
||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
let identifier = store
|
let identifier = state
|
||||||
|
.store
|
||||||
.save_bytes(bytes.clone(), details.media_type())
|
.save_bytes(bytes.clone(), details.media_type())
|
||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
if let Err(VariantAlreadyExists) = repo
|
if let Err(VariantAlreadyExists) = state
|
||||||
|
.repo
|
||||||
.relate_variant_identifier(
|
.relate_variant_identifier(
|
||||||
hash,
|
hash,
|
||||||
thumbnail_path.to_string_lossy().to_string(),
|
thumbnail_path.to_string_lossy().to_string(),
|
||||||
|
@ -184,10 +165,10 @@ async fn process<S: Store + 'static>(
|
||||||
)
|
)
|
||||||
.await?
|
.await?
|
||||||
{
|
{
|
||||||
store.remove(&identifier).await?;
|
state.store.remove(&identifier).await?;
|
||||||
}
|
}
|
||||||
|
|
||||||
repo.relate_details(&identifier, &details).await?;
|
state.repo.relate_details(&identifier, &details).await?;
|
||||||
|
|
||||||
guard.disarm();
|
guard.disarm();
|
||||||
|
|
||||||
|
@ -197,14 +178,10 @@ async fn process<S: Store + 'static>(
|
||||||
#[allow(clippy::too_many_arguments)]
|
#[allow(clippy::too_many_arguments)]
|
||||||
#[tracing::instrument(skip_all)]
|
#[tracing::instrument(skip_all)]
|
||||||
async fn input_identifier<S>(
|
async fn input_identifier<S>(
|
||||||
tmp_dir: &TmpDir,
|
state: &State<S>,
|
||||||
policy_dir: &PolicyDir,
|
|
||||||
repo: &ArcRepo,
|
|
||||||
store: &S,
|
|
||||||
output_format: InputProcessableFormat,
|
output_format: InputProcessableFormat,
|
||||||
hash: Hash,
|
hash: Hash,
|
||||||
original_details: &Details,
|
original_details: &Details,
|
||||||
media: &crate::config::Media,
|
|
||||||
) -> Result<Arc<str>, Error>
|
) -> Result<Arc<str>, Error>
|
||||||
where
|
where
|
||||||
S: Store + 'static,
|
S: Store + 'static,
|
||||||
|
@ -220,11 +197,12 @@ where
|
||||||
};
|
};
|
||||||
|
|
||||||
if should_thumbnail {
|
if should_thumbnail {
|
||||||
if let Some(identifier) = repo.motion_identifier(hash.clone()).await? {
|
if let Some(identifier) = state.repo.motion_identifier(hash.clone()).await? {
|
||||||
return Ok(identifier);
|
return Ok(identifier);
|
||||||
};
|
};
|
||||||
|
|
||||||
let identifier = repo
|
let identifier = state
|
||||||
|
.repo
|
||||||
.identifier(hash.clone())
|
.identifier(hash.clone())
|
||||||
.await?
|
.await?
|
||||||
.ok_or(UploadError::MissingIdentifier)?;
|
.ok_or(UploadError::MissingIdentifier)?;
|
||||||
|
@ -232,24 +210,22 @@ where
|
||||||
let (reader, media_type) = if let Some(processable_format) =
|
let (reader, media_type) = if let Some(processable_format) =
|
||||||
original_details.internal_format().processable_format()
|
original_details.internal_format().processable_format()
|
||||||
{
|
{
|
||||||
let thumbnail_format = media.image.format.unwrap_or(ImageFormat::Webp);
|
let thumbnail_format = state.config.media.image.format.unwrap_or(ImageFormat::Webp);
|
||||||
|
|
||||||
let stream = store.to_stream(&identifier, None, None).await?;
|
let stream = state.store.to_stream(&identifier, None, None).await?;
|
||||||
|
|
||||||
let reader = magick::thumbnail(
|
let reader = magick::thumbnail(
|
||||||
tmp_dir,
|
state,
|
||||||
policy_dir,
|
|
||||||
stream,
|
stream,
|
||||||
processable_format,
|
processable_format,
|
||||||
ProcessableFormat::Image(thumbnail_format),
|
ProcessableFormat::Image(thumbnail_format),
|
||||||
media.image.quality_for(thumbnail_format),
|
config.media.image.quality_for(thumbnail_format),
|
||||||
media.process_timeout,
|
|
||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
(reader, thumbnail_format.media_type())
|
(reader, thumbnail_format.media_type())
|
||||||
} else {
|
} else {
|
||||||
let thumbnail_format = match media.image.format {
|
let thumbnail_format = match state.config.media.image.format {
|
||||||
Some(ImageFormat::Webp | ImageFormat::Avif | ImageFormat::Jxl) => {
|
Some(ImageFormat::Webp | ImageFormat::Avif | ImageFormat::Jxl) => {
|
||||||
ffmpeg::ThumbnailFormat::Webp
|
ffmpeg::ThumbnailFormat::Webp
|
||||||
}
|
}
|
||||||
|
@ -258,14 +234,14 @@ where
|
||||||
};
|
};
|
||||||
|
|
||||||
let reader = ffmpeg::thumbnail(
|
let reader = ffmpeg::thumbnail(
|
||||||
tmp_dir,
|
state.tmp_dir,
|
||||||
store.clone(),
|
state.store.clone(),
|
||||||
identifier,
|
identifier,
|
||||||
original_details
|
original_details
|
||||||
.video_format()
|
.video_format()
|
||||||
.unwrap_or(InternalVideoFormat::Mp4),
|
.unwrap_or(InternalVideoFormat::Mp4),
|
||||||
thumbnail_format,
|
thumbnail_format,
|
||||||
media.process_timeout,
|
state.config.media.process_timeout,
|
||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
|
@ -273,16 +249,20 @@ where
|
||||||
};
|
};
|
||||||
|
|
||||||
let motion_identifier = reader
|
let motion_identifier = reader
|
||||||
.with_stdout(|stdout| async { store.save_async_read(stdout, media_type).await })
|
.with_stdout(|stdout| async { state.store.save_async_read(stdout, media_type).await })
|
||||||
.await??;
|
.await??;
|
||||||
|
|
||||||
repo.relate_motion_identifier(hash, &motion_identifier)
|
state
|
||||||
|
.repo
|
||||||
|
.relate_motion_identifier(hash, &motion_identifier)
|
||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
return Ok(motion_identifier);
|
return Ok(motion_identifier);
|
||||||
}
|
}
|
||||||
|
|
||||||
repo.identifier(hash)
|
state
|
||||||
|
.repo
|
||||||
|
.identifier(hash)
|
||||||
.await?
|
.await?
|
||||||
.ok_or(UploadError::MissingIdentifier)
|
.ok_or(UploadError::MissingIdentifier)
|
||||||
.map_err(From::from)
|
.map_err(From::from)
|
||||||
|
|
|
@ -6,29 +6,29 @@ use crate::{
|
||||||
formats::ProcessableFormat,
|
formats::ProcessableFormat,
|
||||||
magick::{MagickError, PolicyDir, MAGICK_CONFIGURE_PATH, MAGICK_TEMPORARY_PATH},
|
magick::{MagickError, PolicyDir, MAGICK_CONFIGURE_PATH, MAGICK_TEMPORARY_PATH},
|
||||||
process::{Process, ProcessRead},
|
process::{Process, ProcessRead},
|
||||||
|
state::State,
|
||||||
stream::LocalBoxStream,
|
stream::LocalBoxStream,
|
||||||
tmp_file::TmpDir,
|
tmp_file::TmpDir,
|
||||||
};
|
};
|
||||||
|
|
||||||
async fn thumbnail_animation<F, Fut>(
|
async fn thumbnail_animation<S, F, Fut>(
|
||||||
tmp_dir: &TmpDir,
|
state: &State<S>,
|
||||||
policy_dir: &PolicyDir,
|
|
||||||
input_format: ProcessableFormat,
|
input_format: ProcessableFormat,
|
||||||
format: ProcessableFormat,
|
format: ProcessableFormat,
|
||||||
quality: Option<u8>,
|
quality: Option<u8>,
|
||||||
timeout: u64,
|
|
||||||
write_file: F,
|
write_file: F,
|
||||||
) -> Result<ProcessRead, MagickError>
|
) -> Result<ProcessRead, MagickError>
|
||||||
where
|
where
|
||||||
F: FnOnce(crate::file::File) -> Fut,
|
F: FnOnce(crate::file::File) -> Fut,
|
||||||
Fut: std::future::Future<Output = Result<crate::file::File, MagickError>>,
|
Fut: std::future::Future<Output = Result<crate::file::File, MagickError>>,
|
||||||
{
|
{
|
||||||
let temporary_path = tmp_dir
|
let temporary_path = state
|
||||||
|
.tmp_dir
|
||||||
.tmp_folder()
|
.tmp_folder()
|
||||||
.await
|
.await
|
||||||
.map_err(MagickError::CreateTemporaryDirectory)?;
|
.map_err(MagickError::CreateTemporaryDirectory)?;
|
||||||
|
|
||||||
let input_file = tmp_dir.tmp_file(None);
|
let input_file = state.tmp_dir.tmp_file(None);
|
||||||
crate::store::file_store::safe_create_parent(&input_file)
|
crate::store::file_store::safe_create_parent(&input_file)
|
||||||
.await
|
.await
|
||||||
.map_err(MagickError::CreateDir)?;
|
.map_err(MagickError::CreateDir)?;
|
||||||
|
@ -62,10 +62,10 @@ where
|
||||||
|
|
||||||
let envs = [
|
let envs = [
|
||||||
(MAGICK_TEMPORARY_PATH, temporary_path.as_os_str()),
|
(MAGICK_TEMPORARY_PATH, temporary_path.as_os_str()),
|
||||||
(MAGICK_CONFIGURE_PATH, policy_dir.as_os_str()),
|
(MAGICK_CONFIGURE_PATH, state.policy_dir.as_os_str()),
|
||||||
];
|
];
|
||||||
|
|
||||||
let reader = Process::run("magick", &args, &envs, timeout)?
|
let reader = Process::run("magick", &args, &envs, state.config.media.process_timeout)?
|
||||||
.read()
|
.read()
|
||||||
.add_extras(input_file)
|
.add_extras(input_file)
|
||||||
.add_extras(temporary_path);
|
.add_extras(temporary_path);
|
||||||
|
@ -73,22 +73,18 @@ where
|
||||||
Ok(reader)
|
Ok(reader)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(super) async fn thumbnail(
|
pub(super) async fn thumbnail<S>(
|
||||||
tmp_dir: &TmpDir,
|
state: &State<S>,
|
||||||
policy_dir: &PolicyDir,
|
|
||||||
stream: LocalBoxStream<'static, std::io::Result<Bytes>>,
|
stream: LocalBoxStream<'static, std::io::Result<Bytes>>,
|
||||||
input_format: ProcessableFormat,
|
input_format: ProcessableFormat,
|
||||||
format: ProcessableFormat,
|
format: ProcessableFormat,
|
||||||
quality: Option<u8>,
|
quality: Option<u8>,
|
||||||
timeout: u64,
|
|
||||||
) -> Result<ProcessRead, MagickError> {
|
) -> Result<ProcessRead, MagickError> {
|
||||||
thumbnail_animation(
|
thumbnail_animation(
|
||||||
tmp_dir,
|
state,
|
||||||
policy_dir,
|
|
||||||
input_format,
|
input_format,
|
||||||
format,
|
format,
|
||||||
quality,
|
quality,
|
||||||
timeout,
|
|
||||||
|mut tmp_file| async move {
|
|mut tmp_file| async move {
|
||||||
tmp_file
|
tmp_file
|
||||||
.write_from_stream(stream)
|
.write_from_stream(stream)
|
||||||
|
|
110
src/ingest.rs
110
src/ingest.rs
|
@ -8,6 +8,7 @@ use crate::{
|
||||||
future::WithMetrics,
|
future::WithMetrics,
|
||||||
magick::PolicyDir,
|
magick::PolicyDir,
|
||||||
repo::{Alias, ArcRepo, DeleteToken, Hash},
|
repo::{Alias, ArcRepo, DeleteToken, Hash},
|
||||||
|
state::State,
|
||||||
store::Store,
|
store::Store,
|
||||||
tmp_file::TmpDir,
|
tmp_file::TmpDir,
|
||||||
};
|
};
|
||||||
|
@ -19,7 +20,7 @@ use streem::IntoStreamer;
|
||||||
use tracing::{Instrument, Span};
|
use tracing::{Instrument, Span};
|
||||||
|
|
||||||
mod hasher;
|
mod hasher;
|
||||||
use hasher::{Hasher, State};
|
use hasher::Hasher;
|
||||||
|
|
||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
pub(crate) struct Session {
|
pub(crate) struct Session {
|
||||||
|
@ -50,12 +51,17 @@ where
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn process_ingest<S>(
|
async fn process_ingest<S>(
|
||||||
tmp_dir: &TmpDir,
|
state: &State<S>,
|
||||||
policy_dir: &PolicyDir,
|
|
||||||
store: &S,
|
|
||||||
stream: impl Stream<Item = Result<Bytes, Error>> + 'static,
|
stream: impl Stream<Item = Result<Bytes, Error>> + 'static,
|
||||||
media: &crate::config::Media,
|
) -> Result<
|
||||||
) -> Result<(InternalFormat, Arc<str>, Details, Rc<RefCell<State>>), Error>
|
(
|
||||||
|
InternalFormat,
|
||||||
|
Arc<str>,
|
||||||
|
Details,
|
||||||
|
Rc<RefCell<hasher::State>>,
|
||||||
|
),
|
||||||
|
Error,
|
||||||
|
>
|
||||||
where
|
where
|
||||||
S: Store,
|
S: Store,
|
||||||
{
|
{
|
||||||
|
@ -65,43 +71,30 @@ where
|
||||||
|
|
||||||
let permit = crate::process_semaphore().acquire().await?;
|
let permit = crate::process_semaphore().acquire().await?;
|
||||||
|
|
||||||
let prescribed = Validations {
|
|
||||||
image: &media.image,
|
|
||||||
animation: &media.animation,
|
|
||||||
video: &media.video,
|
|
||||||
};
|
|
||||||
|
|
||||||
tracing::trace!("Validating bytes");
|
tracing::trace!("Validating bytes");
|
||||||
let (input_type, process_read) = crate::validate::validate_bytes(
|
let (input_type, process_read) = crate::validate::validate_bytes(state, bytes).await?;
|
||||||
tmp_dir,
|
|
||||||
policy_dir,
|
|
||||||
bytes,
|
|
||||||
prescribed,
|
|
||||||
media.process_timeout,
|
|
||||||
)
|
|
||||||
.await?;
|
|
||||||
|
|
||||||
let process_read = if let Some(operations) = media.preprocess_steps() {
|
let process_read = if let Some(operations) = state.config.media.preprocess_steps() {
|
||||||
if let Some(format) = input_type.processable_format() {
|
if let Some(format) = input_type.processable_format() {
|
||||||
let (_, magick_args) =
|
let (_, magick_args) =
|
||||||
crate::processor::build_chain(operations, format.file_extension())?;
|
crate::processor::build_chain(operations, format.file_extension())?;
|
||||||
|
|
||||||
let quality = match format {
|
let quality = match format {
|
||||||
crate::formats::ProcessableFormat::Image(format) => media.image.quality_for(format),
|
crate::formats::ProcessableFormat::Image(format) => {
|
||||||
|
state.config.media.image.quality_for(format)
|
||||||
|
}
|
||||||
crate::formats::ProcessableFormat::Animation(format) => {
|
crate::formats::ProcessableFormat::Animation(format) => {
|
||||||
media.animation.quality_for(format)
|
state.config.media.animation.quality_for(format)
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
crate::magick::process_image_process_read(
|
crate::magick::process_image_process_read(
|
||||||
tmp_dir,
|
state,
|
||||||
policy_dir,
|
|
||||||
process_read,
|
process_read,
|
||||||
magick_args,
|
magick_args,
|
||||||
format,
|
format,
|
||||||
format,
|
format,
|
||||||
quality,
|
quality,
|
||||||
media.process_timeout,
|
|
||||||
)
|
)
|
||||||
.await?
|
.await?
|
||||||
} else {
|
} else {
|
||||||
|
@ -111,7 +104,7 @@ where
|
||||||
process_read
|
process_read
|
||||||
};
|
};
|
||||||
|
|
||||||
let (state, identifier) = process_read
|
let (hash_state, identifier) = process_read
|
||||||
.with_stdout(|stdout| async move {
|
.with_stdout(|stdout| async move {
|
||||||
let hasher_reader = Hasher::new(stdout);
|
let hasher_reader = Hasher::new(stdout);
|
||||||
let state = hasher_reader.state();
|
let state = hasher_reader.state();
|
||||||
|
@ -119,11 +112,11 @@ where
|
||||||
store
|
store
|
||||||
.save_async_read(hasher_reader, input_type.media_type())
|
.save_async_read(hasher_reader, input_type.media_type())
|
||||||
.await
|
.await
|
||||||
.map(move |identifier| (state, identifier))
|
.map(move |identifier| (hash_state, identifier))
|
||||||
})
|
})
|
||||||
.await??;
|
.await??;
|
||||||
|
|
||||||
let bytes_stream = store.to_bytes(&identifier, None, None).await?;
|
let bytes_stream = state.store.to_bytes(&identifier, None, None).await?;
|
||||||
let details = Details::from_bytes(
|
let details = Details::from_bytes(
|
||||||
tmp_dir,
|
tmp_dir,
|
||||||
policy_dir,
|
policy_dir,
|
||||||
|
@ -134,13 +127,21 @@ where
|
||||||
|
|
||||||
drop(permit);
|
drop(permit);
|
||||||
|
|
||||||
Ok((input_type, identifier, details, state))
|
Ok((input_type, identifier, details, hash_state))
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn dummy_ingest<S>(
|
async fn dummy_ingest<S>(
|
||||||
store: &S,
|
state: &State<S>,
|
||||||
stream: impl Stream<Item = Result<Bytes, Error>> + 'static,
|
stream: impl Stream<Item = Result<Bytes, Error>> + 'static,
|
||||||
) -> Result<(InternalFormat, Arc<str>, Details, Rc<RefCell<State>>), Error>
|
) -> Result<
|
||||||
|
(
|
||||||
|
InternalFormat,
|
||||||
|
Arc<str>,
|
||||||
|
Details,
|
||||||
|
Rc<RefCell<hasher::State>>,
|
||||||
|
),
|
||||||
|
Error,
|
||||||
|
>
|
||||||
where
|
where
|
||||||
S: Store,
|
S: Store,
|
||||||
{
|
{
|
||||||
|
@ -156,7 +157,8 @@ where
|
||||||
|
|
||||||
let input_type = InternalFormat::Image(crate::formats::ImageFormat::Png);
|
let input_type = InternalFormat::Image(crate::formats::ImageFormat::Png);
|
||||||
|
|
||||||
let identifier = store
|
let identifier = state
|
||||||
|
.store
|
||||||
.save_async_read(hasher_reader, input_type.media_type())
|
.save_async_read(hasher_reader, input_type.media_type())
|
||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
|
@ -166,41 +168,37 @@ where
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(clippy::too_many_arguments)]
|
#[allow(clippy::too_many_arguments)]
|
||||||
#[tracing::instrument(skip(tmp_dir, policy_dir, repo, store, client, stream, config))]
|
#[tracing::instrument(skip(state, stream))]
|
||||||
pub(crate) async fn ingest<S>(
|
pub(crate) async fn ingest<S>(
|
||||||
tmp_dir: &TmpDir,
|
state: &State<S>,
|
||||||
policy_dir: &PolicyDir,
|
|
||||||
repo: &ArcRepo,
|
|
||||||
store: &S,
|
|
||||||
client: &ClientWithMiddleware,
|
|
||||||
stream: impl Stream<Item = Result<Bytes, Error>> + 'static,
|
stream: impl Stream<Item = Result<Bytes, Error>> + 'static,
|
||||||
declared_alias: Option<Alias>,
|
declared_alias: Option<Alias>,
|
||||||
config: &crate::config::Configuration,
|
|
||||||
) -> Result<Session, Error>
|
) -> Result<Session, Error>
|
||||||
where
|
where
|
||||||
S: Store,
|
S: Store,
|
||||||
{
|
{
|
||||||
let (input_type, identifier, details, state) = if config.server.danger_dummy_mode {
|
let (input_type, identifier, details, hash_state) = if state.config.server.danger_dummy_mode {
|
||||||
dummy_ingest(store, stream).await?
|
dummy_ingest(state, stream).await?
|
||||||
} else {
|
} else {
|
||||||
process_ingest(tmp_dir, policy_dir, store, stream, &config.media).await?
|
process_ingest(state, stream).await?
|
||||||
};
|
};
|
||||||
|
|
||||||
let mut session = Session {
|
let mut session = Session {
|
||||||
repo: repo.clone(),
|
repo: state.repo.clone(),
|
||||||
delete_token: DeleteToken::generate(),
|
delete_token: DeleteToken::generate(),
|
||||||
hash: None,
|
hash: None,
|
||||||
alias: None,
|
alias: None,
|
||||||
identifier: Some(identifier.clone()),
|
identifier: Some(identifier.clone()),
|
||||||
};
|
};
|
||||||
|
|
||||||
if let Some(endpoint) = &config.media.external_validation {
|
if let Some(endpoint) = &state.config.media.external_validation {
|
||||||
let stream = store.to_stream(&identifier, None, None).await?;
|
let stream = store.to_stream(&identifier, None, None).await?;
|
||||||
|
|
||||||
let response = client
|
let response = state
|
||||||
|
.client
|
||||||
.post(endpoint.as_str())
|
.post(endpoint.as_str())
|
||||||
.timeout(Duration::from_secs(
|
.timeout(Duration::from_secs(
|
||||||
config.media.external_validation_timeout,
|
state.config.media.external_validation_timeout,
|
||||||
))
|
))
|
||||||
.header("Content-Type", input_type.media_type().as_ref())
|
.header("Content-Type", input_type.media_type().as_ref())
|
||||||
.body(Body::wrap_stream(crate::stream::make_send(stream)))
|
.body(Body::wrap_stream(crate::stream::make_send(stream)))
|
||||||
|
@ -214,13 +212,13 @@ where
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
let (hash, size) = state.borrow_mut().finalize_reset();
|
let (hash, size) = hash_state.borrow_mut().finalize_reset();
|
||||||
|
|
||||||
let hash = Hash::new(hash, size, input_type);
|
let hash = Hash::new(hash, size, input_type);
|
||||||
|
|
||||||
save_upload(&mut session, repo, store, hash.clone(), &identifier).await?;
|
save_upload(&mut session, state, hash.clone(), &identifier).await?;
|
||||||
|
|
||||||
repo.relate_details(&identifier, &details).await?;
|
state.repo.relate_details(&identifier, &details).await?;
|
||||||
|
|
||||||
if let Some(alias) = declared_alias {
|
if let Some(alias) = declared_alias {
|
||||||
session.add_existing_alias(hash, alias).await?
|
session.add_existing_alias(hash, alias).await?
|
||||||
|
@ -234,17 +232,21 @@ where
|
||||||
#[tracing::instrument(level = "trace", skip_all)]
|
#[tracing::instrument(level = "trace", skip_all)]
|
||||||
async fn save_upload<S>(
|
async fn save_upload<S>(
|
||||||
session: &mut Session,
|
session: &mut Session,
|
||||||
repo: &ArcRepo,
|
state: &State<S>,
|
||||||
store: &S,
|
|
||||||
hash: Hash,
|
hash: Hash,
|
||||||
identifier: &Arc<str>,
|
identifier: &Arc<str>,
|
||||||
) -> Result<(), Error>
|
) -> Result<(), Error>
|
||||||
where
|
where
|
||||||
S: Store,
|
S: Store,
|
||||||
{
|
{
|
||||||
if repo.create_hash(hash.clone(), identifier).await?.is_err() {
|
if state
|
||||||
|
.repo
|
||||||
|
.create_hash(hash.clone(), identifier)
|
||||||
|
.await?
|
||||||
|
.is_err()
|
||||||
|
{
|
||||||
// duplicate upload
|
// duplicate upload
|
||||||
store.remove(identifier).await?;
|
state.store.remove(identifier).await?;
|
||||||
session.identifier.take();
|
session.identifier.take();
|
||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
}
|
||||||
|
|
891
src/lib.rs
891
src/lib.rs
File diff suppressed because it is too large
Load diff
|
@ -85,27 +85,25 @@ impl MagickError {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(clippy::too_many_arguments)]
|
async fn process_image<S, F, Fut>(
|
||||||
async fn process_image<F, Fut>(
|
state: &State<S>,
|
||||||
tmp_dir: &TmpDir,
|
|
||||||
policy_dir: &PolicyDir,
|
|
||||||
process_args: Vec<String>,
|
process_args: Vec<String>,
|
||||||
input_format: ProcessableFormat,
|
input_format: ProcessableFormat,
|
||||||
format: ProcessableFormat,
|
format: ProcessableFormat,
|
||||||
quality: Option<u8>,
|
quality: Option<u8>,
|
||||||
timeout: u64,
|
|
||||||
write_file: F,
|
write_file: F,
|
||||||
) -> Result<ProcessRead, MagickError>
|
) -> Result<ProcessRead, MagickError>
|
||||||
where
|
where
|
||||||
F: FnOnce(crate::file::File) -> Fut,
|
F: FnOnce(crate::file::File) -> Fut,
|
||||||
Fut: std::future::Future<Output = Result<crate::file::File, MagickError>>,
|
Fut: std::future::Future<Output = Result<crate::file::File, MagickError>>,
|
||||||
{
|
{
|
||||||
let temporary_path = tmp_dir
|
let temporary_path = state
|
||||||
|
.tmp_dir
|
||||||
.tmp_folder()
|
.tmp_folder()
|
||||||
.await
|
.await
|
||||||
.map_err(MagickError::CreateTemporaryDirectory)?;
|
.map_err(MagickError::CreateTemporaryDirectory)?;
|
||||||
|
|
||||||
let input_file = tmp_dir.tmp_file(None);
|
let input_file = state.tmp_dir.tmp_file(None);
|
||||||
crate::store::file_store::safe_create_parent(&input_file)
|
crate::store::file_store::safe_create_parent(&input_file)
|
||||||
.await
|
.await
|
||||||
.map_err(MagickError::CreateDir)?;
|
.map_err(MagickError::CreateDir)?;
|
||||||
|
@ -143,10 +141,10 @@ where
|
||||||
|
|
||||||
let envs = [
|
let envs = [
|
||||||
(MAGICK_TEMPORARY_PATH, temporary_path.as_os_str()),
|
(MAGICK_TEMPORARY_PATH, temporary_path.as_os_str()),
|
||||||
(MAGICK_CONFIGURE_PATH, policy_dir.as_os_str()),
|
(MAGICK_CONFIGURE_PATH, state.policy_dir.as_os_str()),
|
||||||
];
|
];
|
||||||
|
|
||||||
let reader = Process::run("magick", &args, &envs, timeout)?
|
let reader = Process::run("magick", &args, &envs, state.config.media.process_timeout)?
|
||||||
.read()
|
.read()
|
||||||
.add_extras(input_file)
|
.add_extras(input_file)
|
||||||
.add_extras(temporary_path);
|
.add_extras(temporary_path);
|
||||||
|
@ -154,25 +152,20 @@ where
|
||||||
Ok(reader)
|
Ok(reader)
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(clippy::too_many_arguments)]
|
pub(crate) async fn process_image_stream_read<S>(
|
||||||
pub(crate) async fn process_image_stream_read(
|
state: &State<S>,
|
||||||
tmp_dir: &TmpDir,
|
|
||||||
policy_dir: &PolicyDir,
|
|
||||||
stream: LocalBoxStream<'static, std::io::Result<Bytes>>,
|
stream: LocalBoxStream<'static, std::io::Result<Bytes>>,
|
||||||
args: Vec<String>,
|
args: Vec<String>,
|
||||||
input_format: ProcessableFormat,
|
input_format: ProcessableFormat,
|
||||||
format: ProcessableFormat,
|
format: ProcessableFormat,
|
||||||
quality: Option<u8>,
|
quality: Option<u8>,
|
||||||
timeout: u64,
|
|
||||||
) -> Result<ProcessRead, MagickError> {
|
) -> Result<ProcessRead, MagickError> {
|
||||||
process_image(
|
process_image(
|
||||||
tmp_dir,
|
state,
|
||||||
policy_dir,
|
|
||||||
args,
|
args,
|
||||||
input_format,
|
input_format,
|
||||||
format,
|
format,
|
||||||
quality,
|
quality,
|
||||||
timeout,
|
|
||||||
|mut tmp_file| async move {
|
|mut tmp_file| async move {
|
||||||
tmp_file
|
tmp_file
|
||||||
.write_from_stream(stream)
|
.write_from_stream(stream)
|
||||||
|
@ -184,25 +177,20 @@ pub(crate) async fn process_image_stream_read(
|
||||||
.await
|
.await
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(clippy::too_many_arguments)]
|
|
||||||
pub(crate) async fn process_image_process_read(
|
pub(crate) async fn process_image_process_read(
|
||||||
tmp_dir: &TmpDir,
|
state: &State<S>,
|
||||||
policy_dir: &PolicyDir,
|
|
||||||
process_read: ProcessRead,
|
process_read: ProcessRead,
|
||||||
args: Vec<String>,
|
args: Vec<String>,
|
||||||
input_format: ProcessableFormat,
|
input_format: ProcessableFormat,
|
||||||
format: ProcessableFormat,
|
format: ProcessableFormat,
|
||||||
quality: Option<u8>,
|
quality: Option<u8>,
|
||||||
timeout: u64,
|
|
||||||
) -> Result<ProcessRead, MagickError> {
|
) -> Result<ProcessRead, MagickError> {
|
||||||
process_image(
|
process_image(
|
||||||
tmp_dir,
|
state,
|
||||||
policy_dir,
|
|
||||||
args,
|
args,
|
||||||
input_format,
|
input_format,
|
||||||
format,
|
format,
|
||||||
quality,
|
quality,
|
||||||
timeout,
|
|
||||||
|mut tmp_file| async move {
|
|mut tmp_file| async move {
|
||||||
process_read
|
process_read
|
||||||
.with_stdout(|stdout| async {
|
.with_stdout(|stdout| async {
|
||||||
|
|
140
src/queue.rs
140
src/queue.rs
|
@ -188,35 +188,12 @@ pub(crate) async fn queue_generate(
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) async fn process_cleanup<S: Store + 'static>(
|
pub(crate) async fn process_cleanup<S: Store + 'static>(state: State<S>) {
|
||||||
repo: ArcRepo,
|
process_jobs(state, CLEANUP_QUEUE, cleanup::perform).await
|
||||||
store: S,
|
|
||||||
config: Configuration,
|
|
||||||
) {
|
|
||||||
process_jobs(&repo, &store, &config, CLEANUP_QUEUE, cleanup::perform).await
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) async fn process_images<S: Store + 'static>(
|
pub(crate) async fn process_images<S: Store + 'static>(state: State<S>, process_map: ProcessMap) {
|
||||||
tmp_dir: ArcTmpDir,
|
process_image_jobs(state, process_map, PROCESS_QUEUE, process::perform).await
|
||||||
policy_dir: ArcPolicyDir,
|
|
||||||
repo: ArcRepo,
|
|
||||||
store: S,
|
|
||||||
client: ClientWithMiddleware,
|
|
||||||
process_map: ProcessMap,
|
|
||||||
config: Configuration,
|
|
||||||
) {
|
|
||||||
process_image_jobs(
|
|
||||||
&tmp_dir,
|
|
||||||
&policy_dir,
|
|
||||||
&repo,
|
|
||||||
&store,
|
|
||||||
&client,
|
|
||||||
&process_map,
|
|
||||||
&config,
|
|
||||||
PROCESS_QUEUE,
|
|
||||||
process::perform,
|
|
||||||
)
|
|
||||||
.await
|
|
||||||
}
|
}
|
||||||
|
|
||||||
struct MetricsGuard {
|
struct MetricsGuard {
|
||||||
|
@ -250,21 +227,10 @@ impl Drop for MetricsGuard {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn process_jobs<S, F>(
|
async fn process_jobs<S, F>(state: State<S>, queue: &'static str, callback: F)
|
||||||
repo: &ArcRepo,
|
where
|
||||||
store: &S,
|
|
||||||
config: &Configuration,
|
|
||||||
queue: &'static str,
|
|
||||||
callback: F,
|
|
||||||
) where
|
|
||||||
S: Store,
|
S: Store,
|
||||||
for<'a> F: Fn(
|
for<'a> F: Fn(&'a State<S>, serde_json::Value) -> LocalBoxFuture<'a, Result<(), Error>> + Copy,
|
||||||
&'a ArcRepo,
|
|
||||||
&'a S,
|
|
||||||
&'a Configuration,
|
|
||||||
serde_json::Value,
|
|
||||||
) -> LocalBoxFuture<'a, Result<(), Error>>
|
|
||||||
+ Copy,
|
|
||||||
{
|
{
|
||||||
let worker_id = uuid::Uuid::new_v4();
|
let worker_id = uuid::Uuid::new_v4();
|
||||||
|
|
||||||
|
@ -273,7 +239,7 @@ async fn process_jobs<S, F>(
|
||||||
|
|
||||||
tokio::task::yield_now().await;
|
tokio::task::yield_now().await;
|
||||||
|
|
||||||
let res = job_loop(repo, store, config, worker_id, queue, callback).await;
|
let res = job_loop(&state, worker_id, queue, callback).await;
|
||||||
|
|
||||||
if let Err(e) = res {
|
if let Err(e) = res {
|
||||||
tracing::warn!("Error processing jobs: {}", format!("{e}"));
|
tracing::warn!("Error processing jobs: {}", format!("{e}"));
|
||||||
|
@ -291,22 +257,14 @@ async fn process_jobs<S, F>(
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn job_loop<S, F>(
|
async fn job_loop<S, F>(
|
||||||
repo: &ArcRepo,
|
state: &State<S>,
|
||||||
store: &S,
|
|
||||||
config: &Configuration,
|
|
||||||
worker_id: uuid::Uuid,
|
worker_id: uuid::Uuid,
|
||||||
queue: &'static str,
|
queue: &'static str,
|
||||||
callback: F,
|
callback: F,
|
||||||
) -> Result<(), Error>
|
) -> Result<(), Error>
|
||||||
where
|
where
|
||||||
S: Store,
|
S: Store,
|
||||||
for<'a> F: Fn(
|
for<'a> F: Fn(&'a State<S>, serde_json::Value) -> LocalBoxFuture<'a, Result<(), Error>> + Copy,
|
||||||
&'a ArcRepo,
|
|
||||||
&'a S,
|
|
||||||
&'a Configuration,
|
|
||||||
serde_json::Value,
|
|
||||||
) -> LocalBoxFuture<'a, Result<(), Error>>
|
|
||||||
+ Copy,
|
|
||||||
{
|
{
|
||||||
loop {
|
loop {
|
||||||
tracing::trace!("job_loop: looping");
|
tracing::trace!("job_loop: looping");
|
||||||
|
@ -314,20 +272,20 @@ where
|
||||||
tokio::task::yield_now().await;
|
tokio::task::yield_now().await;
|
||||||
|
|
||||||
async {
|
async {
|
||||||
let (job_id, job) = repo.pop(queue, worker_id).await?;
|
let (job_id, job) = state.repo.pop(queue, worker_id).await?;
|
||||||
|
|
||||||
let guard = MetricsGuard::guard(worker_id, queue);
|
let guard = MetricsGuard::guard(worker_id, queue);
|
||||||
|
|
||||||
let res = heartbeat(
|
let res = heartbeat(
|
||||||
repo,
|
&state.repo,
|
||||||
queue,
|
queue,
|
||||||
worker_id,
|
worker_id,
|
||||||
job_id,
|
job_id,
|
||||||
(callback)(repo, store, config, job),
|
(callback)(state, job),
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
|
|
||||||
repo.complete_job(queue, worker_id, job_id).await?;
|
state.repo.complete_job(queue, worker_id, job_id).await?;
|
||||||
|
|
||||||
res?;
|
res?;
|
||||||
|
|
||||||
|
@ -340,29 +298,14 @@ where
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(clippy::too_many_arguments)]
|
|
||||||
async fn process_image_jobs<S, F>(
|
async fn process_image_jobs<S, F>(
|
||||||
tmp_dir: &ArcTmpDir,
|
state: State<S>,
|
||||||
policy_dir: &ArcPolicyDir,
|
process_map: ProcessMap,
|
||||||
repo: &ArcRepo,
|
|
||||||
store: &S,
|
|
||||||
client: &ClientWithMiddleware,
|
|
||||||
process_map: &ProcessMap,
|
|
||||||
config: &Configuration,
|
|
||||||
queue: &'static str,
|
queue: &'static str,
|
||||||
callback: F,
|
callback: F,
|
||||||
) where
|
) where
|
||||||
S: Store,
|
S: Store,
|
||||||
for<'a> F: Fn(
|
for<'a> F: Fn(&'a State<S>, &'a ProcessMap, serde_json::Value) -> LocalBoxFuture<'a, Result<(), Error>>
|
||||||
&'a ArcTmpDir,
|
|
||||||
&'a ArcPolicyDir,
|
|
||||||
&'a ArcRepo,
|
|
||||||
&'a S,
|
|
||||||
&'a ClientWithMiddleware,
|
|
||||||
&'a ProcessMap,
|
|
||||||
&'a Configuration,
|
|
||||||
serde_json::Value,
|
|
||||||
) -> LocalBoxFuture<'a, Result<(), Error>>
|
|
||||||
+ Copy,
|
+ Copy,
|
||||||
{
|
{
|
||||||
let worker_id = uuid::Uuid::new_v4();
|
let worker_id = uuid::Uuid::new_v4();
|
||||||
|
@ -372,19 +315,7 @@ async fn process_image_jobs<S, F>(
|
||||||
|
|
||||||
tokio::task::yield_now().await;
|
tokio::task::yield_now().await;
|
||||||
|
|
||||||
let res = image_job_loop(
|
let res = image_job_loop(&state, &process_map, worker_id, queue, callback).await;
|
||||||
tmp_dir,
|
|
||||||
policy_dir,
|
|
||||||
repo,
|
|
||||||
store,
|
|
||||||
client,
|
|
||||||
process_map,
|
|
||||||
config,
|
|
||||||
worker_id,
|
|
||||||
queue,
|
|
||||||
callback,
|
|
||||||
)
|
|
||||||
.await;
|
|
||||||
|
|
||||||
if let Err(e) = res {
|
if let Err(e) = res {
|
||||||
tracing::warn!("Error processing jobs: {}", format!("{e}"));
|
tracing::warn!("Error processing jobs: {}", format!("{e}"));
|
||||||
|
@ -403,29 +334,15 @@ async fn process_image_jobs<S, F>(
|
||||||
|
|
||||||
#[allow(clippy::too_many_arguments)]
|
#[allow(clippy::too_many_arguments)]
|
||||||
async fn image_job_loop<S, F>(
|
async fn image_job_loop<S, F>(
|
||||||
tmp_dir: &ArcTmpDir,
|
state: &State<S>,
|
||||||
policy_dir: &ArcPolicyDir,
|
|
||||||
repo: &ArcRepo,
|
|
||||||
store: &S,
|
|
||||||
client: &ClientWithMiddleware,
|
|
||||||
process_map: &ProcessMap,
|
process_map: &ProcessMap,
|
||||||
config: &Configuration,
|
|
||||||
worker_id: uuid::Uuid,
|
worker_id: uuid::Uuid,
|
||||||
queue: &'static str,
|
queue: &'static str,
|
||||||
callback: F,
|
callback: F,
|
||||||
) -> Result<(), Error>
|
) -> Result<(), Error>
|
||||||
where
|
where
|
||||||
S: Store,
|
S: Store,
|
||||||
for<'a> F: Fn(
|
for<'a> F: Fn(&'a State<S>, &'a ProcessMap, serde_json::Value) -> LocalBoxFuture<'a, Result<(), Error>>
|
||||||
&'a ArcTmpDir,
|
|
||||||
&'a ArcPolicyDir,
|
|
||||||
&'a ArcRepo,
|
|
||||||
&'a S,
|
|
||||||
&'a ClientWithMiddleware,
|
|
||||||
&'a ProcessMap,
|
|
||||||
&'a Configuration,
|
|
||||||
serde_json::Value,
|
|
||||||
) -> LocalBoxFuture<'a, Result<(), Error>>
|
|
||||||
+ Copy,
|
+ Copy,
|
||||||
{
|
{
|
||||||
loop {
|
loop {
|
||||||
|
@ -434,29 +351,20 @@ where
|
||||||
tokio::task::yield_now().await;
|
tokio::task::yield_now().await;
|
||||||
|
|
||||||
async {
|
async {
|
||||||
let (job_id, job) = repo.pop(queue, worker_id).await?;
|
let (job_id, job) = state.repo.pop(queue, worker_id).await?;
|
||||||
|
|
||||||
let guard = MetricsGuard::guard(worker_id, queue);
|
let guard = MetricsGuard::guard(worker_id, queue);
|
||||||
|
|
||||||
let res = heartbeat(
|
let res = heartbeat(
|
||||||
repo,
|
&state.repo,
|
||||||
queue,
|
queue,
|
||||||
worker_id,
|
worker_id,
|
||||||
job_id,
|
job_id,
|
||||||
(callback)(
|
(callback)(state, process_map, job),
|
||||||
tmp_dir,
|
|
||||||
policy_dir,
|
|
||||||
repo,
|
|
||||||
store,
|
|
||||||
client,
|
|
||||||
process_map,
|
|
||||||
config,
|
|
||||||
job,
|
|
||||||
),
|
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
|
|
||||||
repo.complete_job(queue, worker_id, job_id).await?;
|
state.repo.complete_job(queue, worker_id, job_id).await?;
|
||||||
|
|
||||||
res?;
|
res?;
|
||||||
|
|
||||||
|
|
|
@ -10,13 +10,12 @@ use crate::{
|
||||||
queue::Cleanup,
|
queue::Cleanup,
|
||||||
repo::{Alias, ArcRepo, DeleteToken, Hash},
|
repo::{Alias, ArcRepo, DeleteToken, Hash},
|
||||||
serde_str::Serde,
|
serde_str::Serde,
|
||||||
|
state::State,
|
||||||
store::Store,
|
store::Store,
|
||||||
};
|
};
|
||||||
|
|
||||||
pub(super) fn perform<'a, S>(
|
pub(super) fn perform<'a, S>(
|
||||||
repo: &'a ArcRepo,
|
state: &'a State<S>,
|
||||||
store: &'a S,
|
|
||||||
configuration: &'a Configuration,
|
|
||||||
job: serde_json::Value,
|
job: serde_json::Value,
|
||||||
) -> LocalBoxFuture<'a, Result<(), Error>>
|
) -> LocalBoxFuture<'a, Result<(), Error>>
|
||||||
where
|
where
|
||||||
|
@ -25,26 +24,28 @@ where
|
||||||
Box::pin(async move {
|
Box::pin(async move {
|
||||||
match serde_json::from_value(job) {
|
match serde_json::from_value(job) {
|
||||||
Ok(job) => match job {
|
Ok(job) => match job {
|
||||||
Cleanup::Hash { hash: in_hash } => hash(repo, in_hash).await?,
|
Cleanup::Hash { hash: in_hash } => hash(&state.repo, in_hash).await?,
|
||||||
Cleanup::Identifier {
|
Cleanup::Identifier {
|
||||||
identifier: in_identifier,
|
identifier: in_identifier,
|
||||||
} => identifier(repo, store, Arc::from(in_identifier)).await?,
|
} => identifier(&state.repo, &state.store, Arc::from(in_identifier)).await?,
|
||||||
Cleanup::Alias {
|
Cleanup::Alias {
|
||||||
alias: stored_alias,
|
alias: stored_alias,
|
||||||
token,
|
token,
|
||||||
} => {
|
} => {
|
||||||
alias(
|
alias(
|
||||||
repo,
|
&state.repo,
|
||||||
Serde::into_inner(stored_alias),
|
Serde::into_inner(stored_alias),
|
||||||
Serde::into_inner(token),
|
Serde::into_inner(token),
|
||||||
)
|
)
|
||||||
.await?
|
.await?
|
||||||
}
|
}
|
||||||
Cleanup::Variant { hash, variant } => hash_variant(repo, hash, variant).await?,
|
Cleanup::Variant { hash, variant } => {
|
||||||
Cleanup::AllVariants => all_variants(repo).await?,
|
hash_variant(&state.repo, hash, variant).await?
|
||||||
Cleanup::OutdatedVariants => outdated_variants(repo, configuration).await?,
|
}
|
||||||
Cleanup::OutdatedProxies => outdated_proxies(repo, configuration).await?,
|
Cleanup::AllVariants => all_variants(&state.repo).await?,
|
||||||
Cleanup::Prune => prune(repo, store).await?,
|
Cleanup::OutdatedVariants => outdated_variants(&state.repo, &state.config).await?,
|
||||||
|
Cleanup::OutdatedProxies => outdated_proxies(&state.repo, &state.config).await?,
|
||||||
|
Cleanup::Prune => prune(&state.repo, &state.store).await?,
|
||||||
},
|
},
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
tracing::warn!("Invalid job: {}", format!("{e}"));
|
tracing::warn!("Invalid job: {}", format!("{e}"));
|
||||||
|
|
|
@ -13,20 +13,15 @@ use crate::{
|
||||||
queue::Process,
|
queue::Process,
|
||||||
repo::{Alias, ArcRepo, UploadId, UploadResult},
|
repo::{Alias, ArcRepo, UploadId, UploadResult},
|
||||||
serde_str::Serde,
|
serde_str::Serde,
|
||||||
|
state::State,
|
||||||
store::Store,
|
store::Store,
|
||||||
tmp_file::{ArcTmpDir, TmpDir},
|
tmp_file::{ArcTmpDir, TmpDir},
|
||||||
};
|
};
|
||||||
use std::{path::PathBuf, sync::Arc};
|
use std::{path::PathBuf, sync::Arc};
|
||||||
|
|
||||||
#[allow(clippy::too_many_arguments)]
|
|
||||||
pub(super) fn perform<'a, S>(
|
pub(super) fn perform<'a, S>(
|
||||||
tmp_dir: &'a ArcTmpDir,
|
state: &'a State<S>,
|
||||||
policy_dir: &'a ArcPolicyDir,
|
|
||||||
repo: &'a ArcRepo,
|
|
||||||
store: &'a S,
|
|
||||||
client: &'a ClientWithMiddleware,
|
|
||||||
process_map: &'a ProcessMap,
|
process_map: &'a ProcessMap,
|
||||||
config: &'a Configuration,
|
|
||||||
job: serde_json::Value,
|
job: serde_json::Value,
|
||||||
) -> LocalBoxFuture<'a, Result<(), Error>>
|
) -> LocalBoxFuture<'a, Result<(), Error>>
|
||||||
where
|
where
|
||||||
|
@ -41,15 +36,10 @@ where
|
||||||
declared_alias,
|
declared_alias,
|
||||||
} => {
|
} => {
|
||||||
process_ingest(
|
process_ingest(
|
||||||
tmp_dir,
|
state,
|
||||||
policy_dir,
|
|
||||||
repo,
|
|
||||||
store,
|
|
||||||
client,
|
|
||||||
Arc::from(identifier),
|
Arc::from(identifier),
|
||||||
Serde::into_inner(upload_id),
|
Serde::into_inner(upload_id),
|
||||||
declared_alias.map(Serde::into_inner),
|
declared_alias.map(Serde::into_inner),
|
||||||
config,
|
|
||||||
)
|
)
|
||||||
.await?
|
.await?
|
||||||
}
|
}
|
||||||
|
@ -60,16 +50,12 @@ where
|
||||||
process_args,
|
process_args,
|
||||||
} => {
|
} => {
|
||||||
generate(
|
generate(
|
||||||
tmp_dir,
|
state,
|
||||||
policy_dir,
|
|
||||||
repo,
|
|
||||||
store,
|
|
||||||
process_map,
|
process_map,
|
||||||
target_format,
|
target_format,
|
||||||
Serde::into_inner(source),
|
Serde::into_inner(source),
|
||||||
process_path,
|
process_path,
|
||||||
process_args,
|
process_args,
|
||||||
config,
|
|
||||||
)
|
)
|
||||||
.await?
|
.await?
|
||||||
}
|
}
|
||||||
|
@ -117,18 +103,12 @@ impl Drop for UploadGuard {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(clippy::too_many_arguments)]
|
#[tracing::instrument(skip(state))]
|
||||||
#[tracing::instrument(skip(tmp_dir, policy_dir, repo, store, client, config))]
|
|
||||||
async fn process_ingest<S>(
|
async fn process_ingest<S>(
|
||||||
tmp_dir: &ArcTmpDir,
|
state: &State<S>,
|
||||||
policy_dir: &ArcPolicyDir,
|
|
||||||
repo: &ArcRepo,
|
|
||||||
store: &S,
|
|
||||||
client: &ClientWithMiddleware,
|
|
||||||
unprocessed_identifier: Arc<str>,
|
unprocessed_identifier: Arc<str>,
|
||||||
upload_id: UploadId,
|
upload_id: UploadId,
|
||||||
declared_alias: Option<Alias>,
|
declared_alias: Option<Alias>,
|
||||||
config: &Configuration,
|
|
||||||
) -> Result<(), Error>
|
) -> Result<(), Error>
|
||||||
where
|
where
|
||||||
S: Store + 'static,
|
S: Store + 'static,
|
||||||
|
@ -136,33 +116,18 @@ where
|
||||||
let guard = UploadGuard::guard(upload_id);
|
let guard = UploadGuard::guard(upload_id);
|
||||||
|
|
||||||
let fut = async {
|
let fut = async {
|
||||||
let tmp_dir = tmp_dir.clone();
|
let state2 = state.clone();
|
||||||
let policy_dir = policy_dir.clone();
|
|
||||||
let ident = unprocessed_identifier.clone();
|
|
||||||
let store2 = store.clone();
|
|
||||||
let repo = repo.clone();
|
|
||||||
let client = client.clone();
|
|
||||||
|
|
||||||
let current_span = Span::current();
|
let current_span = Span::current();
|
||||||
let span = tracing::info_span!(parent: current_span, "error_boundary");
|
let span = tracing::info_span!(parent: current_span, "error_boundary");
|
||||||
|
|
||||||
let config = config.clone();
|
|
||||||
let error_boundary = crate::sync::abort_on_drop(crate::sync::spawn(
|
let error_boundary = crate::sync::abort_on_drop(crate::sync::spawn(
|
||||||
"ingest-media",
|
"ingest-media",
|
||||||
async move {
|
async move {
|
||||||
let stream = crate::stream::from_err(store2.to_stream(&ident, None, None).await?);
|
let stream =
|
||||||
|
crate::stream::from_err(state2.store.to_stream(&ident, None, None).await?);
|
||||||
|
|
||||||
let session = crate::ingest::ingest(
|
let session = crate::ingest::ingest(&state2, stream, declared_alias).await?;
|
||||||
&tmp_dir,
|
|
||||||
&policy_dir,
|
|
||||||
&repo,
|
|
||||||
&store2,
|
|
||||||
&client,
|
|
||||||
stream,
|
|
||||||
declared_alias,
|
|
||||||
&config,
|
|
||||||
)
|
|
||||||
.await?;
|
|
||||||
|
|
||||||
Ok(session) as Result<Session, Error>
|
Ok(session) as Result<Session, Error>
|
||||||
}
|
}
|
||||||
|
@ -170,7 +135,7 @@ where
|
||||||
))
|
))
|
||||||
.await;
|
.await;
|
||||||
|
|
||||||
store.remove(&unprocessed_identifier).await?;
|
state.store.remove(&unprocessed_identifier).await?;
|
||||||
|
|
||||||
error_boundary.map_err(|_| UploadError::Canceled)?
|
error_boundary.map_err(|_| UploadError::Canceled)?
|
||||||
};
|
};
|
||||||
|
@ -191,62 +156,46 @@ where
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
repo.complete_upload(upload_id, result).await?;
|
state.repo.complete_upload(upload_id, result).await?;
|
||||||
|
|
||||||
guard.disarm();
|
guard.disarm();
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(clippy::too_many_arguments)]
|
#[tracing::instrument(skip(state, process_map, process_path, process_args))]
|
||||||
#[tracing::instrument(skip(
|
|
||||||
tmp_dir,
|
|
||||||
policy_dir,
|
|
||||||
repo,
|
|
||||||
store,
|
|
||||||
process_map,
|
|
||||||
process_path,
|
|
||||||
process_args,
|
|
||||||
config
|
|
||||||
))]
|
|
||||||
async fn generate<S: Store + 'static>(
|
async fn generate<S: Store + 'static>(
|
||||||
tmp_dir: &TmpDir,
|
state: &State<S>,
|
||||||
policy_dir: &PolicyDir,
|
|
||||||
repo: &ArcRepo,
|
|
||||||
store: &S,
|
|
||||||
process_map: &ProcessMap,
|
process_map: &ProcessMap,
|
||||||
target_format: InputProcessableFormat,
|
target_format: InputProcessableFormat,
|
||||||
source: Alias,
|
source: Alias,
|
||||||
process_path: PathBuf,
|
process_path: PathBuf,
|
||||||
process_args: Vec<String>,
|
process_args: Vec<String>,
|
||||||
config: &Configuration,
|
|
||||||
) -> Result<(), Error> {
|
) -> Result<(), Error> {
|
||||||
let Some(hash) = repo.hash(&source).await? else {
|
let Some(hash) = state.repo.hash(&source).await? else {
|
||||||
// Nothing to do
|
// Nothing to do
|
||||||
return Ok(());
|
return Ok(());
|
||||||
};
|
};
|
||||||
|
|
||||||
let path_string = process_path.to_string_lossy().to_string();
|
let path_string = process_path.to_string_lossy().to_string();
|
||||||
let identifier_opt = repo.variant_identifier(hash.clone(), path_string).await?;
|
let identifier_opt = state
|
||||||
|
.repo
|
||||||
|
.variant_identifier(hash.clone(), path_string)
|
||||||
|
.await?;
|
||||||
|
|
||||||
if identifier_opt.is_some() {
|
if identifier_opt.is_some() {
|
||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
}
|
||||||
|
|
||||||
let original_details =
|
let original_details = crate::ensure_details(state, &source).await?;
|
||||||
crate::ensure_details(tmp_dir, policy_dir, repo, store, config, &source).await?;
|
|
||||||
|
|
||||||
crate::generate::generate(
|
crate::generate::generate(
|
||||||
tmp_dir,
|
state,
|
||||||
policy_dir,
|
|
||||||
repo,
|
|
||||||
store,
|
|
||||||
process_map,
|
process_map,
|
||||||
target_format,
|
target_format,
|
||||||
process_path,
|
process_path,
|
||||||
process_args,
|
process_args,
|
||||||
&original_details,
|
&original_details,
|
||||||
config,
|
|
||||||
hash,
|
hash,
|
||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
|
|
13
src/state.rs
Normal file
13
src/state.rs
Normal file
|
@ -0,0 +1,13 @@
|
||||||
|
use reqwest_middleware::ClientWithMiddleware;
|
||||||
|
|
||||||
|
use crate::{config::Configuration, magick::ArcPolicyDir, repo::ArcRepo, tmp_file::ArcTmpDir};
|
||||||
|
|
||||||
|
#[derive(Clone)]
|
||||||
|
pub(crate) struct State<S> {
|
||||||
|
pub(super) config: Configuration,
|
||||||
|
pub(super) tmp_dir: ArcTmpDir,
|
||||||
|
pub(super) policy_dir: ArcPolicyDir,
|
||||||
|
pub(super) repo: ArcRepo,
|
||||||
|
pub(super) store: S,
|
||||||
|
pub(super) client: ClientWithMiddleware,
|
||||||
|
}
|
108
src/validate.rs
108
src/validate.rs
|
@ -8,10 +8,11 @@ use crate::{
|
||||||
error_code::ErrorCode,
|
error_code::ErrorCode,
|
||||||
formats::{
|
formats::{
|
||||||
AnimationFormat, AnimationOutput, ImageInput, ImageOutput, InputFile, InputVideoFormat,
|
AnimationFormat, AnimationOutput, ImageInput, ImageOutput, InputFile, InputVideoFormat,
|
||||||
InternalFormat, Validations,
|
InternalFormat,
|
||||||
},
|
},
|
||||||
magick::PolicyDir,
|
magick::PolicyDir,
|
||||||
process::ProcessRead,
|
process::ProcessRead,
|
||||||
|
state::State,
|
||||||
tmp_file::TmpDir,
|
tmp_file::TmpDir,
|
||||||
};
|
};
|
||||||
use actix_web::web::Bytes;
|
use actix_web::web::Bytes;
|
||||||
|
@ -57,12 +58,9 @@ impl ValidationError {
|
||||||
const MEGABYTES: usize = 1024 * 1024;
|
const MEGABYTES: usize = 1024 * 1024;
|
||||||
|
|
||||||
#[tracing::instrument(skip_all)]
|
#[tracing::instrument(skip_all)]
|
||||||
pub(crate) async fn validate_bytes(
|
pub(crate) async fn validate_bytes<S>(
|
||||||
tmp_dir: &TmpDir,
|
state: &State<S>,
|
||||||
policy_dir: &PolicyDir,
|
|
||||||
bytes: Bytes,
|
bytes: Bytes,
|
||||||
validations: Validations<'_>,
|
|
||||||
timeout: u64,
|
|
||||||
) -> Result<(InternalFormat, ProcessRead), Error> {
|
) -> Result<(InternalFormat, ProcessRead), Error> {
|
||||||
if bytes.is_empty() {
|
if bytes.is_empty() {
|
||||||
return Err(ValidationError::Empty.into());
|
return Err(ValidationError::Empty.into());
|
||||||
|
@ -77,66 +75,35 @@ pub(crate) async fn validate_bytes(
|
||||||
|
|
||||||
match &input {
|
match &input {
|
||||||
InputFile::Image(input) => {
|
InputFile::Image(input) => {
|
||||||
let (format, process_read) = process_image(
|
let (format, process_read) = process_image(state, bytes, *input, width, height).await?;
|
||||||
tmp_dir,
|
|
||||||
policy_dir,
|
|
||||||
bytes,
|
|
||||||
*input,
|
|
||||||
width,
|
|
||||||
height,
|
|
||||||
validations.image,
|
|
||||||
timeout,
|
|
||||||
)
|
|
||||||
.await?;
|
|
||||||
|
|
||||||
Ok((format, process_read))
|
Ok((format, process_read))
|
||||||
}
|
}
|
||||||
InputFile::Animation(input) => {
|
InputFile::Animation(input) => {
|
||||||
let (format, process_read) = process_animation(
|
let (format, process_read) =
|
||||||
tmp_dir,
|
process_animation(state, bytes, *input, width, height, frames.unwrap_or(1)).await?;
|
||||||
policy_dir,
|
|
||||||
bytes,
|
|
||||||
*input,
|
|
||||||
width,
|
|
||||||
height,
|
|
||||||
frames.unwrap_or(1),
|
|
||||||
validations.animation,
|
|
||||||
timeout,
|
|
||||||
)
|
|
||||||
.await?;
|
|
||||||
|
|
||||||
Ok((format, process_read))
|
Ok((format, process_read))
|
||||||
}
|
}
|
||||||
InputFile::Video(input) => {
|
InputFile::Video(input) => {
|
||||||
let (format, process_read) = process_video(
|
let (format, process_read) =
|
||||||
tmp_dir,
|
process_video(state, bytes, *input, width, height, frames.unwrap_or(1)).await?;
|
||||||
bytes,
|
|
||||||
*input,
|
|
||||||
width,
|
|
||||||
height,
|
|
||||||
frames.unwrap_or(1),
|
|
||||||
validations.video,
|
|
||||||
timeout,
|
|
||||||
)
|
|
||||||
.await?;
|
|
||||||
|
|
||||||
Ok((format, process_read))
|
Ok((format, process_read))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(clippy::too_many_arguments)]
|
#[tracing::instrument(skip(state, bytes))]
|
||||||
#[tracing::instrument(skip(tmp_dir, policy_dir, bytes, validations))]
|
async fn process_image<S>(
|
||||||
async fn process_image(
|
state: &State<S>,
|
||||||
tmp_dir: &TmpDir,
|
|
||||||
policy_dir: &PolicyDir,
|
|
||||||
bytes: Bytes,
|
bytes: Bytes,
|
||||||
input: ImageInput,
|
input: ImageInput,
|
||||||
width: u16,
|
width: u16,
|
||||||
height: u16,
|
height: u16,
|
||||||
validations: &crate::config::Image,
|
|
||||||
timeout: u64,
|
|
||||||
) -> Result<(InternalFormat, ProcessRead), Error> {
|
) -> Result<(InternalFormat, ProcessRead), Error> {
|
||||||
|
let validations = &state.config.media.image;
|
||||||
|
|
||||||
if width > validations.max_width {
|
if width > validations.max_width {
|
||||||
return Err(ValidationError::Width.into());
|
return Err(ValidationError::Width.into());
|
||||||
}
|
}
|
||||||
|
@ -158,16 +125,7 @@ async fn process_image(
|
||||||
let process_read = if needs_transcode {
|
let process_read = if needs_transcode {
|
||||||
let quality = validations.quality_for(format);
|
let quality = validations.quality_for(format);
|
||||||
|
|
||||||
magick::convert_image(
|
magick::convert_image(state, input.format, format, quality, bytes).await?
|
||||||
tmp_dir,
|
|
||||||
policy_dir,
|
|
||||||
input.format,
|
|
||||||
format,
|
|
||||||
quality,
|
|
||||||
timeout,
|
|
||||||
bytes,
|
|
||||||
)
|
|
||||||
.await?
|
|
||||||
} else {
|
} else {
|
||||||
exiftool::clear_metadata_bytes_read(bytes, timeout)?
|
exiftool::clear_metadata_bytes_read(bytes, timeout)?
|
||||||
};
|
};
|
||||||
|
@ -201,19 +159,17 @@ fn validate_animation(
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(clippy::too_many_arguments)]
|
#[tracing::instrument(skip(state, bytes))]
|
||||||
#[tracing::instrument(skip(tmp_dir, policy_dir, bytes, validations))]
|
|
||||||
async fn process_animation(
|
async fn process_animation(
|
||||||
tmp_dir: &TmpDir,
|
state: &State<S>,
|
||||||
policy_dir: &PolicyDir,
|
|
||||||
bytes: Bytes,
|
bytes: Bytes,
|
||||||
input: AnimationFormat,
|
input: AnimationFormat,
|
||||||
width: u16,
|
width: u16,
|
||||||
height: u16,
|
height: u16,
|
||||||
frames: u32,
|
frames: u32,
|
||||||
validations: &crate::config::Animation,
|
|
||||||
timeout: u64,
|
|
||||||
) -> Result<(InternalFormat, ProcessRead), Error> {
|
) -> Result<(InternalFormat, ProcessRead), Error> {
|
||||||
|
let validations = &state.config.media.animation;
|
||||||
|
|
||||||
validate_animation(bytes.len(), width, height, frames, validations)?;
|
validate_animation(bytes.len(), width, height, frames, validations)?;
|
||||||
|
|
||||||
let AnimationOutput {
|
let AnimationOutput {
|
||||||
|
@ -224,10 +180,9 @@ async fn process_animation(
|
||||||
let process_read = if needs_transcode {
|
let process_read = if needs_transcode {
|
||||||
let quality = validations.quality_for(format);
|
let quality = validations.quality_for(format);
|
||||||
|
|
||||||
magick::convert_animation(tmp_dir, policy_dir, input, format, quality, timeout, bytes)
|
magick::convert_animation(state, input, format, quality, bytes).await?
|
||||||
.await?
|
|
||||||
} else {
|
} else {
|
||||||
exiftool::clear_metadata_bytes_read(bytes, timeout)?
|
exiftool::clear_metadata_bytes_read(bytes, state.config.media.process_timeout)?
|
||||||
};
|
};
|
||||||
|
|
||||||
Ok((InternalFormat::Animation(format), process_read))
|
Ok((InternalFormat::Animation(format), process_read))
|
||||||
|
@ -262,18 +217,17 @@ fn validate_video(
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(clippy::too_many_arguments)]
|
#[tracing::instrument(skip(state, bytes))]
|
||||||
#[tracing::instrument(skip(tmp_dir, bytes, validations))]
|
async fn process_video<S>(
|
||||||
async fn process_video(
|
state: &State<S>,
|
||||||
tmp_dir: &TmpDir,
|
|
||||||
bytes: Bytes,
|
bytes: Bytes,
|
||||||
input: InputVideoFormat,
|
input: InputVideoFormat,
|
||||||
width: u16,
|
width: u16,
|
||||||
height: u16,
|
height: u16,
|
||||||
frames: u32,
|
frames: u32,
|
||||||
validations: &crate::config::Video,
|
|
||||||
timeout: u64,
|
|
||||||
) -> Result<(InternalFormat, ProcessRead), Error> {
|
) -> Result<(InternalFormat, ProcessRead), Error> {
|
||||||
|
let validations = &state.config.media.video;
|
||||||
|
|
||||||
validate_video(bytes.len(), width, height, frames, validations)?;
|
validate_video(bytes.len(), width, height, frames, validations)?;
|
||||||
|
|
||||||
let output = input.build_output(
|
let output = input.build_output(
|
||||||
|
@ -284,7 +238,15 @@ async fn process_video(
|
||||||
|
|
||||||
let crf = validations.crf_for(width, height);
|
let crf = validations.crf_for(width, height);
|
||||||
|
|
||||||
let process_read = ffmpeg::transcode_bytes(tmp_dir, input, output, crf, timeout, bytes).await?;
|
let process_read = ffmpeg::transcode_bytes(
|
||||||
|
&state.tmp_dir,
|
||||||
|
input,
|
||||||
|
output,
|
||||||
|
crf,
|
||||||
|
state.config.media.process_timeout,
|
||||||
|
bytes,
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
|
||||||
Ok((
|
Ok((
|
||||||
InternalFormat::Video(output.format.internal_format()),
|
InternalFormat::Video(output.format.internal_format()),
|
||||||
|
|
|
@ -9,67 +9,57 @@ use crate::{
|
||||||
tmp_file::TmpDir,
|
tmp_file::TmpDir,
|
||||||
};
|
};
|
||||||
|
|
||||||
pub(super) async fn convert_image(
|
pub(super) async fn convert_image<S>(
|
||||||
tmp_dir: &TmpDir,
|
state: &State<S>,
|
||||||
policy_dir: &PolicyDir,
|
|
||||||
input: ImageFormat,
|
input: ImageFormat,
|
||||||
output: ImageFormat,
|
output: ImageFormat,
|
||||||
quality: Option<u8>,
|
quality: Option<u8>,
|
||||||
timeout: u64,
|
|
||||||
bytes: Bytes,
|
bytes: Bytes,
|
||||||
) -> Result<ProcessRead, MagickError> {
|
) -> Result<ProcessRead, MagickError> {
|
||||||
convert(
|
convert(
|
||||||
tmp_dir,
|
state,
|
||||||
policy_dir,
|
|
||||||
input.magick_format(),
|
input.magick_format(),
|
||||||
output.magick_format(),
|
output.magick_format(),
|
||||||
false,
|
false,
|
||||||
quality,
|
quality,
|
||||||
timeout,
|
|
||||||
bytes,
|
bytes,
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(super) async fn convert_animation(
|
pub(super) async fn convert_animation<S>(
|
||||||
tmp_dir: &TmpDir,
|
state: &State<S>,
|
||||||
policy_dir: &PolicyDir,
|
|
||||||
input: AnimationFormat,
|
input: AnimationFormat,
|
||||||
output: AnimationFormat,
|
output: AnimationFormat,
|
||||||
quality: Option<u8>,
|
quality: Option<u8>,
|
||||||
timeout: u64,
|
|
||||||
bytes: Bytes,
|
bytes: Bytes,
|
||||||
) -> Result<ProcessRead, MagickError> {
|
) -> Result<ProcessRead, MagickError> {
|
||||||
convert(
|
convert(
|
||||||
tmp_dir,
|
state,
|
||||||
policy_dir,
|
|
||||||
input.magick_format(),
|
input.magick_format(),
|
||||||
output.magick_format(),
|
output.magick_format(),
|
||||||
true,
|
true,
|
||||||
quality,
|
quality,
|
||||||
timeout,
|
timeout,
|
||||||
bytes,
|
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(clippy::too_many_arguments)]
|
|
||||||
async fn convert(
|
async fn convert(
|
||||||
tmp_dir: &TmpDir,
|
state: &State<S>,
|
||||||
policy_dir: &PolicyDir,
|
|
||||||
input: &'static str,
|
input: &'static str,
|
||||||
output: &'static str,
|
output: &'static str,
|
||||||
coalesce: bool,
|
coalesce: bool,
|
||||||
quality: Option<u8>,
|
quality: Option<u8>,
|
||||||
timeout: u64,
|
|
||||||
bytes: Bytes,
|
bytes: Bytes,
|
||||||
) -> Result<ProcessRead, MagickError> {
|
) -> Result<ProcessRead, MagickError> {
|
||||||
let temporary_path = tmp_dir
|
let temporary_path = state
|
||||||
|
.tmp_dir
|
||||||
.tmp_folder()
|
.tmp_folder()
|
||||||
.await
|
.await
|
||||||
.map_err(MagickError::CreateTemporaryDirectory)?;
|
.map_err(MagickError::CreateTemporaryDirectory)?;
|
||||||
|
|
||||||
let input_file = tmp_dir.tmp_file(None);
|
let input_file = state.tmp_dir.tmp_file(None);
|
||||||
|
|
||||||
crate::store::file_store::safe_create_parent(&input_file)
|
crate::store::file_store::safe_create_parent(&input_file)
|
||||||
.await
|
.await
|
||||||
|
@ -104,10 +94,10 @@ async fn convert(
|
||||||
|
|
||||||
let envs = [
|
let envs = [
|
||||||
(MAGICK_TEMPORARY_PATH, temporary_path.as_os_str()),
|
(MAGICK_TEMPORARY_PATH, temporary_path.as_os_str()),
|
||||||
(MAGICK_CONFIGURE_PATH, policy_dir.as_os_str()),
|
(MAGICK_CONFIGURE_PATH, state.policy_dir.as_os_str()),
|
||||||
];
|
];
|
||||||
|
|
||||||
let reader = Process::run("magick", &args, &envs, timeout)?.read();
|
let reader = Process::run("magick", &args, &envs, state.config.media.process_timeout)?.read();
|
||||||
|
|
||||||
let clean_reader = reader.add_extras(input_file).add_extras(temporary_path);
|
let clean_reader = reader.add_extras(input_file).add_extras(temporary_path);
|
||||||
|
|
||||||
|
|
Loading…
Reference in a new issue