From 58d97655945d0a4e06479548e920ef30ee9544b7 Mon Sep 17 00:00:00 2001 From: asonix Date: Tue, 11 Jul 2023 23:11:23 -0500 Subject: [PATCH] WIP: Rework format detection and conversions - Add a bunch of types & methods that aren't used yet. The idea is creating the output from the known input parameters is pure, and works for any uploaded media - Introduce a new kind of media "animation" which is neither an image nor a video. It is likely that animations can be processed properly at the /image/process.{ext} endpoint with a little massaging TODO: - Integrate this into exiftool, ffmpeg, magick commands - Detect apng with ffmpeg (imagemagick treats it as a still png unless it's given a hint) - Infallible conversion from Details to OutputFile - this might be impossible given that we don't currently store more than a mime type and a frame count to give any sort of indication of real type - Try not to break API --- src/formats.rs | 576 +++++++++++++++++++++++++++++++++++++++++++++++++ src/lib.rs | 1 + 2 files changed, 577 insertions(+) create mode 100644 src/formats.rs diff --git a/src/formats.rs b/src/formats.rs new file mode 100644 index 0000000..a18bb36 --- /dev/null +++ b/src/formats.rs @@ -0,0 +1,576 @@ +fn image_apng() -> mime::Mime { + "image/apng".parse().unwrap() +} + +fn image_avif() -> mime::Mime { + "image/avif".parse().unwrap() +} + +fn image_jxl() -> mime::Mime { + "image/jxl".parse().unwrap() +} + +fn image_webp() -> mime::Mime { + "image/webp".parse().unwrap() +} + +fn video_mp4() -> mime::Mime { + "video/mp4".parse().unwrap() +} + +fn video_webm() -> mime::Mime { + "video/webm".parse().unwrap() +} + +#[derive(Clone, Debug)] +pub(crate) struct PrescribedFormats { + image: Option, + animation: Option, + video: Option, + allow_audio: bool, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, serde::Deserialize)] +pub(crate) enum ImageFormat { + #[serde(rename = "avif")] + Avif, + #[serde(rename = "png")] + Png, + #[serde(rename = "jpeg")] + Jpeg, + #[serde(rename = "jxl")] + Jxl, + #[serde(rename = "webp")] + Webp, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, serde::Deserialize)] +pub(crate) enum AnimationFormat { + #[serde(rename = "apng")] + Apng, + #[serde(rename = "avif")] + Avif, + #[serde(rename = "gif")] + Gif, + #[serde(rename = "webp")] + Webp, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub(crate) enum VideoFormat { + Mp4, + Webp { alpha: bool }, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, serde::Deserialize)] +pub(crate) enum VideoCodec { + #[serde(rename = "av1")] + Av1, + #[serde(rename = "h264")] + H264, + #[serde(rename = "h265")] + H265, + #[serde(rename = "vp8")] + Vp8, + #[serde(rename = "vp9")] + Vp9, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, serde::Deserialize)] +pub(crate) enum AudioCodec { + #[serde(rename = "aac")] + Aac, + #[serde(rename = "opus")] + Opus, + #[serde(rename = "vorbis")] + Vorbis, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub(crate) enum OutputVideoFormat { + Mp4 { + video_codec: Mp4Codec, + audio_codec: Option, + }, + Webm { + video_codec: WebmCodec, + audio_codec: Option, + }, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, serde::Deserialize)] +pub(crate) enum Mp4Codec { + #[serde(rename = "h264")] + H264, + #[serde(rename = "h265")] + H265, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, serde::Deserialize)] +pub(crate) enum WebmAlphaCodec { + #[serde(rename = "vp8")] + Vp8, + #[serde(rename = "vp9")] + Vp9, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub(crate) struct AlphaCodec { + pub(crate) alpha: bool, + pub(crate) codec: WebmAlphaCodec, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub(crate) enum WebmCodec { + Av1, + Alpha(AlphaCodec), +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub(crate) enum Mp4AudioCodec { + Aac, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub(crate) enum WebmAudioCodec { + Opus, + Vorbis, +} + +#[derive(Clone, Debug)] +pub(crate) enum InputFile { + Image { + format: ImageFormat, + needs_reorient: bool, + }, + Animation(AnimationFormat), + Video(VideoFormat), +} + +#[derive(Clone, Debug)] +pub(crate) enum OutputFile { + Image { + format: ImageFormat, + needs_transcode: bool, + }, + Animation { + format: AnimationFormat, + needs_transcode: bool, + }, + Video(OutputVideoFormat), +} + +impl InputFile { + const fn file_extension(&self) -> &'static str { + match self { + Self::Image { format, .. } => format.file_extension(), + Self::Animation(format) => format.file_extension(), + Self::Video(format) => format.file_extension(), + } + } + + const fn build_output(&self, prescribed: &PrescribedFormats) -> OutputFile { + match (self, prescribed) { + ( + InputFile::Image { + format, + needs_reorient, + }, + PrescribedFormats { + image: Some(prescribed), + .. + }, + ) => OutputFile::Image { + format: *prescribed, + needs_transcode: *needs_reorient || !format.const_eq(*prescribed), + }, + ( + InputFile::Animation(format), + PrescribedFormats { + animation: Some(prescribed), + .. + }, + ) => OutputFile::Animation { + format: *prescribed, + needs_transcode: !format.const_eq(*prescribed), + }, + ( + InputFile::Video(VideoFormat::Webp { alpha }), + PrescribedFormats { + video: + Some(OutputVideoFormat::Webm { + video_codec: WebmCodec::Alpha(AlphaCodec { codec, .. }), + audio_codec, + }), + .. + }, + ) => OutputFile::Video(OutputVideoFormat::Webm { + video_codec: WebmCodec::Alpha(AlphaCodec { + alpha: *alpha, + codec: *codec, + }), + audio_codec: *audio_codec, + }), + ( + InputFile::Video(_), + PrescribedFormats { + video: Some(prescribed), + .. + }, + ) => OutputFile::Video(*prescribed), + ( + InputFile::Image { + format, + needs_reorient, + }, + PrescribedFormats { image: None, .. }, + ) => OutputFile::Image { + format: *format, + needs_transcode: *needs_reorient, + }, + ( + InputFile::Animation(input), + PrescribedFormats { + animation: None, .. + }, + ) => OutputFile::Animation { + format: *input, + needs_transcode: false, + }, + ( + InputFile::Video(input), + PrescribedFormats { + video: None, + allow_audio: true, + .. + }, + ) => match input { + VideoFormat::Mp4 => OutputFile::Video(OutputVideoFormat::Mp4 { + video_codec: Mp4Codec::H264, + audio_codec: Some(Mp4AudioCodec::Aac), + }), + VideoFormat::Webp { alpha } => OutputFile::Video(OutputVideoFormat::Webm { + video_codec: WebmCodec::Alpha(AlphaCodec { + alpha: *alpha, + codec: WebmAlphaCodec::Vp9, + }), + audio_codec: Some(WebmAudioCodec::Opus), + }), + }, + ( + InputFile::Video(input), + PrescribedFormats { + video: None, + allow_audio: false, + .. + }, + ) => match input { + VideoFormat::Mp4 => OutputFile::Video(OutputVideoFormat::Mp4 { + video_codec: Mp4Codec::H264, + audio_codec: None, + }), + VideoFormat::Webp { alpha } => OutputFile::Video(OutputVideoFormat::Webm { + video_codec: WebmCodec::Alpha(AlphaCodec { + alpha: *alpha, + codec: WebmAlphaCodec::Vp9, + }), + audio_codec: None, + }), + }, + } + } +} + +impl ImageFormat { + const fn const_eq(self, rhs: Self) -> bool { + match (self, rhs) { + (Self::Avif, Self::Avif) + | (Self::Jpeg, Self::Jpeg) + | (Self::Jxl, Self::Jxl) + | (Self::Png, Self::Png) + | (Self::Webp, Self::Webp) => true, + (Self::Avif, _) + | (Self::Jpeg, _) + | (Self::Jxl, _) + | (Self::Png, _) + | (Self::Webp, _) => false, + } + } + + const fn file_extension(self) -> &'static str { + match self { + Self::Avif => ".avif", + Self::Jpeg => ".jpeg", + Self::Jxl => ".jxl", + Self::Png => ".png", + Self::Webp => ".webp", + } + } + + const fn magick_format(self) -> &'static str { + match self { + Self::Avif => "AVIF", + Self::Jpeg => "JPEG", + Self::Jxl => "JXL", + Self::Png => "PNG", + Self::Webp => "Webp", + } + } + + fn media_type(self) -> mime::Mime { + match self { + Self::Avif => image_avif(), + Self::Jpeg => mime::IMAGE_JPEG, + Self::Jxl => image_jxl(), + Self::Png => mime::IMAGE_PNG, + Self::Webp => image_webp(), + } + } +} + +impl AnimationFormat { + const fn const_eq(self, rhs: Self) -> bool { + match (self, rhs) { + (Self::Apng, Self::Apng) + | (Self::Avif, Self::Avif) + | (Self::Gif, Self::Gif) + | (Self::Webp, Self::Webp) => true, + (Self::Apng, _) | (Self::Avif, _) | (Self::Gif, _) | (Self::Webp, _) => false, + } + } + + const fn file_extension(self) -> &'static str { + match self { + Self::Apng => ".apng", + Self::Avif => ".avif", + Self::Gif => ".gif", + Self::Webp => ".webp", + } + } + + const fn magick_format(self) -> &'static str { + match self { + Self::Apng => "APNG", + Self::Avif => "AVIF", + Self::Gif => "GIF", + Self::Webp => "WEBP", + } + } + + fn media_type(self) -> mime::Mime { + match self { + Self::Apng => image_apng(), + Self::Avif => image_avif(), + Self::Gif => mime::IMAGE_GIF, + Self::Webp => image_webp(), + } + } +} + +impl VideoFormat { + const fn file_extension(self) -> &'static str { + match self { + Self::Mp4 => ".mp4", + Self::Webp { .. } => ".webm", + } + } +} + +impl OutputVideoFormat { + const fn from_parts( + video_codec: VideoCodec, + audio_codec: Option, + allow_audio: bool, + ) -> Self { + match (video_codec, audio_codec) { + (VideoCodec::Av1, Some(AudioCodec::Vorbis)) if allow_audio => OutputVideoFormat::Webm { + video_codec: WebmCodec::Av1, + audio_codec: Some(WebmAudioCodec::Vorbis), + }, + (VideoCodec::Av1, _) if allow_audio => OutputVideoFormat::Webm { + video_codec: WebmCodec::Av1, + audio_codec: Some(WebmAudioCodec::Opus), + }, + (VideoCodec::Av1, _) => OutputVideoFormat::Webm { + video_codec: WebmCodec::Av1, + audio_codec: None, + }, + (VideoCodec::H264, _) if allow_audio => OutputVideoFormat::Mp4 { + video_codec: Mp4Codec::H264, + audio_codec: Some(Mp4AudioCodec::Aac), + }, + (VideoCodec::H264, _) => OutputVideoFormat::Mp4 { + video_codec: Mp4Codec::H264, + audio_codec: None, + }, + (VideoCodec::H265, _) if allow_audio => OutputVideoFormat::Mp4 { + video_codec: Mp4Codec::H265, + audio_codec: Some(Mp4AudioCodec::Aac), + }, + (VideoCodec::H265, _) => OutputVideoFormat::Mp4 { + video_codec: Mp4Codec::H265, + audio_codec: None, + }, + (VideoCodec::Vp8, Some(AudioCodec::Vorbis)) if allow_audio => OutputVideoFormat::Webm { + video_codec: WebmCodec::Alpha(AlphaCodec { + alpha: false, + codec: WebmAlphaCodec::Vp8, + }), + audio_codec: Some(WebmAudioCodec::Vorbis), + }, + (VideoCodec::Vp8, _) if allow_audio => OutputVideoFormat::Webm { + video_codec: WebmCodec::Alpha(AlphaCodec { + alpha: false, + codec: WebmAlphaCodec::Vp8, + }), + audio_codec: Some(WebmAudioCodec::Opus), + }, + (VideoCodec::Vp8, _) => OutputVideoFormat::Webm { + video_codec: WebmCodec::Alpha(AlphaCodec { + alpha: false, + codec: WebmAlphaCodec::Vp8, + }), + audio_codec: None, + }, + (VideoCodec::Vp9, Some(AudioCodec::Vorbis)) if allow_audio => OutputVideoFormat::Webm { + video_codec: WebmCodec::Alpha(AlphaCodec { + alpha: false, + codec: WebmAlphaCodec::Vp9, + }), + audio_codec: Some(WebmAudioCodec::Vorbis), + }, + (VideoCodec::Vp9, _) if allow_audio => OutputVideoFormat::Webm { + video_codec: WebmCodec::Alpha(AlphaCodec { + alpha: false, + codec: WebmAlphaCodec::Vp9, + }), + audio_codec: Some(WebmAudioCodec::Opus), + }, + (VideoCodec::Vp9, _) => OutputVideoFormat::Webm { + video_codec: WebmCodec::Alpha(AlphaCodec { + alpha: false, + codec: WebmAlphaCodec::Vp9, + }), + audio_codec: None, + }, + } + } + + const fn file_extension(self) -> &'static str { + match self { + Self::Mp4 { .. } => ".mp4", + Self::Webm { .. } => ".webm", + } + } + + const fn ffmpeg_format(self) -> &'static str { + match self { + Self::Mp4 { .. } => "mp4", + Self::Webm { .. } => "webm", + } + } + + const fn ffmpeg_video_codec(self) -> &'static str { + match self { + Self::Mp4 { video_codec, .. } => video_codec.ffmpeg_codec(), + Self::Webm { video_codec, .. } => video_codec.ffmpeg_codec(), + } + } + + const fn ffmpeg_audio_codec(self) -> Option<&'static str> { + match self { + Self::Mp4 { + audio_codec: Some(audio_codec), + .. + } => Some(audio_codec.ffmpeg_codec()), + Self::Webm { + audio_codec: Some(audio_codec), + .. + } => Some(audio_codec.ffmpeg_codec()), + _ => None, + } + } + + const fn pix_fmt(self) -> &'static str { + match self { + Self::Mp4 { .. } => "yuv420p", + Self::Webm { video_codec, .. } => video_codec.pix_fmt(), + } + } + + fn media_type(self) -> mime::Mime { + match self { + Self::Mp4 { .. } => video_mp4(), + Self::Webm { .. } => video_webm(), + } + } +} + +impl Mp4Codec { + const fn ffmpeg_codec(self) -> &'static str { + match self { + Self::H264 => "h264", + Self::H265 => "hevc", + } + } +} + +impl WebmAlphaCodec { + const fn ffmpeg_codec(self) -> &'static str { + match self { + Self::Vp8 => "vp8", + Self::Vp9 => "vp9", + } + } +} + +impl WebmCodec { + const fn ffmpeg_codec(self) -> &'static str { + match self { + Self::Av1 => "av1", + Self::Alpha(AlphaCodec { codec, .. }) => codec.ffmpeg_codec(), + } + } + + const fn pix_fmt(self) -> &'static str { + match self { + Self::Alpha(AlphaCodec { alpha: true, .. }) => "yuva420p", + _ => "yuv420p", + } + } +} + +impl Mp4AudioCodec { + const fn ffmpeg_codec(self) -> &'static str { + match self { + Self::Aac => "aac", + } + } +} + +impl WebmAudioCodec { + const fn ffmpeg_codec(self) -> &'static str { + match self { + Self::Opus => "libopus", + Self::Vorbis => "vorbis", + } + } +} + +impl OutputFile { + const fn file_extension(&self) -> &'static str { + match self { + Self::Image { format, .. } => format.file_extension(), + Self::Animation { format, .. } => format.file_extension(), + Self::Video(format) => format.file_extension(), + } + } + + fn media_type(&self) -> mime::Mime { + match self { + Self::Image { format, .. } => format.media_type(), + Self::Animation { format, .. } => format.media_type(), + Self::Video(format) => format.media_type(), + } + } +} diff --git a/src/lib.rs b/src/lib.rs index 26c03b1..862a0ef 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -8,6 +8,7 @@ mod error; mod exiftool; mod ffmpeg; mod file; +mod formats; mod generate; mod ingest; mod init_tracing;