From 9ffe2238a7d99e17814137dc48e8f2862b8dd3b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= Date: Mon, 3 Aug 2020 00:21:21 +0300 Subject: [PATCH] examples: Add example that creates a thumbnail of a given position in a stream using the image crate --- examples/Cargo.toml | 5 + examples/src/bin/thumbnail.rs | 251 ++++++++++++++++++++++++++++++++++ 2 files changed, 256 insertions(+) create mode 100644 examples/src/bin/thumbnail.rs diff --git a/examples/Cargo.toml b/examples/Cargo.toml index 68c812f66..d96f95c18 100644 --- a/examples/Cargo.toml +++ b/examples/Cargo.toml @@ -34,6 +34,7 @@ pangocairo = { git = "https://github.com/gtk-rs/pangocairo", optional = true } glutin = { version = "0.21", optional = true } winit = { version = "0.19", optional = true } once_cell = "1.0" +image = { version="0.23", optional = true } [build-dependencies] gl_generator = { version = "0.14", optional = true } @@ -166,3 +167,7 @@ name = "subclass" [[bin]] name = "video_converter" + +[[bin]] +name = "thumbnail" +required-features = ["image"] diff --git a/examples/src/bin/thumbnail.rs b/examples/src/bin/thumbnail.rs new file mode 100644 index 000000000..fa3c03b90 --- /dev/null +++ b/examples/src/bin/thumbnail.rs @@ -0,0 +1,251 @@ +// This example demonstrates how to get a raw video frame at a given position +// and then rescale and store it with the image crate: + +// {uridecodebin} - {videoconvert} - {appsink} + +// The appsink enforces RGBA so that the image crate can use it. The image crate also requires +// tightly packed pixels, which is the case for RGBA by default in GStreamer. + +extern crate gstreamer as gst; +use gst::gst_element_error; +use gst::prelude::*; +extern crate gstreamer_app as gst_app; +extern crate gstreamer_video as gst_video; + +extern crate image; + +use anyhow::Error; +use derive_more::{Display, Error}; + +#[path = "../examples-common.rs"] +mod examples_common; + +#[derive(Debug, Display, Error)] +#[display(fmt = "Missing element {}", _0)] +struct MissingElement(#[error(not(source))] &'static str); + +#[derive(Debug, Display, Error)] +#[display(fmt = "Received error from {}: {} (debug: {:?})", src, error, debug)] +struct ErrorMessage { + src: String, + error: String, + debug: Option, + source: glib::Error, +} + +fn create_pipeline(uri: String, out_path: std::path::PathBuf) -> Result { + gst::init()?; + + // Create our pipeline from a pipeline description string. + let pipeline = gst::parse_launch(&format!( + "uridecodebin uri={} ! videoconvert ! appsink name=sink", + uri + ))? + .downcast::() + .expect("Expected a gst::Pipeline"); + + // Get access to the appsink element. + let appsink = pipeline + .get_by_name("sink") + .expect("Sink element not found") + .downcast::() + .expect("Sink element is expected to be an appsink!"); + + // Don't synchronize on the clock, we only want a snapshot asap. + appsink.set_property("sync", &false).unwrap(); + + // Tell the appsink what format we want. + // This can be set after linking the two objects, because format negotiation between + // both elements will happen during pre-rolling of the pipeline. + appsink.set_caps(Some( + &gst::Caps::builder("video/x-raw") + .field("format", &gst_video::VideoFormat::Rgba.to_str()) + .build(), + )); + + let mut got_snapshot = false; + + // Getting data out of the appsink is done by setting callbacks on it. + // The appsink will then call those handlers, as soon as data is available. + appsink.set_callbacks( + gst_app::AppSinkCallbacks::builder() + // Add a handler to the "new-sample" signal. + .new_sample(move |appsink| { + // Pull the sample in question out of the appsink's buffer. + let sample = appsink.pull_sample().map_err(|_| gst::FlowError::Eos)?; + let buffer = sample.get_buffer().ok_or_else(|| { + gst_element_error!( + appsink, + gst::ResourceError::Failed, + ("Failed to get buffer from appsink") + ); + + gst::FlowError::Error + })?; + + let caps = sample.get_caps().expect("Sample without caps"); + let info = gst_video::VideoInfo::from_caps(&caps).expect("Failed to parse caps"); + + // Make sure that we only get a single buffer + if got_snapshot { + return Err(gst::FlowError::Eos); + } + got_snapshot = true; + + // At this point, buffer is only a reference to an existing memory region somewhere. + // When we want to access its content, we have to map it while requesting the required + // mode of access (read, read/write). + // This type of abstraction is necessary, because the buffer in question might not be + // on the machine's main memory itself, but rather in the GPU's memory. + // So mapping the buffer makes the underlying memory region accessible to us. + // See: https://gstreamer.freedesktop.org/documentation/plugin-development/advanced/allocation.html + let map = buffer.map_readable().map_err(|_| { + gst_element_error!( + appsink, + gst::ResourceError::Failed, + ("Failed to map buffer readable") + ); + + gst::FlowError::Error + })?; + + // We only want to have a single buffer and then have the pipeline terminate + println!("Have video frame"); + + // Calculate a target width/height that keeps the display aspect ratio while having + // a height of 240 pixels + let display_aspect_ratio = (info.width() as f64 * *info.par().numer() as f64) + / (info.height() as f64 * *info.par().denom() as f64); + let target_height = 240; + let target_width = target_height as f64 * display_aspect_ratio; + + // Create an ImageBuffer around the borrowed video frame data from GStreamer. + let img = image::ImageBuffer::, _>::from_raw( + info.width(), + info.height(), + map, + ) + .expect("Failed to create ImageBuffer, probably a stride mismatch"); + + // Scale image to our target dimensions + let scaled_img = + image::imageops::thumbnail(&img, target_width as u32, target_height as u32); + + // Save it at the specific location. This automatically detects the file type + // based on the filename. + scaled_img.save(&out_path).map_err(|err| { + gst_element_error!( + appsink, + gst::ResourceError::Write, + ( + "Failed to write thumbnail file {}: {}", + out_path.display(), + err + ) + ); + + gst::FlowError::Error + })?; + + println!("Wrote thumbnail to {}", out_path.display()); + + Err(gst::FlowError::Eos) + }) + .build(), + ); + + Ok(pipeline) +} + +fn main_loop(pipeline: gst::Pipeline, position: u64) -> Result<(), Error> { + pipeline.set_state(gst::State::Paused)?; + + let bus = pipeline + .get_bus() + .expect("Pipeline without bus. Shouldn't happen!"); + + let mut seeked = false; + + for msg in bus.iter_timed(gst::CLOCK_TIME_NONE) { + use gst::MessageView; + + match msg.view() { + MessageView::AsyncDone(..) => { + if !seeked { + // AsyncDone means that the pipeline has started now and that we can seek + println!("Got AsyncDone message, seeking to {}s", position); + + if pipeline + .seek_simple(gst::SeekFlags::FLUSH, position * gst::SECOND) + .is_err() + { + println!("Failed to seek, taking first frame"); + } + + pipeline.set_state(gst::State::Playing)?; + seeked = true; + } else { + println!("Got second AsyncDone message, seek finished"); + } + } + MessageView::Eos(..) => { + // The End-of-stream message is posted when the stream is done, which in our case + // happens immediately after creating the thumbnail because we return + // gst::FlowError::Eos then. + println!("Got Eos message, done"); + break; + } + MessageView::Error(err) => { + pipeline.set_state(gst::State::Null)?; + return Err(ErrorMessage { + src: msg + .get_src() + .map(|s| String::from(s.get_path_string())) + .unwrap_or_else(|| String::from("None")), + error: err.get_error().to_string(), + debug: err.get_debug(), + source: err.get_error(), + } + .into()); + } + _ => (), + } + } + + pipeline.set_state(gst::State::Null)?; + + Ok(()) +} + +fn example_main() { + use std::env; + + let mut args = env::args(); + + // Parse commandline arguments: input URI, position in seconds, output path + let _arg0 = args.next().unwrap(); + let uri = args + .next() + .expect("No input URI provided on the commandline"); + let position = args + .next() + .expect("No position in second on the commandline"); + let position = position + .parse::() + .expect("Failed to parse position as integer"); + let out_path = args + .next() + .expect("No output path provided on the commandline"); + let out_path = std::path::PathBuf::from(out_path); + + match create_pipeline(uri, out_path).and_then(|pipeline| main_loop(pipeline, position)) { + Ok(r) => r, + Err(e) => eprintln!("Error! {}", e), + } +} + +fn main() { + // tutorials_common::run is only required to set up the application environent on macOS + // (but not necessary in normal Cocoa applications where this is set up autmatically) + examples_common::run(example_main); +}