rafaelcaricio/gstreamer-rs
1
0
Fork 0
forked from mirrors/gstreamer-rs
Code Pull requests Activity

examples: Add a Cairo-based compositor example based on VideoAggregator

Browse source
This commit is contained in:
Sebastian Dröge 2022-03-22 18:59:51 +02:00
parent c251ca5bc3
commit 21dbe86c8e
2 changed files with 692 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
examples/Cargo.toml
View file

@ -186,4 +186,8 @@ required-features = ["image"]
[[bin]]
name = "fd_allocator"
required-features = ["allocators"]
required-features = ["allocators"]
[[bin]]
name = "cairo_compositor"
required-features = ["cairo-rs", "gst-video/v1_18"]

687
examples/src/bin/cairo_compositor.rs Normal file
View file

@ -0,0 +1,687 @@
// This example demonstrates how to implement a custom compositor based on cairo.
#![allow(clippy::non_send_fields_in_send_ty)]
use gst::prelude::*;
use gst_base::prelude::*;
use anyhow::{Context, Error};
use derive_more::{Display, Error};
#[path = "../examples-common.rs"]
mod examples_common;
// Our custom compositor element is defined in this module
mod cairo_compositor {
use super::*;
use gst_base::subclass::prelude::*;
use gst_video::prelude::*;
use gst_video::subclass::prelude::*;
use once_cell::sync::Lazy;
// In the imp submodule we include the actual implementation
mod imp {
use super::*;
use std::sync::Mutex;
// Settings of the compositor
#[derive(Clone)]
struct Settings {
background_color: u32,
}
impl Default for Settings {
fn default() -> Self {
Self {
background_color: 0xff_00_00_00,
}
}
}
// This is the private data of our pad
#[derive(Default)]
pub struct CairoCompositor {
settings: Mutex<Settings>,
}
// This trait registers our type with the GObject object system and
// provides the entry points for creating a new instance and setting
// up the class data
#[glib::object_subclass]
impl ObjectSubclass for CairoCompositor {
const NAME: &'static str = "CairoCompositor";
type Type = super::CairoCompositor;
type ParentType = gst_video::VideoAggregator;
}
// Implementation of glib::Object virtual methods
impl ObjectImpl for CairoCompositor {
fn properties() -> &'static [glib::ParamSpec] {
static PROPERTIES: Lazy<Vec<glib::ParamSpec>> = Lazy::new(|| {
vec![glib::ParamSpecUInt::new(
"background-color",
"Background Color",
"Background color as AARRGGBB",
0,
u32::MAX,
Settings::default().background_color,
glib::ParamFlags::READWRITE,
)]
});
&*PROPERTIES
}
fn set_property(
&self,
_obj: &Self::Type,
_id: usize,
value: &glib::Value,
pspec: &glib::ParamSpec,
) {
let mut settings = self.settings.lock().unwrap();
match pspec.name() {
"background-color" => {
settings.background_color = value.get().unwrap();
}
_ => unimplemented!(),
};
}
fn property(
&self,
_obj: &Self::Type,
_id: usize,
pspec: &glib::ParamSpec,
) -> glib::Value {
let settings = self.settings.lock().unwrap();
match pspec.name() {
"background-color" => settings.background_color.to_value(),
_ => unimplemented!(),
}
}
}
// Implementation of gst::Object virtual methods
impl GstObjectImpl for CairoCompositor {}
// Implementation of gst::Element virtual methods
impl ElementImpl for CairoCompositor {
// The element specific metadata. This information is what is visible from
// gst-inspect-1.0 and can also be programatically retrieved from the gst::Registry
// after initial registration without having to load the plugin in memory.
fn metadata() -> Option<&'static gst::subclass::ElementMetadata> {
static ELEMENT_METADATA: Lazy<gst::subclass::ElementMetadata> = Lazy::new(|| {
gst::subclass::ElementMetadata::new(
"Cairo Compositor",
"Compositor/Video",
"Cairo based compositor",
"Sebastian Dröge <sebastian@centricular.com>",
)
});
Some(&*ELEMENT_METADATA)
}
fn pad_templates() -> &'static [gst::PadTemplate] {
static PAD_TEMPLATES: Lazy<Vec<gst::PadTemplate>> = Lazy::new(|| {
// Create pad templates for our sink and source pad. These are later used for
// actually creating the pads and beforehand already provide information to
// GStreamer about all possible pads that could exist for this type.
// On all pads we can only handle BGRx.
let caps = gst::Caps::builder("video/x-raw")
.field("format", gst_video::VideoFormat::Bgrx.to_str())
.field("width", gst::IntRange::<i32>::new(1, i32::MAX))
.field("height", gst::IntRange::<i32>::new(1, i32::MAX))
.field(
"framerate",
gst::FractionRange::new(
gst::Fraction::new(0, 1),
gst::Fraction::new(i32::MAX, 1),
),
)
.build();
vec![
// The src pad template must be named "src" for aggregator
// and always be there
gst::PadTemplate::with_gtype(
"src",
gst::PadDirection::Src,
gst::PadPresence::Always,
&caps,
super::CairoCompositorPad::static_type(),
)
.unwrap(),
// The sink pad template must be named "sink_%u" by default for aggregator
// and be requested by the application
gst::PadTemplate::with_gtype(
"sink_%u",
gst::PadDirection::Sink,
gst::PadPresence::Request,
&caps,
super::CairoCompositorPad::static_type(),
)
.unwrap(),
]
});
PAD_TEMPLATES.as_ref()
}
}
// Implementation of gst_base::Aggregator virtual methods
impl AggregatorImpl for CairoCompositor {
fn sink_query(
&self,
aggregator: &Self::Type,
aggregator_pad: &gst_base::AggregatorPad,
query: &mut gst::QueryRef,
) -> bool {
use gst::QueryViewMut;
// We can accept any input caps that match the pad template. By default
// videoaggregator only allows caps that have the same format as the output.
match query.view_mut() {
QueryViewMut::Caps(q) => {
let caps = aggregator_pad.pad_template_caps();
let filter = q.filter();
let caps = if let Some(filter) = filter {
filter.intersect_with_mode(&caps, gst::CapsIntersectMode::First)
} else {
caps
};
q.set_result(&caps);
true
}
QueryViewMut::AcceptCaps(q) => {
let caps = q.caps();
let template_caps = aggregator_pad.pad_template_caps();
let res = caps.is_subset(&template_caps);
q.set_result(res);
true
}
_ => self.parent_sink_query(aggregator, aggregator_pad, query),
}
}
}
// Implementation of gst_video::VideoAggregator virtual methods
impl VideoAggregatorImpl for CairoCompositor {
fn find_best_format(
&self,
_element: &Self::Type,
_downstream_caps: &gst::Caps,
) -> Option<(gst_video::VideoInfo, bool)> {
// Let videoaggregator select whatever format downstream wants
None
}
fn aggregate_frames(
&self,
element: &Self::Type,
token: &gst_video::subclass::AggregateFramesToken,
outbuf: &mut gst::BufferRef,
) -> Result<gst::FlowSuccess, gst::FlowError> {
let pads = element.sink_pads();
let out_info = element.video_info().unwrap();
let mut out_frame =
gst_video::VideoFrameRef::from_buffer_ref_writable(outbuf, &out_info).unwrap();
with_frame(&mut out_frame, |ctx| {
let settings = self.settings.lock().unwrap().clone();
let bg = (
((settings.background_color >> 16) & 0xff) as f64 / 255.0,
((settings.background_color >> 8) & 0xff) as f64 / 255.0,
((settings.background_color >> 0) & 0xff) as f64 / 255.0,
);
ctx.set_source_rgb(bg.0, bg.1, bg.2);
ctx.paint().unwrap();
for pad in pads {
let pad = pad.downcast_ref::<CairoCompositorPad>().unwrap();
let settings = pad.imp().settings.lock().unwrap().clone();
if settings.alpha <= 0.0 || settings.scale <= 0.0 {
continue;
}
let frame = match pad.prepared_frame(token) {
Some(frame) => frame,
None => continue,
};
ctx.save().unwrap();
ctx.translate(settings.xpos, settings.ypos);
ctx.scale(settings.scale, settings.scale);
ctx.translate(frame.width() as f64 / 2.0, frame.height() as f64 / 2.0);
ctx.rotate(settings.rotate / 360.0 * 2.0 * std::f64::consts::PI);
ctx.translate(
-(frame.width() as f64 / 2.0),
-(frame.height() as f64 / 2.0),
);
paint_frame(&ctx, &frame, settings.alpha);
ctx.restore().unwrap();
}
});
Ok(gst::FlowSuccess::Ok)
}
}
}
fn with_frame<F: FnOnce(&cairo::Context)>(
frame: &mut gst_video::VideoFrameRef<&mut gst::BufferRef>,
func: F,
) {
// SAFETY: This is the one and only surface reference and it is dropped at the end, meaning
// nothing from cairo is referencing the frame data anymore.
unsafe {
use glib::translate::*;
let surface = cairo::ImageSurface::create_for_data_unsafe(
frame.plane_data_mut(0).unwrap().as_mut_ptr(),
cairo::Format::Rgb24,
frame.width() as i32,
frame.height() as i32,
frame.plane_stride()[0],
)
.unwrap();
let ctx = cairo::Context::new(&surface).unwrap();
func(&ctx);
drop(ctx);
surface.finish();
assert_eq!(
cairo::ffi::cairo_surface_get_reference_count(surface.to_glib_none().0),
1,
);
}
}
fn paint_frame(
ctx: &cairo::Context,
frame: &gst_video::VideoFrameRef<&gst::BufferRef>,
alpha: f64,
) {
// SAFETY: This is the one and only surface reference and it is dropped at the end, meaning
// nothing from cairo is referencing the frame data anymore.
//
// Also nothing is ever writing to the surface from here.
unsafe {
use glib::translate::*;
let surface = cairo::ImageSurface::create_for_data_unsafe(
frame.plane_data(0).unwrap().as_ptr() as *mut u8,
cairo::Format::Rgb24,
frame.width() as i32,
frame.height() as i32,
frame.plane_stride()[0],
)
.unwrap();
ctx.set_source_surface(&surface, 0.0, 0.0).unwrap();
ctx.paint_with_alpha(alpha).unwrap();
ctx.set_source_rgb(0.0, 0.0, 0.0);
assert_eq!(
cairo::ffi::cairo_surface_get_reference_count(surface.to_glib_none().0),
1,
);
}
}
// This here defines the public interface of our element and implements
// the corresponding traits so that it behaves like any other gst::Element
glib::wrapper! {
pub struct CairoCompositor(ObjectSubclass<imp::CairoCompositor>) @extends gst_video::VideoAggregator, gst_base::Aggregator, gst::Element, gst::Object;
}
impl CairoCompositor {
// Creates a new instance of our compositor with the given name
pub fn new(name: Option<&str>) -> Self {
glib::Object::new(&[("name", &name)]).expect("Failed to create cairo compositor")
}
}
// In the imp submodule we include the implementation of the pad subclass
mod imp_pad {
use super::*;
use std::sync::Mutex;
// Settings of our pad
#[derive(Clone)]
pub(super) struct Settings {
pub(super) alpha: f64,
pub(super) scale: f64,
pub(super) rotate: f64,
pub(super) xpos: f64,
pub(super) ypos: f64,
}
impl Default for Settings {
fn default() -> Self {
Self {
alpha: 1.0,
scale: 1.0,
rotate: 0.0,
xpos: 0.0,
ypos: 0.0,
}
}
}
// This is the private data of our pad
#[derive(Default)]
pub struct CairoCompositorPad {
pub(super) settings: Mutex<Settings>,
}
// This trait registers our type with the GObject object system and
// provides the entry points for creating a new instance and setting
// up the class data
#[glib::object_subclass]
impl ObjectSubclass for CairoCompositorPad {
const NAME: &'static str = "CairoCompositorPad";
type Type = super::CairoCompositorPad;
type ParentType = gst_video::VideoAggregatorPad;
}
// Implementation of glib::Object virtual methods
impl ObjectImpl for CairoCompositorPad {
fn properties() -> &'static [glib::ParamSpec] {
static PROPERTIES: Lazy<Vec<glib::ParamSpec>> = Lazy::new(|| {
vec![
glib::ParamSpecDouble::new(
"alpha",
"Alpha",
"Alpha value of the input",
0.0,
1.0,
Settings::default().alpha,
glib::ParamFlags::READWRITE,
),
glib::ParamSpecDouble::new(
"scale",
"Scale",
"Scale factor of the input",
0.0,
f64::MAX,
Settings::default().scale,
glib::ParamFlags::READWRITE,
),
glib::ParamSpecDouble::new(
"rotate",
"Rotate",
"Rotation of the input",
0.0,
360.0,
Settings::default().rotate,
glib::ParamFlags::READWRITE,
),
glib::ParamSpecDouble::new(
"xpos",
"X Position",
"Horizontal position of the input",
0.0,
f64::MAX,
Settings::default().xpos,
glib::ParamFlags::READWRITE,
),
glib::ParamSpecDouble::new(
"ypos",
"Y Position",
"Vertical position of the input",
0.0,
f64::MAX,
Settings::default().ypos,
glib::ParamFlags::READWRITE,
),
]
});
PROPERTIES.as_ref()
}
fn set_property(
&self,
_obj: &Self::Type,
_id: usize,
value: &glib::Value,
pspec: &glib::ParamSpec,
) {
let mut settings = self.settings.lock().unwrap();
match pspec.name() {
"alpha" => {
settings.alpha = value.get().unwrap();
}
"scale" => {
settings.scale = value.get().unwrap();
}
"rotate" => {
settings.rotate = value.get().unwrap();
}
"xpos" => {
settings.xpos = value.get().unwrap();
}
"ypos" => {
settings.ypos = value.get().unwrap();
}
_ => unimplemented!(),
};
}
fn property(
&self,
_obj: &Self::Type,
_id: usize,
pspec: &glib::ParamSpec,
) -> glib::Value {
let settings = self.settings.lock().unwrap();
match pspec.name() {
"alpha" => settings.alpha.to_value(),
"scale" => settings.scale.to_value(),
"rotate" => settings.rotate.to_value(),
"xpos" => settings.xpos.to_value(),
"ypos" => settings.ypos.to_value(),
_ => unimplemented!(),
}
}
}
// Implementation of gst::Object virtual methods
impl GstObjectImpl for CairoCompositorPad {}
// Implementation of gst::Pad virtual methods
impl PadImpl for CairoCompositorPad {}
// Implementation of gst_base::AggregatorPad virtual methods
impl AggregatorPadImpl for CairoCompositorPad {}
// Implementation of gst_video::VideoAggregatorPad virtual methods
impl VideoAggregatorPadImpl for CairoCompositorPad {}
}
// This here defines the public interface of our element and implements
// the corresponding traits so that it behaves like any other gst::Element
glib::wrapper! {
pub struct CairoCompositorPad(ObjectSubclass<imp_pad::CairoCompositorPad>) @extends gst_video::VideoAggregatorPad, gst_base::AggregatorPad, gst::Pad, gst::Object;
}
}
#[derive(Debug, Display, Error)]
#[display(fmt = "Received error from {}: {} (debug: {:?})", src, error, debug)]
struct ErrorMessage {
src: String,
error: String,
debug: Option<String>,
source: glib::Error,
}
fn create_pipeline() -> Result<gst::Pipeline, Error> {
gst::init()?;
// Create our pipeline with the compositor and two input streams
let pipeline = gst::Pipeline::new(None);
let src1 = gst::ElementFactory::make("videotestsrc", None).context("Creating videotestsrc")?;
let src2 = gst::ElementFactory::make("videotestsrc", None).context("Creating videotestsrc")?;
let comp = cairo_compositor::CairoCompositor::new(None);
let conv = gst::ElementFactory::make("videoconvert", None).context("Creating videoconvert")?;
let sink =
gst::ElementFactory::make("autovideosink", None).context("Creating autovideosink")?;
pipeline.add_many(&[&src1, &src2, comp.upcast_ref(), &conv, &sink])?;
src1.set_property_from_str("pattern", "ball");
src2.set_property_from_str("pattern", "smpte");
src1.link_filtered(
&comp,
&gst::Caps::builder("video/x-raw")
.field("width", 320i32)
.field("height", 240i32)
.build(),
)
.context("Linking source 1")?;
src2.link_filtered(
&comp,
&gst::Caps::builder("video/x-raw")
.field("width", 320i32)
.field("height", 240i32)
.build(),
)
.context("Linking source 2")?;
comp.link_filtered(
&conv,
&gst::Caps::builder("video/x-raw")
.field("width", 1280i32)
.field("height", 720i32)
.build(),
)
.context("Linking converter")?;
conv.link(&sink).context("Linking sink")?;
comp.set_property("background-color", 0xff_33_33_33u32);
// Change positions etc of both inputs based on a timer
let xmax = 1280.0 - 320.0f64;
let ymax = 720.0 - 240.0f64;
let sink_0 = comp.static_pad("sink_0").unwrap();
sink_0.set_property("xpos", 0.0f64);
sink_0.set_property("ypos", 0.0f64);
let sink_1 = comp.static_pad("sink_1").unwrap();
sink_1.set_property("xpos", xmax);
sink_1.set_property("ypos", ymax);
comp.set_emit_signals(true);
comp.connect_samples_selected(move |_agg, _seg, pts, _dts, _dur, _info| {
// Position and rotation period is 10s
let pos = (pts.unwrap().nseconds() % gst::ClockTime::from_seconds(10).nseconds()) as f64
/ gst::ClockTime::from_seconds(10).nseconds() as f64;
let xpos = (1.0 + f64::sin(2.0 * std::f64::consts::PI * pos)) * xmax / 2.0;
let ypos = (1.0 + f64::cos(2.0 * std::f64::consts::PI * pos)) * ymax / 2.0;
sink_0.set_property("xpos", xpos);
sink_0.set_property("ypos", ypos);
let xpos = (1.0 + f64::cos(2.0 * std::f64::consts::PI * pos)) * xmax / 2.0;
let ypos = (1.0 + f64::sin(2.0 * std::f64::consts::PI * pos)) * ymax / 2.0;
sink_1.set_property("xpos", xpos);
sink_1.set_property("ypos", ypos);
sink_0.set_property("rotate", pos * 360.0);
sink_1.set_property("rotate", 360.0 - pos * 360.0);
// Alpha period is 2s
let pos = (pts.unwrap().nseconds() % gst::ClockTime::from_seconds(2).nseconds()) as f64
/ gst::ClockTime::from_seconds(2).nseconds() as f64;
if pos < 0.5 {
sink_0.set_property("alpha", 2.0 * pos);
sink_1.set_property("alpha", 1.0 - 2.0 * pos);
} else {
sink_0.set_property("alpha", 1.0 - 2.0 * (pos - 0.5));
sink_1.set_property("alpha", 2.0 * (pos - 0.5));
}
// Scale period is 20s
let pos = (pts.unwrap().nseconds() % gst::ClockTime::from_seconds(20).nseconds()) as f64
/ gst::ClockTime::from_seconds(20).nseconds() as f64;
sink_0.set_property("scale", pos);
sink_1.set_property("scale", 1.0 - pos);
});
Ok(pipeline)
}
fn main_loop(pipeline: gst::Pipeline) -> Result<(), Error> {
pipeline.set_state(gst::State::Playing)?;
let bus = pipeline
.bus()
.expect("Pipeline without bus. Shouldn't happen!");
let main_loop = glib::MainLoop::new(None, false);
bus.add_watch({
let main_loop = main_loop.clone();
move |_bus, msg| {
use gst::MessageView;
match msg.view() {
MessageView::Eos(..) => main_loop.quit(),
MessageView::Error(err) => {
println!(
"Error from {:?}: {} ({:?})",
err.src().map(|s| s.path_string()),
err.error(),
err.debug()
);
main_loop.quit();
}
_ => (),
}
glib::Continue(true)
}
})
.unwrap();
main_loop.run();
pipeline.set_state(gst::State::Null)?;
Ok(())
}
fn example_main() {
match create_pipeline().and_then(main_loop) {
Ok(r) => r,
Err(e) => eprintln!("Error! {}", e),
}
}
fn main() {
// tutorials_common::run is only required to set up the application environment on macOS
// (but not necessary in normal Cocoa applications where this is set up automatically)
examples_common::run(example_main);
}