Implement rounded corners

This plugin takes I420/YUV and appends an alpha plane to give YUVA/A420
to round the corners analogous to the border-radius in CSS. Other video
formats like NV12 not supported yet. Support for other planar formats
will follow.

Not all ways of specifying border-radius as in CSS are implemented at
the moment. Currently, we only support specifying it in pixels and it
gets applied uniformly to all corners.
This commit is contained in:
Sanchayan Maity 2021-09-16 18:00:35 +05:30
parent 86f422592b
commit 2c2cd8c2be
8 changed files with 738 additions and 0 deletions

View file

@ -18,6 +18,7 @@ members = [
"utils/togglerecord",
"video/cdg",
"video/closedcaption",
"video/videofx",
"video/dav1d",
"video/ffv1",
"video/flavors",

View file

@ -57,6 +57,7 @@ plugins = {
# FIXME: libwebp-sys2 will build its bundled version on msvc and apple platforms
# https://github.com/qnighy/libwebp-sys2-rs/issues/4
'gst-plugin-webp': 'libgstrswebp',
'gst-plugin-videofx': 'libgstvideofx',
}
extra_env = {}

58
video/videofx/Cargo.toml Normal file
View file

@ -0,0 +1,58 @@
[package]
name = "gst-plugin-videofx"
version = "0.8.0"
authors = ["Sanchayan Maity <sanchayan@asymptotic.io>"]
repository = "https://gitlab.freedesktop.org/gstreamer/gst-plugins-rs"
license = "MPL-2.0"
description = "Video Effects Plugin"
edition = "2021"
[dependencies]
cairo-rs = { git = "https://github.com/gtk-rs/gtk-rs-core", features=["use_glib"] }
atomic_refcell = "0.1"
once_cell = "1.0"
[dependencies.gst]
git = "https://gitlab.freedesktop.org/gstreamer/gstreamer-rs"
features = ["v1_16"]
package = "gstreamer"
[dependencies.gst-base]
git = "https://gitlab.freedesktop.org/gstreamer/gstreamer-rs"
features = ["v1_16"]
package = "gstreamer-base"
[dependencies.gst-video]
git = "https://gitlab.freedesktop.org/gstreamer/gstreamer-rs"
features = ["v1_16"]
package = "gstreamer-video"
[dev-dependencies.gst-check]
git = "https://gitlab.freedesktop.org/gstreamer/gstreamer-rs"
package = "gstreamer-check"
[lib]
name = "gstvideofx"
crate-type = ["cdylib", "rlib"]
path = "src/lib.rs"
[build-dependencies]
gst-plugin-version-helper = { path="../../version-helper" }
[features]
# GStreamer 1.14 is required for static linking
static = []
capi = []
[package.metadata.capi]
min_version = "0.8.0"
[package.metadata.capi.header]
enabled = false
[package.metadata.capi.library]
install_subdir = "gstreamer-1.0"
versioning = false
[package.metadata.capi.pkg_config]
requires_private = "gstreamer-1.0, gstreamer-base-1.0, gstreamer-video-1.0, gobject-2.0, glib-2.0, cairo-gobject"

View file

@ -0,0 +1 @@
../../LICENSE-MPL-2.0

3
video/videofx/build.rs Normal file
View file

@ -0,0 +1,3 @@
fn main() {
gst_plugin_version_helper::info()
}

View file

@ -0,0 +1,29 @@
// Copyright (C) 2021 Asymptotic Inc.
// Author: Sanchayan Maity <sanchayan@asymptotic.io>
//
// This Source Code Form is subject to the terms of the Mozilla Public License, v2.0.
// If a copy of the MPL was not distributed with this file, You can obtain one at
// <https://mozilla.org/MPL/2.0/>.
//
// SPDX-License-Identifier: MPL-2.0
use gst::glib;
use gst::prelude::*;
pub mod roundedcorners;
glib::wrapper! {
pub struct RoundedCorners(ObjectSubclass<roundedcorners::RoundedCorners>) @extends gst_base::BaseTransform, gst::Element, gst::Object;
}
unsafe impl Send for RoundedCorners {}
unsafe impl Sync for RoundedCorners {}
pub fn register(plugin: &gst::Plugin) -> Result<(), glib::BoolError> {
gst::Element::register(
Some(plugin),
"roundedcorners",
gst::Rank::None,
RoundedCorners::static_type(),
)
}

View file

@ -0,0 +1,619 @@
// Copyright (C) 2021 Asymptotic Inc.
// Author: Sanchayan Maity <sanchayan@asymptotic.io>
//
// This Source Code Form is subject to the terms of the Mozilla Public License, v2.0.
// If a copy of the MPL was not distributed with this file, You can obtain one at
// <https://mozilla.org/MPL/2.0/>.
//
// SPDX-License-Identifier: MPL-2.0
use gst::{glib, gst_debug, gst_info, gst_log, subclass::prelude::*};
use gst_base::{
prelude::*,
subclass::base_transform::{InputBuffer, PrepareOutputBufferSuccess},
};
use gst_video::{subclass::prelude::*, VideoFormat};
use once_cell::sync::Lazy;
use std::sync::Mutex;
const DEFAULT_BORDER_RADIUS: u32 = 0;
static CAT: Lazy<gst::DebugCategory> = Lazy::new(|| {
gst::DebugCategory::new(
"roundedcorners",
gst::DebugColorFlags::empty(),
Some("Rounded corners"),
)
});
#[derive(Debug, Clone, Copy)]
struct Settings {
border_radius_px: u32,
changed: bool,
}
impl Default for Settings {
fn default() -> Self {
Settings {
border_radius_px: DEFAULT_BORDER_RADIUS,
changed: false,
}
}
}
struct State {
alpha_mem: gst::Memory,
out_info: Option<gst_video::VideoInfo>,
}
#[derive(Default)]
pub struct RoundedCorners {
settings: Mutex<Settings>,
state: Mutex<Option<State>>,
}
impl RoundedCorners {
fn draw_rounded_corners(
&self,
cairo_ctx: &cairo::Context,
border_radius_px: u32,
width: f64,
height: f64,
) -> Result<(), cairo::Error> {
let border_radius = border_radius_px as f64;
let degrees = std::f64::consts::PI / 180.0;
// Taken from https://www.cairographics.org/samples/rounded_rectangle/
cairo_ctx.new_sub_path();
cairo_ctx.arc(
width - border_radius,
border_radius,
border_radius,
-90_f64 * degrees,
0 as f64 * degrees,
);
cairo_ctx.arc(
width - border_radius,
height - border_radius,
border_radius,
0 as f64 * degrees,
90_f64 * degrees,
);
cairo_ctx.arc(
border_radius,
height - border_radius,
border_radius,
90_f64 * degrees,
180_f64 * degrees,
);
cairo_ctx.arc(
border_radius,
border_radius,
border_radius,
180_f64 * degrees,
270_f64 * degrees,
);
cairo_ctx.close_path();
cairo_ctx.set_source_rgb(0.0, 0.0, 0.0);
cairo_ctx.fill_preserve()?;
cairo_ctx.set_source_rgba(0.0, 0.0, 0.0, 1.0);
cairo_ctx.set_line_width(1.0);
cairo_ctx.stroke()?;
Ok(())
}
fn generate_alpha_mask(&self, border_radius_px: u32) -> Result<(), gst::LoggableError> {
let mut state_guard = self.state.lock().unwrap();
let state = state_guard.as_mut().unwrap();
let out_info = state.out_info.as_ref().unwrap();
let width = out_info.width() as i32;
let height = out_info.height() as i32;
let alpha_stride = out_info.stride()[3];
let mem = &mut state.alpha_mem;
let mut_mem = mem.get_mut().unwrap();
let mut alpha_mem = mut_mem
.map_writable()
.map_err(|_| gst::loggable_error!(CAT, "Failed to map alpha memory as writable"))?;
if border_radius_px == 0 {
// Border radius is 0 but output needs to have an alpha plane. Attach an opaque alpha
// plane here and just return.
alpha_mem.fill(0xff);
return Ok(());
}
alpha_mem.fill(0);
let surface = unsafe {
cairo::ImageSurface::create_for_data_unsafe(
alpha_mem.as_mut_slice().as_mut_ptr(),
cairo::Format::A8,
width,
height,
alpha_stride,
)
};
if let Err(e) = surface {
return Err(gst::loggable_error!(
CAT,
"Failed to create cairo image surface: {}",
e
));
}
match cairo::Context::new(surface.as_ref().unwrap()) {
Ok(cr) => {
if let Err(e) =
self.draw_rounded_corners(&cr, border_radius_px, width as f64, height as f64)
{
return Err(gst::loggable_error!(
CAT,
"Failed to draw rounded corners: {}",
e
));
};
drop(cr);
unsafe {
assert_eq!(
cairo::ffi::cairo_surface_get_reference_count(
surface.unwrap().to_raw_none()
),
1
);
}
Ok(())
}
Err(e) => Err(gst::loggable_error!(
CAT,
"Failed to create cairo context: {}",
e
)),
}
}
fn add_video_meta(
&self,
buf: &mut gst::BufferRef,
out_info: &gst_video::VideoInfo,
alpha_plane_offset: usize,
input_buffer_writable: bool,
) -> Result<PrepareOutputBufferSuccess, gst::FlowError> {
let mut strides: [i32; 4] = [0; 4];
let mut offsets: [usize; 4] = [0; 4];
match buf.meta_mut::<gst_video::VideoMeta>() {
Some(meta) => {
let video_frame_flags = meta.video_frame_flags();
let n_planes = meta.n_planes() as usize;
offsets[..n_planes].clone_from_slice(&meta.offset()[..n_planes]);
strides[..n_planes].clone_from_slice(&meta.stride()[..n_planes]);
offsets[3] = alpha_plane_offset;
strides[3] = out_info.stride()[3];
match meta.remove() {
Err(_) => {
// We could not remove the meta, probably because it was locked. We need to
// create a new buffer and copy over all memories, metadata (timestamps, etc)
// and metas (except for videometa) of the original buffer into a newly allocated buffer.
let copy_flags = gst::BufferCopyFlags::FLAGS
| gst::BufferCopyFlags::TIMESTAMPS
| gst::BufferCopyFlags::MEMORY;
let mut buf = buf.copy_region(copy_flags, 0, None).unwrap();
let mut_buf = buf.make_mut();
gst_video::VideoMeta::add_full(
mut_buf,
video_frame_flags,
out_info.format(),
out_info.width(),
out_info.height(),
&offsets,
&strides,
)
.unwrap();
Ok(PrepareOutputBufferSuccess::Buffer(mut_buf.to_owned()))
}
Ok(_) => {
gst_video::VideoMeta::add_full(
buf,
video_frame_flags,
out_info.format(),
out_info.width(),
out_info.height(),
&offsets,
&strides,
)
.unwrap();
if input_buffer_writable {
Ok(PrepareOutputBufferSuccess::InputBuffer)
} else {
Ok(PrepareOutputBufferSuccess::Buffer(buf.to_owned()))
}
}
}
}
None => {
let n_planes = out_info.n_planes() as usize;
offsets[..n_planes].clone_from_slice(&out_info.offset()[..n_planes]);
strides[..n_planes].clone_from_slice(&out_info.stride()[..n_planes]);
gst_video::VideoMeta::add_full(
buf,
gst_video::VideoFrameFlags::empty(),
out_info.format(),
out_info.width(),
out_info.height(),
&offsets,
&strides,
)
.unwrap();
if input_buffer_writable {
Ok(PrepareOutputBufferSuccess::InputBuffer)
} else {
Ok(PrepareOutputBufferSuccess::Buffer(buf.to_owned()))
}
}
}
}
}
#[glib::object_subclass]
impl ObjectSubclass for RoundedCorners {
const NAME: &'static str = "RoundedCorners";
type Type = super::RoundedCorners;
type ParentType = gst_base::BaseTransform;
}
impl ObjectImpl for RoundedCorners {
fn properties() -> &'static [glib::ParamSpec] {
static PROPERTIES: Lazy<Vec<glib::ParamSpec>> = Lazy::new(|| {
vec![glib::ParamSpecUInt::new(
"border-radius-px",
"Border radius in pixels",
"Draw rounded corners with given border radius",
0,
u32::MAX,
DEFAULT_BORDER_RADIUS,
glib::ParamFlags::READWRITE | gst::PARAM_FLAG_MUTABLE_PLAYING,
)]
});
PROPERTIES.as_ref()
}
fn set_property(
&self,
obj: &Self::Type,
_id: usize,
value: &glib::Value,
pspec: &glib::ParamSpec,
) {
match pspec.name() {
"border-radius-px" => {
let mut settings = self.settings.lock().unwrap();
let border_radius = value.get().expect("type checked upstream");
if settings.border_radius_px != border_radius {
settings.changed = true;
settings.border_radius_px = border_radius;
gst_info!(
CAT,
obj: obj,
"Changing border radius from {} to {}",
settings.border_radius_px,
border_radius
);
obj.reconfigure_src();
}
}
_ => unimplemented!(),
}
}
fn property(&self, _obj: &Self::Type, _id: usize, pspec: &glib::ParamSpec) -> glib::Value {
match pspec.name() {
"border-radius-px" => {
let settings = self.settings.lock().unwrap();
settings.border_radius_px.to_value()
}
_ => unimplemented!(),
}
}
}
impl GstObjectImpl for RoundedCorners {}
impl ElementImpl for RoundedCorners {
fn metadata() -> Option<&'static gst::subclass::ElementMetadata> {
static ELEMENT_METADATA: Lazy<gst::subclass::ElementMetadata> = Lazy::new(|| {
gst::subclass::ElementMetadata::new(
"Rounded Corners",
"Filter/Effect/Converter/Video",
"Adds rounded corners to video",
"Sanchayan Maity <sanchayan@asymptotic.io>",
)
});
Some(&*ELEMENT_METADATA)
}
fn pad_templates() -> &'static [gst::PadTemplate] {
static PAD_TEMPLATES: Lazy<Vec<gst::PadTemplate>> = Lazy::new(|| {
let sink_caps = gst::Caps::builder("video/x-raw")
.field("format", VideoFormat::I420.to_str())
.field("width", gst::IntRange::new(1, i32::MAX))
.field("height", gst::IntRange::new(1, i32::MAX))
.field(
"framerate",
gst::FractionRange::new(
gst::Fraction::new(0, 1),
gst::Fraction::new(i32::MAX, 1),
),
)
.build();
let sink_pad_template = gst::PadTemplate::new(
"sink",
gst::PadDirection::Sink,
gst::PadPresence::Always,
&sink_caps,
)
.unwrap();
let src_caps = gst::Caps::builder("video/x-raw")
.field(
"format",
gst::List::new([VideoFormat::A420.to_str(), VideoFormat::I420.to_str()]),
)
.field("width", gst::IntRange::new(1, i32::MAX))
.field("height", gst::IntRange::new(1, i32::MAX))
.field(
"framerate",
gst::FractionRange::new(
gst::Fraction::new(0, 1),
gst::Fraction::new(i32::MAX, 1),
),
)
.build();
let src_pad_template = gst::PadTemplate::new(
"src",
gst::PadDirection::Src,
gst::PadPresence::Always,
&src_caps,
)
.unwrap();
vec![sink_pad_template, src_pad_template]
});
PAD_TEMPLATES.as_ref()
}
}
impl BaseTransformImpl for RoundedCorners {
const MODE: gst_base::subclass::BaseTransformMode =
gst_base::subclass::BaseTransformMode::AlwaysInPlace;
const PASSTHROUGH_ON_SAME_CAPS: bool = false;
const TRANSFORM_IP_ON_PASSTHROUGH: bool = false;
fn stop(&self, element: &Self::Type) -> Result<(), gst::ErrorMessage> {
let _ = self.state.lock().unwrap().take();
gst_info!(CAT, obj: element, "Stopped");
Ok(())
}
fn transform_caps(
&self,
element: &Self::Type,
direction: gst::PadDirection,
caps: &gst::Caps,
filter: Option<&gst::Caps>,
) -> Option<gst::Caps> {
let other_caps = if direction == gst::PadDirection::Src {
let mut caps = caps.clone();
for s in caps.make_mut().iter_mut() {
s.set("format", VideoFormat::I420.to_str());
}
caps
} else {
let mut output_caps = gst::Caps::new_empty();
{
let output_caps = output_caps.get_mut().unwrap();
let border_radius = self.settings.lock().unwrap().border_radius_px;
for s in caps.iter() {
let mut s_output = s.to_owned();
if border_radius == 0 {
s_output.set(
"format",
gst::List::new([
VideoFormat::I420.to_str(),
VideoFormat::A420.to_str(),
]),
);
} else {
s_output.set(
"format",
gst::List::new([
VideoFormat::A420.to_str(),
VideoFormat::I420.to_str(),
]),
);
}
output_caps.append_structure(s_output);
}
}
output_caps
};
gst_debug!(
CAT,
obj: element,
"Transformed caps from {} to {} in direction {:?}",
caps,
other_caps,
direction
);
if let Some(filter) = filter {
Some(filter.intersect_with_mode(&other_caps, gst::CapsIntersectMode::First))
} else {
Some(other_caps)
}
}
fn set_caps(
&self,
element: &Self::Type,
incaps: &gst::Caps,
outcaps: &gst::Caps,
) -> Result<(), gst::LoggableError> {
let mut settings = self.settings.lock().unwrap();
let out_info = match gst_video::VideoInfo::from_caps(outcaps) {
Err(_) => return Err(gst::loggable_error!(CAT, "Failed to parse output caps")),
Ok(info) => info,
};
gst_debug!(
CAT,
obj: element,
"Configured for caps {} to {}",
incaps,
outcaps
);
if out_info.format() == VideoFormat::I420 {
element.set_passthrough(true);
return Ok(());
} else {
element.set_passthrough(false);
}
// See "A420" planar 4:4:2:0 AYUV section
// https://gstreamer.freedesktop.org/documentation/additional/design/mediatype-video-raw.html?gi-language=c
let ru2_height = (out_info.height() + 1) & !1;
let alpha_mem_size = (out_info.stride()[3] as u32 * ru2_height) as usize;
*self.state.lock().unwrap() = Some(State {
alpha_mem: gst::Memory::with_size(alpha_mem_size),
out_info: Some(out_info),
});
settings.changed = true;
Ok(())
}
fn prepare_output_buffer(
&self,
element: &Self::Type,
inbuf: InputBuffer,
) -> Result<PrepareOutputBufferSuccess, gst::FlowError> {
if element.is_passthrough() {
return Ok(PrepareOutputBufferSuccess::InputBuffer);
}
let mut settings = self.settings.lock().unwrap();
if settings.changed {
settings.changed = false;
gst_debug!(
CAT,
obj: element,
"Caps or border radius changed, generating alpha mask"
);
let state_guard = self.state.lock().unwrap();
let state = state_guard.as_ref().ok_or_else(|| {
gst::element_error!(element, gst::CoreError::Negotiation, ["Have no state yet"]);
gst::FlowError::NotNegotiated
})?;
match state.out_info.as_ref().unwrap().format() {
VideoFormat::I420 => return Ok(PrepareOutputBufferSuccess::InputBuffer),
VideoFormat::A420 => {
drop(state_guard);
if self.generate_alpha_mask(settings.border_radius_px).is_err() {
gst::element_error!(
element,
gst::CoreError::Negotiation,
["Failed to generate alpha mask"]
);
return Err(gst::FlowError::NotNegotiated);
}
}
_ => unimplemented!(),
}
}
let mut state_guard = self.state.lock().unwrap();
let state = state_guard.as_mut().ok_or_else(|| {
gst::element_error!(element, gst::CoreError::Negotiation, ["Have no state yet"]);
gst::FlowError::NotNegotiated
})?;
let out_info = state.out_info.as_ref().unwrap();
let mem = &state.alpha_mem;
let alpha_mem = mem.clone();
match inbuf {
InputBuffer::Writable(outbuf) => {
gst_log!(
CAT,
obj: element,
"Received writable input buffer of size: {}",
outbuf.size()
);
let alpha_plane_offset = outbuf.size();
outbuf.append_memory(alpha_mem);
self.add_video_meta(outbuf, out_info, alpha_plane_offset, true)
}
InputBuffer::Readable(buf) => {
gst_log!(
CAT,
obj: element,
"Received readable input buffer of size: {}",
buf.size()
);
let alpha_plane_offset = buf.size();
let mut outbuf = buf.copy();
let mut_outbuf = outbuf.make_mut();
mut_outbuf.append_memory(alpha_mem);
self.add_video_meta(mut_outbuf, out_info, alpha_plane_offset, false)
}
}
}
fn transform_ip(
&self,
_element: &Self::Type,
_buf: &mut gst::BufferRef,
) -> Result<gst::FlowSuccess, gst::FlowError> {
Ok(gst::FlowSuccess::Ok)
}
fn propose_allocation(
&self,
element: &Self::Type,
decide_query: Option<gst::query::Allocation<&gst::QueryRef>>,
mut query: gst::query::Allocation<&mut gst::QueryRef>,
) -> Result<(), gst::ErrorMessage> {
query.add_allocation_meta::<gst_video::VideoMeta>(None);
self.parent_propose_allocation(element, decide_query, query)
}
}

26
video/videofx/src/lib.rs Normal file
View file

@ -0,0 +1,26 @@
// Copyright (C) 2021 Asymptotic Inc.
// Author: Sanchayan Maity <sanchayan@asymptotic.io>
//
// This Source Code Form is subject to the terms of the Mozilla Public License, v2.0.
// If a copy of the MPL was not distributed with this file, You can obtain one at
// <https://mozilla.org/MPL/2.0/>.
//
// SPDX-License-Identifier: MPL-2.0
mod border;
fn plugin_init(plugin: &gst::Plugin) -> Result<(), gst::glib::BoolError> {
border::register(plugin)
}
gst::plugin_define!(
videofx,
env!("CARGO_PKG_DESCRIPTION"),
plugin_init,
concat!(env!("CARGO_PKG_VERSION"), "-", env!("COMMIT_ID")),
"MPL-2.0",
env!("CARGO_PKG_NAME"),
env!("CARGO_PKG_NAME"),
env!("CARGO_PKG_REPOSITORY"),
env!("BUILD_REL_DATE")
);