mirrors/gst-plugins-rs
mirrors/gst-plugins-rs
1
1
Fork 0
mirror of https://gitlab.freedesktop.org/gstreamer/gst-plugins-rs.git synced 2024-09-21 19:40:07 +00:00
Code Issues Projects Releases Wiki Activity

videofx: Add colordetect video filter

Browse source 
This new video filter is able to detect the dominant color in a video frame.
When the color has changed from the previous frame the filter posts an Element
message on the bus, the associated structure is named `colordetect` and has two
fields:

* a string field named `dominant-color`
* a list field containing the whole color palette, stored as uint values, sorted
  by dominance, with more dominant colors first
This commit is contained in:
Philippe Normand 2022-04-06 12:48:45 +01:00
parent e82678586f
commit b423febfbe
5 changed files with 447 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
video/videofx/Cargo.toml
View file

@ -11,6 +11,10 @@ edition = "2021"
cairo-rs = { git = "https://github.com/gtk-rs/gtk-rs-core", features=["use_glib"] }
atomic_refcell = "0.1"
once_cell = "1.0"
# color-thief = "0.2"
# https://github.com/RazrFalcon/color-thief-rs/pull/4
color-thief = { git = "https://github.com/philn/color-thief-rs", branch = "max-colors-respect" }
color-name = "1.0.0"
[dependencies.gst]
git = "https://gitlab.freedesktop.org/gstreamer/gstreamer-rs"

348
video/videofx/src/colordetect/imp.rs Normal file
View file

@ -0,0 +1,348 @@
// Copyright (C) 2022, Igalia S.L
// Author: Philippe Normand <philn@igalia.com>
//
// This Source Code Form is subject to the terms of the Mozilla Public License, v2.0.
// If a copy of the MPL was not distributed with this file, You can obtain one at
// <https://mozilla.org/MPL/2.0/>.
//
// SPDX-License-Identifier: MPL-2.0
use atomic_refcell::AtomicRefCell;
use color_name;
use color_thief::{get_palette, Color, ColorFormat};
use gst::{glib, subclass::prelude::*};
use gst_base::prelude::*;
use gst_video::{subclass::prelude::*, VideoFormat};
use once_cell::sync::Lazy;
use std::sync::Mutex;
const DEFAULT_QUALITY: u32 = 10;
const DEFAULT_MAX_COLORS: u32 = 2;
static CAT: Lazy<gst::DebugCategory> = Lazy::new(|| {
gst::DebugCategory::new(
"colordetect",
gst::DebugColorFlags::empty(),
Some("Dominant color detection"),
)
});
#[derive(Debug, Clone, Copy)]
struct Settings {
quality: u32,
max_colors: u32,
}
impl Default for Settings {
fn default() -> Self {
Settings {
quality: DEFAULT_QUALITY,
max_colors: DEFAULT_MAX_COLORS,
}
}
}
struct State {
color_format: ColorFormat,
out_info: gst_video::VideoInfo,
current_color: Option<String>,
}
#[derive(Default)]
pub struct ColorDetect {
settings: Mutex<Settings>,
state: AtomicRefCell<Option<State>>,
}
impl ColorDetect {
fn detect_color(
&self,
element: &super::ColorDetect,
buf: &mut gst::BufferRef,
) -> Result<Option<(String, Vec<Color>)>, gst::FlowError> {
let mut state_guard = self.state.borrow_mut();
let state = state_guard.as_mut().ok_or_else(|| {
gst::element_error!(element, gst::CoreError::Negotiation, ["Have no state yet"]);
gst::FlowError::NotNegotiated
})?;
let settings = *self.settings.lock().unwrap();
let frame =
gst_video::VideoFrameRef::from_buffer_ref_readable(buf, &state.out_info).unwrap();
let palette = get_palette(
frame.plane_data(0).unwrap(),
state.color_format,
settings.quality as u8,
settings.max_colors as u8,
)
.map_err(|_| gst::FlowError::Error)?;
let dominant_color = palette[0];
let dominant_color_name =
color_name::Color::similar([dominant_color.r, dominant_color.g, dominant_color.b])
.to_lowercase();
if state
.current_color
.as_ref()
.map_or(true, |current_color| current_color != &dominant_color_name)
{
let name = dominant_color_name.clone();
state.current_color = Some(dominant_color_name);
return Ok(Some((name, palette)));
}
Ok(None)
}
fn color_changed(
&self,
element: &super::ColorDetect,
dominant_color_name: &str,
palette: Vec<Color>,
) {
gst::debug!(
CAT,
obj: element,
"Dominant color changed to {}",
dominant_color_name
);
let palette_colors = gst::List::new(
palette
.iter()
.map(|c| ((c.r as u32) << 16) | ((c.g as u32) << 8) | (c.b as u32)),
);
element
.post_message(
gst::message::Element::builder(
gst::structure::Structure::builder("colordetect")
.field("dominant-color", dominant_color_name)
.field("palette", palette_colors)
.build(),
)
.build(),
)
.expect("Element without bus. Should not happen!");
}
}
#[glib::object_subclass]
impl ObjectSubclass for ColorDetect {
const NAME: &'static str = "GstColorDetect";
type Type = super::ColorDetect;
type ParentType = gst_base::BaseTransform;
}
impl ObjectImpl for ColorDetect {
fn properties() -> &'static [glib::ParamSpec] {
static PROPERTIES: Lazy<Vec<glib::ParamSpec>> = Lazy::new(|| {
vec![
glib::ParamSpecUInt::new(
"quality",
"Quality of an output colors",
"A step in pixels to improve performance",
0,
10,
DEFAULT_QUALITY,
glib::ParamFlags::READWRITE | gst::PARAM_FLAG_MUTABLE_PLAYING,
),
glib::ParamSpecUInt::new(
"max-colors",
"Number of colors in the output palette",
"Actual colors count can be lower depending on the image",
2,
255,
DEFAULT_MAX_COLORS,
glib::ParamFlags::READWRITE | gst::PARAM_FLAG_MUTABLE_PLAYING,
),
]
});
PROPERTIES.as_ref()
}
fn set_property(
&self,
obj: &Self::Type,
_id: usize,
value: &glib::Value,
pspec: &glib::ParamSpec,
) {
match pspec.name() {
"quality" => {
let mut settings = self.settings.lock().unwrap();
let quality = value.get().expect("type checked upstream");
if settings.quality != quality {
gst::info!(
CAT,
obj: obj,
"Changing quality from {} to {}",
settings.quality,
quality
);
settings.quality = quality;
}
}
"max-colors" => {
let mut settings = self.settings.lock().unwrap();
let max_colors = value.get().expect("type checked upstream");
if settings.max_colors != max_colors {
gst::info!(
CAT,
obj: obj,
"Changing max_colors from {} to {}",
settings.max_colors,
max_colors
);
settings.max_colors = max_colors;
}
}
_ => unimplemented!(),
}
}
fn property(&self, _obj: &Self::Type, _id: usize, pspec: &glib::ParamSpec) -> glib::Value {
match pspec.name() {
"quality" => {
let settings = self.settings.lock().unwrap();
settings.quality.to_value()
}
"max-colors" => {
let settings = self.settings.lock().unwrap();
settings.max_colors.to_value()
}
_ => unimplemented!(),
}
}
}
impl GstObjectImpl for ColorDetect {}
impl ElementImpl for ColorDetect {
fn metadata() -> Option<&'static gst::subclass::ElementMetadata> {
static ELEMENT_METADATA: Lazy<gst::subclass::ElementMetadata> = Lazy::new(|| {
gst::subclass::ElementMetadata::new(
"Dominant color detection",
"Filter/Video",
"Detects the dominant color of a video",
"Philippe Normand <philn@igalia.com>",
)
});
Some(&*ELEMENT_METADATA)
}
fn pad_templates() -> &'static [gst::PadTemplate] {
static PAD_TEMPLATES: Lazy<Vec<gst::PadTemplate>> = Lazy::new(|| {
let formats = gst::List::new([
VideoFormat::Rgb.to_str(),
VideoFormat::Rgba.to_str(),
VideoFormat::Argb.to_str(),
VideoFormat::Bgr.to_str(),
VideoFormat::Bgra.to_str(),
]);
let caps = gst::Caps::builder("video/x-raw")
.field("format", &formats)
.field("width", gst::IntRange::new(1, i32::MAX))
.field("height", gst::IntRange::new(1, i32::MAX))
.field(
"framerate",
gst::FractionRange::new(
gst::Fraction::new(0, 1),
gst::Fraction::new(i32::MAX, 1),
),
)
.build();
let sink_pad_template = gst::PadTemplate::new(
"sink",
gst::PadDirection::Sink,
gst::PadPresence::Always,
&caps,
)
.unwrap();
let src_pad_template = gst::PadTemplate::new(
"src",
gst::PadDirection::Src,
gst::PadPresence::Always,
&caps,
)
.unwrap();
vec![sink_pad_template, src_pad_template]
});
PAD_TEMPLATES.as_ref()
}
}
impl BaseTransformImpl for ColorDetect {
const MODE: gst_base::subclass::BaseTransformMode =
gst_base::subclass::BaseTransformMode::AlwaysInPlace;
const PASSTHROUGH_ON_SAME_CAPS: bool = false;
const TRANSFORM_IP_ON_PASSTHROUGH: bool = false;
fn stop(&self, element: &Self::Type) -> Result<(), gst::ErrorMessage> {
*self.state.borrow_mut() = None;
gst::info!(CAT, obj: element, "Stopped");
Ok(())
}
fn set_caps(
&self,
element: &Self::Type,
incaps: &gst::Caps,
outcaps: &gst::Caps,
) -> Result<(), gst::LoggableError> {
let in_info = match gst_video::VideoInfo::from_caps(incaps) {
Err(_) => return Err(gst::loggable_error!(CAT, "Failed to parse input caps")),
Ok(info) => info,
};
let out_info = match gst_video::VideoInfo::from_caps(outcaps) {
Err(_) => return Err(gst::loggable_error!(CAT, "Failed to parse output caps")),
Ok(info) => info,
};
gst::debug!(
CAT,
obj: element,
"Configured for caps {} to {}",
incaps,
outcaps
);
let color_format = match in_info.format() {
VideoFormat::Rgb => ColorFormat::Rgb,
VideoFormat::Rgba => ColorFormat::Rgba,
VideoFormat::Argb => ColorFormat::Argb,
VideoFormat::Bgr => ColorFormat::Bgr,
VideoFormat::Bgra => ColorFormat::Bgra,
_ => unimplemented!(),
};
let previous_color = match self.state.borrow().as_ref() {
Some(state) => state.current_color.clone(),
None => None,
};
*self.state.borrow_mut() = Some(State {
color_format,
out_info,
current_color: previous_color,
});
Ok(())
}
fn transform_ip(
&self,
element: &Self::Type,
buf: &mut gst::BufferRef,
) -> Result<gst::FlowSuccess, gst::FlowError> {
if let Some((dominant_color_name, palette)) = self.detect_color(element, buf)? {
self.color_changed(element, &dominant_color_name, palette);
}
Ok(gst::FlowSuccess::Ok)
}
}

26
video/videofx/src/colordetect/mod.rs Normal file
View file

@ -0,0 +1,26 @@
// Copyright (C) 2022, Igalia S.L
// Author: Philippe Normand <philn@igalia.com>
//
// This Source Code Form is subject to the terms of the Mozilla Public License, v2.0.
// If a copy of the MPL was not distributed with this file, You can obtain one at
// <https://mozilla.org/MPL/2.0/>.
//
// SPDX-License-Identifier: MPL-2.0
use gst::glib;
use gst::prelude::*;
pub mod imp;
glib::wrapper! {
pub struct ColorDetect(ObjectSubclass<imp::ColorDetect>) @extends gst_base::BaseTransform, gst::Element, gst::Object;
}
pub fn register(plugin: &gst::Plugin) -> Result<(), glib::BoolError> {
gst::Element::register(
Some(plugin),
"colordetect",
gst::Rank::None,
ColorDetect::static_type(),
)
}

4
video/videofx/src/lib.rs
View file

@ -9,9 +9,11 @@
#![allow(clippy::non_send_fields_in_send_ty)]
mod border;
mod colordetect;
fn plugin_init(plugin: &gst::Plugin) -> Result<(), gst::glib::BoolError> {
border::register(plugin)
border::register(plugin)?;
colordetect::register(plugin)
}
gst::plugin_define!(

66
video/videofx/tests/colordetect.rs Normal file
View file

@ -0,0 +1,66 @@
// Copyright (C) 2022 Philippe Normand <philn@igalia.com>
//
// This Source Code Form is subject to the terms of the Mozilla Public License, v2.0.
// If a copy of the MPL was not distributed with this file, You can obtain one at
// <https://mozilla.org/MPL/2.0/>.
//
// SPDX-License-Identifier: MPL-2.0
use gst::prelude::*;
fn init() {
use std::sync::Once;
static INIT: Once = Once::new();
INIT.call_once(|| {
gst::init().unwrap();
gstvideofx::plugin_register_static().expect("Failed to register videofx plugin");
});
}
#[test]
fn test_red_color() {
init();
let pipeline = gst::Pipeline::new(None);
let src = gst::ElementFactory::make("videotestsrc", None).unwrap();
src.set_property_from_str("pattern", "red");
src.set_property("num-buffers", &2i32);
let filter = gst::ElementFactory::make("colordetect", None).unwrap();
let sink = gst::ElementFactory::make("fakevideosink", None).unwrap();
pipeline
.add_many(&[&src, &filter, &sink])
.expect("failed to add elements to the pipeline");
gst::Element::link_many(&[&src, &filter, &sink]).expect("failed to link the elements");
pipeline
.set_state(gst::State::Playing)
.expect("Unable to set the pipeline to the `Playing` state");
let mut detected_color: Option<String> = None;
let bus = pipeline.bus().unwrap();
for msg in bus.iter_timed(gst::ClockTime::NONE) {
use gst::MessageView;
match msg.view() {
MessageView::Element(elt) => {
if let Some(s) = elt.structure() {
if s.name() == "colordetect" {
// The video source emits 2 red frames, but we should
// receive only one message because the dominant color
// doesn't change.
assert_eq!(detected_color.as_deref(), None);
detected_color = Some(s.get::<String>("dominant-color").unwrap());
}
}
}
MessageView::Eos(..) => break,
_ => (),
}
}
pipeline.set_state(gst::State::Null).unwrap();
assert_eq!(detected_color.as_deref(), Some("red"));
}