gstreamer/ges/ges-effect.c
Henry Wilkes 571120dcfb effect: Add support for time effects
Allow the user to register a child property of a base effect as a time
property. This can be used by GES to correctly calculate the
duration-limit of a clip when it has time effects on it. The existing
ges_effect_class_register_rate_property is now used to automatically
register such time effects for rate effects.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-editing-services/-/merge_requests/177>
2020-05-22 19:16:04 +01:00

542 lines
16 KiB
C

/* GStreamer Editing Services
* Copyright (C) 2010 Thibault Saunier <thibault.saunier@collabora.co.uk>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
* Boston, MA 02110-1301, USA.
*/
/**
* SECTION:geseffect
* @title: GESEffect
* @short_description: adds an effect build from a parse-launch style
* bin description to a stream in a GESSourceClip or a GESLayer
*
* Currently we only support effects with 1 sinkpad and 1 sourcepad
* with the exception of `gesaudiomixer` and `gescompositor` which
* can be used as effects.
*/
#ifdef HAVE_CONFIG_H
#include "config.h"
#endif
#include "ges-internal.h"
#include "ges-extractable.h"
#include "ges-track-element.h"
#include "ges-base-effect.h"
#include "ges-effect-asset.h"
#include "ges-effect.h"
static void ges_extractable_interface_init (GESExtractableInterface * iface);
static void ges_effect_dispose (GObject * object);
static void ges_effect_finalize (GObject * object);
static GstElement *ges_effect_create_element (GESTrackElement * self);
struct _GESEffectPrivate
{
gchar *bin_description;
};
enum
{
PROP_0,
PROP_BIN_DESCRIPTION,
};
G_DEFINE_TYPE_WITH_CODE (GESEffect,
ges_effect, GES_TYPE_BASE_EFFECT, G_ADD_PRIVATE (GESEffect)
G_IMPLEMENT_INTERFACE (GES_TYPE_EXTRACTABLE,
ges_extractable_interface_init));
static gchar *
extractable_check_id (GType type, const gchar * id, GError ** error)
{
gchar *bin_desc, *real_id;
GESTrackType ttype;
bin_desc = ges_effect_assect_id_get_type_and_bindesc (id, &ttype, error);
if (bin_desc == NULL)
return NULL;
if (ttype == GES_TRACK_TYPE_AUDIO)
real_id = g_strdup_printf ("audio %s", bin_desc);
else if (ttype == GES_TRACK_TYPE_VIDEO)
real_id = g_strdup_printf ("video %s", bin_desc);
else
g_assert_not_reached ();
g_free (bin_desc);
return real_id;
}
G_GNUC_BEGIN_IGNORE_DEPRECATIONS; /* Start ignoring GParameter deprecation */
static GParameter *
extractable_get_parameters_from_id (const gchar * id, guint * n_params)
{
GParameter *params = g_new0 (GParameter, 3);
gchar *bin_desc;
GESTrackType ttype;
bin_desc = ges_effect_assect_id_get_type_and_bindesc (id, &ttype, NULL);
params[0].name = "bin-description";
g_value_init (&params[0].value, G_TYPE_STRING);
g_value_set_string (&params[0].value, bin_desc);
params[1].name = "track-type";
g_value_init (&params[1].value, GES_TYPE_TRACK_TYPE);
g_value_set_flags (&params[1].value, ttype);
*n_params = 2;
g_free (bin_desc);
return params;
}
G_GNUC_END_IGNORE_DEPRECATIONS; /* End ignoring GParameter deprecation */
static gchar *
extractable_get_id (GESExtractable * self)
{
return g_strdup (GES_EFFECT (self)->priv->bin_description);
}
static void
ges_extractable_interface_init (GESExtractableInterface * iface)
{
iface->asset_type = GES_TYPE_EFFECT_ASSET;
iface->check_id = (GESExtractableCheckId) extractable_check_id;
iface->get_parameters_from_id = extractable_get_parameters_from_id;
iface->get_id = extractable_get_id;
}
static int
property_name_compare (gconstpointer s1, gconstpointer s2)
{
return g_strcmp0 ((const gchar *) s1, (const gchar *) s2);
}
static void
ges_effect_get_property (GObject * object,
guint property_id, GValue * value, GParamSpec * pspec)
{
GESEffectPrivate *priv = GES_EFFECT (object)->priv;
switch (property_id) {
case PROP_BIN_DESCRIPTION:
g_value_set_string (value, priv->bin_description);
break;
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec);
}
}
static void
ges_effect_set_property (GObject * object,
guint property_id, const GValue * value, GParamSpec * pspec)
{
GESEffect *self = GES_EFFECT (object);
switch (property_id) {
case PROP_BIN_DESCRIPTION:
self->priv->bin_description = g_value_dup_string (value);
break;
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec);
}
}
static void
ges_effect_class_init (GESEffectClass * klass)
{
GObjectClass *object_class;
GESTrackElementClass *obj_bg_class;
object_class = G_OBJECT_CLASS (klass);
obj_bg_class = GES_TRACK_ELEMENT_CLASS (klass);
object_class->get_property = ges_effect_get_property;
object_class->set_property = ges_effect_set_property;
object_class->dispose = ges_effect_dispose;
object_class->finalize = ges_effect_finalize;
obj_bg_class->create_element = ges_effect_create_element;
klass->rate_properties = NULL;
ges_effect_class_register_rate_property (klass, "scaletempo", "rate");
ges_effect_class_register_rate_property (klass, "pitch", "tempo");
ges_effect_class_register_rate_property (klass, "pitch", "rate");
ges_effect_class_register_rate_property (klass, "videorate", "rate");
/**
* GESEffect:bin-description:
*
* The description of the effect bin with a gst-launch-style
* pipeline description.
*
* Example: "videobalance saturation=1.5 hue=+0.5"
*/
g_object_class_install_property (object_class, PROP_BIN_DESCRIPTION,
g_param_spec_string ("bin-description",
"bin description",
"Bin description of the effect",
NULL, G_PARAM_READWRITE | G_PARAM_CONSTRUCT_ONLY));
}
static void
ges_effect_init (GESEffect * self)
{
self->priv = ges_effect_get_instance_private (self);
}
static void
ges_effect_dispose (GObject * object)
{
G_OBJECT_CLASS (ges_effect_parent_class)->dispose (object);
}
static void
ges_effect_finalize (GObject * object)
{
GESEffect *self = GES_EFFECT (object);
if (self->priv->bin_description)
g_free (self->priv->bin_description);
G_OBJECT_CLASS (ges_effect_parent_class)->finalize (object);
}
static void
ghost_compatible_pads (GstElement * bin, GstElement * child,
GstCaps * valid_caps, gint * n_src, gint * n_sink)
{
GList *tmp;
for (tmp = child->pads; tmp; tmp = tmp->next) {
GstCaps *caps;
GstPad *pad = tmp->data;
if (GST_PAD_PEER (pad))
continue;
caps = gst_pad_query_caps (pad, NULL);
if (gst_caps_can_intersect (caps, valid_caps)) {
gchar *name =
g_strdup_printf ("%s_%d", GST_PAD_IS_SINK (pad) ? "sink" : "src",
GST_PAD_IS_SINK (pad) ? *n_sink++ : *n_src++);
GST_DEBUG_OBJECT (bin, "Ghosting pad: %" GST_PTR_FORMAT, pad);
gst_element_add_pad (GST_ELEMENT (bin), gst_ghost_pad_new (name, pad));
g_free (name);
} else {
GST_DEBUG_OBJECT (pad, "Can't ghost pad %" GST_PTR_FORMAT, caps);
}
gst_caps_unref (caps);
}
}
static gdouble
_get_rate_factor (GESBaseEffect * effect, GHashTable * rate_values)
{
GHashTableIter iter;
gpointer key, val;
gdouble factor = 1.0;
g_hash_table_iter_init (&iter, rate_values);
while (g_hash_table_iter_next (&iter, &key, &val)) {
GValue *value = val;
gchar *prop_name = key;
gdouble rate = 1.0;
switch (G_VALUE_TYPE (value)) {
case G_TYPE_DOUBLE:
rate = g_value_get_double (value);
break;
case G_TYPE_FLOAT:
rate = g_value_get_float (value);
break;
default:
GST_ERROR_OBJECT (effect, "Rate property %s has neither a gdouble "
"nor gfloat value", prop_name);
break;
}
factor *= rate;
}
return factor;
}
static GstClockTime
_rate_source_to_sink (GESBaseEffect * effect, GstClockTime time,
GHashTable * rate_values, gpointer user_data)
{
/* multiply by rate factor
* E.g. rate=2.0, then the time 30 at the source would become
* 60 at the sink because we are using up twice as much data in a given
* time */
gdouble rate_factor = _get_rate_factor (effect, rate_values);
if (time == 0)
return 0;
if (rate_factor == 0.0) {
GST_ERROR_OBJECT (effect, "The rate effect has a rate of 0");
return 0;
}
return (GstClockTime) (time * rate_factor);
}
static GstClockTime
_rate_sink_to_source (GESBaseEffect * effect, GstClockTime time,
GHashTable * rate_values, gpointer user_data)
{
/* divide by rate factor */
gdouble rate_factor = _get_rate_factor (effect, rate_values);
if (time == 0)
return 0;
if (rate_factor == 0.0) {
GST_ERROR_OBJECT (effect, "The rate effect has a rate of 0");
return GST_CLOCK_TIME_NONE;
}
return (GstClockTime) (time / rate_factor);
}
static GstElement *
ges_effect_create_element (GESTrackElement * object)
{
GESBaseEffect *base_effect = GES_BASE_EFFECT (object);
GESEffectClass *class;
GList *tmp;
GstElement *effect;
gchar *bin_desc;
GstCaps *valid_caps;
gint n_src = 0, n_sink = 0;
gboolean is_rate_effect = FALSE;
GError *error = NULL;
GESEffect *self = GES_EFFECT (object);
const gchar *blacklisted_factories[] =
{ "audioconvert", "audioresample", "videoconvert", NULL };
GESTrackType type = ges_track_element_get_track_type (object);
if (!g_strcmp0 (self->priv->bin_description, "gesaudiomixer") ||
!g_strcmp0 (self->priv->bin_description, "gescompositor"))
return gst_element_factory_make (self->priv->bin_description, NULL);
if (type == GES_TRACK_TYPE_VIDEO) {
bin_desc = g_strconcat ("videoconvert name=pre_video_convert ! ",
self->priv->bin_description, " ! videoconvert name=post_video_convert",
NULL);
valid_caps = gst_caps_from_string ("video/x-raw(ANY)");
} else if (type == GES_TRACK_TYPE_AUDIO) {
bin_desc =
g_strconcat ("audioconvert ! audioresample !",
self->priv->bin_description, NULL);
valid_caps = gst_caps_from_string ("audio/x-raw(ANY)");
} else {
g_assert_not_reached ();
}
effect = gst_parse_bin_from_description (bin_desc, FALSE, &error);
g_free (bin_desc);
if (error != NULL) {
GST_ERROR ("An error occured while creating the GstElement: %s",
error->message);
g_error_free (error);
goto fail;
}
for (tmp = GST_BIN_CHILDREN (effect); tmp; tmp = tmp->next) {
ghost_compatible_pads (effect, tmp->data, valid_caps, &n_src, &n_sink);
if (n_src > 1) {
GST_ERROR ("More than 1 source pad in the effect, that is not possible");
goto fail;
}
}
ges_track_element_add_children_props (object, effect, NULL,
blacklisted_factories, NULL);
class = GES_EFFECT_CLASS (g_type_class_peek (GES_TYPE_EFFECT));
for (tmp = class->rate_properties; tmp; tmp = tmp->next) {
gchar *prop = tmp->data;
if (ges_timeline_element_lookup_child (GES_TIMELINE_ELEMENT (object), prop,
NULL, NULL)) {
if (!ges_base_effect_register_time_property (base_effect, prop))
GST_ERROR_OBJECT (object, "Failed to register rate property %s", prop);
is_rate_effect = TRUE;
}
}
if (is_rate_effect
&& !ges_base_effect_set_time_translation_funcs (base_effect,
_rate_source_to_sink, _rate_sink_to_source, NULL, NULL))
GST_ERROR_OBJECT (object, "Failed to set rate translation functions");
done:
gst_clear_caps (&valid_caps);
return effect;
fail:
gst_clear_object (&effect);
goto done;
}
/**
* ges_effect_new:
* @bin_description: The gst-launch like bin description of the effect
*
* Creates a new #GESEffect from the description of the bin. It should be
* possible to determine the type of the effect through the element
* 'klass' metadata of the GstElements that will be created.
* In that corner case, you should use:
* #ges_asset_request (GES_TYPE_EFFECT, "audio your ! bin ! description", NULL);
* and extract that asset to be in full control.
*
* Returns: (nullable): a newly created #GESEffect, or %NULL if something went
* wrong.
*/
GESEffect *
ges_effect_new (const gchar * bin_description)
{
GESEffect *effect;
GESAsset *asset = ges_asset_request (GES_TYPE_EFFECT,
bin_description, NULL);
g_return_val_if_fail (asset, NULL);
effect = GES_EFFECT (ges_asset_extract (asset, NULL));
gst_object_unref (asset);
return effect;
}
/**
* ges_effect_class_register_rate_property:
* @klass: Instance of the GESEffectClass
* @element_name: The #GstElementFactory name of the element that changes
* the rate
* @property_name: The name of the property that changes the rate
*
* Register an element that can change the rate at which media is playing.
* The property type must be float or double, and must be a factor of the
* rate, i.e. a value of 2.0 must mean that the media plays twice as fast.
* Several properties may be registered for a single element type,
* provided they all contribute to the rate as independent factors. For
* example, this is true for the "GstPitch::rate" and "GstPitch::tempo"
* properties. These are already registered by default in GES, along with
* #videorate:rate for #videorate and #scaletempo:rate for #scaletempo.
*
* If such a rate property becomes a child property of a #GESEffect upon
* its creation (the element is part of its #GESEffect:bin-description),
* it will be automatically registered as a time property (see
* ges_base_effect_register_time_property()) and will have its time
* translation functions set (see
* ges_base_effect_set_time_translation_funcs()) to use the overall rate
* of the rate properties. Note that if an effect contains a rate
* property as well as a non-rate time property, you should ensure to set
* the time translation functions to some other methods using
* ges_base_effect_set_time_translation_funcs().
*
* Note, you can obtain a reference to the GESEffectClass using
*
* ```
* GES_EFFECT_CLASS (g_type_class_ref (GES_TYPE_EFFECT));
* ```
*
* Returns: %TRUE if the rate property was successfully registered. When
* this method returns %FALSE, a warning is emitted with more information.
*/
gboolean
ges_effect_class_register_rate_property (GESEffectClass * klass,
const gchar * element_name, const gchar * property_name)
{
GstElementFactory *element_factory = NULL;
GstElement *element = NULL;
GParamSpec *pspec = NULL;
gchar *full_property_name = NULL;
GType param_type;
gboolean res = FALSE;
element_factory = gst_element_factory_find (element_name);
if (element_factory == NULL) {
GST_WARNING
("Did not add rate property '%s' for element '%s': the element factory could not be found",
property_name, element_name);
goto fail;
}
element = gst_element_factory_create (element_factory, NULL);
if (element == NULL) {
GST_WARNING
("Did not add rate property '%s' for element '%s': the element could not be constructed",
property_name, element_name);
goto fail;
}
pspec =
g_object_class_find_property (G_OBJECT_GET_CLASS (element),
property_name);
if (pspec == NULL) {
GST_WARNING
("Did not add rate property '%s' for element '%s': the element did not have the property name specified",
property_name, element_name);
goto fail;
}
param_type = G_PARAM_SPEC_VALUE_TYPE (pspec);
if (param_type != G_TYPE_FLOAT && param_type != G_TYPE_DOUBLE) {
GST_WARNING
("Did not add rate property '%s' for element '%s': the property is not of float or double type",
property_name, element_name);
goto fail;
}
full_property_name = g_strdup_printf ("%s::%s",
g_type_name (gst_element_factory_get_element_type (element_factory)),
property_name);
if (g_list_find_custom (klass->rate_properties, full_property_name,
property_name_compare) == NULL) {
klass->rate_properties =
g_list_append (klass->rate_properties, full_property_name);
GST_DEBUG ("Added rate property %s", full_property_name);
} else {
g_free (full_property_name);
}
res = TRUE;
fail:
if (element_factory != NULL)
gst_object_unref (element_factory);
if (element != NULL)
gst_object_unref (element);
if (pspec != NULL)
g_param_spec_unref (pspec);
return res;
}