diff --git a/girs/GES-1.0.gir b/girs/GES-1.0.gir index b5151e374b..aac093e295 100644 --- a/girs/GES-1.0.gir +++ b/girs/GES-1.0.gir @@ -3536,10 +3536,23 @@ and [clip management](http://pitivi.org/manual/usingclips.html). - Currently we only support effects with N sinkpads and one single srcpad. -Apart from `gesaudiomixer` and `gescompositor` which can be used as effects -and where sinkpads will be requested as needed based on the timeline topology -GES will always request at most one sinkpad per effect (when required). + Any GStreamer filter can be used as effects in GES. The only restriction we +have is that effects element should have a single [sinkpad](GST_PAD_SINK) +(which will be requested if necessary) and a single [srcpad](GST_PAD_SRC). + +Note that `gesaudiomixer` and `gescompositor` can be used as effects even +though they can have several sinkpads. + +## GES specific effects: + +* **`gesvideoscale`**: GES implements a specific scaling bin that allows + specifying where scaling will happen inside the chain of effects. By + default scaling can happen either in the source (if the source doesn't have + a specific size, like `videotestsrc` or [mixing](ges_track_set_mixing) has + been disabled) or in the mixing element otherwise, when adding that element + as an effect, GES guarantees that the scaling will happen in it. This can + be useful for example if you want to crop the video before scaling or apply + rounding corners to the video after scaling, etc... > Note: GES always adds converters (`audioconvert ! audioresample ! > audioconvert` for audio effects and `videoconvert` for video effects) to diff --git a/subprojects/gst-editing-services/ges/ges-clip.c b/subprojects/gst-editing-services/ges/ges-clip.c index 777b9f9555..f7b0332f1b 100644 --- a/subprojects/gst-editing-services/ges/ges-clip.c +++ b/subprojects/gst-editing-services/ges/ges-clip.c @@ -250,6 +250,7 @@ struct _GESClipPrivate gboolean allow_any_remove; + gint nb_scale_effects; gboolean use_effect_priority; guint32 effect_priority; GError *add_error; @@ -1638,6 +1639,7 @@ _add_child (GESContainer * container, GESTimelineElement * element) GESTimeline *timeline = GES_TIMELINE_ELEMENT_TIMELINE (container); GESClipPrivate *priv = self->priv; GESAsset *asset, *creator_asset; + gboolean adding_scale_effect = FALSE; gboolean prev_prevent = priv->prevent_duration_limit_update; gboolean prev_prevent_outpoint = priv->prevent_children_outpoint_update; GList *tmp; @@ -1775,6 +1777,14 @@ _add_child (GESContainer * container, GESTimelineElement * element) new_prio = MAX (new_prio, _PRIORITY (tmp->data) + 1); } } + + if (GES_IS_EFFECT (element)) { + GESAsset *asset = ges_extractable_get_asset (GES_EXTRACTABLE (element)); + const gchar *bindesc = ges_asset_get_id (asset); + + adding_scale_effect = !strstr (bindesc, "gesvideoscale"); + } + /* make sure higher than core */ for (tmp = container->children; tmp; tmp = tmp->next) { if (_IS_CORE_CHILD (tmp->data)) @@ -1817,9 +1827,16 @@ _add_child (GESContainer * container, GESTimelineElement * element) _update_active_for_track (self, track_el); priv->nb_effects++; + GST_DEBUG_OBJECT (self, "Adding %ith effect: %" GES_FORMAT " Priority %i", priv->nb_effects, GES_ARGS (element), new_prio); + if (adding_scale_effect) { + GST_DEBUG_OBJECT (self, "Adding scaling effect to clip " + "%" GES_FORMAT, GES_ARGS (self)); + priv->nb_scale_effects += 1; + } + /* changing priorities, and updating their offset */ priv->prevent_resort = TRUE; priv->setting_priority = TRUE; @@ -1900,6 +1917,12 @@ ges_clip_set_remove_error (GESClip * clip, GError * error) priv->remove_error = error; } +gboolean +ges_clip_has_scale_effect (GESClip * clip) +{ + return clip->priv->nb_scale_effects > 0; +} + static gboolean _remove_child (GESContainer * container, GESTimelineElement * element) { @@ -1961,6 +1984,17 @@ _remove_child (GESContainer * container, GESTimelineElement * element) * relative priorities */ /* height may have changed */ _compute_height (container); + + if (GES_IS_EFFECT (element)) { + GESAsset *asset = ges_extractable_get_asset (GES_EXTRACTABLE (element)); + const gchar *bindesc = ges_asset_get_id (asset); + + if (bindesc && !strstr (bindesc, "gesvideoscale")) { + GST_DEBUG_OBJECT (self, "Removing scaling effect to clip " + "%" GES_FORMAT, GES_ARGS (self)); + priv->nb_scale_effects -= 1; + } + } } /* duration-limit updated in _child_removed */ return TRUE; diff --git a/subprojects/gst-editing-services/ges/ges-effect.c b/subprojects/gst-editing-services/ges/ges-effect.c index fb6bf782ee..b26c1c91ca 100644 --- a/subprojects/gst-editing-services/ges/ges-effect.c +++ b/subprojects/gst-editing-services/ges/ges-effect.c @@ -23,10 +23,23 @@ * @short_description: adds an effect build from a parse-launch style bin * description to a stream in a GESSourceClip or a GESLayer * - * Currently we only support effects with N sinkpads and one single srcpad. - * Apart from `gesaudiomixer` and `gescompositor` which can be used as effects - * and where sinkpads will be requested as needed based on the timeline topology - * GES will always request at most one sinkpad per effect (when required). + * Any GStreamer filter can be used as effects in GES. The only restriction we + * have is that effects element should have a single [sinkpad](GST_PAD_SINK) + * (which will be requested if necessary) and a single [srcpad](GST_PAD_SRC). + * + * Note that `gesaudiomixer` and `gescompositor` can be used as effects even + * though they can have several sinkpads. + * + * ## GES specific effects: + * + * * **`gesvideoscale`**: GES implements a specific scaling bin that allows + * specifying where scaling will happen inside the chain of effects. By + * default scaling can happen either in the source (if the source doesn't have + * a specific size, like `videotestsrc` or [mixing](ges_track_set_mixing) has + * been disabled) or in the mixing element otherwise, when adding that element + * as an effect, GES guarantees that the scaling will happen in it. This can + * be useful for example if you want to crop the video before scaling or apply + * rounding corners to the video after scaling, etc... * * > Note: GES always adds converters (`audioconvert ! audioresample ! * > audioconvert` for audio effects and `videoconvert` for video effects) to diff --git a/subprojects/gst-editing-services/ges/ges-internal.h b/subprojects/gst-editing-services/ges/ges-internal.h index 785b2f8ddb..2cb5eb0c50 100644 --- a/subprojects/gst-editing-services/ges/ges-internal.h +++ b/subprojects/gst-editing-services/ges/ges-internal.h @@ -462,6 +462,7 @@ G_GNUC_INTERNAL void ges_clip_set_add_error (GESClip * cli G_GNUC_INTERNAL void ges_clip_take_add_error (GESClip * clip, GError ** error); G_GNUC_INTERNAL void ges_clip_set_remove_error (GESClip * clip, GError * error); G_GNUC_INTERNAL void ges_clip_take_remove_error (GESClip * clip, GError ** error); +G_GNUC_INTERNAL gboolean ges_clip_has_scale_effect (GESClip * clip); /**************************************************** * GESLayer * diff --git a/subprojects/gst-editing-services/ges/ges-smart-video-mixer.c b/subprojects/gst-editing-services/ges/ges-smart-video-mixer.c index f71bb24c88..3a8b9d608c 100644 --- a/subprojects/gst-editing-services/ges/ges-smart-video-mixer.c +++ b/subprojects/gst-editing-services/ges/ges-smart-video-mixer.c @@ -240,8 +240,13 @@ set_pad_properties_from_composition_meta (GstPad * mixer_pad, g_object_set (mixer_pad, "alpha", meta->alpha * transalpha, NULL); } - g_object_set (mixer_pad, "xpos", meta->posx, "ypos", - meta->posy, "width", meta->width, "height", meta->height, NULL); + g_object_set (mixer_pad, "xpos", meta->posx, "ypos", meta->posy, NULL); + + if (meta->width >= 0) + g_object_set (mixer_pad, "width", meta->width, NULL); + + if (meta->height >= 0) + g_object_set (mixer_pad, "height", meta->height, NULL); if (self->ABI.abi.has_operator) g_object_set (mixer_pad, "operator", meta->operator, NULL); diff --git a/subprojects/gst-editing-services/ges/ges.c b/subprojects/gst-editing-services/ges/ges.c index d34c4efd79..a2d8c0b5e6 100644 --- a/subprojects/gst-editing-services/ges/ges.c +++ b/subprojects/gst-editing-services/ges/ges.c @@ -54,6 +54,7 @@ G_LOCK_DEFINE_STATIC (init_lock); * between init/deinit */ static GThread *initialized_thread = NULL; +extern GType ges_video_scale_get_type (void); #ifndef GST_DISABLE_GST_DEBUG static gpointer @@ -96,9 +97,10 @@ ges_init_pre (GOptionContext * context, GOptionGroup * group, gpointer data, return TRUE; } + static gboolean -ges_init_post (GOptionContext * context, GOptionGroup * group, gpointer data, - GError ** error) +ges_init_post (GOptionContext * context, GOptionGroup * group, + gpointer data, GError ** error) { GESUriClipAssetClass *uriasset_klass = NULL; GstElementFactory *nlecomposition_factory = NULL; @@ -154,6 +156,7 @@ ges_init_post (GOptionContext * context, GOptionGroup * group, gpointer data, ges_asset_cache_init (); + gst_element_register (NULL, "gesvideoscale", 0, ges_video_scale_get_type ()); gst_element_register (NULL, "gesaudiomixer", 0, GES_TYPE_SMART_ADDER); gst_element_register (NULL, "gescompositor", 0, GES_TYPE_SMART_MIXER); gst_element_register (NULL, "framepositioner", 0, GST_TYPE_FRAME_POSITIONNER); diff --git a/subprojects/gst-editing-services/ges/gesvideoscale.c b/subprojects/gst-editing-services/ges/gesvideoscale.c new file mode 100644 index 0000000000..3ae0810c30 --- /dev/null +++ b/subprojects/gst-editing-services/ges/gesvideoscale.c @@ -0,0 +1,174 @@ +/* GStreamer + * Copyright (C) 2023 Thibault Saunier + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 51 Franklin Street, Suite 500, + * Boston, MA 02110-1335, USA. + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include + +#include "ges-frame-composition-meta.h" + +typedef struct _GESVideoScale GESVideoScale; +typedef struct +{ + GstBinClass parent_class; +} GESVideoScaleClass; + +struct _GESVideoScale +{ + GstBin parent; + + GstPad *sink; + GstElement *capsfilter; + + gint width, height; +}; + +/* *INDENT-OFF* */ +static GstStaticPadTemplate gst_video_scale_sink_template = + GST_STATIC_PAD_TEMPLATE ("sink", + GST_PAD_SINK, + GST_PAD_ALWAYS, + GST_STATIC_CAPS ("ANY") + ); + +static GstStaticPadTemplate gst_video_scale_src_template = + GST_STATIC_PAD_TEMPLATE ("src", + GST_PAD_SRC, + GST_PAD_ALWAYS, + GST_STATIC_CAPS ("ANY") + ); + +GES_DECLARE_TYPE (VideoScale, video_scale, VIDEO_SCALE) +G_DEFINE_TYPE (GESVideoScale, ges_video_scale, GST_TYPE_BIN); +/* *INDENT-ON* */ + +static void +set_dimension (GESVideoScale * self, gint width, gint height) +{ + GstCaps *caps = gst_caps_new_simple ("video/x-raw", + "pixel-aspect-ratio", GST_TYPE_FRACTION, 1, 1, + NULL); + + if (width >= 0) + gst_caps_set_simple (caps, "width", G_TYPE_INT, width, NULL); + if (height >= 0) + gst_caps_set_simple (caps, "height", G_TYPE_INT, height, NULL); + + gst_caps_set_features (caps, 0, gst_caps_features_new_any ()); + g_object_set (self->capsfilter, "caps", caps, NULL); + gst_caps_unref (caps); + + GST_OBJECT_LOCK (self); + self->width = width; + self->height = height; + GST_OBJECT_UNLOCK (self); +} + +static GstFlowReturn +chain (GstPad * pad, GESVideoScale * self, GstBuffer * buffer) +{ + GESFrameCompositionMeta *meta; + + meta = + (GESFrameCompositionMeta *) gst_buffer_get_meta (buffer, + ges_frame_composition_meta_api_get_type ()); + + if (meta) { + GST_OBJECT_LOCK (self); + if (meta->height != self->height || meta->width != self->width) { + GST_OBJECT_UNLOCK (self); + + set_dimension (self, meta->width, meta->height); + } else { + GST_OBJECT_UNLOCK (self); + } + + meta->height = meta->width = -1; + } + + return gst_proxy_pad_chain_default (pad, GST_OBJECT (self), buffer); +} + +static GstStateChangeReturn +change_state (GstElement * element, GstStateChange transition) +{ + GESVideoScale *self = GES_VIDEO_SCALE (element); + GstStateChangeReturn res = + ((GstElementClass *) ges_video_scale_parent_class)->change_state (element, + transition); + + if (transition == GST_STATE_CHANGE_PAUSED_TO_READY) { + GST_OBJECT_LOCK (self); + self->width = 0; + self->height = 0; + GST_OBJECT_UNLOCK (self); + } + + return res; +} + +static void +ges_video_scale_init (GESVideoScale * self) +{ + GstPad *pad; + GstElement *scale; + GstPadTemplate *template = + gst_static_pad_template_get (&gst_video_scale_sink_template); + + scale = gst_element_factory_make ("videoscale", NULL); + g_object_set (scale, "add-borders", FALSE, NULL); + self->capsfilter = gst_element_factory_make ("capsfilter", NULL); + + gst_bin_add_many (GST_BIN (self), scale, self->capsfilter, NULL); + gst_element_link (scale, self->capsfilter); + + self->sink = + gst_ghost_pad_new_from_template ("sink", scale->sinkpads->data, template); + gst_pad_set_chain_function (self->sink, (GstPadChainFunction) chain); + gst_element_add_pad (GST_ELEMENT (self), self->sink); + gst_object_unref (template); + + template = gst_static_pad_template_get (&gst_video_scale_src_template); + pad = + gst_ghost_pad_new_from_template ("src", self->capsfilter->srcpads->data, + template); + gst_element_add_pad (GST_ELEMENT (self), pad); + gst_object_unref (template); +} + +static void +ges_video_scale_class_init (GESVideoScaleClass * klass) +{ + GstElementClass *element_class = GST_ELEMENT_CLASS (klass); + + gst_element_class_set_static_metadata (element_class, + "VideoScale", + "Video/Filter", + "Scaling element usable as a GES effect", + "Thibault Saunier "); + + gst_element_class_add_static_pad_template (element_class, + &gst_video_scale_sink_template); + gst_element_class_add_static_pad_template (element_class, + &gst_video_scale_src_template); + + element_class->change_state = change_state; +} diff --git a/subprojects/gst-editing-services/ges/gstframepositioner.c b/subprojects/gst-editing-services/ges/gstframepositioner.c index 34783baf1f..57c7357358 100644 --- a/subprojects/gst-editing-services/ges/gstframepositioner.c +++ b/subprojects/gst-editing-services/ges/gstframepositioner.c @@ -113,6 +113,24 @@ gst_compositor_operator_get_type_and_default_value (int *default_operator_value) return operator_gtype; } +static gboolean +scales_downstream (GstFramePositioner * self) +{ + if (self->scale_in_compositor) + return TRUE; + + if (!self->track_source) + return self->scale_in_compositor; + + GESTimelineElement *parent = GES_TIMELINE_ELEMENT_PARENT (self->track_source); + + if (!parent || !GES_IS_CLIP (parent)) { + return self->scale_in_compositor; + } + + return ges_clip_has_scale_effect (GES_CLIP (parent)); +} + static void _weak_notify_cb (GstFramePositioner * pos, GObject * old) { @@ -274,7 +292,7 @@ gst_frame_positioner_update_properties (GstFramePositioner * pos, caps = gst_caps_from_string ("video/x-raw(ANY)"); if (pos->track_width && pos->track_height && - (!track_mixing || !pos->scale_in_compositor)) { + (!track_mixing || !scales_downstream (pos))) { gst_caps_set_simple (caps, "width", G_TYPE_INT, pos->track_width, "height", G_TYPE_INT, pos->track_height, NULL); } @@ -321,6 +339,13 @@ gst_frame_positioner_update_properties (GstFramePositioner * pos, reposition_properties (pos, old_track_width, old_track_height); done: + if (scales_downstream (pos) && pos->natural_width && pos->natural_height) { + GST_DEBUG_OBJECT (pos, + "Forcing natural width in source make downstream scaling work"); + gst_caps_set_simple (caps, "width", G_TYPE_INT, pos->natural_width, + "height", G_TYPE_INT, pos->natural_height, NULL); + } + GST_DEBUG_OBJECT (pos, "setting caps %" GST_PTR_FORMAT, caps); g_object_set (pos->capsfilter, "caps", caps, NULL); diff --git a/subprojects/gst-editing-services/ges/meson.build b/subprojects/gst-editing-services/ges/meson.build index 147517947a..7fd16e6764 100644 --- a/subprojects/gst-editing-services/ges/meson.build +++ b/subprojects/gst-editing-services/ges/meson.build @@ -67,6 +67,7 @@ ges_sources = files([ 'ges-structure-parser.c', 'ges-marker-list.c', 'ges-discoverer-manager.c', + 'gesvideoscale.c', 'gstframepositioner.c' ]) diff --git a/subprojects/gst-integration-testsuites/ges/scenarios/videoscale_effect.validatetest b/subprojects/gst-integration-testsuites/ges/scenarios/videoscale_effect.validatetest new file mode 100644 index 0000000000..e745647d00 --- /dev/null +++ b/subprojects/gst-integration-testsuites/ges/scenarios/videoscale_effect.validatetest @@ -0,0 +1,37 @@ +set-globals, media_file="$(test_dir)/../../medias/defaults/matroska/timed_frames_video_only_1fps.mkv" +meta, + tool = "ges-launch-$(gst_api_version)", + handles-states=true, + seek=true, + needs_preroll=true, + args = { + --track-types, video, + --video-caps, "video/x-raw, format=(string)I420, width=(int)1080, height=(int)720, framerate=(fraction)1/1", + --videosink, "$(videosink) name=videosink", + } + +# Add a clip and check that the first frame is displayed +add-clip, name=clip, asset-id="file://$(media_file)", layer-priority=0, type=GESUriClip, name=(string)theclip +set-child-properties, width=100, height=100, posx=10, posy=10, element-name=theclip +container-add-child, container-name=theclip, asset-id="videocrop name=videocrop", child-type=(string)GESEffect, child-name=crop; +set-child-properties, bottom=200, element-name=crop +container-add-child, container-name=theclip, asset-id="gesvideoscale name=videoscale", child-type=(string)GESEffect; + +pause + +# Checking that the 'framepositioner' is forcing caps to the clip "natual size" +check-properties, gesvideourisource0-capsfilter::caps=(GstCaps)"video/x-raw(ANY),framerate=(fraction)1/1,width=(int)1080,height=(int)720" +check-properties, videoscale::parent::parent::priority=3 +check-properties, videocrop::parent::parent::priority=4 +check-current-pad-caps, target-element-name=videocrop, pad=sink, expected-caps=[video/x-raw,width=1080,height=720] +check-current-pad-caps, target-element-name=videocrop, pad=src, expected-caps=[video/x-raw,width=1080,height=520] +check-current-pad-caps, target-element-name=videoscale, pad=sink, expected-caps=[video/x-raw,width=1080,height=520] +check-current-pad-caps, target-element-name=videoscale, pad=src, expected-caps=[video/x-raw,width=100,height=100] + +check-properties, + gessmartmixer0-compositor.sink_0::xpos=10, + gessmartmixer0-compositor.sink_0::ypos=10, + gessmartmixer0-compositor.sink_0::width=-1, + gessmartmixer0-compositor.sink_0::height=-1 + +stop diff --git a/subprojects/gst-integration-testsuites/testsuites/ges.testslist b/subprojects/gst-integration-testsuites/testsuites/ges.testslist index 2850961ed4..c064ac5e3c 100644 --- a/subprojects/gst-integration-testsuites/testsuites/ges.testslist +++ b/subprojects/gst-integration-testsuites/testsuites/ges.testslist @@ -760,3 +760,4 @@ ges.test.edit_deeply_nested_timeline_too_short ges.test.play_deeply_nested_back_to_back ges.test.play_two_nested_back_to_back ges.test.seek_on_stack_change_on_internal_subtimeline +ges.test.videoscale_effect