ges: Fix smart rendering

Smart rendering has been broken since, mostly forever, but some code
was there pretending it was supported... let's try to stop pretending.

We now keep track of the smart rendering state in the timeline, track
and sources to be able to:

 * tell decodebin to stop plugging more (decoding elements) as soon as
   downstream supports the format.

 * avoid plugging converters after the source element when smart
   rendering.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-editing-services/-/merge_requests/198>

ges/ges-enums.h

@@ -344,7 +344,9 @@ GType ges_video_test_pattern_get_type (void);
  * #encodebin:avoid-reencoding property set to %FALSE)
  * @GES_PIPELINE_MODE_SMART_RENDER: Render the #GESPipeline:timeline,
  * avoiding decoding/reencoding (the underlying #encodebin has its
- * #encodebin:avoid-reencoding property set to %TRUE)
+ * #encodebin:avoid-reencoding property set to %TRUE).
+ * > NOTE: Smart rendering can not work in tracks where #GESTrack:mixing
+ * > is enabled.
  *
  * The various modes a #GESPipeline can be configured to.
  */

ges/ges-internal.h

@@ -176,6 +176,12 @@ ges_timeline_add_clip (GESTimeline * timeline, GESClip * clip, GError ** error);
 G_GNUC_INTERNAL void
 ges_timeline_remove_clip (GESTimeline * timeline, GESClip * clip);
 
+G_GNUC_INTERNAL void
+ges_timeline_set_smart_rendering (GESTimeline * timeline, gboolean rendering_smartly);
+
+G_GNUC_INTERNAL gboolean
+ges_timeline_get_smart_rendering (GESTimeline *timeline);
+
 G_GNUC_INTERNAL void
 ges_auto_transition_set_previous_source (GESAutoTransition * self, GESTrackElement * source);
 
@@ -475,6 +481,12 @@ G_GNUC_INTERNAL GstElement* ges_source_create_topbin  (GESSource *source,
                                                        const gchar* bin_name,
                                                        GstElement* sub_element,
                                                        GPtrArray* elements);
+G_GNUC_INTERNAL void ges_source_set_rendering_smartly (GESSource *source,
+                                                       gboolean rendering_smartly);
+G_GNUC_INTERNAL gboolean
+ges_source_get_rendering_smartly                      (GESSource *source);
+
+G_GNUC_INTERNAL void ges_track_set_smart_rendering     (GESTrack* track, gboolean rendering_smartly);
 G_GNUC_INTERNAL GstElement * ges_track_get_composition (GESTrack *track);
 
 

ges/ges-pipeline.c

@@ -1268,6 +1268,11 @@ ges_pipeline_set_mode (GESPipeline * pipeline, GESPipelineFlags mode)
         pipeline->priv->urisink, "sink", GST_PAD_LINK_CHECK_NOTHING);
   }
 
+  if (pipeline->priv->timeline) {
+    ges_timeline_set_smart_rendering (pipeline->priv->timeline,
+        (mode & GES_PIPELINE_MODE_SMART_RENDER) != 0);
+  }
+
   /* FIXUPS */
   /* FIXME
    * If we are rendering, set playsink to sync=False,

ges/ges-source.c

@@ -40,6 +40,7 @@ struct _GESSourcePrivate
   GstElement *last_converter;
   GstPad *ghostpad;
 
+  gboolean is_rendering_smartly;
 };
 
 G_DEFINE_TYPE_WITH_PRIVATE (GESSource, ges_source, GES_TYPE_TRACK_ELEMENT);
@@ -76,11 +77,11 @@ link_elements (GstElement * bin, GPtrArray * elements)
 static void
 _set_ghost_pad_target (GESSource * self, GstPad * srcpad, GstElement * element)
 {
-  GESSourcePrivate *priv = self->priv;
-  gboolean use_converter = FALSE;
   GstPadLinkReturn link_return;
+  GESSourcePrivate *priv = self->priv;
+  gboolean use_converter = ! !priv->first_converter;
 
-  if (priv->first_converter) {
+  if (use_converter && priv->is_rendering_smartly) {
     GstPad *pad = gst_element_get_static_pad (priv->first_converter, "sink");
     use_converter = gst_pad_can_link (srcpad, pad);
     gst_object_unref (pad);
@@ -168,6 +169,30 @@ ges_source_create_topbin (GESSource * source, const gchar * bin_name,
 }
 
 
+void
+ges_source_set_rendering_smartly (GESSource * source,
+    gboolean is_rendering_smartly)
+{
+
+  if (is_rendering_smartly) {
+    GESTrack *track = ges_track_element_get_track (GES_TRACK_ELEMENT (source));
+
+    if (track && ges_track_get_mixing (track)) {
+      GST_DEBUG_OBJECT (source, "Not rendering smartly as track is mixing!");
+
+      source->priv->is_rendering_smartly = FALSE;
+      return;
+    }
+  }
+  source->priv->is_rendering_smartly = is_rendering_smartly;
+}
+
+gboolean
+ges_source_get_rendering_smartly (GESSource * source)
+{
+  return source->priv->is_rendering_smartly;
+}
+
 static void
 ges_source_dispose (GObject * object)
 {

ges/ges-timeline-tree.c

@@ -2481,3 +2481,21 @@ timeline_tree_reset_layer_active (GNode * root, GESLayer * layer)
   g_node_traverse (root, G_PRE_ORDER, G_TRAVERSE_LEAFS, -1,
       (GNodeTraverseFunc) reset_layer_activness, layer);
 }
+
+static gboolean
+set_is_smart_rendering (GNode * node, gboolean * is_rendering_smartly)
+{
+  if (!GES_IS_SOURCE (node->data))
+    return FALSE;
+
+  ges_source_set_rendering_smartly (GES_SOURCE (node->data),
+      *is_rendering_smartly);
+  return FALSE;
+}
+
+void
+timeline_tree_set_smart_rendering (GNode * root, gboolean rendering_smartly)
+{
+  g_node_traverse (root, G_PRE_ORDER, G_TRAVERSE_LEAFS, -1,
+      (GNodeTraverseFunc) set_is_smart_rendering, &rendering_smartly);
+}

ges/ges-timeline-tree.h

@@ -82,5 +82,6 @@ void
 timeline_update_duration                  (GESTimeline * timeline);
 
 void timeline_tree_reset_layer_active     (GNode *root, GESLayer *layer);
+void timeline_tree_set_smart_rendering    (GNode * root, gboolean rendering_smartly);
 
 void timeline_tree_init_debug             (void);

ges/ges-timeline.c

@@ -230,6 +230,8 @@ struct _GESTimelinePrivate
   gboolean disposed;
 
   GstStreamCollection *stream_collection;
+
+  gboolean rendering_smartly;
 };
 
 /* private structure to contain our track-related information */
@@ -2051,6 +2053,10 @@ timeline_add_element (GESTimeline * timeline, GESTimelineElement * element)
       ges_timeline_element_get_name (element), gst_object_ref (element));
 
   timeline_tree_track_element (timeline->priv->tree, element);
+  if (GES_IS_SOURCE (element)) {
+    ges_source_set_rendering_smartly (GES_SOURCE (element),
+        timeline->priv->rendering_smartly);
+  }
 
   return TRUE;
 }
@@ -2085,6 +2091,32 @@ timeline_get_tree (GESTimeline * timeline)
   return timeline->priv->tree;
 }
 
+void
+ges_timeline_set_smart_rendering (GESTimeline * timeline,
+    gboolean rendering_smartly)
+{
+  if (rendering_smartly) {
+    GList *tmp;
+
+    for (tmp = timeline->tracks; tmp; tmp = tmp->next) {
+      if (ges_track_get_mixing (tmp->data)) {
+        GST_INFO_OBJECT (timeline, "Smart rendering will not"
+            " work as track %" GST_PTR_FORMAT " is doing mixing", tmp->data);
+      } else {
+        ges_track_set_smart_rendering (tmp->data, rendering_smartly);
+      }
+    }
+  }
+  timeline_tree_set_smart_rendering (timeline->priv->tree, rendering_smartly);
+  timeline->priv->rendering_smartly = rendering_smartly;
+}
+
+gboolean
+ges_timeline_get_smart_rendering (GESTimeline * timeline)
+{
+  return timeline->priv->rendering_smartly;
+}
+
 /**** API *****/
 /**
  * ges_timeline_new:

ges/ges-track.c

@@ -375,6 +375,15 @@ ges_track_get_composition (GESTrack * track)
   return track->priv->composition;
 }
 
+void
+ges_track_set_smart_rendering (GESTrack * track, gboolean rendering_smartly)
+{
+  GESTrackPrivate *priv = track->priv;
+
+  g_object_set (priv->capsfilter, "caps",
+      rendering_smartly ? NULL : priv->restriction_caps, NULL);
+}
+
 /* FIXME: Find out how to avoid doing this "hack" using the GDestroyNotify
  * function pointer in the trackelements_by_start GSequence
  *
@@ -985,6 +994,9 @@ ges_track_set_caps (GESTrack * track, const GstCaps * caps)
  * @caps: The new restriction-caps for @track
  *
  * Sets the #GESTrack:restriction-caps for the track.
+ *
+ * > **NOTE**: Restriction caps are **not** taken into account when
+ * > using #GESPipeline:mode=#GES_PIPELINE_MODE_SMART_RENDER.
  */
 void
 ges_track_set_restriction_caps (GESTrack * track, const GstCaps * caps)
@@ -1003,7 +1015,9 @@ ges_track_set_restriction_caps (GESTrack * track, const GstCaps * caps)
     gst_caps_unref (priv->restriction_caps);
   priv->restriction_caps = gst_caps_copy (caps);
 
-  g_object_set (priv->capsfilter, "caps", caps, NULL);
+  if (!track->priv->timeline ||
+      !ges_timeline_get_smart_rendering (track->priv->timeline))
+    g_object_set (priv->capsfilter, "caps", caps, NULL);
 
   g_object_notify (G_OBJECT (track), "restriction-caps");
 }
@@ -1105,6 +1119,9 @@ ges_track_set_mixing (GESTrack * track, gboolean mixing)
 notify:
   track->priv->mixing = mixing;
 
+  if (track->priv->timeline)
+    ges_timeline_set_smart_rendering (track->priv->timeline,
+        ges_timeline_get_smart_rendering (track->priv->timeline));
   g_object_notify_by_pspec (G_OBJECT (track), properties[ARG_MIXING]);
 
   GST_DEBUG_OBJECT (track, "The track has been set to mixing = %d", mixing);

ges/ges-uri-source.c

@@ -29,6 +29,36 @@ GST_DEBUG_CATEGORY_STATIC (uri_source_debug);
 #undef GST_CAT_DEFAULT
 #define GST_CAT_DEFAULT uri_source_debug
 
+typedef enum
+{
+  GST_AUTOPLUG_SELECT_TRY,
+  GST_AUTOPLUG_SELECT_EXPOSE,
+  GST_AUTOPLUG_SELECT_SKIP,
+} GstAutoplugSelectResult;
+
+static gint
+autoplug_select_cb (GstElement * bin, GstPad * pad, GstCaps * caps,
+    GstElementFactory * factory, GESUriSource * self)
+{
+  GstElement *nlesrc;
+  GstCaps *downstream_caps;
+  GstAutoplugSelectResult res = GST_AUTOPLUG_SELECT_TRY;
+
+  if (!ges_source_get_rendering_smartly (GES_SOURCE (self->element))) {
+    GST_LOG_OBJECT (self->element, "Not being smart here");
+    return res;
+  }
+
+  nlesrc = ges_track_element_get_nleobject (self->element);
+  downstream_caps = gst_pad_peer_query_caps (nlesrc->srcpads->data, NULL);
+  if (downstream_caps && gst_caps_can_intersect (downstream_caps, caps)) {
+    GST_DEBUG_OBJECT (self, "Exposing %s", GST_OBJECT_NAME (factory));
+    res = GST_AUTOPLUG_SELECT_EXPOSE;
+  }
+  gst_clear_caps (&downstream_caps);
+
+  return res;
+}
 
 GstElement *
 ges_uri_source_create_source (GESUriSource * self)
@@ -48,6 +78,8 @@ ges_uri_source_create_source (GESUriSource * self)
 
   g_object_set (decodebin, "caps", caps,
       "expose-all-streams", FALSE, "uri", self->uri, NULL);
+  g_signal_connect (decodebin, "autoplug-select",
+      G_CALLBACK (autoplug_select_cb), self);
 
   return decodebin;