gstreamer/ges/ges-track.c

/* GStreamer Editing Services
 * Copyright (C) 2009 Edward Hervey <edward.hervey@collabora.co.uk>
 *               2009 Nokia Corporation
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Library General Public
 * License as published by the Free Software Foundation; either
 * version 2 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Library General Public License for more details.
 *
 * You should have received a copy of the GNU Library General Public
 * License along with this library; if not, write to the
 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
 * Boston, MA 02110-1301, USA.
 */

/**
 * SECTION:gestrack
 * @title: GESTrack
 * @short_description: The output source of a #GESTimeline
 *
 * A #GESTrack acts an output source for a #GESTimeline. Each one
 * essentially provides an additional #GstPad for the timeline, with
 * #GESTrack:restriction-caps capabilities. Internally, a track
 * wraps an #nlecomposition filtered by a #capsfilter.
 *
 * A track will contain a number of #GESTrackElement-s, and its role is
 * to select and activate these elements according to their timings when
 * the timeline in played. For example, a track would activate a
 * #GESSource when its #GESTimelineElement:start is reached by outputting
 * its data for its #GESTimelineElement:duration. Similarly, a
 * #GESOperation would be activated by applying its effect to the source
 * data, starting from its #GESTimelineElement:start time and lasting for
 * its #GESTimelineElement:duration.
 *
 * For most users, it will usually be sufficient to add newly created
 * tracks to a timeline, but never directly add an element to a track.
 * Whenever a #GESClip is added to a timeline, the clip adds its
 * elements to the timeline's tracks and assumes responsibility for
 * updating them.
 */
#ifdef HAVE_CONFIG_H
#include "config.h"
#endif

#include "ges-internal.h"
#include "ges-track.h"
#include "ges-track-element.h"
#include "ges-meta-container.h"
#include "ges-video-track.h"
#include "ges-audio-track.h"

#define CHECK_THREAD(track) g_assert(track->priv->valid_thread == g_thread_self())

static GstStaticPadTemplate ges_track_src_pad_template =
GST_STATIC_PAD_TEMPLATE ("src",
    GST_PAD_SRC,
    GST_PAD_ALWAYS,
    GST_STATIC_CAPS_ANY);

/* Structure that represents gaps and keep knowledge
 * of the gaps filled in the track */
typedef struct
{
  GstElement *nleobj;

  GstClockTime start;
  GstClockTime duration;
  GESTrack *track;
} Gap;

struct _GESTrackPrivate
{
  /*< private > */
  GESTimeline *timeline;
  GSequence *trackelements_by_start;
  GHashTable *trackelements_iter;
  GList *gaps;
  gboolean last_gap_disabled;

  guint64 duration;

  GstCaps *caps;
  GstCaps *restriction_caps;

  GstElement *composition;      /* The composition associated with this track */
  GstPad *srcpad;               /* The source GhostPad */

  gboolean updating;

  gboolean mixing;
  GstElement *mixing_operation;
  GstElement *capsfilter;

  /* Virtual method to create GstElement that fill gaps */
  GESCreateElementForGapFunc create_element_for_gaps;

  GThread *valid_thread;
};

enum
{
  ARG_0,
  ARG_CAPS,
  ARG_RESTRICTION_CAPS,
  ARG_TYPE,
  ARG_DURATION,
  ARG_MIXING,
  ARG_ID,
  ARG_LAST,
  TRACK_ELEMENT_ADDED,
  TRACK_ELEMENT_REMOVED,
  COMMITED,
  LAST_SIGNAL
};

static guint ges_track_signals[LAST_SIGNAL] = { 0 };

static GParamSpec *properties[ARG_LAST];

G_DEFINE_TYPE_WITH_CODE (GESTrack, ges_track, GST_TYPE_BIN,
    G_ADD_PRIVATE (GESTrack)
    G_IMPLEMENT_INTERFACE (GES_TYPE_META_CONTAINER, NULL));


static void composition_duration_cb (GstElement * composition, GParamSpec * arg
    G_GNUC_UNUSED, GESTrack * obj);

/* Private methods/functions/callbacks */
static void
add_trackelement_to_list_foreach (GESTrackElement * trackelement, GList ** list)
{
  gst_object_ref (trackelement);
  *list = g_list_prepend (*list, trackelement);
}

static Gap *
gap_new (GESTrack * track, GstClockTime start, GstClockTime duration)
{
  GstElement *nlesrc, *elem;

  Gap *new_gap;

  nlesrc = gst_element_factory_make ("nlesource", NULL);
  elem = track->priv->create_element_for_gaps (track);
  if (G_UNLIKELY (gst_bin_add (GST_BIN (nlesrc), elem) == FALSE)) {
    GST_WARNING_OBJECT (track, "Could not create gap filler");

    if (nlesrc)
      gst_object_unref (nlesrc);

    if (elem)
      gst_object_unref (elem);

    return NULL;
  }

  if (G_UNLIKELY (ges_nle_composition_add_object (track->priv->composition,
              nlesrc) == FALSE)) {
    GST_WARNING_OBJECT (track, "Could not add gap to the composition");

    if (nlesrc)
      gst_object_unref (nlesrc);

    if (elem)
      gst_object_unref (elem);

    return NULL;
  }

  new_gap = g_slice_new (Gap);
  new_gap->start = start;
  new_gap->duration = duration;
  new_gap->track = track;
  new_gap->nleobj = nlesrc;


  g_object_set (nlesrc, "start", new_gap->start, "duration", new_gap->duration,
      "priority", 1, NULL);

  GST_DEBUG_OBJECT (track,
      "Created gap with start %" GST_TIME_FORMAT " duration %" GST_TIME_FORMAT,
      GST_TIME_ARGS (new_gap->start), GST_TIME_ARGS (new_gap->duration));


  return new_gap;
}

static void
free_gap (Gap * gap)
{
  GESTrack *track = gap->track;

  GST_DEBUG_OBJECT (track, "Removed gap with start %" GST_TIME_FORMAT
      " duration %" GST_TIME_FORMAT, GST_TIME_ARGS (gap->start),
      GST_TIME_ARGS (gap->duration));
  ges_nle_composition_remove_object (track->priv->composition, gap->nleobj);

  g_slice_free (Gap, gap);
}

static inline void
update_gaps (GESTrack * track)
{
  Gap *gap;
  GList *gaps;
  GSequenceIter *it;

  GESTrackElement *trackelement;
  GstClockTime start, end, duration = 0, timeline_duration = 0;

  GESTrackPrivate *priv = track->priv;

  if (priv->create_element_for_gaps == NULL) {
    GST_INFO ("Not filling the gaps as no create_element_for_gaps vmethod"
        " provided");
    return;
  }

  gaps = priv->gaps;
  priv->gaps = NULL;

  /* 1- And recalculate gaps */
  for (it = g_sequence_get_begin_iter (priv->trackelements_by_start);
      g_sequence_iter_is_end (it) == FALSE; it = g_sequence_iter_next (it)) {
    trackelement = g_sequence_get (it);

    if (!ges_track_element_is_active (trackelement))
      continue;

    if (priv->timeline) {
      guint32 layer_prio = GES_TIMELINE_ELEMENT_LAYER_PRIORITY (trackelement);

      if (layer_prio != GES_TIMELINE_ELEMENT_NO_LAYER_PRIORITY) {
        GESLayer *layer = g_list_nth_data (priv->timeline->layers, layer_prio);

        if (!ges_layer_get_active_for_track (layer, track))
          continue;
      }
    }

    start = _START (trackelement);
    end = start + _DURATION (trackelement);

    if (start > duration) {
      /* 2- Fill gap */
      gap = gap_new (track, duration, start - duration);

      if (G_LIKELY (gap != NULL))
        priv->gaps = g_list_prepend (priv->gaps, gap);
    }

    duration = MAX (duration, end);
  }

  /* 3- Add a gap at the end of the timeline if needed */
  if (priv->timeline) {
    g_object_get (priv->timeline, "duration", &timeline_duration, NULL);

    if (duration < timeline_duration) {
      gap = gap_new (track, duration, timeline_duration - duration);

      if (G_LIKELY (gap != NULL)) {
        priv->gaps = g_list_prepend (priv->gaps, gap);
      }

      /* FIXME: here the duration is set to the duration of the timeline,
       * but elsewhere it is set to the duration of the composition. Are
       * these always the same? */
      priv->duration = timeline_duration;
    }
  }

  if (!track->priv->last_gap_disabled) {
    GST_DEBUG_OBJECT (track, "Adding a one second gap at the end");
    gap = gap_new (track, timeline_duration, 1);
    priv->gaps = g_list_prepend (priv->gaps, gap);
  }

  /* 4- Remove old gaps */
  g_list_free_full (gaps, (GDestroyNotify) free_gap);
}

void
track_disable_last_gap (GESTrack * track, gboolean disabled)
{
  track->priv->last_gap_disabled = disabled;
  update_gaps (track);
}

void
track_resort_and_fill_gaps (GESTrack * track)
{
  g_sequence_sort (track->priv->trackelements_by_start,
      (GCompareDataFunc) element_start_compare, NULL);

  if (track->priv->updating == TRUE) {
    update_gaps (track);
  }
}

static gboolean
update_field (GQuark field_id, const GValue * value, GstStructure * original)
{
  gst_structure_id_set_value (original, field_id, value);
  return TRUE;
}

/* callbacks */
static void
_ghost_nlecomposition_srcpad (GESTrack * track)
{
  GstPad *capsfilter_sink;
  GstPad *capsfilter_src;
  GESTrackPrivate *priv = track->priv;
  GstPad *pad = gst_element_get_static_pad (priv->composition, "src");

  capsfilter_sink = gst_element_get_static_pad (priv->capsfilter, "sink");

  GST_DEBUG ("track:%p, pad %s:%s", track, GST_DEBUG_PAD_NAME (pad));

  gst_pad_link (pad, capsfilter_sink);
  gst_object_unref (capsfilter_sink);
  gst_object_unref (pad);

  capsfilter_src = gst_element_get_static_pad (priv->capsfilter, "src");
  /* ghost the pad */
  priv->srcpad = gst_ghost_pad_new ("src", capsfilter_src);
  gst_object_unref (capsfilter_src);
  gst_pad_set_active (priv->srcpad, TRUE);
  gst_element_add_pad (GST_ELEMENT (track), priv->srcpad);

  GST_DEBUG ("done");
}

static void
composition_duration_cb (GstElement * composition,
    GParamSpec * arg G_GNUC_UNUSED, GESTrack * track)
{
  guint64 duration;

  g_object_get (composition, "duration", &duration, NULL);

  if (track->priv->duration != duration) {
    GST_DEBUG_OBJECT (track,
        "composition duration : %" GST_TIME_FORMAT " current : %"
        GST_TIME_FORMAT, GST_TIME_ARGS (duration),
        GST_TIME_ARGS (track->priv->duration));

    /* FIXME: here the duration is set to the duration of the composition,
     * but elsewhere it is set to the duration of the timeline. Are these
     * always the same? */
    track->priv->duration = duration;

    g_object_notify_by_pspec (G_OBJECT (track), properties[ARG_DURATION]);
  }
}

static void
composition_commited_cb (GstElement * composition, gboolean changed,
    GESTrack * self)
{
  g_signal_emit (self, ges_track_signals[COMMITED], 0);
}

/* Internal */
GstElement *
ges_track_get_composition (GESTrack * track)
{
  return track->priv->composition;
}

/* FIXME: Find out how to avoid doing this "hack" using the GDestroyNotify
 * function pointer in the trackelements_by_start GSequence
 *
 * Remove @object from @track, but keeps it in the sequence this is needed
 * when finalizing as we can not change a GSequence at the same time we are
 * accessing it
 */
static gboolean
remove_object_internal (GESTrack * track, GESTrackElement * object,
    gboolean emit)
{
  GESTrackPrivate *priv;
  GstElement *nleobject;

  GST_DEBUG_OBJECT (track, "object:%p", object);

  priv = track->priv;

  if (G_UNLIKELY (ges_track_element_get_track (object) != track)) {
    GST_WARNING_OBJECT (track, "Object belongs to another track");
    return FALSE;
  }

  if (!ges_track_element_set_track (object, NULL)) {
    GST_WARNING_OBJECT (track, "Failed to unset the track for %" GES_FORMAT,
        GES_ARGS (object));
    return FALSE;
  }
  ges_timeline_element_set_timeline (GES_TIMELINE_ELEMENT (object), NULL);

  if ((nleobject = ges_track_element_get_nleobject (object))) {
    GST_DEBUG ("Removing NleObject '%s' from composition '%s'",
        GST_ELEMENT_NAME (nleobject), GST_ELEMENT_NAME (priv->composition));

    if (!ges_nle_composition_remove_object (priv->composition, nleobject)) {
      GST_WARNING_OBJECT (track, "Failed to remove nleobject from composition");
      return FALSE;
    }
  }

  if (emit)
    g_signal_emit (track, ges_track_signals[TRACK_ELEMENT_REMOVED], 0,
        GES_TRACK_ELEMENT (object));

  gst_object_unref (object);

  return TRUE;
}

static void
dispose_trackelements_foreach (GESTrackElement * trackelement, GESTrack * track)
{
  remove_object_internal (track, trackelement, TRUE);
}

/* GstElement virtual methods */

static GstStateChangeReturn
ges_track_change_state (GstElement * element, GstStateChange transition)
{
  GESTrack *track = GES_TRACK (element);

  if (transition == GST_STATE_CHANGE_READY_TO_PAUSED &&
      track->priv->valid_thread == g_thread_self ())
    track_resort_and_fill_gaps (GES_TRACK (element));

  return GST_ELEMENT_CLASS (ges_track_parent_class)->change_state (element,
      transition);
}

static void
ges_track_handle_message (GstBin * bin, GstMessage * message)
{
  GESTrack *track = GES_TRACK (bin);

  if (GST_MESSAGE_TYPE (message) == GST_MESSAGE_STREAM_COLLECTION) {
    gint i;
    GList *selected_streams = NULL;
    GstStreamCollection *collection;

    gst_message_parse_stream_collection (message, &collection);

    for (i = 0; i < gst_stream_collection_get_size (collection); i++) {
      GstStream *stream = gst_stream_collection_get_stream (collection, i);
      GstStreamType stype = gst_stream_get_stream_type (stream);

      if ((track->type == GES_TRACK_TYPE_VIDEO
              && stype == GST_STREAM_TYPE_VIDEO)
          || (track->type == GES_TRACK_TYPE_AUDIO
              && stype == GST_STREAM_TYPE_AUDIO)
          || (stype == GST_STREAM_TYPE_UNKNOWN)) {

        selected_streams =
            g_list_append (selected_streams,
            (gchar *) gst_stream_get_stream_id (stream));
      }
    }

    if (selected_streams) {
      gst_element_send_event (GST_ELEMENT (GST_MESSAGE_SRC (message)),
          gst_event_new_select_streams (selected_streams));
      g_list_free (selected_streams);
    }
  }
  gst_element_post_message (GST_ELEMENT_CAST (bin), message);
}

/* GObject virtual methods */
static void
ges_track_get_property (GObject * object, guint property_id,
    GValue * value, GParamSpec * pspec)
{
  GESTrack *track = GES_TRACK (object);

  switch (property_id) {
    case ARG_CAPS:
      gst_value_set_caps (value, track->priv->caps);
      break;
    case ARG_TYPE:
      g_value_set_flags (value, track->type);
      break;
    case ARG_DURATION:
      g_value_set_uint64 (value, track->priv->duration);
      break;
    case ARG_RESTRICTION_CAPS:
      gst_value_set_caps (value, track->priv->restriction_caps);
      break;
    case ARG_MIXING:
      g_value_set_boolean (value, track->priv->mixing);
      break;
    case ARG_ID:
      g_object_get_property (G_OBJECT (track->priv->composition), "id", value);
      break;
    default:
      G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec);
  }
}

static void
ges_track_set_property (GObject * object, guint property_id,
    const GValue * value, GParamSpec * pspec)
{
  GESTrack *track = GES_TRACK (object);

  switch (property_id) {
    case ARG_CAPS:
      ges_track_set_caps (track, gst_value_get_caps (value));
      break;
    case ARG_TYPE:
      track->type = g_value_get_flags (value);
      break;
    case ARG_RESTRICTION_CAPS:
      ges_track_set_restriction_caps (track, gst_value_get_caps (value));
      break;
    case ARG_MIXING:
      ges_track_set_mixing (track, g_value_get_boolean (value));
      break;
    case ARG_ID:
      g_object_set_property (G_OBJECT (track->priv->composition), "id", value);
      break;
    default:
      G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec);
  }
}

static void
ges_track_dispose (GObject * object)
{
  GESTrack *track = (GESTrack *) object;
  GESTrackPrivate *priv = track->priv;

  /* Remove all TrackElements and drop our reference */
  g_hash_table_unref (priv->trackelements_iter);
  g_sequence_foreach (track->priv->trackelements_by_start,
      (GFunc) dispose_trackelements_foreach, track);
  g_sequence_free (priv->trackelements_by_start);
  g_list_free_full (priv->gaps, (GDestroyNotify) free_gap);
  ges_nle_object_commit (track->priv->composition, TRUE);

  gst_clear_object (&track->priv->mixing_operation);
  if (priv->composition) {
    gst_element_remove_pad (GST_ELEMENT (track), priv->srcpad);
    gst_bin_remove (GST_BIN (object), priv->composition);
    priv->composition = NULL;
  }

  if (priv->caps) {
    gst_caps_unref (priv->caps);
    priv->caps = NULL;
  }

  if (priv->restriction_caps) {
    gst_caps_unref (priv->restriction_caps);
    priv->restriction_caps = NULL;
  }

  G_OBJECT_CLASS (ges_track_parent_class)->dispose (object);
}

static void
ges_track_finalize (GObject * object)
{
  G_OBJECT_CLASS (ges_track_parent_class)->finalize (object);
}

static void
ges_track_constructed (GObject * object)
{
  GESTrack *self = GES_TRACK (object);
  gchar *componame = NULL;
  gchar *capsfiltername = NULL;

  if (self->type == GES_TRACK_TYPE_VIDEO) {
    componame =
        g_strdup_printf ("video_%s", GST_OBJECT_NAME (self->priv->composition));
    capsfiltername =
        g_strdup_printf ("video_restriction_%s",
        GST_OBJECT_NAME (self->priv->capsfilter));
  } else if (self->type == GES_TRACK_TYPE_AUDIO) {
    componame =
        g_strdup_printf ("audio_%s", GST_OBJECT_NAME (self->priv->composition));
    capsfiltername =
        g_strdup_printf ("audio_restriction_%s",
        GST_OBJECT_NAME (self->priv->capsfilter));
  }

  if (componame) {
    gst_object_set_name (GST_OBJECT (self->priv->composition), componame);
    gst_object_set_name (GST_OBJECT (self->priv->capsfilter), capsfiltername);

    g_free (componame);
    g_free (capsfiltername);
  }

  if (!gst_bin_add (GST_BIN (self), self->priv->composition))
    GST_ERROR ("Couldn't add composition to bin !");

  if (!gst_bin_add (GST_BIN (self), self->priv->capsfilter))
    GST_ERROR ("Couldn't add capsfilter to bin !");

  _ghost_nlecomposition_srcpad (self);
  if (GES_TRACK_GET_CLASS (self)->get_mixing_element) {
    GstElement *nleobject;
    GstElement *mixer = GES_TRACK_GET_CLASS (self)->get_mixing_element (self);

    if (mixer == NULL) {
      GST_WARNING_OBJECT (self, "Got no element fron get_mixing_element");

      return;
    }

    nleobject = gst_element_factory_make ("nleoperation", "mixing-operation");
    if (!gst_bin_add (GST_BIN (nleobject), mixer)) {
      GST_WARNING_OBJECT (self, "Could not add the mixer to our composition");
      gst_object_unref (mixer);
      gst_object_unref (nleobject);

      return;
    }
    g_object_set (nleobject, "expandable", TRUE, NULL);

    if (self->priv->mixing) {
      if (!ges_nle_composition_add_object (self->priv->composition, nleobject)) {
        GST_WARNING_OBJECT (self, "Could not add the mixer to our composition");
        gst_object_unref (nleobject);

        return;
      }
    }

    self->priv->mixing_operation = gst_object_ref (nleobject);

  } else {
    GST_INFO_OBJECT (self, "No way to create a main mixer");
  }
}

static void
ges_track_class_init (GESTrackClass * klass)
{
  GObjectClass *object_class = G_OBJECT_CLASS (klass);
  GstElementClass *gstelement_class = (GstElementClass *) klass;
  GstBinClass *bin_class = GST_BIN_CLASS (klass);

  gstelement_class->change_state = GST_DEBUG_FUNCPTR (ges_track_change_state);

  bin_class->handle_message = GST_DEBUG_FUNCPTR (ges_track_handle_message);

  object_class->get_property = ges_track_get_property;
  object_class->set_property = ges_track_set_property;
  object_class->dispose = ges_track_dispose;
  object_class->finalize = ges_track_finalize;
  object_class->constructed = ges_track_constructed;

  /**
   * GESTrack:caps:
   *
   * The capabilities used to choose the output of the #GESTrack's
   * elements. Internally, this is used to select output streams when
   * several may be available, by determining whether its #GstPad is
   * compatible (see #nlecomposition:caps for #nlecomposition). As such,
   * this is used as a weaker indication of the desired output type of the
   * track, **before** the #GESTrack:restriction-caps is applied.
   * Therefore, this should be set to a *generic* superset of the
   * #GESTrack:restriction-caps, such as "video/x-raw(ANY)". In addition,
   * it should match with the track's #GESTrack:track-type.
   *
   * Note that when you set this property, the #GstCapsFeatures of all its
   * #GstStructure-s will be automatically set to #GST_CAPS_FEATURES_ANY.
   *
   * Once a track has been added to a #GESTimeline, you should not change
   * this.
   *
   * Default value: #GST_CAPS_ANY.
   */
  properties[ARG_CAPS] = g_param_spec_boxed ("caps", "Caps",
      "Caps used to choose the output stream",
      GST_TYPE_CAPS, G_PARAM_READWRITE | G_PARAM_CONSTRUCT_ONLY);
  g_object_class_install_property (object_class, ARG_CAPS,
      properties[ARG_CAPS]);

  /**
   * GESTrack:restriction-caps:
   *
   * The capabilities that specifies the final output format of the
   * #GESTrack. For example, for a video track, it would specify the
   * height, width, framerate and other properties of the stream.
   *
   * You may change this property after the track has been added to a
   * #GESTimeline, but it must remain compatible with the track's
   * #GESTrack:caps.
   *
   * Default value: #GST_CAPS_ANY.
   */
  properties[ARG_RESTRICTION_CAPS] =
      g_param_spec_boxed ("restriction-caps", "Restriction caps",
      "Caps used as a final filter on the output stream", GST_TYPE_CAPS,
      G_PARAM_READWRITE);
  g_object_class_install_property (object_class, ARG_RESTRICTION_CAPS,
      properties[ARG_RESTRICTION_CAPS]);

  /**
   * GESTrack:duration:
   *
   * Current duration of the track
   *
   * Default value: O
   */
  /* FIXME: is duration the duration of the timeline or the duration of
   * the underlying composition? */
  properties[ARG_DURATION] = g_param_spec_uint64 ("duration", "Duration",
      "The current duration of the track", 0, G_MAXUINT64, GST_SECOND,
      G_PARAM_READABLE);
  g_object_class_install_property (object_class, ARG_DURATION,
      properties[ARG_DURATION]);

  /**
   * GESTrack:track-type:
   *
   * The track type of the track. This controls the type of
   * #GESTrackElement-s that can be added to the track. This should
   * match with the track's #GESTrack:caps.
   *
   * Once a track has been added to a #GESTimeline, you should not change
   * this.
   */
  properties[ARG_TYPE] = g_param_spec_flags ("track-type", "TrackType",
      "Type of stream the track outputs",
      GES_TYPE_TRACK_TYPE, GES_TRACK_TYPE_CUSTOM,
      G_PARAM_READWRITE | G_PARAM_CONSTRUCT_ONLY);
  g_object_class_install_property (object_class, ARG_TYPE,
      properties[ARG_TYPE]);

  /**
   * GESTrack:mixing:
   *
   * Whether the track should support the mixing of #GESLayer data, such
   * as composing the video data of each layer (when part of the video
   * data is transparent, the next layer will become visible) or adding
   * together the audio data. As such, for audio and video tracks, you'll
   * likely want to keep this set to %TRUE.
   */
  properties[ARG_MIXING] = g_param_spec_boolean ("mixing", "Mixing",
      "Whether layer mixing is activated on the track or not",
      TRUE, G_PARAM_READWRITE | G_PARAM_CONSTRUCT | G_PARAM_EXPLICIT_NOTIFY);
  g_object_class_install_property (object_class, ARG_MIXING,
      properties[ARG_MIXING]);

  /**
   * GESTrack:id:
   *
   * The #nlecomposition:id of the underlying #nlecomposition.
   *
   * Since: 1.18
   */
  properties[ARG_ID] =
      g_param_spec_string ("id", "Id", "The stream-id of the composition",
      NULL, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS);
  g_object_class_install_property (object_class, ARG_ID, properties[ARG_ID]);

  gst_element_class_add_static_pad_template (gstelement_class,
      &ges_track_src_pad_template);

  /**
   * GESTrack::track-element-added:
   * @object: The #GESTrack
   * @effect: The element that was added
   *
   * Will be emitted after a track element is added to the track.
   */
  ges_track_signals[TRACK_ELEMENT_ADDED] =
      g_signal_new ("track-element-added", G_TYPE_FROM_CLASS (klass),
      G_SIGNAL_RUN_FIRST, 0, NULL, NULL, NULL,
      G_TYPE_NONE, 1, GES_TYPE_TRACK_ELEMENT);

  /**
   * GESTrack::track-element-removed:
   * @object: The #GESTrack
   * @effect: The element that was removed
   *
   * Will be emitted after a track element is removed from the track.
   */
  ges_track_signals[TRACK_ELEMENT_REMOVED] =
      g_signal_new ("track-element-removed", G_TYPE_FROM_CLASS (klass),
      G_SIGNAL_RUN_FIRST, 0, NULL, NULL, NULL,
      G_TYPE_NONE, 1, GES_TYPE_TRACK_ELEMENT);

  /**
   * GESTrack::commited:
   * @track: The #GESTrack
   *
   * This signal will be emitted once the changes initiated by
   * ges_track_commit() have been executed in the backend. In particular,
   * this will be emitted whenever the underlying #nlecomposition has been
   * committed (see #nlecomposition::commited).
   */
  ges_track_signals[COMMITED] =
      g_signal_new ("commited", G_TYPE_FROM_CLASS (klass),
      G_SIGNAL_RUN_LAST, 0, NULL, NULL, NULL, G_TYPE_NONE, 0);

  klass->get_mixing_element = NULL;
}

static void
ges_track_init (GESTrack * self)
{
  self->priv = ges_track_get_instance_private (self);
  self->priv->valid_thread = g_thread_self ();

  self->priv->composition = gst_element_factory_make ("nlecomposition", NULL);
  self->priv->capsfilter = gst_element_factory_make ("capsfilter", NULL);
  self->priv->updating = TRUE;
  self->priv->trackelements_by_start = g_sequence_new (NULL);
  self->priv->trackelements_iter =
      g_hash_table_new (g_direct_hash, g_direct_equal);
  self->priv->create_element_for_gaps = NULL;
  self->priv->gaps = NULL;
  self->priv->mixing = TRUE;
  self->priv->restriction_caps = NULL;
  self->priv->last_gap_disabled = TRUE;

  g_signal_connect (G_OBJECT (self->priv->composition), "notify::duration",
      G_CALLBACK (composition_duration_cb), self);
  g_signal_connect (G_OBJECT (self->priv->composition), "commited",
      G_CALLBACK (composition_commited_cb), self);
}

/**
 * ges_track_new:
 * @type: The #GESTrack:track-type for the track
 * @caps: (transfer full): The #GESTrack:caps for the track
 *
 * Creates a new track with the given track-type and caps.
 *
 * If @type is #GES_TRACK_TYPE_VIDEO, and @caps is a subset of
 * "video/x-raw(ANY)", then a #GESVideoTrack is created. This will
 * automatically choose a gap creation method suitable for video data. You
 * will likely want to set #GESTrack:restriction-caps separately. You may
 * prefer to use the ges_video_track_new() method instead.
 *
 * If @type is #GES_TRACK_TYPE_AUDIO, and @caps is a subset of
 * "audio/x-raw(ANY)", then a #GESAudioTrack is created. This will
 * automatically choose a gap creation method suitable for audio data, and
 * will set the #GESTrack:restriction-caps to the default for
 * #GESAudioTrack. You may prefer to use the ges_audio_track_new() method
 * instead.
 *
 * Otherwise, a plain #GESTrack is returned. You will likely want to set
 * the #GESTrack:restriction-caps and call
 * ges_track_set_create_element_for_gap_func() on the returned track.
 *
 * Returns: (transfer floating): A new track.
 */
GESTrack *
ges_track_new (GESTrackType type, GstCaps * caps)
{
  GESTrack *track;
  GstCaps *tmpcaps;

  /* TODO Be smarter with well known track types */
  if (type == GES_TRACK_TYPE_VIDEO) {
    tmpcaps = gst_caps_new_empty_simple ("video/x-raw");
    gst_caps_set_features (tmpcaps, 0, gst_caps_features_new_any ());

    if (gst_caps_is_subset (caps, tmpcaps)) {
      track = GES_TRACK (ges_video_track_new ());
      ges_track_set_caps (track, caps);

      gst_caps_unref (tmpcaps);
      /* FIXME: ges_track_set_caps does not take ownership of caps,
       * so we also need to unref caps */
      return track;
    }
    gst_caps_unref (tmpcaps);
  } else if (type == GES_TRACK_TYPE_AUDIO) {
    tmpcaps = gst_caps_new_empty_simple ("audio/x-raw");
    gst_caps_set_features (tmpcaps, 0, gst_caps_features_new_any ());

    if (gst_caps_is_subset (caps, tmpcaps)) {
      track = GES_TRACK (ges_audio_track_new ());
      ges_track_set_caps (track, caps);

      gst_caps_unref (tmpcaps);
      /* FIXME: ges_track_set_caps does not take ownership of caps,
       * so we also need to unref caps */
      return track;
    }

    gst_caps_unref (tmpcaps);
  }

  track = g_object_new (GES_TYPE_TRACK, "caps", caps, "track-type", type, NULL);
  gst_caps_unref (caps);

  return track;
}


/**
 * ges_track_set_timeline:
 * @track: A #GESTrack
 * @timeline (nullable): A #GESTimeline
 *
 * Informs the track that it belongs to the given timeline. Calling this
 * does not actually add the track to the timeline. For that, you should
 * use ges_timeline_add_track(), which will also take care of informing
 * the track that it belongs to the timeline. As such, there is no need
 * for you to call this method.
 */
/* FIXME: this should probably be deprecated and only used internally */
void
ges_track_set_timeline (GESTrack * track, GESTimeline * timeline)
{
  GSequenceIter *it;
  g_return_if_fail (GES_IS_TRACK (track));
  g_return_if_fail (timeline == NULL || GES_IS_TIMELINE (timeline));
  GST_DEBUG ("track:%p, timeline:%p", track, timeline);

  track->priv->timeline = timeline;

  for (it = g_sequence_get_begin_iter (track->priv->trackelements_by_start);
      g_sequence_iter_is_end (it) == FALSE; it = g_sequence_iter_next (it)) {
    GESTimelineElement *trackelement =
        GES_TIMELINE_ELEMENT (g_sequence_get (it));
    ges_timeline_element_set_timeline (trackelement, timeline);
  }
  track_resort_and_fill_gaps (track);
}

/**
 * ges_track_set_caps:
 * @track: A #GESTrack
 * @caps: The new caps for @track
 *
 * Sets the #GESTrack:caps for the track. The new #GESTrack:caps of the
 * track will be a copy of @caps, except its #GstCapsFeatures will be
 * automatically set to #GST_CAPS_FEATURES_ANY.
 */
void
ges_track_set_caps (GESTrack * track, const GstCaps * caps)
{
  GESTrackPrivate *priv;
  gint i;

  g_return_if_fail (GES_IS_TRACK (track));
  CHECK_THREAD (track);

  GST_DEBUG ("track:%p, caps:%" GST_PTR_FORMAT, track, caps);
  g_return_if_fail (GST_IS_CAPS (caps));

  priv = track->priv;

  if (priv->caps)
    gst_caps_unref (priv->caps);
  priv->caps = gst_caps_copy (caps);

  for (i = 0; i < (int) gst_caps_get_size (priv->caps); i++)
    gst_caps_set_features (priv->caps, i, gst_caps_features_new_any ());

  g_object_set (priv->composition, "caps", caps, NULL);
  /* FIXME : update all trackelements ? */
}

/**
 * ges_track_set_restriction_caps:
 * @track: A #GESTrack
 * @caps: The new restriction-caps for @track
 *
 * Sets the #GESTrack:restriction-caps for the track.
 */
void
ges_track_set_restriction_caps (GESTrack * track, const GstCaps * caps)
{
  GESTrackPrivate *priv;

  g_return_if_fail (GES_IS_TRACK (track));
  CHECK_THREAD (track);

  GST_DEBUG ("track:%p, restriction caps:%" GST_PTR_FORMAT, track, caps);
  g_return_if_fail (GST_IS_CAPS (caps));

  priv = track->priv;

  if (priv->restriction_caps)
    gst_caps_unref (priv->restriction_caps);
  priv->restriction_caps = gst_caps_copy (caps);

  g_object_set (priv->capsfilter, "caps", caps, NULL);

  g_object_notify (G_OBJECT (track), "restriction-caps");
}

/**
 * ges_track_update_restriction_caps:
 * @track: A #GESTrack
 * @caps: The caps to update the restriction-caps with
 *
 * Updates the #GESTrack:restriction-caps of the track using the fields
 * found in the given caps. Each of the #GstStructure-s in @caps is
 * compared against the existing structure with the same index in the
 * current #GESTrack:restriction-caps. If there is no corresponding
 * existing structure at that index, then the new structure is simply
 * copied to that index. Otherwise, any fields in the new structure are
 * copied into the existing structure. This will replace existing values,
 * and may introduce new ones, but any fields 'missing' in the new
 * structure are left unchanged in the existing structure.
 *
 * For example, if the existing #GESTrack:restriction-caps are
 * "video/x-raw, width=480, height=360", and the updating caps is
 * "video/x-raw, format=I420, width=500; video/x-bayer, width=400", then
 * the new #GESTrack:restriction-caps after calling this will be
 * "video/x-raw, width=500, height=360, format=I420; video/x-bayer,
 * width=400".
 */
void
ges_track_update_restriction_caps (GESTrack * self, const GstCaps * caps)
{
  guint i;
  GstCaps *new_restriction_caps;

  g_return_if_fail (GES_IS_TRACK (self));
  CHECK_THREAD (self);

  if (!self->priv->restriction_caps) {
    ges_track_set_restriction_caps (self, caps);
    return;
  }

  new_restriction_caps = gst_caps_copy (self->priv->restriction_caps);
  for (i = 0; i < gst_caps_get_size (caps); i++) {
    GstStructure *new = gst_caps_get_structure (caps, i);

    if (gst_caps_get_size (new_restriction_caps) > i) {
      GstStructure *original = gst_caps_get_structure (new_restriction_caps, i);
      gst_structure_foreach (new, (GstStructureForeachFunc) update_field,
          original);
    } else
      gst_caps_append_structure (new_restriction_caps,
          gst_structure_copy (new));
    /* FIXME: maybe appended structure should also have its CapsFeatures
     * copied over? */
  }

  ges_track_set_restriction_caps (self, new_restriction_caps);
  gst_caps_unref (new_restriction_caps);
}

/**
 * ges_track_set_mixing:
 * @track: A #GESTrack
 * @mixing: Whether @track should be mixing
 *
 * Sets the #GESTrack:mixing for the track.
 */
void
ges_track_set_mixing (GESTrack * track, gboolean mixing)
{
  g_return_if_fail (GES_IS_TRACK (track));
  CHECK_THREAD (track);

  if (mixing == track->priv->mixing) {
    GST_DEBUG_OBJECT (track, "Mixing is already set to the same value");

    return;
  }

  if (!track->priv->mixing_operation) {
    GST_DEBUG_OBJECT (track, "Track will be set to mixing = %d", mixing);
    goto notify;
  }

  if (mixing) {
    if (!ges_nle_composition_add_object (track->priv->composition,
            track->priv->mixing_operation)) {
      GST_WARNING_OBJECT (track, "Could not add the mixer to our composition");
      return;
    }
  } else {
    if (!ges_nle_composition_remove_object (track->priv->composition,
            track->priv->mixing_operation)) {
      GST_WARNING_OBJECT (track,
          "Could not remove the mixer from our composition");
      return;
    }
  }

notify:
  track->priv->mixing = mixing;

  g_object_notify_by_pspec (G_OBJECT (track), properties[ARG_MIXING]);

  GST_DEBUG_OBJECT (track, "The track has been set to mixing = %d", mixing);
}

static gboolean
remove_element_internal (GESTrack * track, GESTrackElement * object,
    gboolean emit)
{
  GSequenceIter *it;
  GESTrackPrivate *priv = track->priv;

  GST_DEBUG_OBJECT (track, "Removing %" GST_PTR_FORMAT, object);

  it = g_hash_table_lookup (priv->trackelements_iter, object);
  g_sequence_remove (it);

  if (remove_object_internal (track, object, emit) == TRUE) {
    ges_timeline_element_set_timeline (GES_TIMELINE_ELEMENT (object), NULL);

    return TRUE;
  }

  g_hash_table_insert (track->priv->trackelements_iter, object,
      g_sequence_insert_sorted (track->priv->trackelements_by_start, object,
          (GCompareDataFunc) element_start_compare, NULL));

  return FALSE;
}

/**
 * ges_track_add_element:
 * @track: A #GESTrack
 * @object: (transfer floating): The element to add
 *
 * Adds the given track element to the track, which takes ownership of the
 * element.
 *
 * Note that this can fail if it would break a configuration rule of the
 * track's #GESTimeline.
 *
 * Note that a #GESTrackElement can only be added to one track.
 *
 * Returns: %TRUE if @object was successfully added to @track.
 */
gboolean
ges_track_add_element (GESTrack * track, GESTrackElement * object)
{
  GESTimeline *timeline;
  GESTimelineElement *el;

  g_return_val_if_fail (GES_IS_TRACK (track), FALSE);
  g_return_val_if_fail (GES_IS_TRACK_ELEMENT (object), FALSE);

  el = GES_TIMELINE_ELEMENT (object);

  CHECK_THREAD (track);

  GST_DEBUG ("track:%p, object:%p", track, object);

  if (G_UNLIKELY (ges_track_element_get_track (object) != NULL)) {
    GST_WARNING ("Object already belongs to another track");
    gst_object_ref_sink (object);
    gst_object_unref (object);
    return FALSE;
  }

  if (!ges_track_element_set_track (object, track)) {
    GST_WARNING_OBJECT (track, "Failed to set the track for %" GES_FORMAT,
        GES_ARGS (object));
    gst_object_ref_sink (object);
    gst_object_unref (object);
    return FALSE;
  }
  ges_timeline_element_set_timeline (el, NULL);

  GST_DEBUG ("Adding object %s to ourself %s",
      GST_OBJECT_NAME (ges_track_element_get_nleobject (object)),
      GST_OBJECT_NAME (track->priv->composition));

  if (G_UNLIKELY (!ges_nle_composition_add_object (track->priv->composition,
              ges_track_element_get_nleobject (object)))) {
    GST_WARNING ("Couldn't add object to the NleComposition");
    ges_track_element_set_track (object, NULL);
    gst_object_ref_sink (object);
    gst_object_unref (object);
    return FALSE;
  }

  gst_object_ref_sink (object);
  g_hash_table_insert (track->priv->trackelements_iter, object,
      g_sequence_insert_sorted (track->priv->trackelements_by_start, object,
          (GCompareDataFunc) element_start_compare, NULL));

  timeline = track->priv->timeline;
  ges_timeline_element_set_timeline (el, timeline);
  if (timeline
      && !timeline_tree_can_move_element (timeline_get_tree (timeline), el,
          GES_TIMELINE_ELEMENT_LAYER_PRIORITY (el), el->start, el->duration,
          NULL)) {
    GST_WARNING_OBJECT (track,
        "Could not add the track element %" GES_FORMAT
        " to the track because it breaks the timeline " "configuration rules",
        GES_ARGS (el));
    remove_element_internal (track, object, FALSE);
    return FALSE;
  }

  g_signal_emit (track, ges_track_signals[TRACK_ELEMENT_ADDED], 0,
      GES_TRACK_ELEMENT (object));

  return TRUE;
}

/**
 * ges_track_get_elements:
 * @track: A #GESTrack
 *
 * Gets the track elements contained in the track. The returned list is
 * sorted by the element's #GESTimelineElement:priority and
 * #GESTimelineElement:start.
 *
 * Returns: (transfer full) (element-type GESTrackElement): A list of
 * all the #GESTrackElement-s in @track.
 */
GList *
ges_track_get_elements (GESTrack * track)
{
  GList *ret = NULL;

  g_return_val_if_fail (GES_IS_TRACK (track), NULL);
  CHECK_THREAD (track);

  g_sequence_foreach (track->priv->trackelements_by_start,
      (GFunc) add_trackelement_to_list_foreach, &ret);

  ret = g_list_reverse (ret);
  return ret;
}

/**
 * ges_track_remove_element:
 * @track: A #GESTrack
 * @object: The element to remove
 *
 * Removes the given track element from the track, which revokes
 * ownership of the element.
 *
 * Returns: %TRUE if @object was successfully removed from @track.
 */
gboolean
ges_track_remove_element (GESTrack * track, GESTrackElement * object)
{
  g_return_val_if_fail (GES_IS_TRACK (track), FALSE);
  g_return_val_if_fail (GES_IS_TRACK_ELEMENT (object), FALSE);

  CHECK_THREAD (track);

  return remove_element_internal (track, object, TRUE);
}

/**
 * ges_track_get_caps:
 * @track: A #GESTrack
 *
 * Get the #GESTrack:caps of the track.
 *
 * Returns: The caps of @track.
 */
const GstCaps *
ges_track_get_caps (GESTrack * track)
{
  g_return_val_if_fail (GES_IS_TRACK (track), NULL);
  CHECK_THREAD (track);

  return track->priv->caps;
}

/**
 * ges_track_get_timeline:
 * @track: A #GESTrack
 *
 * Get the timeline this track belongs to.
 *
 * Returns: (nullable): The timeline that @track belongs to, or %NULL if
 * it does not belong to a timeline.
 */
const GESTimeline *
ges_track_get_timeline (GESTrack * track)
{
  g_return_val_if_fail (GES_IS_TRACK (track), NULL);
  CHECK_THREAD (track);

  return track->priv->timeline;
}

/**
 * ges_track_get_mixing:
 * @track: A #GESTrack
 *
 * Gets the #GESTrack:mixing of the track.
 *
 * Returns: Whether @track is mixing.
 */
gboolean
ges_track_get_mixing (GESTrack * track)
{
  g_return_val_if_fail (GES_IS_TRACK (track), FALSE);

  return track->priv->mixing;
}

/**
 * ges_track_commit:
 * @track: A #GESTrack
 *
 * Commits all the pending changes for the elements contained in the
 * track.
 *
 * When changes are made to the timing or priority of elements within a
 * track, they are not directly executed for the underlying
 * #nlecomposition and its children. This method will finally execute
 * these changes so they are reflected in the data output of the track.
 *
 * Any pending changes will be executed in the backend. The
 * #GESTimeline::commited signal will be emitted once this has completed.
 *
 * Note that ges_timeline_commit() will call this method on all of its
 * tracks, so you are unlikely to need to use this directly.
 *
 * Returns: %TRUE if pending changes were committed, or %FALSE if nothing
 * needed to be committed.
 */
gboolean
ges_track_commit (GESTrack * track)
{
  g_return_val_if_fail (GES_IS_TRACK (track), FALSE);
  CHECK_THREAD (track);

  track_resort_and_fill_gaps (track);

  return ges_nle_object_commit (track->priv->composition, TRUE);
}


/**
 * ges_track_set_create_element_for_gap_func:
 * @track: A #GESTrack
 * @func: (scope notified): The function to be used to create a source
 * #GstElement that can fill gaps in @track
 *
 * Sets the function that will be used to create a #GstElement that can be
 * used as a source to fill the gaps of the track. A gap is a timeline
 * region where the track has no #GESTrackElement sources. Therefore, you
 * are likely to want the #GstElement returned by the function to always
 * produce 'empty' content, defined relative to the stream type, such as
 * transparent frames for a video, or mute samples for audio.
 *
 * #GESAudioTrack and #GESVideoTrack objects are created with such a
 * function already set appropriately.
 */
void
ges_track_set_create_element_for_gap_func (GESTrack * track,
    GESCreateElementForGapFunc func)
{
  g_return_if_fail (GES_IS_TRACK (track));
  CHECK_THREAD (track);

  track->priv->create_element_for_gaps = func;
}

/**
 * ges_track_get_restriction_caps:
 * @track: A #GESTrack
 *
 * Gets the #GESTrack:restriction-caps of the track.
 *
 * Returns: (transfer full): The restriction-caps of @track.
 *
 * Since: 1.18
 */
GstCaps *
ges_track_get_restriction_caps (GESTrack * track)
{
  GESTrackPrivate *priv;

  g_return_val_if_fail (GES_IS_TRACK (track), NULL);
  CHECK_THREAD (track);

  priv = track->priv;

  if (priv->restriction_caps)
    return gst_caps_ref (priv->restriction_caps);

  return NULL;
}