gstreamer/ext/gl/gstglbasemixer.c

/* Generic video mixer plugin
 *
 * GStreamer
 * Copyright (C) 2015 Matthew Waters <matthew@centricular.com>
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Library General Public
 * License as published by the Free Software Foundation; either
 * version 2 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Library General Public License for more details.
 *
 * You should have received a copy of the GNU Library General Public
 * License along with this library; if not, write to the
 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
 * Boston, MA 02110-1301, USA.
 */

#ifdef HAVE_CONFIG_H
#include "config.h"
#endif

#include <gst/gst.h>
#include <gst/video/video.h>

#include "gstglbasemixer.h"

#define gst_gl_base_mixer_parent_class parent_class
G_DEFINE_ABSTRACT_TYPE (GstGLBaseMixer, gst_gl_base_mixer,
    GST_TYPE_VIDEO_AGGREGATOR);
static gboolean gst_gl_base_mixer_do_bufferpool (GstGLBaseMixer * mix,
    GstCaps * outcaps);

#define GST_CAT_DEFAULT gst_gl_base_mixer_debug
GST_DEBUG_CATEGORY (gst_gl_base_mixer_debug);

static void gst_gl_base_mixer_pad_get_property (GObject * object, guint prop_id,
    GValue * value, GParamSpec * pspec);
static void gst_gl_base_mixer_pad_set_property (GObject * object, guint prop_id,
    const GValue * value, GParamSpec * pspec);

static void gst_gl_base_mixer_set_context (GstElement * element,
    GstContext * context);
static GstStateChangeReturn gst_gl_base_mixer_change_state (GstElement *
    element, GstStateChange transition);

enum
{
  PROP_PAD_0
};

#define GST_GL_BASE_MIXER_GET_PRIVATE(obj)  \
    (G_TYPE_INSTANCE_GET_PRIVATE ((obj), GST_TYPE_GL_BASE_MIXER, GstGLBaseMixerPrivate))

struct _GstGLBaseMixerPrivate
{
  gboolean negotiated;

  GstGLContext *other_context;

  GstBufferPool *pool;
  GstAllocator *allocator;
  GstAllocationParams params;
  GstQuery *query;
};

G_DEFINE_TYPE (GstGLBaseMixerPad, gst_gl_base_mixer_pad,
    GST_TYPE_VIDEO_AGGREGATOR_PAD);

static void
gst_gl_base_mixer_pad_class_init (GstGLBaseMixerPadClass * klass)
{
  GObjectClass *gobject_class = (GObjectClass *) klass;
  GstVideoAggregatorPadClass *vaggpad_class =
      (GstVideoAggregatorPadClass *) klass;

  gobject_class->set_property = gst_gl_base_mixer_pad_set_property;
  gobject_class->get_property = gst_gl_base_mixer_pad_get_property;

  vaggpad_class->set_info = NULL;
  vaggpad_class->prepare_frame = NULL;
  vaggpad_class->clean_frame = NULL;
}

static void
gst_gl_base_mixer_pad_get_property (GObject * object, guint prop_id,
    GValue * value, GParamSpec * pspec)
{
  switch (prop_id) {
    default:
      G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
      break;
  }
}

static void
gst_gl_base_mixer_pad_set_property (GObject * object, guint prop_id,
    const GValue * value, GParamSpec * pspec)
{
  switch (prop_id) {
    default:
      G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
      break;
  }
}

static gboolean
_negotiated_caps (GstVideoAggregator * vagg, GstCaps * caps)
{
  GstGLBaseMixer *mix = GST_GL_BASE_MIXER (vagg);

  return gst_gl_base_mixer_do_bufferpool (mix, caps);
}

static gboolean
_default_propose_allocation (GstGLBaseMixer * mix, GstGLBaseMixerPad * pad,
    GstQuery * decide_query, GstQuery * query)
{
  return TRUE;
}

static gboolean
gst_gl_base_mixer_sink_query (GstAggregator * agg, GstAggregatorPad * bpad,
    GstQuery * query)
{
  gboolean ret = FALSE;
  GstGLBaseMixer *mix = GST_GL_BASE_MIXER (agg);
  GstGLBaseMixerClass *mix_class = GST_GL_BASE_MIXER_GET_CLASS (mix);
  GstGLBaseMixerPad *pad = GST_GL_BASE_MIXER_PAD (bpad);

  GST_TRACE ("QUERY %" GST_PTR_FORMAT, query);

  switch (GST_QUERY_TYPE (query)) {
    case GST_QUERY_ALLOCATION:
    {
      GstQuery *decide_query = NULL;

      GST_OBJECT_LOCK (mix);
      if (G_UNLIKELY (!mix->priv->negotiated)) {
        GST_DEBUG_OBJECT (mix,
            "not negotiated yet, can't answer ALLOCATION query");
        GST_OBJECT_UNLOCK (mix);
        return FALSE;
      }
      if ((decide_query = mix->priv->query))
        gst_query_ref (decide_query);
      GST_OBJECT_UNLOCK (mix);

      GST_DEBUG_OBJECT (mix,
          "calling propose allocation with query %" GST_PTR_FORMAT,
          decide_query);

      /* pass the query to the propose_allocation vmethod if any */
      if (mix_class->propose_allocation)
        ret = mix_class->propose_allocation (mix, pad, decide_query, query);
      else
        ret = FALSE;

      if (decide_query)
        gst_query_unref (decide_query);

      GST_DEBUG_OBJECT (mix, "ALLOCATION ret %d, %" GST_PTR_FORMAT, ret, query);
      return ret;
    }
    case GST_QUERY_CONTEXT:
    {
      const gchar *context_type;
      GstContext *context, *old_context;

      ret = gst_gl_handle_context_query ((GstElement *) mix, query,
          &mix->display, &mix->priv->other_context);
      if (mix->display)
        gst_gl_display_filter_gl_api (mix->display,
            mix_class->supported_gl_api);
      gst_query_parse_context_type (query, &context_type);

      if (g_strcmp0 (context_type, "gst.gl.local_context") == 0) {
        GstStructure *s;

        gst_query_parse_context (query, &old_context);

        if (old_context)
          context = gst_context_copy (old_context);
        else
          context = gst_context_new ("gst.gl.local_context", FALSE);

        s = gst_context_writable_structure (context);
        gst_structure_set (s, "context", GST_GL_TYPE_CONTEXT, mix->context,
            NULL);
        gst_query_set_context (query, context);
        gst_context_unref (context);

        ret = mix->context != NULL;
      }
      GST_LOG_OBJECT (mix, "context query of type %s %i", context_type, ret);

      if (ret)
        return ret;

      break;
    }
    default:
      break;
  }

  return GST_AGGREGATOR_CLASS (parent_class)->sink_query (agg, bpad, query);;
}

static void
gst_gl_base_mixer_pad_init (GstGLBaseMixerPad * mixerpad)
{
}

/* GLBaseMixer signals and args */
enum
{
  /* FILL ME */
  LAST_SIGNAL
};

enum
{
  PROP_0,
  PROP_CONTEXT
};

static gboolean gst_gl_base_mixer_src_query (GstAggregator * agg,
    GstQuery * query);

static gboolean
gst_gl_base_mixer_src_activate_mode (GstAggregator * aggregator,
    GstPadMode mode, gboolean active);
static gboolean gst_gl_base_mixer_stop (GstAggregator * agg);
static gboolean gst_gl_base_mixer_start (GstAggregator * agg);

static void gst_gl_base_mixer_set_property (GObject * object, guint prop_id,
    const GValue * value, GParamSpec * pspec);
static void gst_gl_base_mixer_get_property (GObject * object, guint prop_id,
    GValue * value, GParamSpec * pspec);

static gboolean gst_gl_base_mixer_decide_allocation (GstGLBaseMixer * mix,
    GstQuery * query);
static gboolean gst_gl_base_mixer_set_allocation (GstGLBaseMixer * mix,
    GstBufferPool * pool, GstAllocator * allocator,
    GstAllocationParams * params, GstQuery * query);

static void gst_gl_base_mixer_finalize (GObject * object);

static void
gst_gl_base_mixer_class_init (GstGLBaseMixerClass * klass)
{
  GObjectClass *gobject_class;
  GstElementClass *element_class;

  GstVideoAggregatorClass *videoaggregator_class =
      (GstVideoAggregatorClass *) klass;
  GstAggregatorClass *agg_class = (GstAggregatorClass *) klass;

  GST_DEBUG_CATEGORY_INIT (GST_CAT_DEFAULT, "glmixer", 0, "opengl mixer");

  gobject_class = (GObjectClass *) klass;
  element_class = GST_ELEMENT_CLASS (klass);

  g_type_class_add_private (klass, sizeof (GstGLBaseMixerPrivate));

  gobject_class->finalize = GST_DEBUG_FUNCPTR (gst_gl_base_mixer_finalize);

  gobject_class->get_property = gst_gl_base_mixer_get_property;
  gobject_class->set_property = gst_gl_base_mixer_set_property;

  element_class->set_context =
      GST_DEBUG_FUNCPTR (gst_gl_base_mixer_set_context);
  element_class->change_state = gst_gl_base_mixer_change_state;

  agg_class->sinkpads_type = GST_TYPE_GL_BASE_MIXER_PAD;
  agg_class->sink_query = gst_gl_base_mixer_sink_query;
  agg_class->src_query = gst_gl_base_mixer_src_query;
  agg_class->src_activate = gst_gl_base_mixer_src_activate_mode;
  agg_class->stop = gst_gl_base_mixer_stop;
  agg_class->start = gst_gl_base_mixer_start;

  videoaggregator_class->negotiated_caps = _negotiated_caps;

  klass->propose_allocation = _default_propose_allocation;

  g_object_class_install_property (gobject_class, PROP_CONTEXT,
      g_param_spec_object ("context",
          "OpenGL context",
          "Get OpenGL context",
          GST_GL_TYPE_CONTEXT, G_PARAM_READABLE | G_PARAM_STATIC_STRINGS));

  /* Register the pad class */
  g_type_class_ref (GST_TYPE_GL_BASE_MIXER_PAD);

  klass->supported_gl_api = GST_GL_API_ANY;
}

static void
gst_gl_base_mixer_reset (GstGLBaseMixer * mix)
{
  /* clean up collect data */
  mix->priv->negotiated = FALSE;
}

static void
gst_gl_base_mixer_init (GstGLBaseMixer * mix)
{
  mix->priv = GST_GL_BASE_MIXER_GET_PRIVATE (mix);

  gst_gl_base_mixer_reset (mix);
}

static void
gst_gl_base_mixer_finalize (GObject * object)
{
  GstGLBaseMixer *mix = GST_GL_BASE_MIXER (object);

  if (mix->priv->other_context) {
    gst_object_unref (mix->priv->other_context);
    mix->priv->other_context = NULL;
  }

  G_OBJECT_CLASS (parent_class)->finalize (object);
}

static void
gst_gl_base_mixer_set_context (GstElement * element, GstContext * context)
{
  GstGLBaseMixer *mix = GST_GL_BASE_MIXER (element);
  GstGLBaseMixerClass *mix_class = GST_GL_BASE_MIXER_GET_CLASS (mix);

  gst_gl_handle_set_context (element, context, &mix->display,
      &mix->priv->other_context);

  if (mix->display)
    gst_gl_display_filter_gl_api (mix->display, mix_class->supported_gl_api);
}

static gboolean
gst_gl_base_mixer_activate (GstGLBaseMixer * mix, gboolean active)
{
  GstGLBaseMixerClass *mix_class = GST_GL_BASE_MIXER_GET_CLASS (mix);
  gboolean result = TRUE;

  if (active) {
    if (!gst_gl_ensure_element_data (mix, &mix->display,
            &mix->priv->other_context))
      return FALSE;

    gst_gl_display_filter_gl_api (mix->display, mix_class->supported_gl_api);
  }

  return result;
}

static gboolean
gst_gl_base_mixer_src_activate_mode (GstAggregator * aggregator,
    GstPadMode mode, gboolean active)
{
  GstGLBaseMixer *mix;
  gboolean result = FALSE;

  mix = GST_GL_BASE_MIXER (aggregator);

  switch (mode) {
    case GST_PAD_MODE_PUSH:
    case GST_PAD_MODE_PULL:
      result = gst_gl_base_mixer_activate (mix, active);
      break;
    default:
      result = TRUE;
      break;
  }
  return result;
}

static gboolean
gst_gl_base_mixer_src_query (GstAggregator * agg, GstQuery * query)
{
  gboolean res = FALSE;
  GstGLBaseMixer *mix = GST_GL_BASE_MIXER (agg);
  GstGLBaseMixerClass *mix_class = GST_GL_BASE_MIXER_GET_CLASS (mix);

  switch (GST_QUERY_TYPE (query)) {
    case GST_QUERY_CONTEXT:
    {
      const gchar *context_type;
      GstContext *context, *old_context;

      res = gst_gl_handle_context_query ((GstElement *) mix, query,
          &mix->display, &mix->priv->other_context);
      if (mix->display)
        gst_gl_display_filter_gl_api (mix->display,
            mix_class->supported_gl_api);
      gst_query_parse_context_type (query, &context_type);

      if (g_strcmp0 (context_type, "gst.gl.local_context") == 0) {
        GstStructure *s;

        gst_query_parse_context (query, &old_context);

        if (old_context)
          context = gst_context_copy (old_context);
        else
          context = gst_context_new ("gst.gl.local_context", FALSE);

        s = gst_context_writable_structure (context);
        gst_structure_set (s, "context", GST_GL_TYPE_CONTEXT, mix->context,
            NULL);
        gst_query_set_context (query, context);
        gst_context_unref (context);

        res = mix->context != NULL;
      }
      GST_LOG_OBJECT (mix, "context query of type %s %i", context_type, res);

      if (res)
        return res;

      break;
    }
    default:
      break;
  }

  return GST_AGGREGATOR_CLASS (parent_class)->src_query (agg, query);
}

static gboolean
_find_local_gl_context (GstGLBaseMixer * mix)
{
  GstQuery *query;
  GstContext *context;
  const GstStructure *s;

  if (mix->context)
    return TRUE;

  query = gst_query_new_context ("gst.gl.local_context");
  if (!mix->context && gst_gl_run_query (GST_ELEMENT (mix), query, GST_PAD_SRC)) {
    gst_query_parse_context (query, &context);
    if (context) {
      s = gst_context_get_structure (context);
      gst_structure_get (s, "context", GST_GL_TYPE_CONTEXT, &mix->context,
          NULL);
    }
  }
  if (!mix->context
      && gst_gl_run_query (GST_ELEMENT (mix), query, GST_PAD_SINK)) {
    gst_query_parse_context (query, &context);
    if (context) {
      s = gst_context_get_structure (context);
      gst_structure_get (s, "context", GST_GL_TYPE_CONTEXT, &mix->context,
          NULL);
    }
  }

  GST_DEBUG_OBJECT (mix, "found local context %p", mix->context);

  gst_query_unref (query);

  if (mix->context)
    return TRUE;

  return FALSE;
}

static gboolean
gst_gl_base_mixer_decide_allocation (GstGLBaseMixer * mix, GstQuery * query)
{
  GstGLBaseMixerClass *mix_class = GST_GL_BASE_MIXER_GET_CLASS (mix);
  GError *error = NULL;
  gboolean ret = TRUE;

  if (!gst_gl_ensure_element_data (mix, &mix->display,
          &mix->priv->other_context))
    return FALSE;

  gst_gl_display_filter_gl_api (mix->display, mix_class->supported_gl_api);

  _find_local_gl_context (mix);

  if (!mix->context) {
    GST_OBJECT_LOCK (mix->display);
    do {
      if (mix->context)
        gst_object_unref (mix->context);
      /* just get a GL context.  we don't care */
      mix->context =
          gst_gl_display_get_gl_context_for_thread (mix->display, NULL);
      if (!mix->context) {
        mix->context = gst_gl_context_new (mix->display);
        if (!gst_gl_context_create (mix->context, mix->priv->other_context,
                &error))
          goto context_error;
      }
    } while (!gst_gl_display_add_context (mix->display, mix->context));
    GST_OBJECT_UNLOCK (mix->display);
  }

  if (mix_class->decide_allocation)
    ret = mix_class->decide_allocation (mix, query);

  return ret;

context_error:
  {
    GST_ELEMENT_ERROR (mix, RESOURCE, NOT_FOUND, ("%s", error->message),
        (NULL));
    return FALSE;
  }
}

/* takes ownership of the pool, allocator and query */
static gboolean
gst_gl_base_mixer_set_allocation (GstGLBaseMixer * mix,
    GstBufferPool * pool, GstAllocator * allocator,
    GstAllocationParams * params, GstQuery * query)
{
  GstAllocator *oldalloc;
  GstBufferPool *oldpool;
  GstQuery *oldquery;
  GstGLBaseMixerPrivate *priv = mix->priv;

  GST_DEBUG ("storing allocation query");

  GST_OBJECT_LOCK (mix);
  oldpool = priv->pool;
  priv->pool = pool;

  oldalloc = priv->allocator;
  priv->allocator = allocator;

  oldquery = priv->query;
  priv->query = query;

  if (params)
    priv->params = *params;
  else
    gst_allocation_params_init (&priv->params);
  GST_OBJECT_UNLOCK (mix);

  if (oldpool) {
    GST_DEBUG_OBJECT (mix, "deactivating old pool %p", oldpool);
    gst_buffer_pool_set_active (oldpool, FALSE);
    gst_object_unref (oldpool);
  }
  if (oldalloc) {
    gst_object_unref (oldalloc);
  }
  if (oldquery) {
    gst_query_unref (oldquery);
  }
  return TRUE;
}

static gboolean
gst_gl_base_mixer_do_bufferpool (GstGLBaseMixer * mix, GstCaps * outcaps)
{
  GstQuery *query;
  gboolean result = TRUE;
  GstBufferPool *pool = NULL;
  GstAllocator *allocator;
  GstAllocationParams params;
  GstAggregator *agg = GST_AGGREGATOR (mix);

  /* find a pool for the negotiated caps now */
  GST_DEBUG_OBJECT (mix, "doing allocation query");
  query = gst_query_new_allocation (outcaps, TRUE);
  if (!gst_pad_peer_query (agg->srcpad, query)) {
    /* not a problem, just debug a little */
    GST_DEBUG_OBJECT (mix, "peer ALLOCATION query failed");
  }

  GST_DEBUG_OBJECT (mix, "calling decide_allocation");
  result = gst_gl_base_mixer_decide_allocation (mix, query);

  GST_DEBUG_OBJECT (mix, "ALLOCATION (%d) params: %" GST_PTR_FORMAT, result,
      query);

  if (!result)
    goto no_decide_allocation;

  /* we got configuration from our peer or the decide_allocation method,
   * parse them */
  if (gst_query_get_n_allocation_params (query) > 0) {
    gst_query_parse_nth_allocation_param (query, 0, &allocator, &params);
  } else {
    allocator = NULL;
    gst_allocation_params_init (&params);
  }

  if (gst_query_get_n_allocation_pools (query) > 0)
    gst_query_parse_nth_allocation_pool (query, 0, &pool, NULL, NULL, NULL);

  /* now store */
  result =
      gst_gl_base_mixer_set_allocation (mix, pool, allocator, &params, query);

  return result;

  /* Errors */
no_decide_allocation:
  {
    GST_WARNING_OBJECT (mix, "Failed to decide allocation");
    gst_query_unref (query);

    return result;
  }
}

GstBufferPool *
gst_gl_base_mixer_get_buffer_pool (GstGLBaseMixer * mix)
{
  GstBufferPool *pool;

  GST_OBJECT_LOCK (mix);
  pool = mix->priv->pool;
  if (pool)
    gst_object_ref (pool);
  GST_OBJECT_UNLOCK (mix);

  return pool;
}

static void
gst_gl_base_mixer_get_property (GObject * object,
    guint prop_id, GValue * value, GParamSpec * pspec)
{
  GstGLBaseMixer *mixer = GST_GL_BASE_MIXER (object);

  switch (prop_id) {
    case PROP_CONTEXT:
      g_value_set_object (value, mixer->context);
      break;
    default:
      G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
      break;
  }
}

static void
gst_gl_base_mixer_set_property (GObject * object,
    guint prop_id, const GValue * value, GParamSpec * pspec)
{
  switch (prop_id) {
    default:
      G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
      break;
  }
}

static gboolean
gst_gl_base_mixer_start (GstAggregator * agg)
{
  return TRUE;
}

static gboolean
gst_gl_base_mixer_stop (GstAggregator * agg)
{
  GstGLBaseMixer *mix = GST_GL_BASE_MIXER (agg);

  if (mix->priv->query) {
    gst_query_unref (mix->priv->query);
    mix->priv->query = NULL;
  }

  if (mix->priv->pool) {
    gst_object_unref (mix->priv->pool);
    mix->priv->pool = NULL;
  }

  if (mix->display) {
    gst_object_unref (mix->display);
    mix->display = NULL;
  }

  if (mix->context) {
    gst_object_unref (mix->context);
    mix->context = NULL;
  }

  gst_gl_base_mixer_reset (mix);

  return TRUE;
}

static GstStateChangeReturn
gst_gl_base_mixer_change_state (GstElement * element, GstStateChange transition)
{
  GstGLBaseMixer *mix = GST_GL_BASE_MIXER (element);
  GstGLBaseMixerClass *mix_class = GST_GL_BASE_MIXER_GET_CLASS (mix);
  GstStateChangeReturn ret = GST_STATE_CHANGE_SUCCESS;

  GST_DEBUG_OBJECT (mix, "changing state: %s => %s",
      gst_element_state_get_name (GST_STATE_TRANSITION_CURRENT (transition)),
      gst_element_state_get_name (GST_STATE_TRANSITION_NEXT (transition)));

  switch (transition) {
    case GST_STATE_CHANGE_NULL_TO_READY:
      if (!gst_gl_ensure_element_data (element, &mix->display,
              &mix->priv->other_context))
        return GST_STATE_CHANGE_FAILURE;

      gst_gl_display_filter_gl_api (mix->display, mix_class->supported_gl_api);
      break;
    default:
      break;
  }

  ret = GST_ELEMENT_CLASS (parent_class)->change_state (element, transition);
  if (ret == GST_STATE_CHANGE_FAILURE)
    return ret;

  switch (transition) {
    default:
      break;
  }

  return ret;
}