gstreamer/subprojects/gst-plugins-bad/sys/mediafoundation/gstmfvideosrc.cpp

/* GStreamer
 * Copyright (C) 2019 Seungha Yang <seungha.yang@navercorp.com>
 * Copyright (C) 2020 Seungha Yang <seungha@centricular.com>
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Library General Public
 * License as published by the Free Software Foundation; either
 * version 2 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Library General Public License for more details.
 *
 * You should have received a copy of the GNU Library General Public
 * License along with this library; if not, write to the
 * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
 * Boston, MA 02110-1301, USA.
 */

/**
 * SECTION:element-mfvideosrc
 * @title: mfvideosrc
 *
 * Provides video capture from the Microsoft Media Foundation API.
 *
 * ## Example pipelines
 * |[
 * gst-launch-1.0 -v mfvideosrc ! fakesink
 * ]| Capture from the default video capture device and render to fakesink.
 *
 * |[
 * gst-launch-1.0 -v mfvideosrc device-index=1 ! fakesink
 * ]| Capture from the second video device (if available) and render to fakesink.
 */

#ifdef HAVE_CONFIG_H
#include "config.h"
#endif

#include "gstmfconfig.h"

#include "gstmfvideosrc.h"
#include "gstmfutils.h"
#include "gstmfsourceobject.h"
#include <string.h>

#if GST_MF_WINAPI_DESKTOP
#include "gstmfcapturedshow.h"
#endif

GST_DEBUG_CATEGORY (gst_mf_video_src_debug);
#define GST_CAT_DEFAULT gst_mf_video_src_debug

#if (GST_MF_WINAPI_APP && !GST_MF_WINAPI_DESKTOP)
/* FIXME: need support JPEG for UWP */
#define SRC_TEMPLATE_CAPS \
    GST_VIDEO_CAPS_MAKE (GST_MF_VIDEO_FORMATS)
#else
#define SRC_TEMPLATE_CAPS \
    GST_VIDEO_CAPS_MAKE (GST_MF_VIDEO_FORMATS) "; " \
        "image/jpeg, width = " GST_VIDEO_SIZE_RANGE ", " \
        "height = " GST_VIDEO_SIZE_RANGE ", " \
        "framerate = " GST_VIDEO_FPS_RANGE
#endif

static GstStaticPadTemplate src_template = GST_STATIC_PAD_TEMPLATE ("src",
    GST_PAD_SRC,
    GST_PAD_ALWAYS,
    GST_STATIC_CAPS (SRC_TEMPLATE_CAPS));

struct _GstMFVideoSrc
{
  GstPushSrc parent;

  GstMFSourceObject *source;
  gboolean started;
  GstVideoInfo info;

  guint64 n_frames;
  GstClockTime latency;

  gboolean use_dshow;

  /* properties */
  gchar *device_path;
  gchar *device_name;
  gint device_index;
  gpointer dispatcher;
};

enum
{
  PROP_0,
  PROP_DEVICE_PATH,
  PROP_DEVICE_NAME,
  PROP_DEVICE_INDEX,
  PROP_DISPATCHER,
};

#define DEFAULT_DEVICE_PATH     nullptr
#define DEFAULT_DEVICE_NAME     nullptr
#define DEFAULT_DEVICE_INDEX    -1

static void gst_mf_video_src_finalize (GObject * object);
static void gst_mf_video_src_get_property (GObject * object, guint prop_id,
    GValue * value, GParamSpec * pspec);
static void gst_mf_video_src_set_property (GObject * object, guint prop_id,
    const GValue * value, GParamSpec * pspec);

static gboolean gst_mf_video_src_start (GstBaseSrc * src);
static gboolean gst_mf_video_src_stop (GstBaseSrc * src);
static gboolean gst_mf_video_src_set_caps (GstBaseSrc * src, GstCaps * caps);
static GstCaps *gst_mf_video_src_get_caps (GstBaseSrc * src, GstCaps * filter);
static GstCaps *gst_mf_video_src_fixate (GstBaseSrc * src, GstCaps * caps);
static gboolean gst_mf_video_src_unlock (GstBaseSrc * src);
static gboolean gst_mf_video_src_unlock_stop (GstBaseSrc * src);
static gboolean gst_mf_video_src_query (GstBaseSrc * src, GstQuery * query);

static GstFlowReturn gst_mf_video_src_create (GstPushSrc * pushsrc,
    GstBuffer ** buffer);

#define gst_mf_video_src_parent_class parent_class
G_DEFINE_TYPE (GstMFVideoSrc, gst_mf_video_src, GST_TYPE_PUSH_SRC);

static void
gst_mf_video_src_class_init (GstMFVideoSrcClass * klass)
{
  GObjectClass *gobject_class = G_OBJECT_CLASS (klass);
  GstElementClass *element_class = GST_ELEMENT_CLASS (klass);
  GstBaseSrcClass *basesrc_class = GST_BASE_SRC_CLASS (klass);
  GstPushSrcClass *pushsrc_class = GST_PUSH_SRC_CLASS (klass);
  GParamFlags flags = (GParamFlags) (G_PARAM_READWRITE |
      GST_PARAM_MUTABLE_READY | G_PARAM_STATIC_STRINGS);

  gobject_class->finalize = gst_mf_video_src_finalize;
  gobject_class->get_property = gst_mf_video_src_get_property;
  gobject_class->set_property = gst_mf_video_src_set_property;

  g_object_class_install_property (gobject_class, PROP_DEVICE_PATH,
      g_param_spec_string ("device-path", "Device Path",
          "The device path", DEFAULT_DEVICE_PATH, flags));
  g_object_class_install_property (gobject_class, PROP_DEVICE_NAME,
      g_param_spec_string ("device-name", "Device Name",
          "The human-readable device name", DEFAULT_DEVICE_NAME, flags));
  g_object_class_install_property (gobject_class, PROP_DEVICE_INDEX,
      g_param_spec_int ("device-index", "Device Index",
          "The zero-based device index", -1, G_MAXINT, DEFAULT_DEVICE_INDEX,
          flags));
#if GST_MF_WINAPI_APP
  /**
   * GstMFVideoSrc:dispatcher:
   *
   * ICoreDispatcher COM object used for activating device from UI thread.
   *
   * Since: 1.18
   */
  g_object_class_install_property (gobject_class, PROP_DISPATCHER,
      g_param_spec_pointer ("dispatcher", "Dispatcher",
          "ICoreDispatcher COM object to use. In order for application to ask "
          "permission of capture device, device activation should be running "
          "on UI thread via ICoreDispatcher. This element will increase "
          "the reference count of given ICoreDispatcher and release it after "
          "use. Therefore, caller does not need to consider additional "
          "reference count management",
          (GParamFlags) (GST_PARAM_CONDITIONALLY_AVAILABLE |
              GST_PARAM_MUTABLE_READY | G_PARAM_WRITABLE |
              G_PARAM_STATIC_STRINGS)));
#endif

  gst_element_class_set_static_metadata (element_class,
      "Media Foundation Video Source",
      "Source/Video/Hardware",
      "Capture video stream through Windows Media Foundation",
      "Seungha Yang <seungha.yang@navercorp.com>");

  gst_element_class_add_static_pad_template (element_class, &src_template);

  basesrc_class->start = GST_DEBUG_FUNCPTR (gst_mf_video_src_start);
  basesrc_class->stop = GST_DEBUG_FUNCPTR (gst_mf_video_src_stop);
  basesrc_class->set_caps = GST_DEBUG_FUNCPTR (gst_mf_video_src_set_caps);
  basesrc_class->get_caps = GST_DEBUG_FUNCPTR (gst_mf_video_src_get_caps);
  basesrc_class->fixate = GST_DEBUG_FUNCPTR (gst_mf_video_src_fixate);
  basesrc_class->unlock = GST_DEBUG_FUNCPTR (gst_mf_video_src_unlock);
  basesrc_class->unlock_stop = GST_DEBUG_FUNCPTR (gst_mf_video_src_unlock_stop);
  basesrc_class->query = GST_DEBUG_FUNCPTR (gst_mf_video_src_query);

  pushsrc_class->create = GST_DEBUG_FUNCPTR (gst_mf_video_src_create);

  GST_DEBUG_CATEGORY_INIT (gst_mf_video_src_debug, "mfvideosrc", 0,
      "mfvideosrc");
}

static void
gst_mf_video_src_init (GstMFVideoSrc * self)
{
  gst_base_src_set_format (GST_BASE_SRC (self), GST_FORMAT_TIME);
  gst_base_src_set_live (GST_BASE_SRC (self), TRUE);

  self->device_index = DEFAULT_DEVICE_INDEX;
}

static void
gst_mf_video_src_finalize (GObject * object)
{
  GstMFVideoSrc *self = GST_MF_VIDEO_SRC (object);

  g_free (self->device_name);
  g_free (self->device_path);

  G_OBJECT_CLASS (parent_class)->finalize (object);
}

static void
gst_mf_video_src_get_property (GObject * object, guint prop_id, GValue * value,
    GParamSpec * pspec)
{
  GstMFVideoSrc *self = GST_MF_VIDEO_SRC (object);

  switch (prop_id) {
    case PROP_DEVICE_PATH:
      g_value_set_string (value, self->device_path);
      break;
    case PROP_DEVICE_NAME:
      g_value_set_string (value, self->device_name);
      break;
    case PROP_DEVICE_INDEX:
      g_value_set_int (value, self->device_index);
      break;
    default:
      G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
      break;
  }
}

static void
gst_mf_video_src_set_property (GObject * object, guint prop_id,
    const GValue * value, GParamSpec * pspec)
{
  GstMFVideoSrc *self = GST_MF_VIDEO_SRC (object);

  switch (prop_id) {
    case PROP_DEVICE_PATH:
      g_free (self->device_path);
      self->device_path = g_value_dup_string (value);
      break;
    case PROP_DEVICE_NAME:
      g_free (self->device_name);
      self->device_name = g_value_dup_string (value);
      break;
    case PROP_DEVICE_INDEX:
      self->device_index = g_value_get_int (value);
      break;
#if GST_MF_WINAPI_APP
    case PROP_DISPATCHER:
      self->dispatcher = g_value_get_pointer (value);
      break;
#endif
    default:
      G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
      break;
  }
}

static gboolean
gst_mf_video_src_start (GstBaseSrc * src)
{
  GstMFVideoSrc *self = GST_MF_VIDEO_SRC (src);

  GST_DEBUG_OBJECT (self, "Start");

  self->source = gst_mf_source_object_new (GST_MF_SOURCE_TYPE_VIDEO,
      self->device_index, self->device_name, self->device_path, nullptr);

  self->n_frames = 0;
  self->latency = 0;
  self->use_dshow = FALSE;

  if (!self->source) {
#if GST_MF_WINAPI_DESKTOP
    self->use_dshow = TRUE;
    self->source = gst_mf_capture_dshow_new (GST_MF_SOURCE_TYPE_VIDEO,
        self->device_index, self->device_name, self->device_path);
#endif
  }

  if (!self->source) {
    GST_ERROR_OBJECT (self, "Couldn't create capture object");
    return FALSE;
  }

  gst_mf_source_object_set_client (self->source, GST_ELEMENT (self));

  return TRUE;
}

static gboolean
gst_mf_video_src_stop (GstBaseSrc * src)
{
  GstMFVideoSrc *self = GST_MF_VIDEO_SRC (src);

  GST_DEBUG_OBJECT (self, "Stop");

  if (self->source) {
    gst_mf_source_object_stop (self->source);
    gst_object_unref (self->source);
    self->source = nullptr;
  }

  self->started = FALSE;

  return TRUE;
}

static gboolean
gst_mf_video_src_set_caps (GstBaseSrc * src, GstCaps * caps)
{
  GstMFVideoSrc *self = GST_MF_VIDEO_SRC (src);

  GST_DEBUG_OBJECT (self, "Set caps %" GST_PTR_FORMAT, caps);

  if (!self->source) {
    GST_ERROR_OBJECT (self, "No capture engine yet");
    return FALSE;
  }

  if (!gst_mf_source_object_set_caps (self->source, caps)) {
    GST_ERROR_OBJECT (self, "CaptureEngine couldn't accept caps");
    return FALSE;
  }

  gst_video_info_from_caps (&self->info, caps);
  if (GST_VIDEO_INFO_FORMAT (&self->info) != GST_VIDEO_FORMAT_ENCODED)
    gst_base_src_set_blocksize (src, GST_VIDEO_INFO_SIZE (&self->info));

  return TRUE;
}

static GstCaps *
gst_mf_video_src_get_caps (GstBaseSrc * src, GstCaps * filter)
{
  GstMFVideoSrc *self = GST_MF_VIDEO_SRC (src);
  GstCaps *caps = nullptr;

  if (self->source)
    caps = gst_mf_source_object_get_caps (self->source);

  if (!caps)
    caps = gst_pad_get_pad_template_caps (GST_BASE_SRC_PAD (src));

  if (filter) {
    GstCaps *filtered =
        gst_caps_intersect_full (filter, caps, GST_CAPS_INTERSECT_FIRST);
    gst_caps_unref (caps);
    caps = filtered;
  }

  GST_DEBUG_OBJECT (self, "Returning caps %" GST_PTR_FORMAT, caps);

  return caps;
}

static GstCaps *
gst_mf_video_src_fixate (GstBaseSrc * src, GstCaps * caps)
{
  GstStructure *structure;
  GstCaps *fixated_caps;
  gint i;

  fixated_caps = gst_caps_make_writable (caps);

  for (i = 0; i < gst_caps_get_size (fixated_caps); ++i) {
    structure = gst_caps_get_structure (fixated_caps, i);
    gst_structure_fixate_field_nearest_int (structure, "width", G_MAXINT);
    gst_structure_fixate_field_nearest_int (structure, "height", G_MAXINT);
    gst_structure_fixate_field_nearest_fraction (structure, "framerate",
        G_MAXINT, 1);
  }

  fixated_caps = gst_caps_fixate (fixated_caps);

  return fixated_caps;
}

static gboolean
gst_mf_video_src_unlock (GstBaseSrc * src)
{
  GstMFVideoSrc *self = GST_MF_VIDEO_SRC (src);

  if (self->source)
    gst_mf_source_object_set_flushing (self->source, TRUE);

  return TRUE;
}

static gboolean
gst_mf_video_src_unlock_stop (GstBaseSrc * src)
{
  GstMFVideoSrc *self = GST_MF_VIDEO_SRC (src);

  if (self->source)
    gst_mf_source_object_set_flushing (self->source, FALSE);

  return TRUE;
}

static gboolean
gst_mf_video_src_query (GstBaseSrc * src, GstQuery * query)
{
  GstMFVideoSrc *self = GST_MF_VIDEO_SRC (src);

  switch (GST_QUERY_TYPE (query)) {
    case GST_QUERY_LATENCY:
      if (self->started) {
        gst_query_set_latency (query, TRUE, 0, self->latency);

        return TRUE;
      }
      break;
    default:
      break;
  }

  return GST_BASE_SRC_CLASS (parent_class)->query (src, query);
}

static GstFlowReturn
gst_mf_video_src_create (GstPushSrc * pushsrc, GstBuffer ** buffer)
{
  GstMFVideoSrc *self = GST_MF_VIDEO_SRC (pushsrc);
  GstFlowReturn ret = GST_FLOW_OK;
  GstBuffer *buf = nullptr;
  GstSample *sample = nullptr;
  GstClock *clock;
  GstClockTime running_time = GST_CLOCK_TIME_NONE;
  GstClockTimeDiff diff;

  if (!self->started) {
    if (!gst_mf_source_object_start (self->source)) {
      GST_ERROR_OBJECT (self, "Failed to start capture object");

      return GST_FLOW_ERROR;
    }

    self->started = TRUE;
  }

  if (self->use_dshow) {
    ret = gst_mf_source_object_get_sample (self->source, &sample);
  } else if (GST_VIDEO_INFO_FORMAT (&self->info) != GST_VIDEO_FORMAT_ENCODED) {
    ret = GST_BASE_SRC_CLASS (parent_class)->alloc (GST_BASE_SRC (self), 0,
        GST_VIDEO_INFO_SIZE (&self->info), &buf);

    if (ret != GST_FLOW_OK) {
      gst_clear_buffer (&buf);
      return ret;
    }

    ret = gst_mf_source_object_fill (self->source, buf);
  } else {
    ret = gst_mf_source_object_create (self->source, &buf);
  }

  if (ret != GST_FLOW_OK) {
    gst_clear_buffer (&buf);
    return ret;
  }

  /* DirectShow capture object will set caps if it's got updated */
  if (sample) {
    if (gst_sample_get_caps (sample)) {
      if (!gst_base_src_negotiate (GST_BASE_SRC (self))) {
        GST_ERROR_OBJECT (self, "Failed to negotiate with new caps");
        gst_sample_unref (sample);
        return GST_FLOW_NOT_NEGOTIATED;
      } else {
        GST_DEBUG_OBJECT (self, "Renegotiated");
      }
    }

    buf = gst_sample_get_buffer (sample);
    gst_buffer_ref (buf);
    gst_sample_unref (sample);
  }

  GST_BUFFER_OFFSET (buf) = self->n_frames;
  GST_BUFFER_OFFSET_END (buf) = GST_BUFFER_OFFSET (buf) + 1;
  self->n_frames++;

  GST_LOG_OBJECT (self,
      "Captured buffer timestamp %" GST_TIME_FORMAT ", duration %"
      GST_TIME_FORMAT, GST_TIME_ARGS (GST_BUFFER_TIMESTAMP (buf)),
      GST_TIME_ARGS (GST_BUFFER_DURATION (buf)));

  /* Update latency */
  clock = gst_element_get_clock (GST_ELEMENT_CAST (self));
  if (clock) {
    GstClockTime now;

    now = gst_clock_get_time (clock);
    running_time = now - GST_ELEMENT_CAST (self)->base_time;
    gst_object_unref (clock);
  }

  diff = GST_CLOCK_DIFF (GST_BUFFER_PTS (buf), running_time);
  if (diff > self->latency) {
    self->latency = (GstClockTime) diff;
    GST_DEBUG_OBJECT (self, "Updated latency value %" GST_TIME_FORMAT,
        GST_TIME_ARGS (self->latency));
  }

  *buffer = buf;

  return GST_FLOW_OK;
}