gstreamer/subprojects/gst-plugins-bad/sys/mediafoundation/gstmfvideosrc.cpp

512 lines
15 KiB
C++
Raw Normal View History

/* GStreamer
* Copyright (C) 2019 Seungha Yang <seungha.yang@navercorp.com>
* Copyright (C) 2020 Seungha Yang <seungha@centricular.com>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
* Boston, MA 02110-1301, USA.
*/
/**
* SECTION:element-mfvideosrc
* @title: mfvideosrc
*
* Provides video capture from the Microsoft Media Foundation API.
*
* ## Example pipelines
* |[
* gst-launch-1.0 -v mfvideosrc ! fakesink
* ]| Capture from the default video capture device and render to fakesink.
*
* |[
* gst-launch-1.0 -v mfvideosrc device-index=1 ! fakesink
* ]| Capture from the second video device (if available) and render to fakesink.
*/
#ifdef HAVE_CONFIG_H
#include "config.h"
#endif
#include "gstmfconfig.h"
#include "gstmfvideosrc.h"
#include "gstmfutils.h"
#include "gstmfsourceobject.h"
#include <string.h>
#if GST_MF_WINAPI_DESKTOP
#include "gstmfcapturedshow.h"
#endif
GST_DEBUG_CATEGORY (gst_mf_video_src_debug);
#define GST_CAT_DEFAULT gst_mf_video_src_debug
#if (GST_MF_WINAPI_APP && !GST_MF_WINAPI_DESKTOP)
/* FIXME: need support JPEG for UWP */
#define SRC_TEMPLATE_CAPS \
GST_VIDEO_CAPS_MAKE (GST_MF_VIDEO_FORMATS)
#else
#define SRC_TEMPLATE_CAPS \
GST_VIDEO_CAPS_MAKE (GST_MF_VIDEO_FORMATS) "; " \
"image/jpeg, width = " GST_VIDEO_SIZE_RANGE ", " \
"height = " GST_VIDEO_SIZE_RANGE ", " \
"framerate = " GST_VIDEO_FPS_RANGE
#endif
static GstStaticPadTemplate src_template = GST_STATIC_PAD_TEMPLATE ("src",
GST_PAD_SRC,
GST_PAD_ALWAYS,
GST_STATIC_CAPS (SRC_TEMPLATE_CAPS));
struct _GstMFVideoSrc
{
GstPushSrc parent;
GstMFSourceObject *source;
gboolean started;
GstVideoInfo info;
guint64 n_frames;
GstClockTime latency;
gboolean use_dshow;
/* properties */
gchar *device_path;
gchar *device_name;
gint device_index;
gpointer dispatcher;
};
enum
{
PROP_0,
PROP_DEVICE_PATH,
PROP_DEVICE_NAME,
PROP_DEVICE_INDEX,
PROP_DISPATCHER,
};
#define DEFAULT_DEVICE_PATH nullptr
#define DEFAULT_DEVICE_NAME nullptr
#define DEFAULT_DEVICE_INDEX -1
static void gst_mf_video_src_finalize (GObject * object);
static void gst_mf_video_src_get_property (GObject * object, guint prop_id,
GValue * value, GParamSpec * pspec);
static void gst_mf_video_src_set_property (GObject * object, guint prop_id,
const GValue * value, GParamSpec * pspec);
static gboolean gst_mf_video_src_start (GstBaseSrc * src);
static gboolean gst_mf_video_src_stop (GstBaseSrc * src);
static gboolean gst_mf_video_src_set_caps (GstBaseSrc * src, GstCaps * caps);
static GstCaps *gst_mf_video_src_get_caps (GstBaseSrc * src, GstCaps * filter);
static GstCaps *gst_mf_video_src_fixate (GstBaseSrc * src, GstCaps * caps);
static gboolean gst_mf_video_src_unlock (GstBaseSrc * src);
static gboolean gst_mf_video_src_unlock_stop (GstBaseSrc * src);
static gboolean gst_mf_video_src_query (GstBaseSrc * src, GstQuery * query);
static GstFlowReturn gst_mf_video_src_create (GstPushSrc * pushsrc,
GstBuffer ** buffer);
#define gst_mf_video_src_parent_class parent_class
G_DEFINE_TYPE (GstMFVideoSrc, gst_mf_video_src, GST_TYPE_PUSH_SRC);
static void
gst_mf_video_src_class_init (GstMFVideoSrcClass * klass)
{
GObjectClass *gobject_class = G_OBJECT_CLASS (klass);
GstElementClass *element_class = GST_ELEMENT_CLASS (klass);
GstBaseSrcClass *basesrc_class = GST_BASE_SRC_CLASS (klass);
GstPushSrcClass *pushsrc_class = GST_PUSH_SRC_CLASS (klass);
GParamFlags flags = (GParamFlags) (G_PARAM_READWRITE |
GST_PARAM_MUTABLE_READY | G_PARAM_STATIC_STRINGS);
gobject_class->finalize = gst_mf_video_src_finalize;
gobject_class->get_property = gst_mf_video_src_get_property;
gobject_class->set_property = gst_mf_video_src_set_property;
g_object_class_install_property (gobject_class, PROP_DEVICE_PATH,
g_param_spec_string ("device-path", "Device Path",
"The device path", DEFAULT_DEVICE_PATH, flags));
g_object_class_install_property (gobject_class, PROP_DEVICE_NAME,
g_param_spec_string ("device-name", "Device Name",
"The human-readable device name", DEFAULT_DEVICE_NAME, flags));
g_object_class_install_property (gobject_class, PROP_DEVICE_INDEX,
g_param_spec_int ("device-index", "Device Index",
"The zero-based device index", -1, G_MAXINT, DEFAULT_DEVICE_INDEX,
flags));
#if GST_MF_WINAPI_APP
/**
* GstMFVideoSrc:dispatcher:
*
* ICoreDispatcher COM object used for activating device from UI thread.
*
* Since: 1.18
*/
g_object_class_install_property (gobject_class, PROP_DISPATCHER,
g_param_spec_pointer ("dispatcher", "Dispatcher",
"ICoreDispatcher COM object to use. In order for application to ask "
"permission of capture device, device activation should be running "
"on UI thread via ICoreDispatcher. This element will increase "
"the reference count of given ICoreDispatcher and release it after "
"use. Therefore, caller does not need to consider additional "
"reference count management",
(GParamFlags) (GST_PARAM_CONDITIONALLY_AVAILABLE |
GST_PARAM_MUTABLE_READY | G_PARAM_WRITABLE |
G_PARAM_STATIC_STRINGS)));
#endif
gst_element_class_set_static_metadata (element_class,
"Media Foundation Video Source",
"Source/Video/Hardware",
"Capture video stream through Windows Media Foundation",
"Seungha Yang <seungha.yang@navercorp.com>");
gst_element_class_add_static_pad_template (element_class, &src_template);
basesrc_class->start = GST_DEBUG_FUNCPTR (gst_mf_video_src_start);
basesrc_class->stop = GST_DEBUG_FUNCPTR (gst_mf_video_src_stop);
basesrc_class->set_caps = GST_DEBUG_FUNCPTR (gst_mf_video_src_set_caps);
basesrc_class->get_caps = GST_DEBUG_FUNCPTR (gst_mf_video_src_get_caps);
basesrc_class->fixate = GST_DEBUG_FUNCPTR (gst_mf_video_src_fixate);
basesrc_class->unlock = GST_DEBUG_FUNCPTR (gst_mf_video_src_unlock);
basesrc_class->unlock_stop = GST_DEBUG_FUNCPTR (gst_mf_video_src_unlock_stop);
basesrc_class->query = GST_DEBUG_FUNCPTR (gst_mf_video_src_query);
pushsrc_class->create = GST_DEBUG_FUNCPTR (gst_mf_video_src_create);
GST_DEBUG_CATEGORY_INIT (gst_mf_video_src_debug, "mfvideosrc", 0,
"mfvideosrc");
}
static void
gst_mf_video_src_init (GstMFVideoSrc * self)
{
gst_base_src_set_format (GST_BASE_SRC (self), GST_FORMAT_TIME);
gst_base_src_set_live (GST_BASE_SRC (self), TRUE);
self->device_index = DEFAULT_DEVICE_INDEX;
}
static void
gst_mf_video_src_finalize (GObject * object)
{
GstMFVideoSrc *self = GST_MF_VIDEO_SRC (object);
g_free (self->device_name);
g_free (self->device_path);
G_OBJECT_CLASS (parent_class)->finalize (object);
}
static void
gst_mf_video_src_get_property (GObject * object, guint prop_id, GValue * value,
GParamSpec * pspec)
{
GstMFVideoSrc *self = GST_MF_VIDEO_SRC (object);
switch (prop_id) {
case PROP_DEVICE_PATH:
g_value_set_string (value, self->device_path);
break;
case PROP_DEVICE_NAME:
g_value_set_string (value, self->device_name);
break;
case PROP_DEVICE_INDEX:
g_value_set_int (value, self->device_index);
break;
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
break;
}
}
static void
gst_mf_video_src_set_property (GObject * object, guint prop_id,
const GValue * value, GParamSpec * pspec)
{
GstMFVideoSrc *self = GST_MF_VIDEO_SRC (object);
switch (prop_id) {
case PROP_DEVICE_PATH:
g_free (self->device_path);
self->device_path = g_value_dup_string (value);
break;
case PROP_DEVICE_NAME:
g_free (self->device_name);
self->device_name = g_value_dup_string (value);
break;
case PROP_DEVICE_INDEX:
self->device_index = g_value_get_int (value);
break;
#if GST_MF_WINAPI_APP
case PROP_DISPATCHER:
self->dispatcher = g_value_get_pointer (value);
break;
#endif
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
break;
}
}
static gboolean
gst_mf_video_src_start (GstBaseSrc * src)
{
GstMFVideoSrc *self = GST_MF_VIDEO_SRC (src);
GST_DEBUG_OBJECT (self, "Start");
self->source = gst_mf_source_object_new (GST_MF_SOURCE_TYPE_VIDEO,
self->device_index, self->device_name, self->device_path, nullptr);
self->n_frames = 0;
self->latency = 0;
self->use_dshow = FALSE;
if (!self->source) {
#if GST_MF_WINAPI_DESKTOP
self->use_dshow = TRUE;
self->source = gst_mf_capture_dshow_new (GST_MF_SOURCE_TYPE_VIDEO,
self->device_index, self->device_name, self->device_path);
#endif
}
if (!self->source) {
GST_ERROR_OBJECT (self, "Couldn't create capture object");
return FALSE;
}
gst_mf_source_object_set_client (self->source, GST_ELEMENT (self));
return TRUE;
}
static gboolean
gst_mf_video_src_stop (GstBaseSrc * src)
{
GstMFVideoSrc *self = GST_MF_VIDEO_SRC (src);
GST_DEBUG_OBJECT (self, "Stop");
if (self->source) {
gst_mf_source_object_stop (self->source);
gst_object_unref (self->source);
self->source = nullptr;
}
self->started = FALSE;
return TRUE;
}
static gboolean
gst_mf_video_src_set_caps (GstBaseSrc * src, GstCaps * caps)
{
GstMFVideoSrc *self = GST_MF_VIDEO_SRC (src);
GST_DEBUG_OBJECT (self, "Set caps %" GST_PTR_FORMAT, caps);
if (!self->source) {
GST_ERROR_OBJECT (self, "No capture engine yet");
return FALSE;
}
if (!gst_mf_source_object_set_caps (self->source, caps)) {
GST_ERROR_OBJECT (self, "CaptureEngine couldn't accept caps");
return FALSE;
}
gst_video_info_from_caps (&self->info, caps);
if (GST_VIDEO_INFO_FORMAT (&self->info) != GST_VIDEO_FORMAT_ENCODED)
gst_base_src_set_blocksize (src, GST_VIDEO_INFO_SIZE (&self->info));
return TRUE;
}
static GstCaps *
gst_mf_video_src_get_caps (GstBaseSrc * src, GstCaps * filter)
{
GstMFVideoSrc *self = GST_MF_VIDEO_SRC (src);
GstCaps *caps = nullptr;
if (self->source)
caps = gst_mf_source_object_get_caps (self->source);
if (!caps)
caps = gst_pad_get_pad_template_caps (GST_BASE_SRC_PAD (src));
if (filter) {
GstCaps *filtered =
gst_caps_intersect_full (filter, caps, GST_CAPS_INTERSECT_FIRST);
gst_caps_unref (caps);
caps = filtered;
}
GST_DEBUG_OBJECT (self, "Returning caps %" GST_PTR_FORMAT, caps);
return caps;
}
static GstCaps *
gst_mf_video_src_fixate (GstBaseSrc * src, GstCaps * caps)
{
GstStructure *structure;
GstCaps *fixated_caps;
gint i;
fixated_caps = gst_caps_make_writable (caps);
for (i = 0; i < gst_caps_get_size (fixated_caps); ++i) {
structure = gst_caps_get_structure (fixated_caps, i);
gst_structure_fixate_field_nearest_int (structure, "width", G_MAXINT);
gst_structure_fixate_field_nearest_int (structure, "height", G_MAXINT);
gst_structure_fixate_field_nearest_fraction (structure, "framerate",
G_MAXINT, 1);
}
fixated_caps = gst_caps_fixate (fixated_caps);
return fixated_caps;
}
static gboolean
gst_mf_video_src_unlock (GstBaseSrc * src)
{
GstMFVideoSrc *self = GST_MF_VIDEO_SRC (src);
if (self->source)
gst_mf_source_object_set_flushing (self->source, TRUE);
return TRUE;
}
static gboolean
gst_mf_video_src_unlock_stop (GstBaseSrc * src)
{
GstMFVideoSrc *self = GST_MF_VIDEO_SRC (src);
if (self->source)
gst_mf_source_object_set_flushing (self->source, FALSE);
return TRUE;
}
static gboolean
gst_mf_video_src_query (GstBaseSrc * src, GstQuery * query)
{
GstMFVideoSrc *self = GST_MF_VIDEO_SRC (src);
switch (GST_QUERY_TYPE (query)) {
case GST_QUERY_LATENCY:
if (self->started) {
gst_query_set_latency (query, TRUE, 0, self->latency);
return TRUE;
}
break;
default:
break;
}
return GST_BASE_SRC_CLASS (parent_class)->query (src, query);
}
static GstFlowReturn
gst_mf_video_src_create (GstPushSrc * pushsrc, GstBuffer ** buffer)
{
GstMFVideoSrc *self = GST_MF_VIDEO_SRC (pushsrc);
GstFlowReturn ret = GST_FLOW_OK;
GstBuffer *buf = nullptr;
GstSample *sample = nullptr;
GstClock *clock;
GstClockTime running_time = GST_CLOCK_TIME_NONE;
GstClockTimeDiff diff;
if (!self->started) {
if (!gst_mf_source_object_start (self->source)) {
GST_ERROR_OBJECT (self, "Failed to start capture object");
return GST_FLOW_ERROR;
}
self->started = TRUE;
}
if (self->use_dshow) {
ret = gst_mf_source_object_get_sample (self->source, &sample);
} else if (GST_VIDEO_INFO_FORMAT (&self->info) != GST_VIDEO_FORMAT_ENCODED) {
ret = GST_BASE_SRC_CLASS (parent_class)->alloc (GST_BASE_SRC (self), 0,
GST_VIDEO_INFO_SIZE (&self->info), &buf);
if (ret != GST_FLOW_OK)
return ret;
ret = gst_mf_source_object_fill (self->source, buf);
} else {
ret = gst_mf_source_object_create (self->source, &buf);
}
if (ret != GST_FLOW_OK)
return ret;
/* DirectShow capture object will set caps if it's got updated */
if (sample) {
if (gst_sample_get_caps (sample)) {
if (!gst_base_src_negotiate (GST_BASE_SRC (self))) {
GST_ERROR_OBJECT (self, "Failed to negotiate with new caps");
gst_sample_unref (sample);
return GST_FLOW_NOT_NEGOTIATED;
} else {
GST_DEBUG_OBJECT (self, "Renegotiated");
}
}
buf = gst_sample_get_buffer (sample);
gst_buffer_ref (buf);
gst_sample_unref (sample);
}
GST_BUFFER_OFFSET (buf) = self->n_frames;
GST_BUFFER_OFFSET_END (buf) = GST_BUFFER_OFFSET (buf) + 1;
self->n_frames++;
GST_LOG_OBJECT (self,
"Captured buffer timestamp %" GST_TIME_FORMAT ", duration %"
GST_TIME_FORMAT, GST_TIME_ARGS (GST_BUFFER_TIMESTAMP (buf)),
GST_TIME_ARGS (GST_BUFFER_DURATION (buf)));
/* Update latency */
clock = gst_element_get_clock (GST_ELEMENT_CAST (self));
if (clock) {
GstClockTime now;
now = gst_clock_get_time (clock);
running_time = now - GST_ELEMENT_CAST (self)->base_time;
gst_object_unref (clock);
}
diff = GST_CLOCK_DIFF (GST_BUFFER_PTS (buf), running_time);
if (diff > self->latency) {
self->latency = (GstClockTime) diff;
GST_DEBUG_OBJECT (self, "Updated latency value %" GST_TIME_FORMAT,
GST_TIME_ARGS (self->latency));
}
*buffer = buf;
return GST_FLOW_OK;
}