msdk: vpp : Add frame rate control

https://bugzilla.gnome.org/show_bug.cgi?id=793705
This commit is contained in:
Sreerenj Balachandran 2018-03-22 16:04:02 +00:00
parent c0ea4bdafb
commit e4b4f09496
5 changed files with 144 additions and 35 deletions

View file

@ -78,6 +78,7 @@ enum
PROP_MIRRORING, PROP_MIRRORING,
PROP_SCALING_MODE, PROP_SCALING_MODE,
PROP_FORCE_ASPECT_RATIO, PROP_FORCE_ASPECT_RATIO,
PROP_FRC_ALGORITHM,
PROP_N, PROP_N,
}; };
@ -95,6 +96,7 @@ enum
#define PROP_MIRRORING_DEFAULT MFX_MIRRORING_DISABLED #define PROP_MIRRORING_DEFAULT MFX_MIRRORING_DISABLED
#define PROP_SCALING_MODE_DEFAULT MFX_SCALING_MODE_DEFAULT #define PROP_SCALING_MODE_DEFAULT MFX_SCALING_MODE_DEFAULT
#define PROP_FORCE_ASPECT_RATIO_DEFAULT TRUE #define PROP_FORCE_ASPECT_RATIO_DEFAULT TRUE
#define PROP_FRC_ALGORITHM_DEFAULT _MFX_FRC_ALGORITHM_NONE
#define gst_msdkvpp_parent_class parent_class #define gst_msdkvpp_parent_class parent_class
G_DEFINE_TYPE (GstMsdkVPP, gst_msdkvpp, GST_TYPE_BASE_TRANSFORM); G_DEFINE_TYPE (GstMsdkVPP, gst_msdkvpp, GST_TYPE_BASE_TRANSFORM);
@ -511,12 +513,16 @@ gst_msdkvpp_transform (GstBaseTransform * trans, GstBuffer * inbuf,
GstBuffer * outbuf) GstBuffer * outbuf)
{ {
GstMsdkVPP *thiz = GST_MSDKVPP (trans); GstMsdkVPP *thiz = GST_MSDKVPP (trans);
GstClockTime timestamp;
GstFlowReturn ret = GST_FLOW_OK;
mfxSession session; mfxSession session;
mfxSyncPoint sync_point = NULL; mfxSyncPoint sync_point = NULL;
mfxStatus status; mfxStatus status;
MsdkSurface *in_surface = NULL; MsdkSurface *in_surface = NULL;
MsdkSurface *out_surface = NULL; MsdkSurface *out_surface = NULL;
timestamp = GST_BUFFER_TIMESTAMP (inbuf);
in_surface = get_msdk_surface_from_input_buffer (thiz, inbuf); in_surface = get_msdk_surface_from_input_buffer (thiz, inbuf);
if (!in_surface) if (!in_surface)
return GST_FLOW_ERROR; return GST_FLOW_ERROR;
@ -530,13 +536,16 @@ gst_msdkvpp_transform (GstBaseTransform * trans, GstBuffer * inbuf,
} }
session = gst_msdk_context_get_session (thiz->context); session = gst_msdk_context_get_session (thiz->context);
/* outer loop is for handling FrameRate Control and deinterlace use cases */
do {
for (;;) { for (;;) {
status = status =
MFXVideoVPP_RunFrameVPPAsync (session, in_surface->surface, MFXVideoVPP_RunFrameVPPAsync (session, in_surface->surface,
out_surface->surface, NULL, &sync_point); out_surface->surface, NULL, &sync_point);
if (status != MFX_WRN_DEVICE_BUSY) if (status != MFX_WRN_DEVICE_BUSY)
break; break;
/* If device is busy, wait 1ms and retry, as per MSDK's recomendation */ /* If device is busy, wait 1ms and retry, as per MSDK's recommendation */
g_usleep (1000); g_usleep (1000);
}; };
@ -547,17 +556,27 @@ gst_msdkvpp_transform (GstBaseTransform * trans, GstBuffer * inbuf,
/* No output generated */ /* No output generated */
if (status == MFX_ERR_MORE_DATA) if (status == MFX_ERR_MORE_DATA)
goto error_more_data; goto error_more_data;
if (sync_point) if (sync_point)
MFXVideoCORE_SyncOperation (session, sync_point, 10000); MFXVideoCORE_SyncOperation (session, sync_point, 10000);
/* More than one output buffers are generated */ /* More than one output buffers are generated */
if (status == MFX_ERR_MORE_SURFACE) if (status == MFX_ERR_MORE_SURFACE) {
status = MFX_ERR_NONE; GST_BUFFER_TIMESTAMP (outbuf) = timestamp;
GST_BUFFER_DURATION (outbuf) = thiz->buffer_duration;
gst_buffer_copy_into (outbuf, inbuf, GST_BUFFER_COPY_TIMESTAMPS, 0, -1); timestamp += thiz->buffer_duration;
ret = gst_pad_push (GST_BASE_TRANSFORM_SRC_PAD (trans), outbuf);
if (ret != GST_FLOW_OK)
goto error_push_buffer;
outbuf = create_output_buffer (thiz);
} else {
GST_BUFFER_TIMESTAMP (outbuf) = timestamp;
GST_BUFFER_DURATION (outbuf) = thiz->buffer_duration;
}
} while (status == MFX_ERR_MORE_SURFACE);
free_msdk_surface (in_surface); free_msdk_surface (in_surface);
return GST_FLOW_OK; return ret;
vpp_error: vpp_error:
GST_ERROR_OBJECT (thiz, "MSDK Failed to do VPP"); GST_ERROR_OBJECT (thiz, "MSDK Failed to do VPP");
@ -571,6 +590,15 @@ error_more_data:
"Retruning FLOW_DROPPED since no output buffer was generated"); "Retruning FLOW_DROPPED since no output buffer was generated");
free_msdk_surface (in_surface); free_msdk_surface (in_surface);
return GST_BASE_TRANSFORM_FLOW_DROPPED; return GST_BASE_TRANSFORM_FLOW_DROPPED;
error_push_buffer:
{
free_msdk_surface (in_surface);
free_msdk_surface (out_surface);
GST_DEBUG_OBJECT (thiz, "failed to push output buffer: %s",
gst_flow_get_name (ret));
return ret;
}
} }
static void static void
@ -600,7 +628,7 @@ gst_msdkvpp_close (GstMsdkVPP * thiz)
gst_object_unref (thiz->srcpad_buffer_pool); gst_object_unref (thiz->srcpad_buffer_pool);
thiz->srcpad_buffer_pool = NULL; thiz->srcpad_buffer_pool = NULL;
thiz->field_duration = GST_CLOCK_TIME_NONE; thiz->buffer_duration = GST_CLOCK_TIME_NONE;
gst_video_info_init (&thiz->sinkpad_info); gst_video_info_init (&thiz->sinkpad_info);
gst_video_info_init (&thiz->srcpad_info); gst_video_info_init (&thiz->srcpad_info);
} }
@ -700,6 +728,18 @@ ensure_filters (GstMsdkVPP * thiz)
thiz->max_filter_algorithms[n_filters] = MFX_EXTBUFF_VPP_SCALING; thiz->max_filter_algorithms[n_filters] = MFX_EXTBUFF_VPP_SCALING;
n_filters++; n_filters++;
} }
/* FRC */
if (thiz->flags & GST_MSDK_FLAG_FRC) {
mfxExtVPPFrameRateConversion *mfx_frc = &thiz->mfx_frc;
mfx_frc->Header.BufferId = MFX_EXTBUFF_VPP_FRAME_RATE_CONVERSION;
mfx_frc->Header.BufferSz = sizeof (mfxExtVPPFrameRateConversion);
mfx_frc->Algorithm = thiz->frc_algm;
gst_msdkvpp_add_extra_param (thiz, (mfxExtBuffer *) mfx_frc);
thiz->max_filter_algorithms[n_filters] =
MFX_EXTBUFF_VPP_FRAME_RATE_CONVERSION;
n_filters++;
}
} }
static void static void
@ -756,10 +796,23 @@ gst_msdkvpp_initialize (GstMsdkVPP * thiz)
/* update output video attributes, only CSC and Scaling are supported for now */ /* update output video attributes, only CSC and Scaling are supported for now */
gst_msdk_set_mfx_frame_info_from_video_info (&thiz->param.vpp.Out, gst_msdk_set_mfx_frame_info_from_video_info (&thiz->param.vpp.Out,
&thiz->srcpad_info); &thiz->srcpad_info);
thiz->param.vpp.Out.FrameRateExtN =
GST_VIDEO_INFO_FPS_N (&thiz->sinkpad_info); /* use msdk frame rarte control if there is a mismatch in In & OUt fps */
thiz->param.vpp.Out.FrameRateExtD = if (GST_VIDEO_INFO_FPS_N (&thiz->srcpad_info) &&
GST_VIDEO_INFO_FPS_D (&thiz->sinkpad_info); (GST_VIDEO_INFO_FPS_N (&thiz->sinkpad_info) !=
GST_VIDEO_INFO_FPS_N (&thiz->srcpad_info)
|| GST_VIDEO_INFO_FPS_D (&thiz->sinkpad_info) !=
GST_VIDEO_INFO_FPS_D (&thiz->srcpad_info))) {
thiz->flags |= GST_MSDK_FLAG_FRC;
/* So far this is the only algorithm which is working somewhat good */
thiz->frc_algm = MFX_FRCALGM_PRESERVE_TIMESTAMP;
}
/* work-around to avoid zero fps in msdk structure */
if (!thiz->param.vpp.In.FrameRateExtN)
thiz->param.vpp.In.FrameRateExtN = 30;
if (!thiz->param.vpp.Out.FrameRateExtN)
thiz->param.vpp.Out.FrameRateExtN = thiz->param.vpp.In.FrameRateExtN;
/* set vpp out picstruct as progressive if deinterlacing enabled */ /* set vpp out picstruct as progressive if deinterlacing enabled */
if (thiz->flags & GST_MSDK_FLAG_DEINTERLACE) if (thiz->flags & GST_MSDK_FLAG_DEINTERLACE)
@ -861,9 +914,10 @@ gst_msdkvpp_set_caps (GstBaseTransform * trans, GstCaps * caps,
deinterlace = gst_msdkvpp_is_deinterlace_enabled (thiz, &in_info); deinterlace = gst_msdkvpp_is_deinterlace_enabled (thiz, &in_info);
if (deinterlace) if (deinterlace)
thiz->flags |= GST_MSDK_FLAG_DEINTERLACE; thiz->flags |= GST_MSDK_FLAG_DEINTERLACE;
thiz->field_duration = GST_VIDEO_INFO_FPS_N (&in_info) > 0 ?
gst_util_uint64_scale (GST_SECOND, GST_VIDEO_INFO_FPS_D (&in_info), thiz->buffer_duration = GST_VIDEO_INFO_FPS_N (&out_info) > 0 ?
(1 + deinterlace) * GST_VIDEO_INFO_FPS_N (&in_info)) : 0; gst_util_uint64_scale (GST_SECOND, GST_VIDEO_INFO_FPS_D (&out_info),
GST_VIDEO_INFO_FPS_N (&out_info)) : 0;
if (!gst_msdkvpp_initialize (thiz)) if (!gst_msdkvpp_initialize (thiz))
return FALSE; return FALSE;
@ -1009,6 +1063,9 @@ gst_msdkvpp_set_property (GObject * object, guint prop_id,
case PROP_FORCE_ASPECT_RATIO: case PROP_FORCE_ASPECT_RATIO:
thiz->keep_aspect = g_value_get_boolean (value); thiz->keep_aspect = g_value_get_boolean (value);
break; break;
case PROP_FRC_ALGORITHM:
thiz->frc_algm = g_value_get_enum (value);
break;
default: default:
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
break; break;
@ -1064,6 +1121,9 @@ gst_msdkvpp_get_property (GObject * object, guint prop_id,
case PROP_FORCE_ASPECT_RATIO: case PROP_FORCE_ASPECT_RATIO:
g_value_set_boolean (value, thiz->keep_aspect); g_value_set_boolean (value, thiz->keep_aspect);
break; break;
case PROP_FRC_ALGORITHM:
g_value_set_enum (value, thiz->frc_algm);
break;
default: default:
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
break; break;
@ -1209,6 +1269,12 @@ gst_msdkvpp_class_init (GstMsdkVPPClass * klass)
PROP_FORCE_ASPECT_RATIO_DEFAULT, PROP_FORCE_ASPECT_RATIO_DEFAULT,
G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS); G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS);
obj_properties[PROP_FRC_ALGORITHM] =
g_param_spec_enum ("frc-algorithm", "FrameRateControl Algorithm",
"The Framerate Control Alogorithm to use",
gst_msdkvpp_frc_algorithm_get_type (), PROP_FRC_ALGORITHM_DEFAULT,
G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS);
g_object_class_install_properties (gobject_class, PROP_N, obj_properties); g_object_class_install_properties (gobject_class, PROP_N, obj_properties);
} }
@ -1221,7 +1287,7 @@ gst_msdkvpp_init (GstMsdkVPP * thiz)
thiz->rotation = PROP_ROTATION_DEFAULT; thiz->rotation = PROP_ROTATION_DEFAULT;
thiz->deinterlace_mode = PROP_DEINTERLACE_MODE_DEFAULT; thiz->deinterlace_mode = PROP_DEINTERLACE_MODE_DEFAULT;
thiz->deinterlace_method = PROP_DEINTERLACE_METHOD_DEFAULT; thiz->deinterlace_method = PROP_DEINTERLACE_METHOD_DEFAULT;
thiz->field_duration = GST_CLOCK_TIME_NONE; thiz->buffer_duration = GST_CLOCK_TIME_NONE;
thiz->hue = PROP_HUE_DEFAULT; thiz->hue = PROP_HUE_DEFAULT;
thiz->saturation = PROP_SATURATION_DEFAULT; thiz->saturation = PROP_SATURATION_DEFAULT;
thiz->brightness = PROP_BRIGHTNESS_DEFAULT; thiz->brightness = PROP_BRIGHTNESS_DEFAULT;
@ -1230,6 +1296,7 @@ gst_msdkvpp_init (GstMsdkVPP * thiz)
thiz->mirroring = PROP_MIRRORING_DEFAULT; thiz->mirroring = PROP_MIRRORING_DEFAULT;
thiz->scaling_mode = PROP_SCALING_MODE_DEFAULT; thiz->scaling_mode = PROP_SCALING_MODE_DEFAULT;
thiz->keep_aspect = PROP_FORCE_ASPECT_RATIO_DEFAULT; thiz->keep_aspect = PROP_FORCE_ASPECT_RATIO_DEFAULT;
thiz->frc_algm = PROP_FRC_ALGORITHM_DEFAULT;
gst_video_info_init (&thiz->sinkpad_info); gst_video_info_init (&thiz->sinkpad_info);
gst_video_info_init (&thiz->srcpad_info); gst_video_info_init (&thiz->srcpad_info);
} }

View file

@ -68,6 +68,7 @@ typedef enum {
GST_MSDK_FLAG_DETAIL = 1 << 7, GST_MSDK_FLAG_DETAIL = 1 << 7,
GST_MSDK_FLAG_MIRRORING = 1 << 8, GST_MSDK_FLAG_MIRRORING = 1 << 8,
GST_MSDK_FLAG_SCALING_MODE = 1 << 9, GST_MSDK_FLAG_SCALING_MODE = 1 << 9,
GST_MSDK_FLAG_FRC = 1 << 10,
} GstMsdkVppFlags; } GstMsdkVppFlags;
struct _GstMsdkVPP struct _GstMsdkVPP
@ -114,8 +115,9 @@ struct _GstMsdkVPP
guint mirroring; guint mirroring;
guint scaling_mode; guint scaling_mode;
gboolean keep_aspect; gboolean keep_aspect;
guint frc_algm;
GstClockTime field_duration; GstClockTime buffer_duration;
/* MFX Filters */ /* MFX Filters */
mfxExtVPPDoUse mfx_vpp_douse; mfxExtVPPDoUse mfx_vpp_douse;
@ -127,6 +129,7 @@ struct _GstMsdkVPP
mfxExtVPPDetail mfx_detail; mfxExtVPPDetail mfx_detail;
mfxExtVPPMirroring mfx_mirroring; mfxExtVPPMirroring mfx_mirroring;
mfxExtVPPScaling mfx_scaling; mfxExtVPPScaling mfx_scaling;
mfxExtVPPFrameRateConversion mfx_frc;
/* Extended buffers */ /* Extended buffers */
mfxExtBuffer *extra_params[MAX_EXTRA_PARAMS]; mfxExtBuffer *extra_params[MAX_EXTRA_PARAMS];

View file

@ -429,10 +429,19 @@ overflow_error:
static gboolean static gboolean
fixate_frame_rate (GstMsdkVPP * thiz, GstVideoInfo * vinfo, GstStructure * outs) fixate_frame_rate (GstMsdkVPP * thiz, GstVideoInfo * vinfo, GstStructure * outs)
{ {
gint fps_n, fps_d; gint fps_n = 0, fps_d;
/* fixate the srcpad fps */
if (gst_structure_fixate_field (outs, "framerate"))
gst_structure_get (outs, "framerate", GST_TYPE_FRACTION, &fps_n, &fps_d,
NULL);
/* if we don't have a fixed non-zero fps_n, use the sinkpad fps */
if (!fps_n) {
fps_n = GST_VIDEO_INFO_FPS_N (vinfo); fps_n = GST_VIDEO_INFO_FPS_N (vinfo);
fps_d = GST_VIDEO_INFO_FPS_D (vinfo); fps_d = GST_VIDEO_INFO_FPS_D (vinfo);
}
if (gst_msdkvpp_is_deinterlace_enabled (thiz, vinfo)) { if (gst_msdkvpp_is_deinterlace_enabled (thiz, vinfo)) {
/* Fixme: set double framerate?: /* Fixme: set double framerate?:
* msdk is not outputting double framerate for bob or adv deinterlace */ * msdk is not outputting double framerate for bob or adv deinterlace */

View file

@ -278,3 +278,29 @@ gst_msdkvpp_scaling_mode_get_type (void)
} }
return type; return type;
} }
GType
gst_msdkvpp_frc_algorithm_get_type (void)
{
static GType type = 0;
static const GEnumValue values[] = {
{_MFX_FRC_ALGORITHM_NONE, "No FrameRate Control algorithm", "none"},
{MFX_FRCALGM_PRESERVE_TIMESTAMP,
"Frame dropping/repetition, Preserve timestamp", "preserve-ts"},
{MFX_FRCALGM_DISTRIBUTED_TIMESTAMP,
"Frame dropping/repetition, Distribute timestamp", "distribute-ts"},
{MFX_FRCALGM_FRAME_INTERPOLATION, "Frame interpolation", "interpolate"},
{MFX_FRCALGM_FRAME_INTERPOLATION | MFX_FRCALGM_PRESERVE_TIMESTAMP,
"Frame interpolation, Preserve timestamp", "interpolate-preserve-ts"},
{MFX_FRCALGM_FRAME_INTERPOLATION | MFX_FRCALGM_DISTRIBUTED_TIMESTAMP,
"Frame interpolation, Distribute timestamp",
"interpolate-distribute-ts"},
{0, NULL, NULL}
};
if (!type) {
type = g_enum_register_static ("GstMsdkVPPFrcAlgorithm", values);
}
return type;
}

View file

@ -84,5 +84,9 @@ gst_msdkvpp_mirroring_get_type (void);
GType GType
gst_msdkvpp_scaling_mode_get_type (void); gst_msdkvpp_scaling_mode_get_type (void);
#define _MFX_FRC_ALGORITHM_NONE 0
GType
gst_msdkvpp_frc_algorithm_get_type (void);
G_END_DECLS G_END_DECLS
#endif #endif