From e4b4f0949613f53f3cd84e9ca6d5c7dfbec8f188 Mon Sep 17 00:00:00 2001 From: Sreerenj Balachandran Date: Thu, 22 Mar 2018 16:04:02 +0000 Subject: [PATCH] msdk: vpp : Add frame rate control https://bugzilla.gnome.org/show_bug.cgi?id=793705 --- sys/msdk/gstmsdkvpp.c | 129 +++++++++++++++++++++++++++++--------- sys/msdk/gstmsdkvpp.h | 5 +- sys/msdk/gstmsdkvpputil.c | 15 ++++- sys/msdk/msdk-enums.c | 26 ++++++++ sys/msdk/msdk-enums.h | 4 ++ 5 files changed, 144 insertions(+), 35 deletions(-) diff --git a/sys/msdk/gstmsdkvpp.c b/sys/msdk/gstmsdkvpp.c index 2fd5eee85a..3369cf12d9 100644 --- a/sys/msdk/gstmsdkvpp.c +++ b/sys/msdk/gstmsdkvpp.c @@ -78,6 +78,7 @@ enum PROP_MIRRORING, PROP_SCALING_MODE, PROP_FORCE_ASPECT_RATIO, + PROP_FRC_ALGORITHM, PROP_N, }; @@ -95,6 +96,7 @@ enum #define PROP_MIRRORING_DEFAULT MFX_MIRRORING_DISABLED #define PROP_SCALING_MODE_DEFAULT MFX_SCALING_MODE_DEFAULT #define PROP_FORCE_ASPECT_RATIO_DEFAULT TRUE +#define PROP_FRC_ALGORITHM_DEFAULT _MFX_FRC_ALGORITHM_NONE #define gst_msdkvpp_parent_class parent_class G_DEFINE_TYPE (GstMsdkVPP, gst_msdkvpp, GST_TYPE_BASE_TRANSFORM); @@ -511,12 +513,16 @@ gst_msdkvpp_transform (GstBaseTransform * trans, GstBuffer * inbuf, GstBuffer * outbuf) { GstMsdkVPP *thiz = GST_MSDKVPP (trans); + GstClockTime timestamp; + GstFlowReturn ret = GST_FLOW_OK; mfxSession session; mfxSyncPoint sync_point = NULL; mfxStatus status; MsdkSurface *in_surface = NULL; MsdkSurface *out_surface = NULL; + timestamp = GST_BUFFER_TIMESTAMP (inbuf); + in_surface = get_msdk_surface_from_input_buffer (thiz, inbuf); if (!in_surface) return GST_FLOW_ERROR; @@ -530,34 +536,47 @@ gst_msdkvpp_transform (GstBaseTransform * trans, GstBuffer * inbuf, } session = gst_msdk_context_get_session (thiz->context); - for (;;) { - status = - MFXVideoVPP_RunFrameVPPAsync (session, in_surface->surface, - out_surface->surface, NULL, &sync_point); - if (status != MFX_WRN_DEVICE_BUSY) - break; - /* If device is busy, wait 1ms and retry, as per MSDK's recomendation */ - g_usleep (1000); - }; - if (status != MFX_ERR_NONE && status != MFX_ERR_MORE_DATA - && status != MFX_ERR_MORE_SURFACE) - goto vpp_error; + /* outer loop is for handling FrameRate Control and deinterlace use cases */ + do { + for (;;) { + status = + MFXVideoVPP_RunFrameVPPAsync (session, in_surface->surface, + out_surface->surface, NULL, &sync_point); + if (status != MFX_WRN_DEVICE_BUSY) + break; + /* If device is busy, wait 1ms and retry, as per MSDK's recommendation */ + g_usleep (1000); + }; - /* No output generated */ - if (status == MFX_ERR_MORE_DATA) - goto error_more_data; - if (sync_point) - MFXVideoCORE_SyncOperation (session, sync_point, 10000); + if (status != MFX_ERR_NONE && status != MFX_ERR_MORE_DATA + && status != MFX_ERR_MORE_SURFACE) + goto vpp_error; - /* More than one output buffers are generated */ - if (status == MFX_ERR_MORE_SURFACE) - status = MFX_ERR_NONE; + /* No output generated */ + if (status == MFX_ERR_MORE_DATA) + goto error_more_data; - gst_buffer_copy_into (outbuf, inbuf, GST_BUFFER_COPY_TIMESTAMPS, 0, -1); + if (sync_point) + MFXVideoCORE_SyncOperation (session, sync_point, 10000); + + /* More than one output buffers are generated */ + if (status == MFX_ERR_MORE_SURFACE) { + GST_BUFFER_TIMESTAMP (outbuf) = timestamp; + GST_BUFFER_DURATION (outbuf) = thiz->buffer_duration; + timestamp += thiz->buffer_duration; + ret = gst_pad_push (GST_BASE_TRANSFORM_SRC_PAD (trans), outbuf); + if (ret != GST_FLOW_OK) + goto error_push_buffer; + outbuf = create_output_buffer (thiz); + } else { + GST_BUFFER_TIMESTAMP (outbuf) = timestamp; + GST_BUFFER_DURATION (outbuf) = thiz->buffer_duration; + } + } while (status == MFX_ERR_MORE_SURFACE); free_msdk_surface (in_surface); - return GST_FLOW_OK; + return ret; vpp_error: GST_ERROR_OBJECT (thiz, "MSDK Failed to do VPP"); @@ -571,6 +590,15 @@ error_more_data: "Retruning FLOW_DROPPED since no output buffer was generated"); free_msdk_surface (in_surface); return GST_BASE_TRANSFORM_FLOW_DROPPED; + +error_push_buffer: + { + free_msdk_surface (in_surface); + free_msdk_surface (out_surface); + GST_DEBUG_OBJECT (thiz, "failed to push output buffer: %s", + gst_flow_get_name (ret)); + return ret; + } } static void @@ -600,7 +628,7 @@ gst_msdkvpp_close (GstMsdkVPP * thiz) gst_object_unref (thiz->srcpad_buffer_pool); thiz->srcpad_buffer_pool = NULL; - thiz->field_duration = GST_CLOCK_TIME_NONE; + thiz->buffer_duration = GST_CLOCK_TIME_NONE; gst_video_info_init (&thiz->sinkpad_info); gst_video_info_init (&thiz->srcpad_info); } @@ -700,6 +728,18 @@ ensure_filters (GstMsdkVPP * thiz) thiz->max_filter_algorithms[n_filters] = MFX_EXTBUFF_VPP_SCALING; n_filters++; } + + /* FRC */ + if (thiz->flags & GST_MSDK_FLAG_FRC) { + mfxExtVPPFrameRateConversion *mfx_frc = &thiz->mfx_frc; + mfx_frc->Header.BufferId = MFX_EXTBUFF_VPP_FRAME_RATE_CONVERSION; + mfx_frc->Header.BufferSz = sizeof (mfxExtVPPFrameRateConversion); + mfx_frc->Algorithm = thiz->frc_algm; + gst_msdkvpp_add_extra_param (thiz, (mfxExtBuffer *) mfx_frc); + thiz->max_filter_algorithms[n_filters] = + MFX_EXTBUFF_VPP_FRAME_RATE_CONVERSION; + n_filters++; + } } static void @@ -756,10 +796,23 @@ gst_msdkvpp_initialize (GstMsdkVPP * thiz) /* update output video attributes, only CSC and Scaling are supported for now */ gst_msdk_set_mfx_frame_info_from_video_info (&thiz->param.vpp.Out, &thiz->srcpad_info); - thiz->param.vpp.Out.FrameRateExtN = - GST_VIDEO_INFO_FPS_N (&thiz->sinkpad_info); - thiz->param.vpp.Out.FrameRateExtD = - GST_VIDEO_INFO_FPS_D (&thiz->sinkpad_info); + + /* use msdk frame rarte control if there is a mismatch in In & OUt fps */ + if (GST_VIDEO_INFO_FPS_N (&thiz->srcpad_info) && + (GST_VIDEO_INFO_FPS_N (&thiz->sinkpad_info) != + GST_VIDEO_INFO_FPS_N (&thiz->srcpad_info) + || GST_VIDEO_INFO_FPS_D (&thiz->sinkpad_info) != + GST_VIDEO_INFO_FPS_D (&thiz->srcpad_info))) { + thiz->flags |= GST_MSDK_FLAG_FRC; + /* So far this is the only algorithm which is working somewhat good */ + thiz->frc_algm = MFX_FRCALGM_PRESERVE_TIMESTAMP; + } + + /* work-around to avoid zero fps in msdk structure */ + if (!thiz->param.vpp.In.FrameRateExtN) + thiz->param.vpp.In.FrameRateExtN = 30; + if (!thiz->param.vpp.Out.FrameRateExtN) + thiz->param.vpp.Out.FrameRateExtN = thiz->param.vpp.In.FrameRateExtN; /* set vpp out picstruct as progressive if deinterlacing enabled */ if (thiz->flags & GST_MSDK_FLAG_DEINTERLACE) @@ -861,9 +914,10 @@ gst_msdkvpp_set_caps (GstBaseTransform * trans, GstCaps * caps, deinterlace = gst_msdkvpp_is_deinterlace_enabled (thiz, &in_info); if (deinterlace) thiz->flags |= GST_MSDK_FLAG_DEINTERLACE; - thiz->field_duration = GST_VIDEO_INFO_FPS_N (&in_info) > 0 ? - gst_util_uint64_scale (GST_SECOND, GST_VIDEO_INFO_FPS_D (&in_info), - (1 + deinterlace) * GST_VIDEO_INFO_FPS_N (&in_info)) : 0; + + thiz->buffer_duration = GST_VIDEO_INFO_FPS_N (&out_info) > 0 ? + gst_util_uint64_scale (GST_SECOND, GST_VIDEO_INFO_FPS_D (&out_info), + GST_VIDEO_INFO_FPS_N (&out_info)) : 0; if (!gst_msdkvpp_initialize (thiz)) return FALSE; @@ -1009,6 +1063,9 @@ gst_msdkvpp_set_property (GObject * object, guint prop_id, case PROP_FORCE_ASPECT_RATIO: thiz->keep_aspect = g_value_get_boolean (value); break; + case PROP_FRC_ALGORITHM: + thiz->frc_algm = g_value_get_enum (value); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; @@ -1064,6 +1121,9 @@ gst_msdkvpp_get_property (GObject * object, guint prop_id, case PROP_FORCE_ASPECT_RATIO: g_value_set_boolean (value, thiz->keep_aspect); break; + case PROP_FRC_ALGORITHM: + g_value_set_enum (value, thiz->frc_algm); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); break; @@ -1209,6 +1269,12 @@ gst_msdkvpp_class_init (GstMsdkVPPClass * klass) PROP_FORCE_ASPECT_RATIO_DEFAULT, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS); + obj_properties[PROP_FRC_ALGORITHM] = + g_param_spec_enum ("frc-algorithm", "FrameRateControl Algorithm", + "The Framerate Control Alogorithm to use", + gst_msdkvpp_frc_algorithm_get_type (), PROP_FRC_ALGORITHM_DEFAULT, + G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS); + g_object_class_install_properties (gobject_class, PROP_N, obj_properties); } @@ -1221,7 +1287,7 @@ gst_msdkvpp_init (GstMsdkVPP * thiz) thiz->rotation = PROP_ROTATION_DEFAULT; thiz->deinterlace_mode = PROP_DEINTERLACE_MODE_DEFAULT; thiz->deinterlace_method = PROP_DEINTERLACE_METHOD_DEFAULT; - thiz->field_duration = GST_CLOCK_TIME_NONE; + thiz->buffer_duration = GST_CLOCK_TIME_NONE; thiz->hue = PROP_HUE_DEFAULT; thiz->saturation = PROP_SATURATION_DEFAULT; thiz->brightness = PROP_BRIGHTNESS_DEFAULT; @@ -1230,6 +1296,7 @@ gst_msdkvpp_init (GstMsdkVPP * thiz) thiz->mirroring = PROP_MIRRORING_DEFAULT; thiz->scaling_mode = PROP_SCALING_MODE_DEFAULT; thiz->keep_aspect = PROP_FORCE_ASPECT_RATIO_DEFAULT; + thiz->frc_algm = PROP_FRC_ALGORITHM_DEFAULT; gst_video_info_init (&thiz->sinkpad_info); gst_video_info_init (&thiz->srcpad_info); } diff --git a/sys/msdk/gstmsdkvpp.h b/sys/msdk/gstmsdkvpp.h index abf8abc979..57cf829044 100644 --- a/sys/msdk/gstmsdkvpp.h +++ b/sys/msdk/gstmsdkvpp.h @@ -68,6 +68,7 @@ typedef enum { GST_MSDK_FLAG_DETAIL = 1 << 7, GST_MSDK_FLAG_MIRRORING = 1 << 8, GST_MSDK_FLAG_SCALING_MODE = 1 << 9, + GST_MSDK_FLAG_FRC = 1 << 10, } GstMsdkVppFlags; struct _GstMsdkVPP @@ -114,8 +115,9 @@ struct _GstMsdkVPP guint mirroring; guint scaling_mode; gboolean keep_aspect; + guint frc_algm; - GstClockTime field_duration; + GstClockTime buffer_duration; /* MFX Filters */ mfxExtVPPDoUse mfx_vpp_douse; @@ -127,6 +129,7 @@ struct _GstMsdkVPP mfxExtVPPDetail mfx_detail; mfxExtVPPMirroring mfx_mirroring; mfxExtVPPScaling mfx_scaling; + mfxExtVPPFrameRateConversion mfx_frc; /* Extended buffers */ mfxExtBuffer *extra_params[MAX_EXTRA_PARAMS]; diff --git a/sys/msdk/gstmsdkvpputil.c b/sys/msdk/gstmsdkvpputil.c index 03506f992d..eef85ed179 100644 --- a/sys/msdk/gstmsdkvpputil.c +++ b/sys/msdk/gstmsdkvpputil.c @@ -429,10 +429,19 @@ overflow_error: static gboolean fixate_frame_rate (GstMsdkVPP * thiz, GstVideoInfo * vinfo, GstStructure * outs) { - gint fps_n, fps_d; + gint fps_n = 0, fps_d; + + /* fixate the srcpad fps */ + if (gst_structure_fixate_field (outs, "framerate")) + gst_structure_get (outs, "framerate", GST_TYPE_FRACTION, &fps_n, &fps_d, + NULL); + + /* if we don't have a fixed non-zero fps_n, use the sinkpad fps */ + if (!fps_n) { + fps_n = GST_VIDEO_INFO_FPS_N (vinfo); + fps_d = GST_VIDEO_INFO_FPS_D (vinfo); + } - fps_n = GST_VIDEO_INFO_FPS_N (vinfo); - fps_d = GST_VIDEO_INFO_FPS_D (vinfo); if (gst_msdkvpp_is_deinterlace_enabled (thiz, vinfo)) { /* Fixme: set double framerate?: * msdk is not outputting double framerate for bob or adv deinterlace */ diff --git a/sys/msdk/msdk-enums.c b/sys/msdk/msdk-enums.c index 2001b62908..03ce6375b9 100644 --- a/sys/msdk/msdk-enums.c +++ b/sys/msdk/msdk-enums.c @@ -278,3 +278,29 @@ gst_msdkvpp_scaling_mode_get_type (void) } return type; } + +GType +gst_msdkvpp_frc_algorithm_get_type (void) +{ + static GType type = 0; + + static const GEnumValue values[] = { + {_MFX_FRC_ALGORITHM_NONE, "No FrameRate Control algorithm", "none"}, + {MFX_FRCALGM_PRESERVE_TIMESTAMP, + "Frame dropping/repetition, Preserve timestamp", "preserve-ts"}, + {MFX_FRCALGM_DISTRIBUTED_TIMESTAMP, + "Frame dropping/repetition, Distribute timestamp", "distribute-ts"}, + {MFX_FRCALGM_FRAME_INTERPOLATION, "Frame interpolation", "interpolate"}, + {MFX_FRCALGM_FRAME_INTERPOLATION | MFX_FRCALGM_PRESERVE_TIMESTAMP, + "Frame interpolation, Preserve timestamp", "interpolate-preserve-ts"}, + {MFX_FRCALGM_FRAME_INTERPOLATION | MFX_FRCALGM_DISTRIBUTED_TIMESTAMP, + "Frame interpolation, Distribute timestamp", + "interpolate-distribute-ts"}, + {0, NULL, NULL} + }; + + if (!type) { + type = g_enum_register_static ("GstMsdkVPPFrcAlgorithm", values); + } + return type; +} diff --git a/sys/msdk/msdk-enums.h b/sys/msdk/msdk-enums.h index 75e97c32fa..444a3877ec 100644 --- a/sys/msdk/msdk-enums.h +++ b/sys/msdk/msdk-enums.h @@ -84,5 +84,9 @@ gst_msdkvpp_mirroring_get_type (void); GType gst_msdkvpp_scaling_mode_get_type (void); +#define _MFX_FRC_ALGORITHM_NONE 0 +GType +gst_msdkvpp_frc_algorithm_get_type (void); + G_END_DECLS #endif