/* GStreamer Intel MSDK plugin * Copyright (c) 2016, Oblong Industries, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * * 2. Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. * * 3. Neither the name of the copyright holder nor the names of its contributors * may be used to endorse or promote products derived from this software * without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ /* TODO: * - Add support for interlaced content * - Add support for MVC AVC * - Wrap more configuration options and maybe move properties to derived */ #ifdef HAVE_CONFIG_H # include #endif #ifdef _WIN32 # include #endif #include #include "gstmsdkenc.h" #include "gstmsdkbufferpool.h" #include "gstmsdkvideomemory.h" #include "gstmsdksystemmemory.h" #include "gstmsdkcontextutil.h" #include "mfxjpeg.h" #ifndef _WIN32 #include "gstmsdkallocator_libva.h" #endif static inline void * _aligned_alloc (size_t alignment, size_t size) { #ifdef _WIN32 return _aligned_malloc (size, alignment); #else void *out; if (posix_memalign (&out, alignment, size) != 0) out = NULL; return out; #endif } #ifndef _WIN32 #define _aligned_free free #endif static void gst_msdkenc_close_encoder (GstMsdkEnc * thiz); GST_DEBUG_CATEGORY_EXTERN (gst_msdkenc_debug); #define GST_CAT_DEFAULT gst_msdkenc_debug static GstStaticPadTemplate sink_factory = GST_STATIC_PAD_TEMPLATE ("sink", GST_PAD_SINK, GST_PAD_ALWAYS, GST_STATIC_CAPS (GST_MSDK_CAPS_STR ("{ NV12, I420, YV12, YUY2, UYVY, BGRA }", "NV12")) ); #define PROP_HARDWARE_DEFAULT TRUE #define PROP_ASYNC_DEPTH_DEFAULT 4 #define PROP_TARGET_USAGE_DEFAULT (MFX_TARGETUSAGE_BALANCED) #define PROP_RATE_CONTROL_DEFAULT (MFX_RATECONTROL_CBR) #define PROP_BITRATE_DEFAULT (2 * 1024) #define PROP_QPI_DEFAULT 0 #define PROP_QPP_DEFAULT 0 #define PROP_QPB_DEFAULT 0 #define PROP_GOP_SIZE_DEFAULT 256 #define PROP_REF_FRAMES_DEFAULT 1 #define PROP_I_FRAMES_DEFAULT 0 #define PROP_B_FRAMES_DEFAULT 0 #define PROP_NUM_SLICES_DEFAULT 0 #define PROP_AVBR_ACCURACY_DEFAULT 0 #define PROP_AVBR_CONVERGENCE_DEFAULT 0 #define PROP_RC_LOOKAHEAD_DEPTH_DEFAULT 10 #define PROP_MAX_VBV_BITRATE_DEFAULT 0 #define PROP_MAX_FRAME_SIZE_DEFAULT 0 #define PROP_MBBRC_DEFAULT MFX_CODINGOPTION_OFF #define PROP_ADAPTIVE_I_DEFAULT MFX_CODINGOPTION_OFF #define PROP_ADAPTIVE_B_DEFAULT MFX_CODINGOPTION_OFF /* External coding properties */ #define EC_PROPS_STRUCT_NAME "props" #define EC_PROPS_EXTBRC "extbrc" #define gst_msdkenc_parent_class parent_class G_DEFINE_TYPE (GstMsdkEnc, gst_msdkenc, GST_TYPE_VIDEO_ENCODER); typedef struct { mfxFrameSurface1 *surface; GstBuffer *buf; } MsdkSurface; void gst_msdkenc_add_extra_param (GstMsdkEnc * thiz, mfxExtBuffer * param) { if (thiz->num_extra_params < MAX_EXTRA_PARAMS) { thiz->extra_params[thiz->num_extra_params] = param; thiz->num_extra_params++; } } static void gst_msdkenc_set_context (GstElement * element, GstContext * context) { GstMsdkContext *msdk_context = NULL; GstMsdkEnc *thiz = GST_MSDKENC (element); if (gst_msdk_context_get_context (context, &msdk_context)) { gst_object_replace ((GstObject **) & thiz->context, (GstObject *) msdk_context); gst_object_unref (msdk_context); } GST_ELEMENT_CLASS (parent_class)->set_context (element, context); } static void ensure_bitrate_control (GstMsdkEnc * thiz) { mfxInfoMFX *mfx = &thiz->param.mfx; mfxExtCodingOption2 *option2 = &thiz->option2; mfxExtCodingOption3 *option3 = &thiz->option3; GST_DEBUG_OBJECT (thiz, "set target bitrate: %u kbit/sec", thiz->bitrate); mfx->RateControlMethod = thiz->rate_control; /* No effect in CQP variant algorithms */ if ((mfx->RateControlMethod != MFX_RATECONTROL_CQP) && (thiz->bitrate > G_MAXUINT16 || thiz->max_vbv_bitrate > G_MAXUINT16)) { mfxU32 max_val = MAX (thiz->max_vbv_bitrate, thiz->bitrate); mfx->BRCParamMultiplier = (mfxU16) ((max_val + 0x10000) / 0x10000); mfx->TargetKbps = (mfxU16) (thiz->bitrate / mfx->BRCParamMultiplier); mfx->MaxKbps = (mfxU16) (thiz->max_vbv_bitrate / mfx->BRCParamMultiplier); mfx->BufferSizeInKB = (mfxU16) (mfx->BufferSizeInKB / mfx->BRCParamMultiplier); /* Currently InitialDelayInKB is not used in this plugin */ mfx->InitialDelayInKB = (mfxU16) (mfx->InitialDelayInKB / mfx->BRCParamMultiplier); } else { mfx->TargetKbps = thiz->bitrate; mfx->MaxKbps = thiz->max_vbv_bitrate; mfx->BRCParamMultiplier = 1; } switch (mfx->RateControlMethod) { case MFX_RATECONTROL_CQP: mfx->QPI = thiz->qpi; mfx->QPP = thiz->qpp; mfx->QPB = thiz->qpb; break; case MFX_RATECONTROL_LA_ICQ: option2->LookAheadDepth = thiz->lookahead_depth; case MFX_RATECONTROL_ICQ: mfx->ICQQuality = CLAMP (thiz->qpi, 1, 51); break; case MFX_RATECONTROL_LA: /* VBR with LA. Only supported in H264?? */ case MFX_RATECONTROL_LA_HRD: /* VBR with LA, HRD compliant */ option2->LookAheadDepth = thiz->lookahead_depth; break; case MFX_RATECONTROL_QVBR: option3->QVBRQuality = CLAMP (thiz->qpi, 1, 51); thiz->enable_extopt3 = TRUE; break; case MFX_RATECONTROL_AVBR: mfx->Accuracy = thiz->accuracy; mfx->Convergence = thiz->convergence; break; case MFX_RATECONTROL_VBR: option2->MaxFrameSize = thiz->max_frame_size * 1000; break; case MFX_RATECONTROL_VCM: /*Non HRD compliant mode with no B-frame and interlaced support */ thiz->param.mfx.GopRefDist = 0; break; case MFX_RATECONTROL_CBR: break; default: GST_ERROR ("Unsupported RateControl!"); break; } } static gint16 coding_option_get_value (const gchar * key, const gchar * nickname) { if (!g_strcmp0 (nickname, "on")) { return MFX_CODINGOPTION_ON; } else if (!g_strcmp0 (nickname, "off")) { return MFX_CODINGOPTION_OFF; } else if (!g_strcmp0 (nickname, "auto")) { return MFX_CODINGOPTION_UNKNOWN; } GST_ERROR ("\"%s\" illegal option \"%s\", set to \"off\"", key, nickname); return MFX_CODINGOPTION_OFF; } static gboolean structure_transform (const GstStructure * src, GstStructure * dst) { guint len; GValue dst_value = G_VALUE_INIT; gboolean ret = TRUE; g_return_val_if_fail (src != NULL, FALSE); g_return_val_if_fail (dst != NULL, FALSE); len = gst_structure_n_fields (src); for (guint i = 0; i < len; i++) { const gchar *key = gst_structure_nth_field_name (src, i); const GValue *src_value = gst_structure_get_value (src, key); if (!gst_structure_has_field (dst, key)) { GST_ERROR ("structure \"%s\" does not support \"%s\"", gst_structure_get_name (dst), key); ret = FALSE; continue; } g_value_init (&dst_value, gst_structure_get_field_type (dst, key)); if (g_value_transform (src_value, &dst_value)) { gst_structure_set_value (dst, key, &dst_value); } else { GST_ERROR ("\"%s\" transform %s to %s failed", key, G_VALUE_TYPE_NAME (src_value), G_VALUE_TYPE_NAME (&dst_value)); ret = FALSE; } g_value_unset (&dst_value); } return ret; } /* Supported types: gchar*, gboolean, gint, guint, gfloat, gdouble */ static gboolean structure_get_value (const GstStructure * s, const gchar * key, gpointer value) { const GValue *gvalue = gst_structure_get_value (s, key); if (!gvalue) { GST_ERROR ("structure \"%s\" does not support \"%s\"", gst_structure_get_name (s), key); return FALSE; } switch (G_VALUE_TYPE (gvalue)) { case G_TYPE_STRING:{ const gchar **val = (const gchar **) value; *val = g_value_get_string (gvalue); break; } case G_TYPE_BOOLEAN:{ gboolean *val = (gboolean *) value; *val = g_value_get_boolean (gvalue); break; } case G_TYPE_INT:{ gint *val = (gint *) value; *val = g_value_get_int (gvalue); break; } case G_TYPE_UINT:{ guint *val = (guint *) value; *val = g_value_get_uint (gvalue); break; } case G_TYPE_FLOAT:{ gfloat *val = (gfloat *) value; *val = g_value_get_float (gvalue); break; } case G_TYPE_DOUBLE:{ gdouble *val = (gdouble *) value; *val = g_value_get_double (gvalue); break; } default: GST_ERROR ("\"%s\" unsupported type %s", key, G_VALUE_TYPE_NAME (gvalue)); return FALSE; } return TRUE; } static gboolean ext_coding_props_get_value (GstMsdkEnc * thiz, const gchar * key, gpointer value) { gboolean ret; if (!(ret = structure_get_value (thiz->ext_coding_props, key, value))) { GST_ERROR_OBJECT (thiz, "structure \"%s\" failed to get value for \"%s\"", gst_structure_get_name (thiz->ext_coding_props), key); } return ret; } void gst_msdkenc_ensure_extended_coding_options (GstMsdkEnc * thiz) { mfxExtCodingOption2 *option2 = &thiz->option2; mfxExtCodingOption3 *option3 = &thiz->option3; gchar *extbrc; ext_coding_props_get_value (thiz, EC_PROPS_EXTBRC, &extbrc); /* Fill ExtendedCodingOption2, set non-zero defaults too */ option2->Header.BufferId = MFX_EXTBUFF_CODING_OPTION2; option2->Header.BufferSz = sizeof (thiz->option2); option2->MBBRC = thiz->mbbrc; option2->ExtBRC = coding_option_get_value (EC_PROPS_EXTBRC, extbrc); option2->AdaptiveI = thiz->adaptive_i; option2->AdaptiveB = thiz->adaptive_b; option2->BitrateLimit = MFX_CODINGOPTION_OFF; option2->EnableMAD = MFX_CODINGOPTION_OFF; option2->UseRawRef = MFX_CODINGOPTION_OFF; gst_msdkenc_add_extra_param (thiz, (mfxExtBuffer *) option2); if (thiz->enable_extopt3) { option3->Header.BufferId = MFX_EXTBUFF_CODING_OPTION3; option3->Header.BufferSz = sizeof (thiz->option3); gst_msdkenc_add_extra_param (thiz, (mfxExtBuffer *) option3); } } /* Return TRUE if ROI is changed and update ROI parameters in encoder_roi */ gboolean gst_msdkenc_get_roi_params (GstMsdkEnc * thiz, GstVideoCodecFrame * frame, mfxExtEncoderROI * encoder_roi) { GstBuffer *input; guint num_roi, i, num_valid_roi = 0; gushort roi_mode = G_MAXUINT16; gpointer state = NULL; mfxExtEncoderROI *curr_roi = encoder_roi; mfxExtEncoderROI *prev_roi = encoder_roi + 1; if (!frame || !frame->input_buffer) return FALSE; memset (curr_roi, 0, sizeof (mfxExtEncoderROI)); input = frame->input_buffer; num_roi = gst_buffer_get_n_meta (input, GST_VIDEO_REGION_OF_INTEREST_META_API_TYPE); if (num_roi == 0) goto end; curr_roi->Header.BufferId = MFX_EXTBUFF_ENCODER_ROI; curr_roi->Header.BufferSz = sizeof (mfxExtEncoderROI); for (i = 0; i < num_roi && num_valid_roi < 256; i++) { GstVideoRegionOfInterestMeta *roi; GstStructure *s; roi = (GstVideoRegionOfInterestMeta *) gst_buffer_iterate_meta_filtered (input, &state, GST_VIDEO_REGION_OF_INTEREST_META_API_TYPE); if (!roi) continue; /* ignore roi if overflow */ if ((roi->x > G_MAXINT16) || (roi->y > G_MAXINT16) || (roi->w > G_MAXUINT16) || (roi->h > G_MAXUINT16)) { GST_DEBUG_OBJECT (thiz, "Ignoring ROI... ROI overflow"); continue; } GST_LOG ("Input buffer ROI: type=%s id=%d (%d, %d) %dx%d", g_quark_to_string (roi->roi_type), roi->id, roi->x, roi->y, roi->w, roi->h); curr_roi->ROI[num_valid_roi].Left = roi->x; curr_roi->ROI[num_valid_roi].Top = roi->y; curr_roi->ROI[num_valid_roi].Right = roi->x + roi->w; curr_roi->ROI[num_valid_roi].Bottom = roi->y + roi->h; s = gst_video_region_of_interest_meta_get_param (roi, "roi/msdk"); if (s) { int value = 0; if (roi_mode == G_MAXUINT16) { if (gst_structure_get_int (s, "delta-qp", &value)) { #if (MFX_VERSION >= 1022) roi_mode = MFX_ROI_MODE_QP_DELTA; curr_roi->ROI[num_valid_roi].DeltaQP = CLAMP (value, -51, 51); GST_LOG ("Use delta-qp %d", value); #else GST_WARNING ("Ignore delta QP because the MFX doesn't support delta QP mode"); #endif } else if (gst_structure_get_int (s, "priority", &value)) { roi_mode = MFX_ROI_MODE_PRIORITY; curr_roi->ROI[num_valid_roi].Priority = CLAMP (value, -3, 3); GST_LOG ("Use priority %d", value); } else continue; #if (MFX_VERSION >= 1022) } else if (roi_mode == MFX_ROI_MODE_QP_DELTA && gst_structure_get_int (s, "delta-qp", &value)) { curr_roi->ROI[num_valid_roi].DeltaQP = CLAMP (value, -51, 51); #endif } else if (roi_mode == MFX_ROI_MODE_PRIORITY && gst_structure_get_int (s, "priority", &value)) { curr_roi->ROI[num_valid_roi].Priority = CLAMP (value, -3, 3); } else continue; num_valid_roi++; } } #if (MFX_VERSION >= 1022) curr_roi->ROIMode = roi_mode; #endif curr_roi->NumROI = num_valid_roi; end: if (curr_roi->NumROI == 0 && prev_roi->NumROI == 0) return FALSE; if (curr_roi->NumROI != prev_roi->NumROI || memcmp (curr_roi, prev_roi, sizeof (mfxExtEncoderROI)) != 0) { *prev_roi = *curr_roi; return TRUE; } return FALSE; } static gboolean gst_msdkenc_init_encoder (GstMsdkEnc * thiz) { GstMsdkEncClass *klass = GST_MSDKENC_GET_CLASS (thiz); GstVideoInfo *info; mfxSession session; mfxStatus status; mfxFrameAllocRequest request[2]; guint i; gboolean need_vpp = TRUE; GstVideoFormat encoder_input_fmt; mfxExtVideoSignalInfo ext_vsi; if (thiz->initialized) { GST_DEBUG_OBJECT (thiz, "Already initialized"); return TRUE; } if (!thiz->context) { GST_WARNING_OBJECT (thiz, "No MSDK Context"); return FALSE; } if (!thiz->input_state) { GST_DEBUG_OBJECT (thiz, "Have no input state yet"); return FALSE; } info = &thiz->input_state->info; GST_OBJECT_LOCK (thiz); session = gst_msdk_context_get_session (thiz->context); thiz->codename = msdk_get_platform_codename (session); thiz->has_vpp = FALSE; if (thiz->use_video_memory) gst_msdk_set_frame_allocator (thiz->context); encoder_input_fmt = GST_VIDEO_INFO_FORMAT (info); need_vpp = klass->need_conversion (thiz, info, &encoder_input_fmt); if (need_vpp) { switch (GST_VIDEO_INFO_FORMAT (info)) { case GST_VIDEO_FORMAT_YV12: case GST_VIDEO_FORMAT_I420: thiz->vpp_param.vpp.In.FourCC = MFX_FOURCC_YV12; thiz->vpp_param.vpp.In.ChromaFormat = MFX_CHROMAFORMAT_YUV420; break; case GST_VIDEO_FORMAT_YUY2: thiz->vpp_param.vpp.In.FourCC = MFX_FOURCC_YUY2; thiz->vpp_param.vpp.In.ChromaFormat = MFX_CHROMAFORMAT_YUV422; break; case GST_VIDEO_FORMAT_UYVY: thiz->vpp_param.vpp.In.FourCC = MFX_FOURCC_UYVY; thiz->vpp_param.vpp.In.ChromaFormat = MFX_CHROMAFORMAT_YUV422; break; case GST_VIDEO_FORMAT_BGRA: thiz->vpp_param.vpp.In.FourCC = MFX_FOURCC_RGB4; thiz->vpp_param.vpp.In.ChromaFormat = MFX_CHROMAFORMAT_YUV444; break; default: g_assert_not_reached (); break; } if (thiz->use_video_memory) thiz->vpp_param.IOPattern = MFX_IOPATTERN_IN_VIDEO_MEMORY | MFX_IOPATTERN_OUT_VIDEO_MEMORY; else thiz->vpp_param.IOPattern = MFX_IOPATTERN_IN_SYSTEM_MEMORY | MFX_IOPATTERN_OUT_SYSTEM_MEMORY; thiz->vpp_param.vpp.In.Width = GST_ROUND_UP_16 (info->width); thiz->vpp_param.vpp.In.Height = GST_ROUND_UP_32 (info->height); thiz->vpp_param.vpp.In.CropW = info->width; thiz->vpp_param.vpp.In.CropH = info->height; thiz->vpp_param.vpp.In.FrameRateExtN = info->fps_n; thiz->vpp_param.vpp.In.FrameRateExtD = info->fps_d; thiz->vpp_param.vpp.In.AspectRatioW = info->par_n; thiz->vpp_param.vpp.In.AspectRatioH = info->par_d; thiz->vpp_param.vpp.In.PicStruct = MFX_PICSTRUCT_PROGRESSIVE; /* work-around to avoid zero fps in msdk structure */ if (0 == thiz->vpp_param.vpp.In.FrameRateExtN) thiz->vpp_param.vpp.In.FrameRateExtN = 30; thiz->vpp_param.vpp.Out = thiz->vpp_param.vpp.In; switch (encoder_input_fmt) { case GST_VIDEO_FORMAT_P010_10LE: thiz->vpp_param.vpp.Out.FourCC = MFX_FOURCC_P010; thiz->vpp_param.vpp.Out.ChromaFormat = MFX_CHROMAFORMAT_YUV420; break; case GST_VIDEO_FORMAT_YUY2: thiz->vpp_param.vpp.Out.FourCC = MFX_FOURCC_YUY2; thiz->vpp_param.vpp.Out.ChromaFormat = MFX_CHROMAFORMAT_YUV422; break; default: thiz->vpp_param.vpp.Out.FourCC = MFX_FOURCC_NV12; thiz->vpp_param.vpp.Out.ChromaFormat = MFX_CHROMAFORMAT_YUV420; break; } /* validate parameters and allow MFX to make adjustments */ status = MFXVideoVPP_Query (session, &thiz->vpp_param, &thiz->vpp_param); if (status < MFX_ERR_NONE) { GST_ERROR_OBJECT (thiz, "Video VPP Query failed (%s)", msdk_status_to_string (status)); goto failed; } else if (status > MFX_ERR_NONE) { GST_WARNING_OBJECT (thiz, "Video VPP Query returned: %s", msdk_status_to_string (status)); } status = MFXVideoVPP_QueryIOSurf (session, &thiz->vpp_param, request); if (status < MFX_ERR_NONE) { GST_ERROR_OBJECT (thiz, "VPP Query IO surfaces failed (%s)", msdk_status_to_string (status)); goto failed; } else if (status > MFX_ERR_NONE) { GST_WARNING_OBJECT (thiz, "VPP Query IO surfaces returned: %s", msdk_status_to_string (status)); } if (thiz->use_video_memory) request[0].NumFrameSuggested += gst_msdk_context_get_shared_async_depth (thiz->context); thiz->num_vpp_surfaces = request[0].NumFrameSuggested; if (thiz->use_video_memory) gst_msdk_frame_alloc (thiz->context, &(request[0]), &thiz->vpp_alloc_resp); status = MFXVideoVPP_Init (session, &thiz->vpp_param); if (status < MFX_ERR_NONE) { GST_ERROR_OBJECT (thiz, "Init failed (%s)", msdk_status_to_string (status)); goto no_vpp_free_resource; } else if (status > MFX_ERR_NONE) { GST_WARNING_OBJECT (thiz, "Init returned: %s", msdk_status_to_string (status)); } status = MFXVideoVPP_GetVideoParam (session, &thiz->vpp_param); if (status < MFX_ERR_NONE) { mfxStatus status1; GST_ERROR_OBJECT (thiz, "Get VPP Parameters failed (%s)", msdk_status_to_string (status)); status1 = MFXVideoVPP_Close (session); if (status1 != MFX_ERR_NONE && status1 != MFX_ERR_NOT_INITIALIZED) GST_WARNING_OBJECT (thiz, "VPP close failed (%s)", msdk_status_to_string (status1)); goto no_vpp_free_resource; } else if (status > MFX_ERR_NONE) { GST_WARNING_OBJECT (thiz, "Get VPP Parameters returned: %s", msdk_status_to_string (status)); } thiz->has_vpp = TRUE; } thiz->param.AsyncDepth = thiz->async_depth; if (thiz->use_video_memory) thiz->param.IOPattern = MFX_IOPATTERN_IN_VIDEO_MEMORY; else thiz->param.IOPattern = MFX_IOPATTERN_IN_SYSTEM_MEMORY; thiz->param.mfx.TargetUsage = thiz->target_usage; thiz->param.mfx.GopPicSize = thiz->gop_size; thiz->param.mfx.GopRefDist = thiz->b_frames + 1; thiz->param.mfx.IdrInterval = thiz->i_frames; thiz->param.mfx.NumSlice = thiz->num_slices; thiz->param.mfx.NumRefFrame = thiz->ref_frames; thiz->param.mfx.EncodedOrder = 0; /* Take input frames in display order */ thiz->param.mfx.FrameInfo.Width = GST_ROUND_UP_16 (info->width); thiz->param.mfx.FrameInfo.Height = GST_ROUND_UP_32 (info->height); thiz->param.mfx.FrameInfo.CropW = info->width; thiz->param.mfx.FrameInfo.CropH = info->height; thiz->param.mfx.FrameInfo.FrameRateExtN = info->fps_n; thiz->param.mfx.FrameInfo.FrameRateExtD = info->fps_d; thiz->param.mfx.FrameInfo.AspectRatioW = info->par_n; thiz->param.mfx.FrameInfo.AspectRatioH = info->par_d; thiz->param.mfx.FrameInfo.PicStruct = MFX_PICSTRUCT_PROGRESSIVE; thiz->param.mfx.FrameInfo.ChromaFormat = MFX_CHROMAFORMAT_YUV420; switch (encoder_input_fmt) { case GST_VIDEO_FORMAT_P010_10LE: thiz->param.mfx.FrameInfo.FourCC = MFX_FOURCC_P010; thiz->param.mfx.FrameInfo.BitDepthLuma = 10; thiz->param.mfx.FrameInfo.BitDepthChroma = 10; thiz->param.mfx.FrameInfo.Shift = 1; break; case GST_VIDEO_FORMAT_VUYA: thiz->param.mfx.FrameInfo.FourCC = MFX_FOURCC_AYUV; thiz->param.mfx.FrameInfo.ChromaFormat = MFX_CHROMAFORMAT_YUV444; thiz->param.mfx.FrameInfo.BitDepthLuma = 8; thiz->param.mfx.FrameInfo.BitDepthChroma = 8; break; #if (MFX_VERSION >= 1027) case GST_VIDEO_FORMAT_Y410: thiz->param.mfx.FrameInfo.FourCC = MFX_FOURCC_Y410; thiz->param.mfx.FrameInfo.ChromaFormat = MFX_CHROMAFORMAT_YUV444; thiz->param.mfx.FrameInfo.BitDepthLuma = 10; thiz->param.mfx.FrameInfo.BitDepthChroma = 10; break; case GST_VIDEO_FORMAT_Y210: thiz->param.mfx.FrameInfo.FourCC = MFX_FOURCC_Y210; thiz->param.mfx.FrameInfo.ChromaFormat = MFX_CHROMAFORMAT_YUV422; thiz->param.mfx.FrameInfo.BitDepthLuma = 10; thiz->param.mfx.FrameInfo.BitDepthChroma = 10; thiz->param.mfx.FrameInfo.Shift = 1; break; #endif case GST_VIDEO_FORMAT_BGRA: thiz->param.mfx.FrameInfo.FourCC = MFX_FOURCC_RGB4; thiz->param.mfx.FrameInfo.ChromaFormat = MFX_CHROMAFORMAT_YUV444; thiz->param.mfx.FrameInfo.BitDepthLuma = 8; thiz->param.mfx.FrameInfo.BitDepthChroma = 8; break; case GST_VIDEO_FORMAT_BGR10A2_LE: thiz->param.mfx.FrameInfo.FourCC = MFX_FOURCC_A2RGB10; thiz->param.mfx.FrameInfo.ChromaFormat = MFX_CHROMAFORMAT_YUV444; thiz->param.mfx.FrameInfo.BitDepthLuma = 10; thiz->param.mfx.FrameInfo.BitDepthChroma = 10; break; case GST_VIDEO_FORMAT_YUY2: thiz->param.mfx.FrameInfo.FourCC = MFX_FOURCC_YUY2; thiz->param.mfx.FrameInfo.ChromaFormat = MFX_CHROMAFORMAT_YUV422; thiz->param.mfx.FrameInfo.BitDepthLuma = 8; thiz->param.mfx.FrameInfo.BitDepthChroma = 8; break; #if (MFX_VERSION >= 1031) case GST_VIDEO_FORMAT_P012_LE: thiz->param.mfx.FrameInfo.FourCC = MFX_FOURCC_P016; thiz->param.mfx.FrameInfo.ChromaFormat = MFX_CHROMAFORMAT_YUV420; thiz->param.mfx.FrameInfo.BitDepthLuma = 12; thiz->param.mfx.FrameInfo.BitDepthChroma = 12; thiz->param.mfx.FrameInfo.Shift = 1; break; #endif default: thiz->param.mfx.FrameInfo.FourCC = MFX_FOURCC_NV12; thiz->param.mfx.FrameInfo.BitDepthLuma = 8; thiz->param.mfx.FrameInfo.BitDepthChroma = 8; } /* work-around to avoid zero fps in msdk structure */ if (0 == thiz->param.mfx.FrameInfo.FrameRateExtN) thiz->param.mfx.FrameInfo.FrameRateExtN = 30; /* ensure bitrate control parameters */ ensure_bitrate_control (thiz); /* allow subclass configure further */ if (klass->configure) { if (!klass->configure (thiz)) goto failed; } /* If color properties are available from upstream, set it and pass to MediaSDK here. * MJPEG and VP9 are excluded as MediaSDK does not support to handle video param * extbuff with buffer id equals to MFX_EXTBUFF_VIDEO_SIGNAL_INFO. */ if (thiz->param.mfx.CodecId != MFX_CODEC_JPEG && thiz->param.mfx.CodecId != MFX_CODEC_VP9 && (info->colorimetry.primaries || info->colorimetry.transfer || info->colorimetry.matrix)) { memset (&ext_vsi, 0, sizeof (ext_vsi)); ext_vsi.Header.BufferId = MFX_EXTBUFF_VIDEO_SIGNAL_INFO; ext_vsi.Header.BufferSz = sizeof (ext_vsi); ext_vsi.ColourDescriptionPresent = 1; ext_vsi.ColourPrimaries = gst_video_color_primaries_to_iso (info->colorimetry.primaries); ext_vsi.TransferCharacteristics = gst_video_transfer_function_to_iso (info->colorimetry.transfer); ext_vsi.MatrixCoefficients = gst_video_color_matrix_to_iso (info->colorimetry.matrix); gst_msdkenc_add_extra_param (thiz, (mfxExtBuffer *) & ext_vsi); } if (thiz->num_extra_params) { thiz->param.NumExtParam = thiz->num_extra_params; thiz->param.ExtParam = thiz->extra_params; } /* validate parameters and allow MFX to make adjustments */ status = MFXVideoENCODE_Query (session, &thiz->param, &thiz->param); if (status < MFX_ERR_NONE) { GST_ERROR_OBJECT (thiz, "Video Encode Query failed (%s)", msdk_status_to_string (status)); goto failed; } else if (status > MFX_ERR_NONE) { GST_WARNING_OBJECT (thiz, "Video Encode Query returned: %s", msdk_status_to_string (status)); } status = MFXVideoENCODE_QueryIOSurf (session, &thiz->param, request); if (status < MFX_ERR_NONE) { GST_ERROR_OBJECT (thiz, "Encode Query IO surfaces failed (%s)", msdk_status_to_string (status)); goto failed; } else if (status > MFX_ERR_NONE) { GST_WARNING_OBJECT (thiz, "Encode Query IO surfaces returned: %s", msdk_status_to_string (status)); } request[0].NumFrameSuggested += thiz->num_extra_frames; if (thiz->has_vpp) request[0].NumFrameSuggested += thiz->num_vpp_surfaces + 1 - 4; if (thiz->use_video_memory) { if (thiz->use_dmabuf && !thiz->has_vpp) request[0].Type |= MFX_MEMTYPE_EXPORT_FRAME; gst_msdk_frame_alloc (thiz->context, &(request[0]), &thiz->alloc_resp); } /* Maximum of VPP output and encoder input, if using VPP */ if (thiz->has_vpp) request[0].NumFrameSuggested = MAX (request[0].NumFrameSuggested, request[1].NumFrameSuggested); if (request[0].NumFrameSuggested < thiz->param.AsyncDepth) { GST_ERROR_OBJECT (thiz, "Required %d surfaces (%d suggested), async %d", request[0].NumFrameMin, request[0].NumFrameSuggested, thiz->param.AsyncDepth); goto failed; } /* This is VPP output (if any) and encoder input */ thiz->num_surfaces = request[0].NumFrameSuggested; GST_DEBUG_OBJECT (thiz, "Required %d surfaces (%d suggested), allocated %d", request[0].NumFrameMin, request[0].NumFrameSuggested, thiz->num_surfaces); status = MFXVideoENCODE_Init (session, &thiz->param); if (status < MFX_ERR_NONE) { GST_ERROR_OBJECT (thiz, "Init failed (%s)", msdk_status_to_string (status)); goto failed; } else if (status > MFX_ERR_NONE) { GST_WARNING_OBJECT (thiz, "Init returned: %s", msdk_status_to_string (status)); } status = MFXVideoENCODE_GetVideoParam (session, &thiz->param); if (status < MFX_ERR_NONE) { GST_ERROR_OBJECT (thiz, "Get Video Parameters failed (%s)", msdk_status_to_string (status)); goto failed; } else if (status > MFX_ERR_NONE) { GST_WARNING_OBJECT (thiz, "Get Video Parameters returned: %s", msdk_status_to_string (status)); } thiz->num_tasks = thiz->param.AsyncDepth; thiz->tasks = g_new0 (MsdkEncTask, thiz->num_tasks); for (i = 0; i < thiz->num_tasks; i++) { thiz->tasks[i].output_bitstream.Data = _aligned_alloc (32, thiz->param.mfx.BufferSizeInKB * thiz->param.mfx.BRCParamMultiplier * 1024); if (!thiz->tasks[i].output_bitstream.Data) { GST_ERROR_OBJECT (thiz, "Memory allocation failed"); goto failed; } thiz->tasks[i].output_bitstream.MaxLength = thiz->param.mfx.BufferSizeInKB * thiz->param.mfx.BRCParamMultiplier * 1024; } thiz->next_task = 0; thiz->reconfig = FALSE; thiz->initialized = TRUE; GST_OBJECT_UNLOCK (thiz); return TRUE; no_vpp_free_resource: if (thiz->use_video_memory) gst_msdk_frame_free (thiz->context, &thiz->vpp_alloc_resp); failed: GST_OBJECT_UNLOCK (thiz); return FALSE; } static void gst_msdkenc_close_encoder (GstMsdkEnc * thiz) { guint i; mfxStatus status; if (!thiz->context || !thiz->initialized) return; GST_DEBUG_OBJECT (thiz, "Closing encoder with context %" GST_PTR_FORMAT, thiz->context); gst_clear_object (&thiz->msdk_pool); gst_clear_object (&thiz->msdk_converted_pool); if (thiz->use_video_memory) gst_msdk_frame_free (thiz->context, &thiz->alloc_resp); status = MFXVideoENCODE_Close (gst_msdk_context_get_session (thiz->context)); if (status != MFX_ERR_NONE && status != MFX_ERR_NOT_INITIALIZED) { GST_WARNING_OBJECT (thiz, "Encoder close failed (%s)", msdk_status_to_string (status)); } if (thiz->tasks) { for (i = 0; i < thiz->num_tasks; i++) { MsdkEncTask *task = &thiz->tasks[i]; if (task->output_bitstream.Data) { _aligned_free (task->output_bitstream.Data); } } } g_free (thiz->tasks); thiz->tasks = NULL; /* Close VPP before freeing the surfaces. They are shared between encoder * and VPP */ if (thiz->has_vpp) { if (thiz->use_video_memory) gst_msdk_frame_free (thiz->context, &thiz->vpp_alloc_resp); status = MFXVideoVPP_Close (gst_msdk_context_get_session (thiz->context)); if (status != MFX_ERR_NONE && status != MFX_ERR_NOT_INITIALIZED) { GST_WARNING_OBJECT (thiz, "VPP close failed (%s)", msdk_status_to_string (status)); } } memset (&thiz->param, 0, sizeof (thiz->param)); thiz->num_extra_params = 0; thiz->initialized = FALSE; } typedef struct { GstVideoCodecFrame *frame; MsdkSurface *frame_surface; MsdkSurface *converted_surface; } FrameData; static FrameData * gst_msdkenc_queue_frame (GstMsdkEnc * thiz, GstVideoCodecFrame * frame, GstVideoInfo * info) { FrameData *fdata; fdata = g_slice_new (FrameData); fdata->frame = gst_video_codec_frame_ref (frame); thiz->pending_frames = g_list_prepend (thiz->pending_frames, fdata); return fdata; } static MsdkSurface * gst_msdkenc_create_surface (mfxFrameSurface1 * surface, GstBuffer * buf) { MsdkSurface *msdk_surface; msdk_surface = g_slice_new0 (MsdkSurface); msdk_surface->surface = surface; msdk_surface->buf = buf; return msdk_surface; } static void gst_msdkenc_free_surface (MsdkSurface * surface) { if (surface->buf) gst_buffer_unref (surface->buf); g_slice_free (MsdkSurface, surface); } static void gst_msdkenc_free_frame_data (GstMsdkEnc * thiz, FrameData * fdata) { if (fdata->frame_surface) gst_msdkenc_free_surface (fdata->frame_surface); if (thiz->has_vpp) gst_msdkenc_free_surface (fdata->converted_surface); gst_video_codec_frame_unref (fdata->frame); g_slice_free (FrameData, fdata); } static void gst_msdkenc_dequeue_frame (GstMsdkEnc * thiz, GstVideoCodecFrame * frame) { GList *l; for (l = thiz->pending_frames; l;) { FrameData *fdata = l->data; GList *l1 = l; l = l->next; if (fdata->frame != frame) continue; gst_msdkenc_free_frame_data (thiz, fdata); thiz->pending_frames = g_list_delete_link (thiz->pending_frames, l1); return; } } static void gst_msdkenc_dequeue_all_frames (GstMsdkEnc * thiz) { GList *l; for (l = thiz->pending_frames; l; l = l->next) { FrameData *fdata = l->data; gst_msdkenc_free_frame_data (thiz, fdata); } g_list_free (thiz->pending_frames); thiz->pending_frames = NULL; } static MsdkEncTask * gst_msdkenc_get_free_task (GstMsdkEnc * thiz) { MsdkEncTask *tasks = thiz->tasks; guint size = thiz->num_tasks; guint start = thiz->next_task; guint i; if (tasks) { for (i = 0; i < size; i++) { guint t = (start + i) % size; if (tasks[t].sync_point == NULL) return &tasks[t]; } } return NULL; } static void gst_msdkenc_reset_task (MsdkEncTask * task) { task->output_bitstream.DataLength = 0; task->sync_point = NULL; } static GstVideoCodecFrame * gst_msdkenc_find_best_frame (GstMsdkEnc * thiz, GList * frames, mfxBitstream * bitstream) { GList *iter; GstVideoCodecFrame *ret = NULL; GstClockTime pts; GstClockTimeDiff best_diff = GST_CLOCK_STIME_NONE; if (!bitstream) return NULL; if (bitstream->TimeStamp == MFX_TIMESTAMP_UNKNOWN) { pts = GST_CLOCK_TIME_NONE; } else { pts = gst_util_uint64_scale (bitstream->TimeStamp, GST_SECOND, 90000); } for (iter = frames; iter; iter = g_list_next (iter)) { GstVideoCodecFrame *frame = (GstVideoCodecFrame *) iter->data; /* if we don't know the time stamp, find the first frame which * has unknown timestamp */ if (!GST_CLOCK_TIME_IS_VALID (pts)) { if (!GST_CLOCK_TIME_IS_VALID (frame->pts)) { ret = frame; break; } } else { GstClockTimeDiff abs_diff = ABS (GST_CLOCK_DIFF (frame->pts, pts)); if (abs_diff == 0) { ret = frame; break; } if (!GST_CLOCK_STIME_IS_VALID (best_diff) || abs_diff < best_diff) { ret = frame; best_diff = abs_diff; } } } if (ret) gst_video_codec_frame_ref (ret); return ret; } static GstFlowReturn gst_msdkenc_finish_frame (GstMsdkEnc * thiz, MsdkEncTask * task, gboolean discard) { GstVideoCodecFrame *frame; GList *list; if (!task->sync_point) return GST_FLOW_OK; list = gst_video_encoder_get_frames (GST_VIDEO_ENCODER (thiz)); if (!list) { GST_ERROR_OBJECT (thiz, "failed to get list of frame"); return GST_FLOW_ERROR; } /* Wait for encoding operation to complete, the magic number 300000 below * is used in MSDK samples * #define MSDK_ENC_WAIT_INTERVAL 300000 */ if (MFXVideoCORE_SyncOperation (gst_msdk_context_get_session (thiz->context), task->sync_point, 300000) != MFX_ERR_NONE) GST_WARNING_OBJECT (thiz, "failed to do sync operation"); if (!discard && task->output_bitstream.DataLength) { GstBuffer *out_buf = NULL; guint8 *data = task->output_bitstream.Data + task->output_bitstream.DataOffset; gsize size = task->output_bitstream.DataLength; frame = gst_msdkenc_find_best_frame (thiz, list, &task->output_bitstream); if (!frame) { /* just pick the oldest one */ frame = gst_video_encoder_get_oldest_frame (GST_VIDEO_ENCODER (thiz)); } out_buf = gst_buffer_new_allocate (NULL, size, NULL); gst_buffer_fill (out_buf, 0, data, size); frame->output_buffer = out_buf; frame->pts = gst_util_uint64_scale (task->output_bitstream.TimeStamp, GST_SECOND, 90000); frame->dts = gst_util_uint64_scale (task->output_bitstream.DecodeTimeStamp, GST_SECOND, 90000); if ((task->output_bitstream.FrameType & MFX_FRAMETYPE_IDR) != 0 || (task->output_bitstream.FrameType & MFX_FRAMETYPE_xIDR) != 0) { GST_VIDEO_CODEC_FRAME_SET_SYNC_POINT (frame); } /* Mark task as available */ gst_msdkenc_reset_task (task); } else { frame = gst_video_encoder_get_oldest_frame (GST_VIDEO_ENCODER (thiz)); } g_list_free_full (list, (GDestroyNotify) gst_video_codec_frame_unref); gst_video_codec_frame_unref (frame); gst_msdkenc_dequeue_frame (thiz, frame); return gst_video_encoder_finish_frame (GST_VIDEO_ENCODER (thiz), frame); } static GstFlowReturn gst_msdkenc_encode_frame (GstMsdkEnc * thiz, mfxFrameSurface1 * surface, GstVideoCodecFrame * input_frame) { mfxSession session; MsdkEncTask *task; mfxStatus status; if (G_UNLIKELY (thiz->context == NULL)) { gst_msdkenc_dequeue_frame (thiz, input_frame); gst_video_encoder_finish_frame (GST_VIDEO_ENCODER (thiz), input_frame); return GST_FLOW_NOT_NEGOTIATED; } session = gst_msdk_context_get_session (thiz->context); task = gst_msdkenc_get_free_task (thiz); for (;;) { /* Force key-frame if needed */ if (GST_VIDEO_CODEC_FRAME_IS_FORCE_KEYFRAME (input_frame)) thiz->enc_cntrl.FrameType = MFX_FRAMETYPE_I | MFX_FRAMETYPE_IDR | MFX_FRAMETYPE_REF; else thiz->enc_cntrl.FrameType = MFX_FRAMETYPE_UNKNOWN; status = MFXVideoENCODE_EncodeFrameAsync (session, &thiz->enc_cntrl, surface, &task->output_bitstream, &task->sync_point); if (status != MFX_WRN_DEVICE_BUSY) break; /* If device is busy, wait 1ms and retry, as per MSDK's recomendation */ g_usleep (1000); }; if (status != MFX_ERR_NONE && status != MFX_ERR_MORE_DATA) { GST_ELEMENT_ERROR (thiz, STREAM, ENCODE, ("Encode frame failed."), ("MSDK encode error (%s)", msdk_status_to_string (status))); gst_msdkenc_dequeue_frame (thiz, input_frame); gst_video_encoder_finish_frame (GST_VIDEO_ENCODER (thiz), input_frame); return GST_FLOW_ERROR; } if (task->sync_point) { thiz->next_task = ((task - thiz->tasks) + 1) % thiz->num_tasks; } else if (status == MFX_ERR_MORE_DATA) { gst_msdkenc_dequeue_frame (thiz, input_frame); } /* Ensure that next task is available */ task = thiz->tasks + thiz->next_task; return gst_msdkenc_finish_frame (thiz, task, FALSE); } static guint gst_msdkenc_maximum_delayed_frames (GstMsdkEnc * thiz) { return thiz->num_tasks; } static void gst_msdkenc_set_latency (GstMsdkEnc * thiz) { GstVideoInfo *info = &thiz->input_state->info; gint max_delayed_frames; GstClockTime latency; max_delayed_frames = gst_msdkenc_maximum_delayed_frames (thiz); if (info->fps_n) { latency = gst_util_uint64_scale_ceil (GST_SECOND * info->fps_d, max_delayed_frames, info->fps_n); } else { /* FIXME: Assume 25fps. This is better than reporting no latency at * all and then later failing in live pipelines */ latency = gst_util_uint64_scale_ceil (GST_SECOND * 1, max_delayed_frames, 25); } GST_INFO_OBJECT (thiz, "Updating latency to %" GST_TIME_FORMAT " (%d frames)", GST_TIME_ARGS (latency), max_delayed_frames); gst_video_encoder_set_latency (GST_VIDEO_ENCODER (thiz), latency, latency); } static void gst_msdkenc_flush_frames (GstMsdkEnc * thiz, gboolean discard) { mfxStatus status; mfxSession session; MsdkEncTask *task; guint i, t; if (!thiz->tasks) return; GST_DEBUG_OBJECT (thiz, "flush frames"); session = gst_msdk_context_get_session (thiz->context); for (;;) { task = thiz->tasks + thiz->next_task; gst_msdkenc_finish_frame (thiz, task, FALSE); status = MFXVideoENCODE_EncodeFrameAsync (session, NULL, NULL, &task->output_bitstream, &task->sync_point); if (status != MFX_ERR_NONE && status != MFX_ERR_MORE_DATA) { GST_ELEMENT_ERROR (thiz, STREAM, ENCODE, ("Encode frame failed."), ("MSDK encode error (%s)", msdk_status_to_string (status))); break; } if (task->sync_point) { thiz->next_task = ((task - thiz->tasks) + 1) % thiz->num_tasks; } else if (status == MFX_ERR_MORE_DATA) { break; } }; t = thiz->next_task; for (i = 0; i < thiz->num_tasks; i++) { gst_msdkenc_finish_frame (thiz, &thiz->tasks[t], discard); t = (t + 1) % thiz->num_tasks; } } static gboolean gst_msdkenc_set_src_caps (GstMsdkEnc * thiz) { GstMsdkEncClass *klass = GST_MSDKENC_GET_CLASS (thiz); GstCaps *outcaps = NULL; GstVideoCodecState *state; GstTagList *tags; if (klass->set_src_caps) outcaps = klass->set_src_caps (thiz); if (!outcaps) return FALSE; state = gst_video_encoder_set_output_state (GST_VIDEO_ENCODER (thiz), outcaps, thiz->input_state); GST_DEBUG_OBJECT (thiz, "output caps: %" GST_PTR_FORMAT, state->caps); gst_video_codec_state_unref (state); tags = gst_tag_list_new_empty (); gst_tag_list_add (tags, GST_TAG_MERGE_REPLACE, GST_TAG_ENCODER, "msdkenc", GST_TAG_MAXIMUM_BITRATE, thiz->bitrate * 1024, GST_TAG_NOMINAL_BITRATE, thiz->bitrate * 1024, NULL); gst_video_encoder_merge_tags (GST_VIDEO_ENCODER (thiz), tags, GST_TAG_MERGE_REPLACE); gst_tag_list_unref (tags); return TRUE; } static GstBufferPool * gst_msdkenc_create_buffer_pool (GstMsdkEnc * thiz, GstCaps * caps, guint num_buffers, gboolean set_align) { GstBufferPool *pool = NULL; GstStructure *config; GstAllocator *allocator = NULL; GstVideoInfo info; GstVideoAlignment align; GstAllocationParams params = { 0, 31, 0, 0, }; mfxFrameAllocResponse *alloc_resp = NULL; if (thiz->has_vpp) alloc_resp = set_align ? &thiz->vpp_alloc_resp : &thiz->alloc_resp; else alloc_resp = &thiz->alloc_resp; pool = gst_msdk_buffer_pool_new (thiz->context, alloc_resp); if (!pool) goto error_no_pool; if (!gst_video_info_from_caps (&info, caps)) { GST_INFO_OBJECT (thiz, "failed to get video info"); return NULL; } gst_msdk_set_video_alignment (&info, 0, 0, &align); gst_video_info_align (&info, &align); if (thiz->use_dmabuf) allocator = gst_msdk_dmabuf_allocator_new (thiz->context, &info, alloc_resp); else if (thiz->use_video_memory) allocator = gst_msdk_video_allocator_new (thiz->context, &info, alloc_resp); else allocator = gst_msdk_system_allocator_new (&info); if (!allocator) goto error_no_allocator; config = gst_buffer_pool_get_config (GST_BUFFER_POOL_CAST (pool)); gst_buffer_pool_config_set_params (config, caps, info.size, num_buffers, 0); gst_buffer_pool_config_add_option (config, GST_BUFFER_POOL_OPTION_VIDEO_META); gst_buffer_pool_config_add_option (config, GST_BUFFER_POOL_OPTION_VIDEO_ALIGNMENT); if (thiz->use_video_memory) { gst_buffer_pool_config_add_option (config, GST_BUFFER_POOL_OPTION_MSDK_USE_VIDEO_MEMORY); if (thiz->use_dmabuf) gst_buffer_pool_config_add_option (config, GST_BUFFER_POOL_OPTION_MSDK_USE_DMABUF); } gst_buffer_pool_config_set_video_alignment (config, &align); gst_buffer_pool_config_set_allocator (config, allocator, ¶ms); gst_object_unref (allocator); if (!gst_buffer_pool_set_config (pool, config)) goto error_pool_config; if (set_align) thiz->aligned_info = info; return pool; error_no_pool: { GST_INFO_OBJECT (thiz, "failed to create bufferpool"); return NULL; } error_no_allocator: { GST_INFO_OBJECT (thiz, "failed to create allocator"); gst_object_unref (pool); return NULL; } error_pool_config: { GST_INFO_OBJECT (thiz, "failed to set config"); gst_object_unref (pool); gst_object_unref (allocator); return NULL; } } /* Fixme: Common routine used by all msdk elements, should be * moved to a common util file */ static gboolean _gst_caps_has_feature (const GstCaps * caps, const gchar * feature) { guint i; for (i = 0; i < gst_caps_get_size (caps); i++) { GstCapsFeatures *const features = gst_caps_get_features (caps, i); /* Skip ANY features, we need an exact match for correct evaluation */ if (gst_caps_features_is_any (features)) continue; if (gst_caps_features_contains (features, feature)) return TRUE; } return FALSE; } static gboolean sinkpad_can_dmabuf (GstMsdkEnc * thiz) { gboolean ret = FALSE; GstCaps *caps, *allowed_caps; GstPad *sinkpad; sinkpad = GST_VIDEO_ENCODER_SINK_PAD (thiz); caps = gst_pad_get_pad_template_caps (sinkpad); allowed_caps = gst_pad_peer_query_caps (sinkpad, caps); if (!allowed_caps) goto done; if (gst_caps_is_any (allowed_caps) || gst_caps_is_empty (allowed_caps) || allowed_caps == caps) goto done; if (_gst_caps_has_feature (allowed_caps, GST_CAPS_FEATURE_MEMORY_DMABUF)) ret = TRUE; done: if (caps) gst_caps_unref (caps); if (allowed_caps) gst_caps_unref (allowed_caps); return ret; } static gboolean gst_msdkenc_set_format (GstVideoEncoder * encoder, GstVideoCodecState * state) { GstMsdkEnc *thiz = GST_MSDKENC (encoder); GstMsdkEncClass *klass = GST_MSDKENC_GET_CLASS (thiz); if (state) { if (thiz->input_state) { if (!gst_video_info_is_equal (&thiz->input_state->info, &state->info)) { GST_INFO_OBJECT (thiz, "Re-init the encoder as info changed"); gst_msdkenc_flush_frames (thiz, FALSE); gst_msdkenc_close_encoder (thiz); } gst_video_codec_state_unref (thiz->input_state); } thiz->input_state = gst_video_codec_state_ref (state); } /* TODO: Currently d3d allocator is not implemented. * So encoder uses system memory by default on Windows. */ #ifndef _WIN32 thiz->use_video_memory = TRUE; #else thiz->use_video_memory = FALSE; #endif GST_INFO_OBJECT (encoder, "This MSDK encoder uses %s memory", thiz->use_video_memory ? "video" : "system"); if (klass->set_format) { if (!klass->set_format (thiz)) return FALSE; } /* If upstream supports DMABufCapsfeatures, then we request for the dmabuf * based pipeline usage. Ideally we should have dmabuf support even with * raw-caps negotiation, but we don't have dmabuf-import support in msdk * plugin yet */ if (sinkpad_can_dmabuf (thiz)) { thiz->input_state->caps = gst_caps_make_writable (thiz->input_state->caps); gst_caps_set_features (thiz->input_state->caps, 0, gst_caps_features_new (GST_CAPS_FEATURE_MEMORY_DMABUF, NULL)); thiz->use_dmabuf = TRUE; } if (!gst_msdkenc_init_encoder (thiz)) return FALSE; if (!gst_msdkenc_set_src_caps (thiz)) { gst_msdkenc_close_encoder (thiz); return FALSE; } if (!thiz->msdk_pool) { guint num_buffers = gst_msdkenc_maximum_delayed_frames (thiz) + 1; thiz->msdk_pool = gst_msdkenc_create_buffer_pool (thiz, thiz->input_state->caps, num_buffers, TRUE); } gst_msdkenc_set_latency (thiz); /* Create another bufferpool if VPP requires */ if (thiz->has_vpp) { GstVideoInfo *info = &thiz->input_state->info; GstVideoInfo out_info; GstVideoFormat out_fmt; GstCaps *caps; GstBufferPool *pool = NULL; gst_video_info_init (&out_info); out_fmt = gst_msdk_get_video_format_from_mfx_fourcc (thiz->vpp_param.vpp. Out.FourCC); gst_video_info_set_format (&out_info, out_fmt, info->width, info->height); caps = gst_video_info_to_caps (&out_info); /* If there's an existing pool try to reuse it when is compatible */ if (thiz->msdk_converted_pool) { GstStructure *config; GstCaps *pool_caps; gboolean is_pool_compatible = FALSE; config = gst_buffer_pool_get_config (thiz->msdk_converted_pool); gst_buffer_pool_config_get_params (config, &pool_caps, NULL, NULL, NULL); if (caps && pool_caps) is_pool_compatible = gst_caps_is_equal (caps, pool_caps); gst_structure_free (config); /* If caps are the same then we are done */ if (is_pool_compatible) { gst_caps_unref (caps); goto done; } /* Release current pool because we are going to create a new one */ gst_clear_object (&thiz->msdk_converted_pool); } /* Otherwise create a new pool */ pool = gst_msdkenc_create_buffer_pool (thiz, caps, thiz->num_surfaces, FALSE); thiz->msdk_converted_pool = pool; gst_caps_unref (caps); } done: return TRUE; } static MsdkSurface * gst_msdkenc_get_surface_from_pool (GstMsdkEnc * thiz, GstBufferPool * pool, GstBufferPoolAcquireParams * params) { GstBuffer *new_buffer; mfxFrameSurface1 *new_surface; MsdkSurface *msdk_surface; if (!gst_buffer_pool_is_active (pool) && !gst_buffer_pool_set_active (pool, TRUE)) { GST_ERROR_OBJECT (pool, "failed to activate buffer pool"); return NULL; } if (gst_buffer_pool_acquire_buffer (pool, &new_buffer, params) != GST_FLOW_OK) { GST_ERROR_OBJECT (pool, "failed to acquire a buffer from pool"); return NULL; } if (gst_msdk_is_msdk_buffer (new_buffer)) new_surface = gst_msdk_get_surface_from_buffer (new_buffer); else { GST_ERROR_OBJECT (pool, "the acquired memory is not MSDK memory"); return NULL; } msdk_surface = gst_msdkenc_create_surface (new_surface, new_buffer); return msdk_surface; } #ifndef _WIN32 static gboolean import_dmabuf_to_msdk_surface (GstMsdkEnc * thiz, GstBuffer * buf, MsdkSurface * msdk_surface) { GstMemory *mem = NULL; GstVideoInfo vinfo; GstVideoMeta *vmeta; GstMsdkMemoryID *msdk_mid = NULL; mfxFrameSurface1 *mfx_surface = NULL; gint fd, i; mem = gst_buffer_peek_memory (buf, 0); fd = gst_dmabuf_memory_get_fd (mem); if (fd < 0) return FALSE; vinfo = thiz->input_state->info; /* Update offset/stride/size if there is VideoMeta attached to * the buffer */ vmeta = gst_buffer_get_video_meta (buf); if (vmeta) { if (GST_VIDEO_INFO_FORMAT (&vinfo) != vmeta->format || GST_VIDEO_INFO_WIDTH (&vinfo) != vmeta->width || GST_VIDEO_INFO_HEIGHT (&vinfo) != vmeta->height || GST_VIDEO_INFO_N_PLANES (&vinfo) != vmeta->n_planes) { GST_ERROR_OBJECT (thiz, "VideoMeta attached to buffer is not matching" "the negotiated width/height/format"); return FALSE; } for (i = 0; i < GST_VIDEO_INFO_N_PLANES (&vinfo); ++i) { GST_VIDEO_INFO_PLANE_OFFSET (&vinfo, i) = vmeta->offset[i]; GST_VIDEO_INFO_PLANE_STRIDE (&vinfo, i) = vmeta->stride[i]; } GST_VIDEO_INFO_SIZE (&vinfo) = gst_buffer_get_size (buf); } /* Upstream neither accepted the msdk pool nor the msdk buffer size restrictions. * Current media-driver and GMMLib will fail due to strict memory size restrictions. * Ideally, media-driver should accept what ever memory coming from other drivers * in case of dmabuf-import and this is how the intel-vaapi-driver works. * For now, in order to avoid any crash we check the buffer size and fallback * to copy frame method. * * See this: https://github.com/intel/media-driver/issues/169 * */ if (GST_VIDEO_INFO_SIZE (&vinfo) < GST_VIDEO_INFO_SIZE (&thiz->aligned_info)) return FALSE; mfx_surface = msdk_surface->surface; msdk_mid = (GstMsdkMemoryID *) mfx_surface->Data.MemId; /* release the internal memory storage of associated mfxSurface */ gst_msdk_replace_mfx_memid (thiz->context, mfx_surface, VA_INVALID_ID); /* export dmabuf to vasurface */ if (!gst_msdk_export_dmabuf_to_vasurface (thiz->context, &vinfo, fd, msdk_mid->surface)) return FALSE; return TRUE; } #endif static MsdkSurface * gst_msdkenc_get_surface_from_frame (GstMsdkEnc * thiz, GstVideoCodecFrame * frame) { GstVideoFrame src_frame, out_frame; MsdkSurface *msdk_surface; GstBuffer *inbuf; #ifndef _WIN32 GstMemory *mem = NULL; #endif inbuf = frame->input_buffer; if (gst_msdk_is_msdk_buffer (inbuf)) { msdk_surface = g_slice_new0 (MsdkSurface); msdk_surface->surface = gst_msdk_get_surface_from_buffer (inbuf); return msdk_surface; } /* If upstream hasn't accpeted the proposed msdk bufferpool, * just copy frame (if not dmabuf backed )to msdk buffer and take a surface from it. */ if (!(msdk_surface = gst_msdkenc_get_surface_from_pool (thiz, thiz->msdk_pool, NULL))) goto error; #ifndef _WIN32 /************ dmabuf-import ************* */ /* if upstream provided a dmabuf backed memory, but not an msdk * buffer, we could try to export the dmabuf to underlined vasurface */ mem = gst_buffer_peek_memory (inbuf, 0); if (gst_is_dmabuf_memory (mem)) { if (import_dmabuf_to_msdk_surface (thiz, inbuf, msdk_surface)) return msdk_surface; else GST_INFO_OBJECT (thiz, "Upstream dmabuf-backed memory is not imported" "to the msdk surface, fall back to the copy input frame method"); } #endif if (!gst_video_frame_map (&src_frame, &thiz->input_state->info, inbuf, GST_MAP_READ)) { GST_ERROR_OBJECT (thiz, "failed to map the frame for source"); goto error; } if (!gst_video_frame_map (&out_frame, &thiz->aligned_info, msdk_surface->buf, GST_MAP_WRITE)) { GST_ERROR_OBJECT (thiz, "failed to map the frame for destination"); gst_video_frame_unmap (&src_frame); goto error; } if (!gst_video_frame_copy (&out_frame, &src_frame)) { GST_ERROR_OBJECT (thiz, "failed to copy frame"); gst_video_frame_unmap (&out_frame); gst_video_frame_unmap (&src_frame); goto error; } gst_video_frame_unmap (&out_frame); gst_video_frame_unmap (&src_frame); gst_buffer_replace (&frame->input_buffer, msdk_surface->buf); gst_buffer_unref (msdk_surface->buf); msdk_surface->buf = NULL; return msdk_surface; error: if (msdk_surface) { if (msdk_surface->buf) gst_buffer_unref (msdk_surface->buf); g_slice_free (MsdkSurface, msdk_surface); } return NULL; } static GstFlowReturn gst_msdkenc_handle_frame (GstVideoEncoder * encoder, GstVideoCodecFrame * frame) { GstMsdkEnc *thiz = GST_MSDKENC (encoder); GstMsdkEncClass *klass = GST_MSDKENC_GET_CLASS (thiz); GstVideoInfo *info = &thiz->input_state->info; FrameData *fdata; MsdkSurface *surface; if (thiz->reconfig || klass->need_reconfig (thiz, frame)) { gst_msdkenc_flush_frames (thiz, FALSE); gst_msdkenc_close_encoder (thiz); klass->set_extra_params (thiz, frame); // This will reinitialized the encoder but keep same input format. gst_msdkenc_set_format (encoder, NULL); } if (G_UNLIKELY (thiz->context == NULL)) goto not_inited; if (thiz->has_vpp) { MsdkSurface *vpp_surface; GstVideoFrame vframe; mfxSession session; mfxSyncPoint vpp_sync_point = NULL; mfxStatus status; vpp_surface = gst_msdkenc_get_surface_from_frame (thiz, frame); if (!vpp_surface) goto invalid_surface; surface = gst_msdkenc_get_surface_from_pool (thiz, thiz->msdk_converted_pool, NULL); if (!surface) goto invalid_surface; if (!gst_video_frame_map (&vframe, info, frame->input_buffer, GST_MAP_READ)) goto invalid_frame; if (frame->pts != GST_CLOCK_TIME_NONE) { vpp_surface->surface->Data.TimeStamp = gst_util_uint64_scale (frame->pts, 90000, GST_SECOND); surface->surface->Data.TimeStamp = gst_util_uint64_scale (frame->pts, 90000, GST_SECOND); } else { vpp_surface->surface->Data.TimeStamp = MFX_TIMESTAMP_UNKNOWN; surface->surface->Data.TimeStamp = MFX_TIMESTAMP_UNKNOWN; } session = gst_msdk_context_get_session (thiz->context); for (;;) { status = MFXVideoVPP_RunFrameVPPAsync (session, vpp_surface->surface, surface->surface, NULL, &vpp_sync_point); if (status != MFX_WRN_DEVICE_BUSY) break; /* If device is busy, wait 1ms and retry, as per MSDK's recomendation */ g_usleep (1000); }; gst_video_frame_unmap (&vframe); if (status != MFX_ERR_NONE && status != MFX_ERR_MORE_DATA) { GST_ELEMENT_ERROR (thiz, STREAM, ENCODE, ("Converting frame failed."), ("MSDK VPP error (%s)", msdk_status_to_string (status))); gst_video_encoder_finish_frame (GST_VIDEO_ENCODER (thiz), frame); return GST_FLOW_ERROR; } fdata = g_slice_new0 (FrameData); fdata->frame = gst_video_codec_frame_ref (frame); fdata->frame_surface = vpp_surface; fdata->converted_surface = surface; thiz->pending_frames = g_list_prepend (thiz->pending_frames, fdata); } else { surface = gst_msdkenc_get_surface_from_frame (thiz, frame); if (!surface) goto invalid_surface; fdata = gst_msdkenc_queue_frame (thiz, frame, info); if (!fdata) goto invalid_frame; fdata->frame_surface = surface; if (frame->pts != GST_CLOCK_TIME_NONE) { surface->surface->Data.TimeStamp = gst_util_uint64_scale (frame->pts, 90000, GST_SECOND); } else { surface->surface->Data.TimeStamp = MFX_TIMESTAMP_UNKNOWN; } } return gst_msdkenc_encode_frame (thiz, surface->surface, frame); /* ERRORS */ not_inited: { GST_WARNING_OBJECT (encoder, "Got buffer before set_caps was called"); return GST_FLOW_NOT_NEGOTIATED; } invalid_surface: { GST_ERROR_OBJECT (encoder, "Surface pool is full"); return GST_FLOW_ERROR; } invalid_frame: { GST_WARNING_OBJECT (encoder, "Failed to map frame"); return GST_FLOW_OK; } } static gboolean gst_msdkenc_context_prepare (GstMsdkEnc * thiz) { /* Try to find an existing context from the pipeline. This may (indirectly) * invoke gst_msdkenc_set_context, which will set thiz->context. */ if (!gst_msdk_context_find (GST_ELEMENT_CAST (thiz), &thiz->context)) return FALSE; if (thiz->context == thiz->old_context) { GST_INFO_OBJECT (thiz, "Found old context %" GST_PTR_FORMAT ", reusing as-is", thiz->context); return TRUE; } GST_INFO_OBJECT (thiz, "Found context %" GST_PTR_FORMAT " from neighbour", thiz->context); /* Check GST_MSDK_JOB_VPP and GST_MSDK_JOB_ENCODER together to avoid sharing context * between VPP and ENCODER * Example: * gst-launch-1.0 videotestsrc ! video/x-raw,format=I420 ! msdkh264enc ! \ * msdkh264dec ! msdkvpp ! video/x-raw,format=YUY2 ! fakesink */ if (!(gst_msdk_context_get_job_type (thiz->context) & (GST_MSDK_JOB_VPP | GST_MSDK_JOB_ENCODER))) { gst_msdk_context_add_job_type (thiz->context, GST_MSDK_JOB_ENCODER); return TRUE; } /* Found an existing context that's already being used as an encoder, clone * the MFX session inside it to create a new one */ { GstMsdkContext *parent_context, *msdk_context; GST_INFO_OBJECT (thiz, "Creating new context %" GST_PTR_FORMAT " with " "joined session", thiz->context); parent_context = thiz->context; msdk_context = gst_msdk_context_new_with_parent (parent_context); if (!msdk_context) { GST_ERROR_OBJECT (thiz, "Failed to create a context with parent context " "as %" GST_PTR_FORMAT, parent_context); return FALSE; } thiz->context = msdk_context; gst_object_unref (parent_context); } return TRUE; } static gboolean gst_msdkenc_start (GstVideoEncoder * encoder) { GstMsdkEnc *thiz = GST_MSDKENC (encoder); if (!gst_msdkenc_context_prepare (thiz)) { if (!gst_msdk_context_ensure_context (GST_ELEMENT_CAST (thiz), thiz->hardware, GST_MSDK_JOB_ENCODER)) return FALSE; GST_INFO_OBJECT (thiz, "Creating new context %" GST_PTR_FORMAT, thiz->context); } /* Save the current context in a separate field so that we know whether it * has changed between calls to _start() */ gst_object_replace ((GstObject **) & thiz->old_context, (GstObject *) thiz->context); gst_msdk_context_add_shared_async_depth (thiz->context, thiz->async_depth); /* Set the minimum pts to some huge value (1000 hours). This keeps the dts at the start of the stream from needing to be negative. */ gst_video_encoder_set_min_pts (encoder, GST_SECOND * 60 * 60 * 1000); return TRUE; } static gboolean gst_msdkenc_stop (GstVideoEncoder * encoder) { GstMsdkEnc *thiz = GST_MSDKENC (encoder); gst_msdkenc_flush_frames (thiz, TRUE); gst_msdkenc_close_encoder (thiz); gst_msdkenc_dequeue_all_frames (thiz); if (thiz->input_state) gst_video_codec_state_unref (thiz->input_state); thiz->input_state = NULL; gst_clear_object (&thiz->context); return TRUE; } static gboolean gst_msdkenc_flush (GstVideoEncoder * encoder) { GstMsdkEnc *thiz = GST_MSDKENC (encoder); GST_DEBUG_OBJECT (encoder, "flush and close encoder"); gst_msdkenc_flush_frames (thiz, TRUE); gst_msdkenc_close_encoder (thiz); gst_msdkenc_dequeue_all_frames (thiz); gst_msdkenc_init_encoder (thiz); return TRUE; } static GstFlowReturn gst_msdkenc_finish (GstVideoEncoder * encoder) { GstMsdkEnc *thiz = GST_MSDKENC (encoder); gst_msdkenc_flush_frames (thiz, FALSE); return GST_FLOW_OK; } static gboolean gst_msdkenc_propose_allocation (GstVideoEncoder * encoder, GstQuery * query) { GstMsdkEnc *thiz = GST_MSDKENC (encoder); GstVideoInfo info; GstBufferPool *pool = NULL; GstAllocator *allocator = NULL; GstCaps *caps; guint num_buffers; if (!thiz->input_state) return FALSE; gst_query_parse_allocation (query, &caps, NULL); if (!caps) { GST_INFO_OBJECT (encoder, "failed to get caps"); return FALSE; } if (!gst_video_info_from_caps (&info, caps)) { GST_INFO_OBJECT (encoder, "failed to get video info"); return FALSE; } /* if upstream allocation query supports dmabuf-capsfeatures, * we do allocate dmabuf backed memory */ if (_gst_caps_has_feature (caps, GST_CAPS_FEATURE_MEMORY_DMABUF)) { GST_INFO_OBJECT (thiz, "MSDK VPP srcpad uses DMABuf memory"); thiz->use_dmabuf = TRUE; } num_buffers = gst_msdkenc_maximum_delayed_frames (thiz) + 1; pool = gst_msdkenc_create_buffer_pool (thiz, caps, num_buffers, TRUE); gst_query_add_allocation_pool (query, pool, GST_VIDEO_INFO_SIZE (&info), num_buffers, 0); gst_query_add_allocation_meta (query, GST_VIDEO_META_API_TYPE, NULL); if (pool) { GstStructure *config; GstAllocationParams params = { 0, 31, 0, 0, }; config = gst_buffer_pool_get_config (GST_BUFFER_POOL_CAST (pool)); if (gst_buffer_pool_config_get_allocator (config, &allocator, NULL)) gst_query_add_allocation_param (query, allocator, ¶ms); gst_structure_free (config); } gst_object_unref (pool); return GST_VIDEO_ENCODER_CLASS (parent_class)->propose_allocation (encoder, query); } static void gst_msdkenc_dispose (GObject * object) { GstMsdkEnc *thiz = GST_MSDKENC (object); if (thiz->input_state) gst_video_codec_state_unref (thiz->input_state); thiz->input_state = NULL; gst_clear_object (&thiz->msdk_pool); gst_clear_object (&thiz->msdk_converted_pool); gst_clear_object (&thiz->old_context); gst_clear_structure (&thiz->ext_coding_props); G_OBJECT_CLASS (parent_class)->dispose (object); } static gboolean gst_msdkenc_need_conversion (GstMsdkEnc * encoder, GstVideoInfo * info, GstVideoFormat * out_format) { switch (GST_VIDEO_INFO_FORMAT (info)) { case GST_VIDEO_FORMAT_NV12: case GST_VIDEO_FORMAT_P010_10LE: case GST_VIDEO_FORMAT_VUYA: #if (MFX_VERSION >= 1027) case GST_VIDEO_FORMAT_Y410: case GST_VIDEO_FORMAT_Y210: #endif return FALSE; default: if (GST_VIDEO_INFO_COMP_DEPTH (info, 0) == 10) *out_format = GST_VIDEO_FORMAT_P010_10LE; else *out_format = GST_VIDEO_FORMAT_NV12; return TRUE; } } static gboolean gst_msdkenc_need_reconfig (GstMsdkEnc * encoder, GstVideoCodecFrame * frame) { return FALSE; } static void gst_msdkenc_set_extra_params (GstMsdkEnc * encoder, GstVideoCodecFrame * frame) { /* Do nothing */ } static void gst_msdkenc_class_init (GstMsdkEncClass * klass) { GObjectClass *gobject_class; GstElementClass *element_class; GstVideoEncoderClass *gstencoder_class; gobject_class = G_OBJECT_CLASS (klass); element_class = GST_ELEMENT_CLASS (klass); gstencoder_class = GST_VIDEO_ENCODER_CLASS (klass); klass->need_conversion = gst_msdkenc_need_conversion; klass->need_reconfig = gst_msdkenc_need_reconfig; klass->set_extra_params = gst_msdkenc_set_extra_params; klass->qp_max = 51; klass->qp_min = 0; gobject_class->dispose = gst_msdkenc_dispose; element_class->set_context = gst_msdkenc_set_context; gstencoder_class->set_format = GST_DEBUG_FUNCPTR (gst_msdkenc_set_format); gstencoder_class->handle_frame = GST_DEBUG_FUNCPTR (gst_msdkenc_handle_frame); gstencoder_class->start = GST_DEBUG_FUNCPTR (gst_msdkenc_start); gstencoder_class->stop = GST_DEBUG_FUNCPTR (gst_msdkenc_stop); gstencoder_class->flush = GST_DEBUG_FUNCPTR (gst_msdkenc_flush); gstencoder_class->finish = GST_DEBUG_FUNCPTR (gst_msdkenc_finish); gstencoder_class->propose_allocation = GST_DEBUG_FUNCPTR (gst_msdkenc_propose_allocation); gst_element_class_add_static_pad_template (element_class, &sink_factory); } static void gst_msdkenc_init (GstMsdkEnc * thiz) { thiz->hardware = PROP_HARDWARE_DEFAULT; thiz->async_depth = PROP_ASYNC_DEPTH_DEFAULT; thiz->target_usage = PROP_TARGET_USAGE_DEFAULT; thiz->rate_control = PROP_RATE_CONTROL_DEFAULT; thiz->bitrate = PROP_BITRATE_DEFAULT; thiz->max_frame_size = PROP_MAX_FRAME_SIZE_DEFAULT; thiz->max_vbv_bitrate = PROP_MAX_VBV_BITRATE_DEFAULT; thiz->accuracy = PROP_AVBR_ACCURACY_DEFAULT; thiz->convergence = PROP_AVBR_ACCURACY_DEFAULT; thiz->lookahead_depth = PROP_RC_LOOKAHEAD_DEPTH_DEFAULT; thiz->qpi = PROP_QPI_DEFAULT; thiz->qpp = PROP_QPP_DEFAULT; thiz->qpb = PROP_QPB_DEFAULT; thiz->gop_size = PROP_GOP_SIZE_DEFAULT; thiz->ref_frames = PROP_REF_FRAMES_DEFAULT; thiz->i_frames = PROP_I_FRAMES_DEFAULT; thiz->b_frames = PROP_B_FRAMES_DEFAULT; thiz->num_slices = PROP_NUM_SLICES_DEFAULT; thiz->mbbrc = PROP_MBBRC_DEFAULT; thiz->adaptive_i = PROP_ADAPTIVE_I_DEFAULT; thiz->adaptive_b = PROP_ADAPTIVE_B_DEFAULT; thiz->ext_coding_props = gst_structure_new (EC_PROPS_STRUCT_NAME, EC_PROPS_EXTBRC, G_TYPE_STRING, "off", NULL); } /* gst_msdkenc_set_common_property: * * This is a helper function to set the common property * of base encoder from subclass implementation. */ gboolean gst_msdkenc_set_common_property (GObject * object, guint prop_id, const GValue * value, GParamSpec * pspec) { GstMsdkEnc *thiz = GST_MSDKENC (object); GstState state; gboolean ret = TRUE; GST_OBJECT_LOCK (thiz); state = GST_STATE (thiz); if ((state != GST_STATE_READY && state != GST_STATE_NULL) && !(pspec->flags & GST_PARAM_MUTABLE_PLAYING)) { ret = FALSE; goto wrong_state; } switch (prop_id) { case GST_MSDKENC_PROP_HARDWARE: thiz->hardware = g_value_get_boolean (value); break; case GST_MSDKENC_PROP_ASYNC_DEPTH: thiz->async_depth = g_value_get_uint (value); break; case GST_MSDKENC_PROP_TARGET_USAGE: thiz->target_usage = g_value_get_uint (value); break; case GST_MSDKENC_PROP_RATE_CONTROL: thiz->rate_control = g_value_get_enum (value); break; case GST_MSDKENC_PROP_BITRATE: { guint bitrate = g_value_get_uint (value); /* Ensure that bitrate changed before triggering a reconfig */ if (bitrate != thiz->bitrate) { thiz->bitrate = bitrate; thiz->reconfig = TRUE; GST_DEBUG_OBJECT (thiz, "changed bitrate to %u", bitrate); } break; } case GST_MSDKENC_PROP_MAX_FRAME_SIZE: thiz->max_frame_size = g_value_get_uint (value); break; case GST_MSDKENC_PROP_MAX_VBV_BITRATE: thiz->max_vbv_bitrate = g_value_get_uint (value); break; case GST_MSDKENC_PROP_AVBR_ACCURACY: thiz->accuracy = g_value_get_uint (value); break; case GST_MSDKENC_PROP_AVBR_CONVERGENCE: thiz->convergence = g_value_get_uint (value); break; case GST_MSDKENC_PROP_RC_LOOKAHEAD_DEPTH: thiz->lookahead_depth = g_value_get_uint (value); break; case GST_MSDKENC_PROP_QPI: thiz->qpi = g_value_get_uint (value); break; case GST_MSDKENC_PROP_QPP: thiz->qpp = g_value_get_uint (value); break; case GST_MSDKENC_PROP_QPB: thiz->qpb = g_value_get_uint (value); break; case GST_MSDKENC_PROP_GOP_SIZE: thiz->gop_size = g_value_get_uint (value); break; case GST_MSDKENC_PROP_REF_FRAMES: thiz->ref_frames = g_value_get_uint (value); break; case GST_MSDKENC_PROP_I_FRAMES: thiz->i_frames = g_value_get_uint (value); break; case GST_MSDKENC_PROP_B_FRAMES: thiz->b_frames = g_value_get_uint (value); break; case GST_MSDKENC_PROP_NUM_SLICES: thiz->num_slices = g_value_get_uint (value); break; case GST_MSDKENC_PROP_MBBRC: thiz->mbbrc = g_value_get_enum (value); break; case GST_MSDKENC_PROP_ADAPTIVE_I: thiz->adaptive_i = g_value_get_enum (value); break; case GST_MSDKENC_PROP_ADAPTIVE_B: thiz->adaptive_b = g_value_get_enum (value); break; case GST_MSDKENC_PROP_EXT_CODING_PROPS: { const GstStructure *s = gst_value_get_structure (value); const gchar *name = gst_structure_get_name (s); gst_structure_set_name (thiz->ext_coding_props, name); if (!structure_transform (s, thiz->ext_coding_props)) { GST_ERROR_OBJECT (thiz, "failed to transform structure"); } break; } default: ret = FALSE; break; } GST_OBJECT_UNLOCK (thiz); return ret; /* ERROR */ wrong_state: { GST_WARNING_OBJECT (thiz, "setting property in wrong state"); GST_OBJECT_UNLOCK (thiz); return ret; } } /* gst_msdkenc_get_common_property: * * This is a helper function to get the common property * of base encoder from subclass implementation. */ gboolean gst_msdkenc_get_common_property (GObject * object, guint prop_id, GValue * value, GParamSpec * pspec) { GstMsdkEnc *thiz = GST_MSDKENC (object); gboolean ret = TRUE; GST_OBJECT_LOCK (thiz); switch (prop_id) { case GST_MSDKENC_PROP_HARDWARE: g_value_set_boolean (value, thiz->hardware); break; case GST_MSDKENC_PROP_ASYNC_DEPTH: g_value_set_uint (value, thiz->async_depth); break; case GST_MSDKENC_PROP_TARGET_USAGE: g_value_set_uint (value, thiz->target_usage); break; case GST_MSDKENC_PROP_RATE_CONTROL: g_value_set_enum (value, thiz->rate_control); break; case GST_MSDKENC_PROP_BITRATE: g_value_set_uint (value, thiz->bitrate); break; case GST_MSDKENC_PROP_MAX_FRAME_SIZE: g_value_set_uint (value, thiz->max_frame_size); break; case GST_MSDKENC_PROP_MAX_VBV_BITRATE: g_value_set_uint (value, thiz->max_vbv_bitrate); break; case GST_MSDKENC_PROP_AVBR_ACCURACY: g_value_set_uint (value, thiz->accuracy); break; case GST_MSDKENC_PROP_AVBR_CONVERGENCE: g_value_set_uint (value, thiz->convergence); break; case GST_MSDKENC_PROP_RC_LOOKAHEAD_DEPTH: g_value_set_uint (value, thiz->lookahead_depth); break; case GST_MSDKENC_PROP_QPI: g_value_set_uint (value, thiz->qpi); break; case GST_MSDKENC_PROP_QPP: g_value_set_uint (value, thiz->qpp); break; case GST_MSDKENC_PROP_QPB: g_value_set_uint (value, thiz->qpb); break; case GST_MSDKENC_PROP_GOP_SIZE: g_value_set_uint (value, thiz->gop_size); break; case GST_MSDKENC_PROP_REF_FRAMES: g_value_set_uint (value, thiz->ref_frames); break; case GST_MSDKENC_PROP_I_FRAMES: g_value_set_uint (value, thiz->i_frames); break; case GST_MSDKENC_PROP_B_FRAMES: g_value_set_uint (value, thiz->b_frames); break; case GST_MSDKENC_PROP_NUM_SLICES: g_value_set_uint (value, thiz->num_slices); break; case GST_MSDKENC_PROP_MBBRC: g_value_set_enum (value, thiz->mbbrc); break; case GST_MSDKENC_PROP_ADAPTIVE_I: g_value_set_enum (value, thiz->adaptive_i); break; case GST_MSDKENC_PROP_ADAPTIVE_B: g_value_set_enum (value, thiz->adaptive_b); break; case GST_MSDKENC_PROP_EXT_CODING_PROPS: gst_value_set_structure (value, thiz->ext_coding_props); break; default: ret = FALSE; break; } GST_OBJECT_UNLOCK (thiz); return ret; } /* gst_msdkenc_install_common_properties: * @thiz: a #GstMsdkEnc * * This is a helper function to install common properties * of base encoder from subclass implementation. * Encoders like jpeg do't require all the common properties * and they can avoid installing it into base gobject. */ void gst_msdkenc_install_common_properties (GstMsdkEncClass * klass) { GObjectClass *gobject_class = G_OBJECT_CLASS (klass); GParamSpec *obj_properties[GST_MSDKENC_PROP_MAX] = { NULL, }; guint qp_range_max = klass->qp_max; guint qp_range_min = klass->qp_min; obj_properties[GST_MSDKENC_PROP_HARDWARE] = g_param_spec_boolean ("hardware", "Hardware", "Enable hardware encoders", PROP_HARDWARE_DEFAULT, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS); obj_properties[GST_MSDKENC_PROP_ASYNC_DEPTH] = g_param_spec_uint ("async-depth", "Async Depth", "Depth of asynchronous pipeline", 1, 20, PROP_ASYNC_DEPTH_DEFAULT, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS); obj_properties[GST_MSDKENC_PROP_TARGET_USAGE] = g_param_spec_uint ("target-usage", "Target Usage", "1: Best quality, 4: Balanced, 7: Best speed", 1, 7, PROP_TARGET_USAGE_DEFAULT, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS); obj_properties[GST_MSDKENC_PROP_RATE_CONTROL] = g_param_spec_enum ("rate-control", "Rate Control", "Rate control method", gst_msdkenc_rate_control_get_type (), PROP_RATE_CONTROL_DEFAULT, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS); obj_properties[GST_MSDKENC_PROP_BITRATE] = g_param_spec_uint ("bitrate", "Bitrate", "Bitrate in kbit/sec", 1, 2000 * 1024, PROP_BITRATE_DEFAULT, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS | GST_PARAM_MUTABLE_PLAYING); obj_properties[GST_MSDKENC_PROP_MAX_FRAME_SIZE] = g_param_spec_uint ("max-frame-size", "Max Frame Size", "Maximum possible size (in kbyte) of any compressed frames (0: auto-calculate)", 0, G_MAXUINT16, PROP_MAX_FRAME_SIZE_DEFAULT, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS); /* Set the same upper bound with bitrate */ obj_properties[GST_MSDKENC_PROP_MAX_VBV_BITRATE] = g_param_spec_uint ("max-vbv-bitrate", "Max VBV Bitrate", "Maximum bitrate(kbit/sec) at which data enters Video Buffering Verifier (0: auto-calculate)", 0, 2000 * 1024, PROP_MAX_VBV_BITRATE_DEFAULT, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS); obj_properties[GST_MSDKENC_PROP_AVBR_ACCURACY] = g_param_spec_uint ("accuracy", "Accuracy", "The AVBR Accuracy in " "the unit of tenth of percent", 0, G_MAXUINT16, PROP_AVBR_ACCURACY_DEFAULT, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS); obj_properties[GST_MSDKENC_PROP_AVBR_CONVERGENCE] = g_param_spec_uint ("convergence", "Convergence", "The AVBR Convergence in the unit of 100 frames", 0, G_MAXUINT16, PROP_AVBR_CONVERGENCE_DEFAULT, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS); obj_properties[GST_MSDKENC_PROP_RC_LOOKAHEAD_DEPTH] = g_param_spec_uint ("rc-lookahead", "Look-ahead depth", "Number of frames to look ahead for Rate control", 10, 100, PROP_RC_LOOKAHEAD_DEPTH_DEFAULT, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS); obj_properties[GST_MSDKENC_PROP_QPI] = g_param_spec_uint ("qpi", "QPI", "Constant quantizer for I frames (0 unlimited). Also used as " "ICQQuality or QVBRQuality for different RateControl methods", qp_range_min, qp_range_max, PROP_QPI_DEFAULT, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS); obj_properties[GST_MSDKENC_PROP_QPP] = g_param_spec_uint ("qpp", "QPP", "Constant quantizer for P frames (0 unlimited)", qp_range_min, qp_range_max, PROP_QPP_DEFAULT, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS); obj_properties[GST_MSDKENC_PROP_QPB] = g_param_spec_uint ("qpb", "QPB", "Constant quantizer for B frames (0 unlimited)", qp_range_min, qp_range_max, PROP_QPB_DEFAULT, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS); obj_properties[GST_MSDKENC_PROP_GOP_SIZE] = g_param_spec_uint ("gop-size", "GOP Size", "GOP Size", 0, G_MAXINT, PROP_GOP_SIZE_DEFAULT, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS); obj_properties[GST_MSDKENC_PROP_REF_FRAMES] = g_param_spec_uint ("ref-frames", "Reference Frames", "Number of reference frames", 0, G_MAXINT, PROP_REF_FRAMES_DEFAULT, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS); obj_properties[GST_MSDKENC_PROP_I_FRAMES] = g_param_spec_uint ("i-frames", "I Frames", "Number of I frames between IDR frames", 0, G_MAXINT, PROP_I_FRAMES_DEFAULT, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS); obj_properties[GST_MSDKENC_PROP_B_FRAMES] = g_param_spec_uint ("b-frames", "B Frames", "Number of B frames between I and P frames", 0, G_MAXINT, PROP_B_FRAMES_DEFAULT, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS); obj_properties[GST_MSDKENC_PROP_NUM_SLICES] = g_param_spec_uint ("num-slices", "Number of Slices", "Number of slices per frame, Zero tells the encoder to " "choose any slice partitioning allowed by the codec standard", 0, G_MAXINT, PROP_NUM_SLICES_DEFAULT, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS); obj_properties[GST_MSDKENC_PROP_MBBRC] = g_param_spec_enum ("mbbrc", "MB level bitrate control", "Macroblock level bitrate control", gst_msdkenc_mbbrc_get_type (), PROP_MBBRC_DEFAULT, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS); obj_properties[GST_MSDKENC_PROP_ADAPTIVE_I] = g_param_spec_enum ("i-adapt", "Adaptive I-Frame Insertion", "Adaptive I-Frame Insertion control", gst_msdkenc_adaptive_i_get_type (), PROP_ADAPTIVE_I_DEFAULT, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS); obj_properties[GST_MSDKENC_PROP_ADAPTIVE_B] = g_param_spec_enum ("b-adapt", "Adaptive B-Frame Insertion", "Adaptive B-Frame Insertion control", gst_msdkenc_adaptive_b_get_type (), PROP_ADAPTIVE_B_DEFAULT, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS); /** * GstMsdkEnc:ext-coding-props * * The properties for the external coding. * * Supported properties: * ``` * extbrc : External bitrate control * String. Range: { auto, on, off } Default: off * ``` * * Example: * ``` * ext-coding-props="props,extbrc=on" * ``` * * Since: 1.20 * */ obj_properties[GST_MSDKENC_PROP_EXT_CODING_PROPS] = g_param_spec_boxed ("ext-coding-props", "External coding properties", "The properties for the external coding, refer to the hotdoc for the " "supported properties", GST_TYPE_STRUCTURE, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS); g_object_class_install_properties (gobject_class, GST_MSDKENC_PROP_MAX, obj_properties); }