From fed69c48b60cceef50e8be452f1184886f3471ac Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Wed, 20 Oct 2021 02:19:33 +0530 Subject: [PATCH] vtenc: Add FieldDetail properties for interlaced input Standard interlace handling: * If we have interlace-mode=interleaved and the field order, we just set it when creating the session * If we have interlace-mode=(interleaved|mixed) and no field order, we set the field order on the first buffer The encoder session does not support changing the FieldDetail after it has started encoding frames, so we cannot support mixed streams correctly. Part-of: --- .../gst-plugins-bad/sys/applemedia/vtenc.c | 80 +++++++++++++++++++ .../gst-plugins-bad/sys/applemedia/vtenc.h | 1 + 2 files changed, 81 insertions(+) diff --git a/subprojects/gst-plugins-bad/sys/applemedia/vtenc.c b/subprojects/gst-plugins-bad/sys/applemedia/vtenc.c index 9f27cc9913..c3b90af67b 100644 --- a/subprojects/gst-plugins-bad/sys/applemedia/vtenc.c +++ b/subprojects/gst-plugins-bad/sys/applemedia/vtenc.c @@ -195,6 +195,23 @@ gst_vtenc_base_init (GstVTEncClass * klass) "height", GST_TYPE_INT_RANGE, min_height, max_height, "framerate", GST_TYPE_FRACTION_RANGE, min_fps_n, min_fps_d, max_fps_n, max_fps_d, NULL); + + /* Signal our limited interlace support */ + { + G_GNUC_BEGIN_IGNORE_DEPRECATIONS; + GValueArray *arr = g_value_array_new (2); + GValue val = G_VALUE_INIT; + + g_value_init (&val, G_TYPE_STRING); + g_value_set_string (&val, "progressive"); + arr = g_value_array_append (arr, &val); + g_value_set_string (&val, "interleaved"); + arr = g_value_array_append (arr, &val); + G_GNUC_END_IGNORE_DEPRECATIONS; + gst_structure_set_list (gst_caps_get_structure (src_caps, 0), + "interlace-mode", arr); + } + switch (codec_details->format_id) { case kCMVideoCodecType_H264: gst_structure_set (gst_caps_get_structure (src_caps, 0), @@ -319,6 +336,7 @@ gst_vtenc_init (GstVTEnc * self) self->latency_frames = -1; self->session = NULL; self->profile_level = NULL; + self->have_field_order = TRUE; self->keyframe_props = CFDictionaryCreate (NULL, (const void **) keyframe_props_keys, @@ -1119,6 +1137,41 @@ gst_vtenc_create_session (GstVTEnc * self) gst_vtenc_set_colorimetry (self, session); + /* Interlacing */ + switch (GST_VIDEO_INFO_INTERLACE_MODE (&self->video_info)) { + case GST_VIDEO_INTERLACE_MODE_PROGRESSIVE: + gst_vtenc_session_configure_property_int (self, session, + kVTCompressionPropertyKey_FieldCount, 1); + break; + case GST_VIDEO_INTERLACE_MODE_INTERLEAVED: + gst_vtenc_session_configure_property_int (self, session, + kVTCompressionPropertyKey_FieldCount, 2); + switch (GST_VIDEO_INFO_FIELD_ORDER (&self->video_info)) { + case GST_VIDEO_FIELD_ORDER_TOP_FIELD_FIRST: + status = VTSessionSetProperty (session, + kVTCompressionPropertyKey_FieldDetail, + kCMFormatDescriptionFieldDetail_TemporalTopFirst); + GST_DEBUG_OBJECT (self, "kVTCompressionPropertyKey_FieldDetail " + "TemporalTopFirst => %d", (int) status); + break; + case GST_VIDEO_FIELD_ORDER_BOTTOM_FIELD_FIRST: + status = VTSessionSetProperty (session, + kVTCompressionPropertyKey_FieldDetail, + kCMFormatDescriptionFieldDetail_TemporalBottomFirst); + GST_DEBUG_OBJECT (self, "kVTCompressionPropertyKey_FieldDetail " + "TemporalBottomFirst => %d", (int) status); + break; + case GST_VIDEO_FIELD_ORDER_UNKNOWN: + GST_INFO_OBJECT (self, "Unknown field order for interleaved content, " + "will check first buffer"); + self->have_field_order = FALSE; + } + break; + default: + /* Caps negotiation should prevent this */ + g_assert_not_reached (); + } + gst_vtenc_session_configure_realtime (self, session, gst_vtenc_get_realtime (self)); gst_vtenc_session_configure_allow_frame_reordering (self, session, @@ -1371,6 +1424,33 @@ gst_vtenc_encode_frame (GstVTEnc * self, GstVideoCodecFrame * frame) else duration = kCMTimeInvalid; + /* If we don't have field order, we need to pick it up from the first buffer + * that has that information. The encoder session also cannot be reconfigured + * with a new field detail after it has been set, so we encode mixed streams + * with whatever the first buffer's field order is. */ + if (!self->have_field_order) { + CFStringRef field_detail = NULL; + + if (GST_VIDEO_BUFFER_IS_TOP_FIELD (frame->input_buffer)) + field_detail = kCMFormatDescriptionFieldDetail_TemporalTopFirst; + else if (GST_VIDEO_BUFFER_IS_BOTTOM_FIELD (frame->input_buffer)) + field_detail = kCMFormatDescriptionFieldDetail_TemporalBottomFirst; + + if (field_detail) { + vt_status = VTSessionSetProperty (self->session, + kVTCompressionPropertyKey_FieldDetail, field_detail); + GST_DEBUG_OBJECT (self, "kVTCompressionPropertyKey_FieldDetail => %d", + (int) vt_status); + } else { + GST_WARNING_OBJECT (self, "have interlaced content, but don't know field " + "order yet, skipping buffer"); + gst_video_codec_frame_unref (frame); + return GST_FLOW_OK; + } + + self->have_field_order = TRUE; + } + meta = gst_buffer_get_core_media_meta (frame->input_buffer); if (meta != NULL) { pbuf = gst_core_media_buffer_get_pixel_buffer (frame->input_buffer); diff --git a/subprojects/gst-plugins-bad/sys/applemedia/vtenc.h b/subprojects/gst-plugins-bad/sys/applemedia/vtenc.h index 6f6ccb3b07..073ba75792 100644 --- a/subprojects/gst-plugins-bad/sys/applemedia/vtenc.h +++ b/subprojects/gst-plugins-bad/sys/applemedia/vtenc.h @@ -76,6 +76,7 @@ struct _GstVTEnc gint negotiated_fps_n, negotiated_fps_d; gint caps_width, caps_height; gint caps_fps_n, caps_fps_d; + gboolean have_field_order; GstVideoCodecState *input_state; GstVideoInfo video_info; VTCompressionSessionRef session;