From 80bb69ba668f5a6cb8ff87a3ef24f49f74590c74 Mon Sep 17 00:00:00 2001 From: Nirbheek Chauhan Date: Fri, 4 Oct 2024 23:37:35 +0530 Subject: [PATCH] vtdec: Set input formats when we get incomplete caps In some cases, decodebin3 will send us incomplete caps (not containing codec_data), and then a GAP event, which will force a negotiation. This segfaults due to a null pointer deref because self->input_state is NULL. The only possible fix is to avoid negotiating when we get incomplete caps (to avoid re-negotiationg immediately afterwards, which isn't supported by some muxers), but also set as much input state as possible so that a renegotiation triggered by a GAP event can complete successfully. Part-of: --- .../gst-plugins-bad/sys/applemedia/vtdec.c | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/subprojects/gst-plugins-bad/sys/applemedia/vtdec.c b/subprojects/gst-plugins-bad/sys/applemedia/vtdec.c index 57fcbf9928..7438dd313f 100644 --- a/subprojects/gst-plugins-bad/sys/applemedia/vtdec.c +++ b/subprojects/gst-plugins-bad/sys/applemedia/vtdec.c @@ -685,6 +685,7 @@ gst_vtdec_negotiate (GstVideoDecoder * decoder) static gboolean gst_vtdec_set_format (GstVideoDecoder * decoder, GstVideoCodecState * state) { + gboolean negotiate_now = TRUE; GstStructure *structure; CMVideoCodecType cm_format = 0; CMFormatDescriptionRef format_description = NULL; @@ -718,15 +719,20 @@ gst_vtdec_set_format (GstVideoDecoder * decoder, GstVideoCodecState * state) if ((cm_format == kCMVideoCodecType_H264 || cm_format == kCMVideoCodecType_HEVC) && state->codec_data == NULL) { - GST_INFO_OBJECT (vtdec, "no codec data, wait for one"); - return TRUE; + GST_INFO_OBJECT (vtdec, "waiting for codec_data before negotiation"); + negotiate_now = FALSE; } gst_video_info_from_caps (&vtdec->video_info, state->caps); - if (!gst_vtdec_compute_dpb_size (vtdec, cm_format, state->codec_data)) + if (negotiate_now && + !gst_vtdec_compute_dpb_size (vtdec, cm_format, state->codec_data)) { + GST_INFO_OBJECT (vtdec, "Failed to compute DPB size"); return FALSE; - gst_vtdec_set_latency (vtdec); + } + + if (negotiate_now) + gst_vtdec_set_latency (vtdec); if (state->codec_data) { format_description = create_format_description_from_codec_data (vtdec, @@ -743,7 +749,7 @@ gst_vtdec_set_format (GstVideoDecoder * decoder, GstVideoCodecState * state) gst_video_codec_state_unref (vtdec->input_state); vtdec->input_state = gst_video_codec_state_ref (state); - return gst_vtdec_negotiate (decoder); + return negotiate_now ? gst_vtdec_negotiate (decoder) : TRUE; } static gboolean