From 82294fd6656979ef2b3b7a8cf15a3ea2a9697eff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Brzezi=C5=84ski?= Date: Fri, 22 Mar 2024 16:14:24 +0100 Subject: [PATCH] vtdec: Fix caps criticals during negotiation Calling gst_pad_peer_query_caps() without a filter can give us EMPTY caps, whereas all the code below assumes that's not the case. Replacing query+intersect with a filtered query ensures we always get a subset of the template caps back. Part-of: --- .../gst-plugins-bad/sys/applemedia/vtdec.c | 25 +++++++++++++------ 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/subprojects/gst-plugins-bad/sys/applemedia/vtdec.c b/subprojects/gst-plugins-bad/sys/applemedia/vtdec.c index c015baafa7..078b1304a6 100644 --- a/subprojects/gst-plugins-bad/sys/applemedia/vtdec.c +++ b/subprojects/gst-plugins-bad/sys/applemedia/vtdec.c @@ -482,7 +482,22 @@ gst_vtdec_negotiate (GstVideoDecoder * decoder) gst_video_codec_state_unref (output_state); } - peercaps = gst_pad_peer_query_caps (GST_VIDEO_DECODER_SRC_PAD (vtdec), NULL); + templcaps = + gst_pad_get_pad_template_caps (GST_VIDEO_DECODER_SRC_PAD (decoder)); + peercaps = + gst_pad_peer_query_caps (GST_VIDEO_DECODER_SRC_PAD (vtdec), templcaps); + gst_caps_unref (templcaps); + + if (gst_caps_is_empty (peercaps)) { + GST_INFO_OBJECT (vtdec, "empty peer caps, can't negotiate"); + + gst_caps_unref (peercaps); + if (prevcaps) + gst_caps_unref (prevcaps); + + return FALSE; + } + if (prevcaps && gst_caps_can_intersect (prevcaps, peercaps)) { /* The hardware decoder can become (temporarily) unavailable across * VTDecompressionSessionCreate/Destroy calls. So if the currently configured @@ -492,14 +507,10 @@ gst_vtdec_negotiate (GstVideoDecoder * decoder) GST_INFO_OBJECT (vtdec, "current and peer caps are compatible, keeping current caps"); caps = gst_caps_ref (prevcaps); + gst_caps_unref (peercaps); } else { - templcaps = - gst_pad_get_pad_template_caps (GST_VIDEO_DECODER_SRC_PAD (decoder)); - caps = - gst_caps_intersect_full (peercaps, templcaps, GST_CAPS_INTERSECT_FIRST); - gst_caps_unref (templcaps); + caps = peercaps; } - gst_caps_unref (peercaps); caps = gst_caps_truncate (gst_caps_make_writable (caps));