decoder: sanitize codec-data decoding.

Add a new GstVaapiDecoder::decode_codec_data() hook to actually decode
codec-data in the decoder sub-class. Provide a common shared helper
function to do the actual work and delegating further to the sub-class.

gst-libs/gst/vaapi/gstvaapidecoder.c

@@ -956,3 +956,28 @@ gst_vaapi_decoder_flush(GstVaapiDecoder *decoder)
 
     return do_flush(decoder);
 }
+
+GstVaapiDecoderStatus
+gst_vaapi_decoder_decode_codec_data(GstVaapiDecoder *decoder)
+{
+    GstVaapiDecoderClass * const klass = GST_VAAPI_DECODER_GET_CLASS(decoder);
+    GstBuffer * const codec_data = GST_VAAPI_DECODER_CODEC_DATA(decoder);
+    GstVaapiDecoderStatus status;
+    const guchar *buf;
+    guint buf_size;
+
+    if (!codec_data)
+        return GST_VAAPI_DECODER_STATUS_SUCCESS;
+
+    /* FIXME: add a meaningful error code? */
+    if (!klass->decode_codec_data)
+        return GST_VAAPI_DECODER_STATUS_SUCCESS;
+
+    buf      = GST_BUFFER_DATA(codec_data);
+    buf_size = GST_BUFFER_SIZE(codec_data);
+    if (!buf || buf_size == 0)
+        return GST_VAAPI_DECODER_STATUS_SUCCESS;
+
+    status = klass->decode_codec_data(decoder, buf, buf_size);
+    return status;
+}

gst-libs/gst/vaapi/gstvaapidecoder.h

@@ -124,6 +124,8 @@ struct _GstVaapiDecoderClass {
         struct _GstVaapiDecoderUnit *unit);
     GstVaapiDecoderStatus (*end_frame)(GstVaapiDecoder *decoder);
     GstVaapiDecoderStatus (*flush)(GstVaapiDecoder *decoder);
+    GstVaapiDecoderStatus (*decode_codec_data)(GstVaapiDecoder *decoder,
+        const guchar *buf, guint buf_size);
 };
 
 GType

gst-libs/gst/vaapi/gstvaapidecoder_h264.c

@@ -2763,24 +2763,20 @@ decode_unit(GstVaapiDecoderH264 *decoder, GstVaapiDecoderUnit *unit)
 }
 
 static GstVaapiDecoderStatus
-decode_codec_data(GstVaapiDecoderH264 *decoder, GstBuffer *buffer)
+gst_vaapi_decoder_h264_decode_codec_data(GstVaapiDecoder *base_decoder,
+    const guchar *buf, guint buf_size)
 {
+    GstVaapiDecoderH264 * const decoder =
+        GST_VAAPI_DECODER_H264_CAST(base_decoder);
     GstVaapiDecoderH264Private * const priv = decoder->priv;
     GstVaapiDecoderStatus status;
     GstVaapiDecoderUnit unit;
     GstVaapiParserInfoH264 pi;
     GstH264ParserResult result;
-    guchar *buf;
-    guint buf_size;
     guint i, ofs, num_sps, num_pps;
 
     unit.parsed_info = π
 
-    buf      = GST_BUFFER_DATA(buffer);
-    buf_size = GST_BUFFER_SIZE(buffer);
-    if (!buf || buf_size == 0)
-        return GST_VAAPI_DECODER_STATUS_SUCCESS;
-
     if (buf_size < 8)
         return GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA;
 
@@ -2836,7 +2832,6 @@ ensure_decoder(GstVaapiDecoderH264 *decoder)
 {
     GstVaapiDecoderH264Private * const priv = decoder->priv;
     GstVaapiDecoderStatus status;
-    GstBuffer *codec_data;
 
     g_return_val_if_fail(priv->is_constructed,
                          GST_VAAPI_DECODER_STATUS_ERROR_INIT_FAILED);
@@ -2846,13 +2841,11 @@ ensure_decoder(GstVaapiDecoderH264 *decoder)
         if (!priv->is_opened)
             return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_CODEC;
 
-        codec_data = GST_VAAPI_DECODER_CODEC_DATA(decoder);
+        status = gst_vaapi_decoder_decode_codec_data(
-        if (codec_data) {
+            GST_VAAPI_DECODER_CAST(decoder));
-            status = decode_codec_data(decoder, codec_data);
         if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
             return status;
     }
-    }
     return GST_VAAPI_DECODER_STATUS_SUCCESS;
 }
 
@@ -3091,6 +3084,9 @@ gst_vaapi_decoder_h264_class_init(GstVaapiDecoderH264Class *klass)
     decoder_class->start_frame  = gst_vaapi_decoder_h264_start_frame;
     decoder_class->end_frame    = gst_vaapi_decoder_h264_end_frame;
     decoder_class->flush        = gst_vaapi_decoder_h264_flush;
+
+    decoder_class->decode_codec_data =
+        gst_vaapi_decoder_h264_decode_codec_data;
 }
 
 static void

gst-libs/gst/vaapi/gstvaapidecoder_mpeg4.c

@@ -41,6 +41,9 @@ G_DEFINE_TYPE(GstVaapiDecoderMpeg4,
               gst_vaapi_decoder_mpeg4,
               GST_VAAPI_TYPE_DECODER)
 
+#define GST_VAAPI_DECODER_MPEG4_CAST(decoder) \
+    ((GstVaapiDecoderMpeg4 *)(decoder))
+
 #define GST_VAAPI_DECODER_MPEG4_GET_PRIVATE(obj)                \
     (G_TYPE_INSTANCE_GET_PRIVATE((obj),                         \
                                  GST_VAAPI_TYPE_DECODER_MPEG4,  \
@@ -921,14 +924,15 @@ decode_buffer(GstVaapiDecoderMpeg4 *decoder, const guchar *buf, guint buf_size)
 }
 
 static GstVaapiDecoderStatus
-decode_codec_data(GstVaapiDecoderMpeg4 *decoder, GstBuffer *buffer)
+gst_vaapi_decoder_mpeg4_decode_codec_data(GstVaapiDecoder *base_decoder,
+    const guchar *_buf, guint _buf_size)
 {
+    GstVaapiDecoderMpeg4 * const decoder =
+        GST_VAAPI_DECODER_MPEG4_CAST(base_decoder);
     GstVaapiDecoderStatus status;
-    guchar *buf, *_buf;
+    guchar *buf;
-    guint pos, buf_size, _buf_size;
+    guint pos, buf_size;
 
-    _buf      = GST_BUFFER_DATA(buffer);
-    _buf_size = GST_BUFFER_SIZE(buffer);
     // add additional 0x000001b2 to enclose the last header
     buf_size = _buf_size + 4;
     buf = malloc(buf_size);
@@ -965,7 +969,6 @@ ensure_decoder(GstVaapiDecoderMpeg4 *decoder)
 {
     GstVaapiDecoderMpeg4Private * const priv = decoder->priv;
     GstVaapiDecoderStatus status;
-    GstBuffer *codec_data;
 
     g_return_val_if_fail(priv->is_constructed,
                          GST_VAAPI_DECODER_STATUS_ERROR_INIT_FAILED);
@@ -975,13 +978,11 @@ ensure_decoder(GstVaapiDecoderMpeg4 *decoder)
         if (!priv->is_opened)
             return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_CODEC;
 
-        codec_data = GST_VAAPI_DECODER_CODEC_DATA(decoder);
+        status = gst_vaapi_decoder_decode_codec_data(
-        if (codec_data) {
+            GST_VAAPI_DECODER_CAST(decoder));
-            status = decode_codec_data(decoder, codec_data);
         if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
             return status;
     }
-    }
     return GST_VAAPI_DECODER_STATUS_SUCCESS;
 }
 
@@ -1137,6 +1138,9 @@ gst_vaapi_decoder_mpeg4_class_init(GstVaapiDecoderMpeg4Class *klass)
 
     decoder_class->parse        = gst_vaapi_decoder_mpeg4_parse;
     decoder_class->decode       = gst_vaapi_decoder_mpeg4_decode;
+
+    decoder_class->decode_codec_data =
+        gst_vaapi_decoder_mpeg4_decode_codec_data;
 }
 
 static void

gst-libs/gst/vaapi/gstvaapidecoder_priv.h

@@ -222,6 +222,10 @@ G_GNUC_INTERNAL
 GstVaapiDecoderStatus
 gst_vaapi_decoder_check_status(GstVaapiDecoder *decoder);
 
+G_GNUC_INTERNAL
+GstVaapiDecoderStatus
+gst_vaapi_decoder_decode_codec_data(GstVaapiDecoder *decoder);
+
 G_END_DECLS
 
 #endif /* GST_VAAPI_DECODER_PRIV_H */

gst-libs/gst/vaapi/gstvaapidecoder_vc1.c

@@ -42,6 +42,9 @@ G_DEFINE_TYPE(GstVaapiDecoderVC1,
               gst_vaapi_decoder_vc1,
               GST_VAAPI_TYPE_DECODER)
 
+#define GST_VAAPI_DECODER_VC1_CAST(decoder) \
+    ((GstVaapiDecoderVC1 *)(decoder))
+
 #define GST_VAAPI_DECODER_VC1_GET_PRIVATE(obj)                  \
     (G_TYPE_INSTANCE_GET_PRIVATE((obj),                         \
                                  GST_VAAPI_TYPE_DECODER_VC1,    \
@@ -1061,8 +1064,11 @@ decode_buffer(GstVaapiDecoderVC1 *decoder, guchar *buf, guint buf_size)
 }
 
 static GstVaapiDecoderStatus
-decode_codec_data(GstVaapiDecoderVC1 *decoder, GstBuffer *buffer)
+gst_vaapi_decoder_vc1_decode_codec_data(GstVaapiDecoder *base_decoder,
+    const guchar *buf, guint buf_size)
 {
+    GstVaapiDecoderVC1 * const decoder =
+        GST_VAAPI_DECODER_VC1_CAST(base_decoder);
     GstVaapiDecoderVC1Private * const priv = decoder->priv;
     GstVC1SeqHdr * const seq_hdr = &priv->seq_hdr;
     GstVaapiDecoderStatus status;
@@ -1070,16 +1076,12 @@ decode_codec_data(GstVaapiDecoderVC1 *decoder, GstBuffer *buffer)
     GstVC1BDU ebdu;
     GstCaps *caps;
     GstStructure *structure;
-    guchar *buf;
+    guint ofs;
-    guint buf_size, ofs;
     gint width, height;
     guint32 format;
     gint version;
 
-    buf      = GST_BUFFER_DATA(buffer);
+    priv->has_codec_data = TRUE;
-    buf_size = GST_BUFFER_SIZE(buffer);
-    if (!buf || buf_size == 0)
-        return GST_VAAPI_DECODER_STATUS_SUCCESS;
 
     width = GST_VAAPI_DECODER_WIDTH(decoder);
     height = GST_VAAPI_DECODER_HEIGHT(decoder);
@@ -1109,7 +1111,7 @@ decode_codec_data(GstVaapiDecoderVC1 *decoder, GstBuffer *buffer)
         ebdu.size      = buf_size;
         ebdu.sc_offset = 0;
         ebdu.offset    = 0;
-        ebdu.data      = buf;
+        ebdu.data      = (guint8 *)buf;
         return decode_ebdu(decoder, &ebdu);
     }
 
@@ -1149,7 +1151,6 @@ ensure_decoder(GstVaapiDecoderVC1 *decoder)
 {
     GstVaapiDecoderVC1Private * const priv = decoder->priv;
     GstVaapiDecoderStatus status;
-    GstBuffer *codec_data;
 
     g_return_val_if_fail(priv->is_constructed,
                          GST_VAAPI_DECODER_STATUS_ERROR_INIT_FAILED);
@@ -1159,13 +1160,10 @@ ensure_decoder(GstVaapiDecoderVC1 *decoder)
         if (!priv->is_opened)
             return GST_VAAPI_DECODER_STATUS_ERROR_UNSUPPORTED_CODEC;
 
-        codec_data = GST_VAAPI_DECODER_CODEC_DATA(decoder);
+        status = gst_vaapi_decoder_decode_codec_data(
-        if (codec_data) {
+            GST_VAAPI_DECODER_CAST(decoder));
-            status = decode_codec_data(decoder, codec_data);
         if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
             return status;
-            priv->has_codec_data = TRUE;
-        }
     }
     return GST_VAAPI_DECODER_STATUS_SUCCESS;
 }
@@ -1355,6 +1353,9 @@ gst_vaapi_decoder_vc1_class_init(GstVaapiDecoderVC1Class *klass)
     decoder_class->start_frame  = gst_vaapi_decoder_vc1_start_frame;
     decoder_class->end_frame    = gst_vaapi_decoder_vc1_end_frame;
     decoder_class->flush        = gst_vaapi_decoder_vc1_flush;
+
+    decoder_class->decode_codec_data =
+        gst_vaapi_decoder_vc1_decode_codec_data;
 }
 
 static void