h2645parser: Catch overflows in AVC/HEVC NAL unit length calculations

Offset and size are stored as 32 bit guint and might overflow when
adding the nal_length_size, so let's avoid that.

For the size this would happen if the AVC/HEVC NAL unit size happens to
be stored in 4 bytes and is 4294967292 or higher, which is likely
corrupted data anyway.

For the offset this is something for the caller of these functions to
take care of but is unlikely to happen as it would require parsing on a
>4GB buffer.

Allowing these overflows causes all kinds of follow-up bugs in the
h2645parse elements, ranging from infinite loops and memory leaks to
potential memory corruptions.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/-/merge_requests/2103>

gst-libs/gst/codecparsers/gsth264parser.c

@@ -1556,6 +1556,14 @@ gst_h264_parser_identify_nalu_avc (GstH264NalParser * nalparser,
 
   memset (nalu, 0, sizeof (*nalu));
 
+  /* Would overflow guint below otherwise: the callers needs to ensure that
+   * this never happens */
+  if (offset > G_MAXUINT32 - nal_length_size) {
+    GST_WARNING ("offset + nal_length_size overflow");
+    nalu->size = 0;
+    return GST_H264_PARSER_BROKEN_DATA;
+  }
+
   if (size < offset + nal_length_size) {
     GST_DEBUG ("Can't parse, buffer has too small size %" G_GSIZE_FORMAT
         ", offset %u", size, offset);
@@ -1570,7 +1578,13 @@ gst_h264_parser_identify_nalu_avc (GstH264NalParser * nalparser,
   nalu->sc_offset = offset;
   nalu->offset = offset + nal_length_size;
 
-  if (size < nalu->size + nal_length_size) {
+  if (nalu->size > G_MAXUINT32 - nal_length_size) {
+    GST_WARNING ("NALU size + nal_length_size overflow");
+    nalu->size = 0;
+    return GST_H264_PARSER_BROKEN_DATA;
+  }
+
+  if (size < (gsize) nalu->size + nal_length_size) {
     nalu->size = 0;
 
     return GST_H264_PARSER_NO_NAL_END;

gst-libs/gst/codecparsers/gsth265parser.c

@@ -1531,6 +1531,14 @@ gst_h265_parser_identify_nalu_hevc (GstH265Parser * parser,
 
   memset (nalu, 0, sizeof (*nalu));
 
+  /* Would overflow guint below otherwise: the callers needs to ensure that
+   * this never happens */
+  if (offset > G_MAXUINT32 - nal_length_size) {
+    GST_WARNING ("offset + nal_length_size overflow");
+    nalu->size = 0;
+    return GST_H265_PARSER_BROKEN_DATA;
+  }
+
   if (size < offset + nal_length_size) {
     GST_DEBUG ("Can't parse, buffer has too small size %" G_GSIZE_FORMAT
         ", offset %u", size, offset);
@@ -1545,7 +1553,13 @@ gst_h265_parser_identify_nalu_hevc (GstH265Parser * parser,
   nalu->sc_offset = offset;
   nalu->offset = offset + nal_length_size;
 
-  if (size < nalu->size + nal_length_size) {
+  if (nalu->size > G_MAXUINT32 - nal_length_size) {
+    GST_WARNING ("NALU size + nal_length_size overflow");
+    nalu->size = 0;
+    return GST_H265_PARSER_BROKEN_DATA;
+  }
+
+  if (size < (gsize) nalu->size + nal_length_size) {
     nalu->size = 0;
 
     return GST_H265_PARSER_NO_NAL_END;

tests/check/libs/h264parser.c

@@ -229,6 +229,65 @@ GST_START_TEST (test_h264_parse_slice_5bytes)
 
 GST_END_TEST;
 
+GST_START_TEST (test_h264_parse_identify_nalu_avc)
+{
+  GstH264ParserResult res;
+  GstH264NalUnit nalu;
+  GstH264NalParser *const parser = gst_h264_nal_parser_new ();
+  /* Skip 3 bytes for the start code */
+  const gsize nal_size = sizeof (slice_dpa) - 3;
+  const gsize buf_size = 4 + nal_size;
+  guint8 *buf = g_new (guint8, buf_size);
+
+  memcpy (buf + 4, slice_dpa + 3, nal_size);
+
+  GST_WRITE_UINT16_BE (buf + 2, nal_size);
+  res = gst_h264_parser_identify_nalu_avc (parser, buf, 2, buf_size, 2, &nalu);
+
+  assert_equals_int (res, GST_H264_PARSER_OK);
+  assert_equals_int (nalu.type, GST_H264_NAL_SLICE_DPA);
+  assert_equals_int (nalu.offset, 4);
+  assert_equals_int (nalu.size, nal_size);
+
+  GST_WRITE_UINT32_BE (buf, nal_size);
+  res = gst_h264_parser_identify_nalu_avc (parser, buf, 0, buf_size, 4, &nalu);
+
+  assert_equals_int (res, GST_H264_PARSER_OK);
+  assert_equals_int (nalu.type, GST_H264_NAL_SLICE_DPA);
+  assert_equals_int (nalu.offset, 4);
+  assert_equals_int (nalu.size, nal_size);
+
+  GST_WRITE_UINT32_BE (buf, G_MAXUINT32);
+  res = gst_h264_parser_identify_nalu_avc (parser, buf, 0, buf_size, 4, &nalu);
+
+  assert_equals_int (res, GST_H264_PARSER_BROKEN_DATA);
+
+  GST_WRITE_UINT32_BE (buf, G_MAXUINT32 - 2);
+  res = gst_h264_parser_identify_nalu_avc (parser, buf, 0, buf_size, 4, &nalu);
+
+  assert_equals_int (res, GST_H264_PARSER_BROKEN_DATA);
+
+  GST_WRITE_UINT32_BE (buf, G_MAXUINT32 - 3);
+  res = gst_h264_parser_identify_nalu_avc (parser, buf, 0, buf_size, 4, &nalu);
+
+  assert_equals_int (res, GST_H264_PARSER_BROKEN_DATA);
+
+  GST_WRITE_UINT32_BE (buf, G_MAXUINT32 - 4);
+  res = gst_h264_parser_identify_nalu_avc (parser, buf, 0, buf_size, 4, &nalu);
+
+  assert_equals_int (res, GST_H264_PARSER_NO_NAL_END);
+
+  GST_WRITE_UINT32_BE (buf, G_MAXUINT32 - 6);
+  res = gst_h264_parser_identify_nalu_avc (parser, buf, 0, buf_size, 4, &nalu);
+
+  assert_equals_int (res, GST_H264_PARSER_NO_NAL_END);
+
+  g_free (buf);
+  gst_h264_nal_parser_free (parser);
+}
+
+GST_END_TEST;
+
 static guint8 nalu_sps_with_vui[] = {
   0x00, 0x00, 0x00, 0x01, 0x67, 0x64, 0x00, 0x28,
   0xac, 0xd9, 0x40, 0x78, 0x04, 0x4f, 0xde, 0x03,
@@ -666,6 +725,7 @@ h264parser_suite (void)
   tcase_add_test (tc_chain, test_h264_parse_slice_dpa);
   tcase_add_test (tc_chain, test_h264_parse_slice_eoseq_slice);
   tcase_add_test (tc_chain, test_h264_parse_slice_5bytes);
+  tcase_add_test (tc_chain, test_h264_parse_identify_nalu_avc);
   tcase_add_test (tc_chain, test_h264_parse_invalid_sei);
   tcase_add_test (tc_chain, test_h264_create_sei);
 

tests/check/libs/h265parser.c

@@ -255,6 +255,65 @@ GST_START_TEST (test_h265_parse_slice_6bytes)
 
 GST_END_TEST;
 
+GST_START_TEST (test_h265_parse_identify_nalu_hevc)
+{
+  GstH265ParserResult res;
+  GstH265NalUnit nalu;
+  GstH265Parser *parser = gst_h265_parser_new ();
+  /* Skip 4 bytes for the start code */
+  const gsize nal_size = sizeof (slice_eos_slice_eob) - 4;
+  const gsize buf_size = 4 + nal_size;
+  guint8 *buf = g_new (guint8, buf_size);
+
+  memcpy (buf + 4, slice_eos_slice_eob + 4, nal_size);
+
+  GST_WRITE_UINT16_BE (buf + 2, nal_size);
+  res = gst_h265_parser_identify_nalu_hevc (parser, buf, 2, buf_size, 2, &nalu);
+
+  assert_equals_int (res, GST_H265_PARSER_OK);
+  assert_equals_int (nalu.type, GST_H265_NAL_SLICE_IDR_W_RADL);
+  assert_equals_int (nalu.offset, 4);
+  assert_equals_int (nalu.size, nal_size);
+
+  GST_WRITE_UINT32_BE (buf, nal_size);
+  res = gst_h265_parser_identify_nalu_hevc (parser, buf, 0, buf_size, 4, &nalu);
+
+  assert_equals_int (res, GST_H265_PARSER_OK);
+  assert_equals_int (nalu.type, GST_H265_NAL_SLICE_IDR_W_RADL);
+  assert_equals_int (nalu.offset, 4);
+  assert_equals_int (nalu.size, nal_size);
+
+  GST_WRITE_UINT32_BE (buf, G_MAXUINT32);
+  res = gst_h265_parser_identify_nalu_hevc (parser, buf, 0, buf_size, 4, &nalu);
+
+  assert_equals_int (res, GST_H265_PARSER_BROKEN_DATA);
+
+  GST_WRITE_UINT32_BE (buf, G_MAXUINT32 - 2);
+  res = gst_h265_parser_identify_nalu_hevc (parser, buf, 0, buf_size, 4, &nalu);
+
+  assert_equals_int (res, GST_H265_PARSER_BROKEN_DATA);
+
+  GST_WRITE_UINT32_BE (buf, G_MAXUINT32 - 3);
+  res = gst_h265_parser_identify_nalu_hevc (parser, buf, 0, buf_size, 4, &nalu);
+
+  assert_equals_int (res, GST_H265_PARSER_BROKEN_DATA);
+
+  GST_WRITE_UINT32_BE (buf, G_MAXUINT32 - 4);
+  res = gst_h265_parser_identify_nalu_hevc (parser, buf, 0, buf_size, 4, &nalu);
+
+  assert_equals_int (res, GST_H265_PARSER_NO_NAL_END);
+
+  GST_WRITE_UINT32_BE (buf, G_MAXUINT32 - 6);
+  res = gst_h265_parser_identify_nalu_hevc (parser, buf, 0, buf_size, 4, &nalu);
+
+  assert_equals_int (res, GST_H265_PARSER_NO_NAL_END);
+
+  g_free (buf);
+  gst_h265_parser_free (parser);
+}
+
+GST_END_TEST;
+
 GST_START_TEST (test_h265_base_profiles)
 {
   GstH265ProfileTierLevel ptl;
@@ -1118,6 +1177,7 @@ h265parser_suite (void)
   tcase_add_test (tc_chain, test_h265_parse_slice_eos_slice_eob);
   tcase_add_test (tc_chain, test_h265_parse_pic_timing);
   tcase_add_test (tc_chain, test_h265_parse_slice_6bytes);
+  tcase_add_test (tc_chain, test_h265_parse_identify_nalu_hevc);
   tcase_add_test (tc_chain, test_h265_base_profiles);
   tcase_add_test (tc_chain, test_h265_base_profiles_compat);
   tcase_add_test (tc_chain, test_h265_format_range_profiles_exact_match);