flagset: Fail parsing on overflowing hex strings

This adds code to detect when the hex form of the string we are to
parse exceeds the number of bytes that would form a 32bit flag. This will
avoid treating as flagset anything above then the expected 32 bits and also
stop treading DRM format with modifiers as flagset (like
drm-format=AB24:0x0100000000000002).

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/4775>

subprojects/gstreamer/gst/gstvalue.c

@@ -7900,6 +7900,16 @@ gst_value_deserialize_flagset (GValue * dest, const gchar * s)
   if (G_UNLIKELY ((mask == 0 && errno == EINVAL) || cur == next))
     goto try_as_flags_string;
 
+  if (g_str_has_prefix (cur, "0x") || g_str_has_prefix (cur, "0X"))
+    cur += 2;
+
+  /* Flagsets are 32 bits hex numbers, so do not accept any number that has more
+   * then 8 characters. strtoul() accepts unlimited number of leading zeros and
+   * 64bit numbers on 64bit platforms.
+   */
+  if ((next - cur) > 8)
+    return FALSE;
+
   /* Next char should be NULL terminator, or a ':'. If ':', we need the flag string after */
   if (G_UNLIKELY (next[0] == 0)) {
     res = TRUE;

subprojects/gstreamer/tests/check/gst/gstvalue.c

@@ -773,6 +773,16 @@ GST_START_TEST (test_flagset)
   g_value_unset (&dest);
   g_value_unset (&value);
   g_value_unset (&value2);
+
+  /* Check that we reject flagset looking string that holds 64 bit integers. */
+  g_value_init (&value, GST_TYPE_FLAG_SET);
+  string = g_strdup ("AB24:0x0100000000000002");
+
+  fail_if (gst_value_deserialize (&value, string),
+      "matched something that isn't a flagset %s", string);
+
+  g_free (string);
+  g_value_unset (&value);
 }
 
 GST_END_TEST;