vaapih264dec: Add a property to assume constrained-baseline

When baseline-as-constrained is set, the decoder will expose support
for baseline decoding and assume that the baseline content is
constrained-baseline. This can be handy to decode streams in hardware
that would otherwise not be possible to decode. A lot of baseline
content is in fact constrained.

gst-libs/gst/vaapi/gstvaapidecoder_h264.c

@@ -537,6 +537,7 @@ struct _GstVaapiDecoderH264Private
 
   gboolean force_low_latency;
   gboolean base_only;
+  gboolean baseline_as_constrained;
 };
 
 /**
@@ -1473,7 +1474,7 @@ get_profile (GstVaapiDecoderH264 * decoder, GstH264SPS * sps, guint dpb_size)
   fill_profiles (profiles, &n_profiles, profile);
   switch (profile) {
     case GST_VAAPI_PROFILE_H264_BASELINE:
-      if (sps->constraint_set1_flag) {  // A.2.2 (main profile)
+      if (priv->baseline_as_constrained || sps->constraint_set1_flag) { // A.2.2 (main profile)
         fill_profiles (profiles, &n_profiles,
             GST_VAAPI_PROFILE_H264_CONSTRAINED_BASELINE);
         fill_profiles (profiles, &n_profiles, GST_VAAPI_PROFILE_H264_MAIN);
@@ -4800,6 +4801,24 @@ gst_vaapi_decoder_h264_set_base_only (GstVaapiDecoderH264 * decoder,
   decoder->priv.base_only = base_only;
 }
 
+/**
+ * gst_vaapi_decoder_h264_set_baseline_as_constrained:
+ * @decoder: a #GstVaapiDecoderH264
+ * @baseline_as_constrained: %TRUE to assume all baseline is constrained
+ *
+ * This is a small hack that makes the decoder assumes that baseline contents
+ * is already constrained. This may allow decoding some streams that would
+ * otherwise fails to negotiation.
+ */
+void
+gst_vaapi_decoder_h264_set_baseline_as_constrained (GstVaapiDecoderH264 *
+    decoder, gboolean baseline_as_constrained)
+{
+  g_return_if_fail (decoder != NULL);
+
+  decoder->priv.baseline_as_constrained = baseline_as_constrained;
+}
+
 /**
  * gst_vaapi_decoder_h264_set_low_latency:
  * @decoder: a #GstVaapiDecoderH264

gst-libs/gst/vaapi/gstvaapidecoder_h264.h

@@ -73,6 +73,10 @@ void
 gst_vaapi_decoder_h264_set_base_only(GstVaapiDecoderH264 * decoder,
     gboolean base_only);
 
+void
+gst_vaapi_decoder_h264_set_baseline_as_constrained(GstVaapiDecoderH264 * decoder,
+    gboolean baseline_as_constrained);
+
 G_END_DECLS
 
 #endif /* GST_VAAPI_DECODER_H264_H */

gst/vaapi/gstvaapidecode.c

@@ -912,6 +912,9 @@ gst_vaapidecode_create (GstVaapiDecode * decode, GstCaps * caps)
               (decode->decoder), priv->is_low_latency);
           gst_vaapi_decoder_h264_set_base_only (GST_VAAPI_DECODER_H264
               (decode->decoder), priv->base_only);
+          gst_vaapi_decoder_h264_set_baseline_as_constrained
+              (GST_VAAPI_DECODER_H264 (decode->decoder),
+              priv->baseline_as_constrained);
         }
       }
       break;
@@ -1233,6 +1236,8 @@ gst_vaapidecode_ensure_allowed_sinkpad_caps (GstVaapiDecode * decode)
   gboolean have_high = FALSE;
   gboolean have_mvc = FALSE;
   gboolean have_svc = FALSE;
+  GstVaapiDecodeH264Private *priv =
+      gst_vaapi_decode_h264_get_instance_private (decode);
 
   profiles = gst_vaapi_display_get_decode_profiles (display);
   if (!profiles)
@@ -1275,6 +1280,11 @@ gst_vaapidecode_ensure_allowed_sinkpad_caps (GstVaapiDecode * decode)
     have_mvc |= is_mvc_profile (profile);
     have_svc |= is_svc_profile (profile);
     have_high |= profile == GST_VAAPI_PROFILE_H264_HIGH;
+
+    if (priv && priv->baseline_as_constrained &&
+        profile == GST_VAAPI_PROFILE_H264_CONSTRAINED_BASELINE)
+      allowed_sinkpad_caps =
+          add_h264_profile_in_caps (allowed_sinkpad_caps, "baseline");
   }
 
   if (have_high) {

gst/vaapi/gstvaapidecode_props.c

@@ -29,7 +29,8 @@
 enum
 {
   GST_VAAPI_DECODER_H264_PROP_FORCE_LOW_LATENCY = 1,
-  GST_VAAPI_DECODER_H264_PROP_BASE_ONLY
+  GST_VAAPI_DECODER_H264_PROP_BASE_ONLY,
+  GST_VAAPI_DECODER_H264_PROP_BASELINE_AS_CONSTRAINED,
 };
 
 static gint h264_private_offset;
@@ -49,6 +50,9 @@ gst_vaapi_decode_h264_get_property (GObject * object, guint prop_id,
     case GST_VAAPI_DECODER_H264_PROP_BASE_ONLY:
       g_value_set_boolean (value, priv->base_only);
       break;
+    case GST_VAAPI_DECODER_H264_PROP_BASELINE_AS_CONSTRAINED:
+      g_value_set_boolean (value, priv->baseline_as_constrained);
+      break;
     default:
       G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
       break;
@@ -77,6 +81,13 @@ gst_vaapi_decode_h264_set_property (GObject * object, guint prop_id,
       if (decoder)
         gst_vaapi_decoder_h264_set_base_only (decoder, priv->base_only);
       break;
+    case GST_VAAPI_DECODER_H264_PROP_BASELINE_AS_CONSTRAINED:
+      priv->baseline_as_constrained = g_value_get_boolean (value);
+      decoder = GST_VAAPI_DECODER_H264 (GST_VAAPIDECODE (object)->decoder);
+      if (decoder)
+        gst_vaapi_decoder_h264_set_baseline_as_constrained (decoder,
+            priv->baseline_as_constrained);
+      break;
     default:
       G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
       break;
@@ -103,6 +114,13 @@ gst_vaapi_decode_h264_install_properties (GObjectClass * klass)
       g_param_spec_boolean ("base-only", "Decode base view only",
           "Drop any NAL unit not defined in Annex.A", FALSE,
           G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
+
+  g_object_class_install_property (klass,
+      GST_VAAPI_DECODER_H264_PROP_BASELINE_AS_CONSTRAINED,
+      g_param_spec_boolean ("baseline-as-constrained",
+          "Baseline as Constrained",
+          "Assume all baseline content is also constrained.", FALSE,
+          G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
 }
 
 GstVaapiDecodeH264Private *

gst/vaapi/gstvaapidecode_props.h

@@ -34,6 +34,7 @@ struct _GstVaapiDecodeH264Private
 {
   gboolean is_low_latency;
   gboolean base_only;
+  gboolean baseline_as_constrained;
 };
 
 void