vaapidecode: update internal decoder sink caps

When a new sink caps arrive the internal decoder state is updated
and, if it is, request a downstream renegotiation.

Previously, when new caps arrived the whole decoder where destroyed
and recreated. Now, if the caps are compatible or has the same codec,
the internal decoder is kept, but a downstream renegotiation is
requested.

https://bugzilla.gnome.org/show_bug.cgi?id=776979

gst-libs/gst/vaapi/gstvaapidecoder.c

@@ -418,6 +418,8 @@ set_caps (GstVaapiDecoder * decoder, const GstCaps * caps)
   if (!gst_video_info_from_caps (&codec_state->info, caps))
     return FALSE;
 
+  if (codec_state->caps)
+    gst_caps_unref (codec_state->caps);
   codec_state->caps = gst_caps_copy (caps);
 
   v_codec_data = gst_structure_get_value (structure, "codec_data");
@@ -1085,3 +1087,50 @@ gst_vaapi_decoder_get_surface_formats (GstVaapiDecoder * decoder)
     return gst_vaapi_context_get_surface_formats (decoder->context);
   return NULL;
 }
+
+/**
+ * gst_vaapi_decoder_update_caps:
+ * @decoder: a #GstVaapiDecoder
+ * @caps: a #GstCaps
+ *
+ * If @caps is compatible with the current caps, or they have the same
+ * codec, the caps are updated internally.
+ *
+ * This method will not call codec_state_changed() callback, since
+ * this function is intended to run sync and during the set_format()
+ * vmethod.
+ *
+ * Returns: %TRUE if the caps were updated internally.
+ **/
+gboolean
+gst_vaapi_decoder_update_caps (GstVaapiDecoder * decoder, GstCaps * caps)
+{
+  GstCaps *decoder_caps;
+  GstVaapiProfile profile;
+  GstVaapiCodec codec;
+
+  g_return_val_if_fail (decoder != NULL, FALSE);
+  g_return_val_if_fail (caps != NULL, FALSE);
+
+  decoder_caps = get_caps (decoder);
+  if (!decoder_caps)
+    return FALSE;
+
+  if (gst_caps_is_always_compatible (caps, decoder_caps)) {
+    set_caps (decoder, caps);
+    return TRUE;
+  }
+
+  profile = gst_vaapi_profile_from_caps (caps);
+  if (profile == GST_VAAPI_PROFILE_UNKNOWN)
+    return FALSE;
+  codec = gst_vaapi_profile_get_codec (profile);
+  if (codec == 0)
+    return FALSE;
+  if (codec == decoder->codec) {
+    set_caps (decoder, caps);
+    return TRUE;
+  }
+
+  return FALSE;
+}

gst-libs/gst/vaapi/gstvaapidecoder.h

@@ -135,6 +135,9 @@ gst_vaapi_decoder_flush (GstVaapiDecoder * decoder);
 GstVaapiDecoderStatus
 gst_vaapi_decoder_check_status (GstVaapiDecoder * decoder);
 
+gboolean
+gst_vaapi_decoder_update_caps (GstVaapiDecoder * decoder, GstCaps * caps);
+
 G_END_DECLS
 
 #endif /* GST_VAAPI_DECODER_H */

gst/vaapi/gstvaapidecode.c

@@ -134,7 +134,8 @@ static gboolean gst_vaapidecode_update_sink_caps (GstVaapiDecode * decode,
 static gboolean gst_vaapi_decode_input_state_replace (GstVaapiDecode * decode,
     const GstVideoCodecState * new_state);
 
-/* get invoked only if actural VASurface size (not the cropped values) changed */
+/* invoked if actual VASurface size (not the cropped values)
+ * changed */
 static void
 gst_vaapi_decoder_state_changed (GstVaapiDecoder * decoder,
     const GstVideoCodecState * codec_state, gpointer user_data)
@@ -511,8 +512,9 @@ gst_vaapidecode_push_decoded_frame (GstVideoDecoder * vdec,
     caps_renegotiate = is_display_resolution_changed (decode, crop_rect);
 
     if (gst_pad_needs_reconfigure (GST_VIDEO_DECODER_SRC_PAD (vdec))
-        || alloc_renegotiate || caps_renegotiate) {
+        || alloc_renegotiate || caps_renegotiate || decode->do_renego) {
 
+      g_atomic_int_set (&decode->do_renego, FALSE);
       if (!gst_vaapidecode_negotiate (decode))
         return GST_FLOW_ERROR;
     }
@@ -878,7 +880,6 @@ gst_vaapidecode_create (GstVaapiDecode * decode, GstCaps * caps)
   gst_vaapi_decoder_set_codec_state_changed_func (decode->decoder,
       gst_vaapi_decoder_state_changed, decode);
 
-  decode->decoder_caps = gst_caps_ref (caps);
   return TRUE;
 }
 
@@ -915,7 +916,6 @@ gst_vaapidecode_destroy (GstVaapiDecode * decode)
   gst_vaapidecode_purge (decode);
 
   gst_vaapi_decoder_replace (&decode->decoder, NULL);
-  gst_caps_replace (&decode->decoder_caps, NULL);
 
   gst_vaapidecode_release (gst_object_ref (decode));
 }
@@ -924,17 +924,14 @@ static gboolean
 gst_vaapidecode_reset_full (GstVaapiDecode * decode, GstCaps * caps,
     gboolean hard)
 {
-  GstVaapiCodec codec;
-
   /* Reset tracked frame size */
   decode->current_frame_size = 0;
 
-  if (!hard && decode->decoder && decode->decoder_caps) {
+  if (!hard && decode->decoder) {
-    if (gst_caps_is_always_compatible (caps, decode->decoder_caps))
+    if (gst_vaapi_decoder_update_caps (decode->decoder, caps)) {
-      return TRUE;
+      g_atomic_int_set (&decode->do_renego, TRUE);
-    codec = gst_vaapi_codec_from_caps (caps);
-    if (codec == gst_vaapi_decoder_get_codec (decode->decoder))
       return TRUE;
+    }
   }
 
   gst_vaapidecode_destroy (decode);
@@ -1009,7 +1006,6 @@ gst_vaapidecode_stop (GstVideoDecoder * vdec)
   gst_vaapidecode_purge (decode);
   gst_vaapi_decode_input_state_replace (decode, NULL);
   gst_vaapi_decoder_replace (&decode->decoder, NULL);
-  gst_caps_replace (&decode->decoder_caps, NULL);
   gst_caps_replace (&decode->sinkpad_caps, NULL);
   gst_caps_replace (&decode->srcpad_caps, NULL);
   return TRUE;
@@ -1049,7 +1045,7 @@ gst_vaapidecode_set_format (GstVideoDecoder * vdec, GstVideoCodecState * state)
     return FALSE;
   if (!gst_vaapi_plugin_base_set_caps (plugin, decode->sinkpad_caps, NULL))
     return FALSE;
-  if (!gst_vaapidecode_reset_full (decode, decode->sinkpad_caps, TRUE))
+  if (!gst_vaapidecode_reset_full (decode, decode->sinkpad_caps, FALSE))
     return FALSE;
 
   return TRUE;

gst/vaapi/gstvaapidecode.h

@@ -45,7 +45,6 @@ struct _GstVaapiDecode {
     GstVaapiDecoder    *decoder;
     GMutex              surface_ready_mutex;
     GCond               surface_ready;
-    GstCaps            *decoder_caps;
     GstCaps            *allowed_sinkpad_caps;
     GstCaps            *allowed_srcpad_caps;
     guint               current_frame_size;
@@ -56,6 +55,8 @@ struct _GstVaapiDecode {
 
     GstVideoCodecState *input_state;
     GstSegment          in_segment;
+
+    gboolean            do_renego;
 };
 
 struct _GstVaapiDecodeClass {