diff --git a/gst-libs/gst/vaapi/gstvaapidecoder.c b/gst-libs/gst/vaapi/gstvaapidecoder.c
index 9a8cda2a84..6af1372fdd 100644
--- a/gst-libs/gst/vaapi/gstvaapidecoder.c
+++ b/gst-libs/gst/vaapi/gstvaapidecoder.c
@@ -418,6 +418,8 @@ set_caps (GstVaapiDecoder * decoder, const GstCaps * caps)
   if (!gst_video_info_from_caps (&codec_state->info, caps))
     return FALSE;
 
+  if (codec_state->caps)
+    gst_caps_unref (codec_state->caps);
   codec_state->caps = gst_caps_copy (caps);
 
   v_codec_data = gst_structure_get_value (structure, "codec_data");
@@ -1085,3 +1087,50 @@ gst_vaapi_decoder_get_surface_formats (GstVaapiDecoder * decoder)
     return gst_vaapi_context_get_surface_formats (decoder->context);
   return NULL;
 }
+
+/**
+ * gst_vaapi_decoder_update_caps:
+ * @decoder: a #GstVaapiDecoder
+ * @caps: a #GstCaps
+ *
+ * If @caps is compatible with the current caps, or they have the same
+ * codec, the caps are updated internally.
+ *
+ * This method will not call codec_state_changed() callback, since
+ * this function is intended to run sync and during the set_format()
+ * vmethod.
+ *
+ * Returns: %TRUE if the caps were updated internally.
+ **/
+gboolean
+gst_vaapi_decoder_update_caps (GstVaapiDecoder * decoder, GstCaps * caps)
+{
+  GstCaps *decoder_caps;
+  GstVaapiProfile profile;
+  GstVaapiCodec codec;
+
+  g_return_val_if_fail (decoder != NULL, FALSE);
+  g_return_val_if_fail (caps != NULL, FALSE);
+
+  decoder_caps = get_caps (decoder);
+  if (!decoder_caps)
+    return FALSE;
+
+  if (gst_caps_is_always_compatible (caps, decoder_caps)) {
+    set_caps (decoder, caps);
+    return TRUE;
+  }
+
+  profile = gst_vaapi_profile_from_caps (caps);
+  if (profile == GST_VAAPI_PROFILE_UNKNOWN)
+    return FALSE;
+  codec = gst_vaapi_profile_get_codec (profile);
+  if (codec == 0)
+    return FALSE;
+  if (codec == decoder->codec) {
+    set_caps (decoder, caps);
+    return TRUE;
+  }
+
+  return FALSE;
+}
diff --git a/gst-libs/gst/vaapi/gstvaapidecoder.h b/gst-libs/gst/vaapi/gstvaapidecoder.h
index 4b75f2d2da..a8ca5aa9ec 100644
--- a/gst-libs/gst/vaapi/gstvaapidecoder.h
+++ b/gst-libs/gst/vaapi/gstvaapidecoder.h
@@ -135,6 +135,9 @@ gst_vaapi_decoder_flush (GstVaapiDecoder * decoder);
 GstVaapiDecoderStatus
 gst_vaapi_decoder_check_status (GstVaapiDecoder * decoder);
 
+gboolean
+gst_vaapi_decoder_update_caps (GstVaapiDecoder * decoder, GstCaps * caps);
+
 G_END_DECLS
 
 #endif /* GST_VAAPI_DECODER_H */
diff --git a/gst/vaapi/gstvaapidecode.c b/gst/vaapi/gstvaapidecode.c
index 6b490176b2..b7570025e4 100644
--- a/gst/vaapi/gstvaapidecode.c
+++ b/gst/vaapi/gstvaapidecode.c
@@ -134,7 +134,8 @@ static gboolean gst_vaapidecode_update_sink_caps (GstVaapiDecode * decode,
 static gboolean gst_vaapi_decode_input_state_replace (GstVaapiDecode * decode,
     const GstVideoCodecState * new_state);
 
-/* get invoked only if actural VASurface size (not the cropped values) changed */
+/* invoked if actual VASurface size (not the cropped values)
+ * changed */
 static void
 gst_vaapi_decoder_state_changed (GstVaapiDecoder * decoder,
     const GstVideoCodecState * codec_state, gpointer user_data)
@@ -511,8 +512,9 @@ gst_vaapidecode_push_decoded_frame (GstVideoDecoder * vdec,
     caps_renegotiate = is_display_resolution_changed (decode, crop_rect);
 
     if (gst_pad_needs_reconfigure (GST_VIDEO_DECODER_SRC_PAD (vdec))
-        || alloc_renegotiate || caps_renegotiate) {
+        || alloc_renegotiate || caps_renegotiate || decode->do_renego) {
 
+      g_atomic_int_set (&decode->do_renego, FALSE);
       if (!gst_vaapidecode_negotiate (decode))
         return GST_FLOW_ERROR;
     }
@@ -878,7 +880,6 @@ gst_vaapidecode_create (GstVaapiDecode * decode, GstCaps * caps)
   gst_vaapi_decoder_set_codec_state_changed_func (decode->decoder,
       gst_vaapi_decoder_state_changed, decode);
 
-  decode->decoder_caps = gst_caps_ref (caps);
   return TRUE;
 }
 
@@ -915,7 +916,6 @@ gst_vaapidecode_destroy (GstVaapiDecode * decode)
   gst_vaapidecode_purge (decode);
 
   gst_vaapi_decoder_replace (&decode->decoder, NULL);
-  gst_caps_replace (&decode->decoder_caps, NULL);
 
   gst_vaapidecode_release (gst_object_ref (decode));
 }
@@ -924,17 +924,14 @@ static gboolean
 gst_vaapidecode_reset_full (GstVaapiDecode * decode, GstCaps * caps,
     gboolean hard)
 {
-  GstVaapiCodec codec;
-
   /* Reset tracked frame size */
   decode->current_frame_size = 0;
 
-  if (!hard && decode->decoder && decode->decoder_caps) {
-    if (gst_caps_is_always_compatible (caps, decode->decoder_caps))
-      return TRUE;
-    codec = gst_vaapi_codec_from_caps (caps);
-    if (codec == gst_vaapi_decoder_get_codec (decode->decoder))
+  if (!hard && decode->decoder) {
+    if (gst_vaapi_decoder_update_caps (decode->decoder, caps)) {
+      g_atomic_int_set (&decode->do_renego, TRUE);
       return TRUE;
+    }
   }
 
   gst_vaapidecode_destroy (decode);
@@ -1009,7 +1006,6 @@ gst_vaapidecode_stop (GstVideoDecoder * vdec)
   gst_vaapidecode_purge (decode);
   gst_vaapi_decode_input_state_replace (decode, NULL);
   gst_vaapi_decoder_replace (&decode->decoder, NULL);
-  gst_caps_replace (&decode->decoder_caps, NULL);
   gst_caps_replace (&decode->sinkpad_caps, NULL);
   gst_caps_replace (&decode->srcpad_caps, NULL);
   return TRUE;
@@ -1049,7 +1045,7 @@ gst_vaapidecode_set_format (GstVideoDecoder * vdec, GstVideoCodecState * state)
     return FALSE;
   if (!gst_vaapi_plugin_base_set_caps (plugin, decode->sinkpad_caps, NULL))
     return FALSE;
-  if (!gst_vaapidecode_reset_full (decode, decode->sinkpad_caps, TRUE))
+  if (!gst_vaapidecode_reset_full (decode, decode->sinkpad_caps, FALSE))
     return FALSE;
 
   return TRUE;
diff --git a/gst/vaapi/gstvaapidecode.h b/gst/vaapi/gstvaapidecode.h
index cb2a2f1d81..8c841901dd 100644
--- a/gst/vaapi/gstvaapidecode.h
+++ b/gst/vaapi/gstvaapidecode.h
@@ -45,7 +45,6 @@ struct _GstVaapiDecode {
     GstVaapiDecoder    *decoder;
     GMutex              surface_ready_mutex;
     GCond               surface_ready;
-    GstCaps            *decoder_caps;
     GstCaps            *allowed_sinkpad_caps;
     GstCaps            *allowed_srcpad_caps;
     guint               current_frame_size;
@@ -56,6 +55,8 @@ struct _GstVaapiDecode {
 
     GstVideoCodecState *input_state;
     GstSegment          in_segment;
+
+    gboolean            do_renego;
 };
 
 struct _GstVaapiDecodeClass {