Implement decoder reset on flush, rather than recreating

Clear decoders out on a flush but keep the same instance,
rather than completely recreating them. That avoids
unecessarily freeing and recreating surface pools
and contexts, which can be quite expensive

https://bugzilla.gnome.org/show_bug.cgi?id=781142

gst-libs/gst/vaapi/gstvaapidecoder.c

@@ -40,6 +40,26 @@
 
 static void drop_frame (GstVaapiDecoder * decoder, GstVideoCodecFrame * frame);
 
+static void
+parser_state_reset (GstVaapiParserState * ps)
+{
+
+  if (ps->input_adapter)
+    gst_adapter_clear (ps->input_adapter);
+  if (ps->output_adapter)
+    gst_adapter_clear (ps->output_adapter);
+  ps->current_adapter = NULL;
+
+  if (ps->next_unit_pending) {
+    gst_vaapi_decoder_unit_clear (&ps->next_unit);
+    ps->next_unit_pending = FALSE;
+  }
+
+  ps->current_frame_number = 0;
+  ps->input_offset1 = ps->input_offset2 = 0;
+  ps->at_eos = FALSE;
+}
+
 static void
 parser_state_finalize (GstVaapiParserState * ps)
 {
@@ -266,16 +286,6 @@ do_decode (GstVaapiDecoder * decoder, GstVideoCodecFrame * base_frame)
   return status;
 }
 
-static inline GstVaapiDecoderStatus
-do_flush (GstVaapiDecoder * decoder)
-{
-  GstVaapiDecoderClass *const klass = GST_VAAPI_DECODER_GET_CLASS (decoder);
-
-  if (klass->flush)
-    return klass->flush (decoder);
-  return GST_VAAPI_DECODER_STATUS_SUCCESS;
-}
-
 static GstVaapiDecoderStatus
 decode_step (GstVaapiDecoder * decoder)
 {
@@ -1029,13 +1039,69 @@ gst_vaapi_decoder_decode (GstVaapiDecoder * decoder, GstVideoCodecFrame * frame)
   return do_decode (decoder, frame);
 }
 
+/* This function really marks the end of input,
+ * so that the decoder will drain out any pending
+ * frames on calls to gst_vaapi_decoder_get_frame_with_timeout() */
 GstVaapiDecoderStatus
 gst_vaapi_decoder_flush (GstVaapiDecoder * decoder)
 {
+  GstVaapiDecoderClass *klass;
+
   g_return_val_if_fail (decoder != NULL,
       GST_VAAPI_DECODER_STATUS_ERROR_INVALID_PARAMETER);
 
-  return do_flush (decoder);
+  klass = GST_VAAPI_DECODER_GET_CLASS (decoder);
+
+  if (klass->flush)
+    return klass->flush (decoder);
+
+  return GST_VAAPI_DECODER_STATUS_SUCCESS;
+}
+
+/* Reset the decoder instance to a clean state,
+ * clearing any pending decode state, without
+ * reallocating the entire decoder */
+GstVaapiDecoderStatus
+gst_vaapi_decoder_reset (GstVaapiDecoder * decoder)
+{
+  GstVaapiDecoderClass *klass;
+  GstVaapiDecoderStatus ret = GST_VAAPI_DECODER_STATUS_SUCCESS;
+
+  g_return_val_if_fail (decoder != NULL,
+      GST_VAAPI_DECODER_STATUS_ERROR_INVALID_PARAMETER);
+
+  klass = GST_VAAPI_DECODER_GET_CLASS (decoder);
+
+  GST_DEBUG ("Resetting decoder");
+
+  if (klass->reset) {
+    ret = klass->reset (decoder);
+  } else {
+    if (klass->destroy)
+      klass->destroy (decoder);
+    if (klass->create)
+      if (!klass->create (decoder))
+        ret = GST_VAAPI_DECODER_STATUS_ERROR_UNKNOWN;
+  }
+
+  if (ret != GST_VAAPI_DECODER_STATUS_SUCCESS)
+    return ret;
+
+  /* Clear any buffers and frame in the queues */
+  {
+    GstVideoCodecFrame *frame;
+    GstBuffer *buffer;
+
+    while ((frame = g_async_queue_try_pop (decoder->frames)) != NULL)
+      gst_video_codec_frame_unref (frame);
+
+    while ((buffer = g_async_queue_try_pop (decoder->buffers)) != NULL)
+      gst_buffer_unref (buffer);
+  }
+
+  parser_state_reset (&decoder->parser_state);
+
+  return GST_VAAPI_DECODER_STATUS_SUCCESS;
 }
 
 GstVaapiDecoderStatus

gst-libs/gst/vaapi/gstvaapidecoder.h

@@ -132,6 +132,9 @@ gst_vaapi_decoder_decode (GstVaapiDecoder * decoder,
 GstVaapiDecoderStatus
 gst_vaapi_decoder_flush (GstVaapiDecoder * decoder);
 
+GstVaapiDecoderStatus
+gst_vaapi_decoder_reset (GstVaapiDecoder * decoder);
+
 GstVaapiDecoderStatus
 gst_vaapi_decoder_check_status (GstVaapiDecoder * decoder);
 

gst-libs/gst/vaapi/gstvaapidecoder_h264.c

@@ -1221,10 +1221,11 @@ gst_vaapi_decoder_h264_destroy (GstVaapiDecoder * base_decoder)
   guint i;
 
   gst_vaapi_decoder_h264_close (decoder);
+  priv->is_opened = FALSE;
 
   g_free (priv->dpb);
   priv->dpb = NULL;
-  priv->dpb_size = 0;
+  priv->dpb_size_max = priv->dpb_size = 0;
 
   g_free (priv->prev_ref_frames);
   priv->prev_ref_frames = NULL;

gst-libs/gst/vaapi/gstvaapidecoder_h265.c

@@ -957,6 +957,8 @@ gst_vaapi_decoder_h265_close (GstVaapiDecoderH265 * decoder)
     gst_h265_parser_free (priv->parser);
     priv->parser = NULL;
   }
+
+  priv->is_opened = FALSE;
 }
 
 static gboolean
@@ -982,7 +984,8 @@ gst_vaapi_decoder_h265_destroy (GstVaapiDecoder * base_decoder)
   gst_vaapi_decoder_h265_close (decoder);
   g_free (priv->dpb);
   priv->dpb = NULL;
-  priv->dpb_size = 0;
+  priv->dpb_count = priv->dpb_size_max = priv->dpb_size = 0;
+
   for (i = 0; i < G_N_ELEMENTS (priv->pps); i++)
     gst_vaapi_parser_info_h265_replace (&priv->pps[i], NULL);
   gst_vaapi_parser_info_h265_replace (&priv->active_pps, NULL);

gst-libs/gst/vaapi/gstvaapidecoder_mpeg2.c

@@ -317,6 +317,8 @@ gst_vaapi_decoder_mpeg2_close (GstVaapiDecoderMpeg2 * decoder)
   priv->state = 0;
 
   gst_vaapi_dpb_replace (&priv->dpb, NULL);
+
+  priv->is_opened = FALSE;
 }
 
 static gboolean

gst-libs/gst/vaapi/gstvaapidecoder_priv.h

@@ -225,6 +225,7 @@ struct _GstVaapiDecoderClass
       struct _GstVaapiDecoderUnit * unit);
   GstVaapiDecoderStatus (*end_frame) (GstVaapiDecoder * decoder);
   GstVaapiDecoderStatus (*flush) (GstVaapiDecoder * decoder);
+  GstVaapiDecoderStatus (*reset) (GstVaapiDecoder * decoder);
   GstVaapiDecoderStatus (*decode_codec_data) (GstVaapiDecoder * decoder,
       const guchar * buf, guint buf_size);
 };

gst-libs/gst/vaapi/gstvaapidecoder_vc1.c

@@ -67,7 +67,6 @@ struct _GstVaapiDecoderVC1Private
   guint8 rndctrl;
   guint rbdu_buffer_size;
   guint is_opened:1;
-  guint is_first_field:1;
   guint has_codec_data:1;
   guint has_entrypoint:1;
   guint size_changed:1;
@@ -134,6 +133,7 @@ gst_vaapi_decoder_vc1_close (GstVaapiDecoderVC1 * decoder)
     gst_vc1_bitplanes_free (priv->bitplanes);
     priv->bitplanes = NULL;
   }
+  priv->is_opened = FALSE;
 }
 
 static gboolean
@@ -150,6 +150,11 @@ gst_vaapi_decoder_vc1_open (GstVaapiDecoderVC1 * decoder)
   priv->bitplanes = gst_vc1_bitplanes_new ();
   if (!priv->bitplanes)
     return FALSE;
+
+  memset (&priv->seq_hdr, 0, sizeof (GstVC1SeqHdr));
+  memset (&priv->entrypoint_hdr, 0, sizeof (GstVC1EntryPointHdr));
+  memset (&priv->frame_hdr, 0, sizeof (GstVC1FrameHdr));
+
   return TRUE;
 }
 
@@ -174,8 +179,13 @@ gst_vaapi_decoder_vc1_create (GstVaapiDecoder * base_decoder)
   GstVaapiDecoderVC1 *const decoder = GST_VAAPI_DECODER_VC1_CAST (base_decoder);
   GstVaapiDecoderVC1Private *const priv = &decoder->priv;
 
+  priv->has_codec_data = priv->has_entrypoint =
+      priv->size_changed = priv->profile_changed =
+      priv->closed_entry = priv->broken_link = FALSE;
+
   priv->profile = (GstVaapiProfile) 0;
   priv->rndctrl = 0;
+  priv->width = priv->height = 0;
   return TRUE;
 }
 
@@ -1357,6 +1367,9 @@ gst_vaapi_decoder_vc1_start_frame (GstVaapiDecoder * base_decoder,
     GST_ERROR ("failed to reset context");
     return status;
   }
+  status = ensure_decoder (decoder);
+  if (status != GST_VAAPI_DECODER_STATUS_SUCCESS)
+    return status;
 
   picture = GST_VAAPI_PICTURE_NEW (VC1, decoder);
   if (!picture) {

gst/vaapi/gstvaapidecode.c

@@ -953,20 +953,21 @@ gst_vaapidecode_destroy (GstVaapiDecode * decode)
 }
 
 static gboolean
-gst_vaapidecode_reset_full (GstVaapiDecode * decode, GstCaps * caps,
+gst_vaapidecode_reset (GstVaapiDecode * decode, GstCaps * caps,
-    gboolean hard)
+    gboolean force_reset)
 {
   /* Reset tracked frame size */
   decode->current_frame_size = 0;
 
-  if (!hard && decode->decoder) {
+  if (decode->decoder) {
     if (gst_vaapi_decoder_update_caps (decode->decoder, caps)) {
       g_atomic_int_set (&decode->do_renego, TRUE);
-      return TRUE;
+      if (!force_reset)
+        return TRUE;
     }
+    return gst_vaapi_decoder_reset (decode->decoder);
   }
 
-  gst_vaapidecode_destroy (decode);
   return gst_vaapidecode_create (decode, caps);
 }
 
@@ -1047,8 +1048,6 @@ static gboolean
 gst_vaapidecode_flush (GstVideoDecoder * vdec)
 {
   GstVaapiDecode *const decode = GST_VAAPIDECODE (vdec);
-  gboolean reverse;
-
   if (!decode->decoder)
     return FALSE;
 
@@ -1056,13 +1055,9 @@ gst_vaapidecode_flush (GstVideoDecoder * vdec)
 
   gst_vaapidecode_purge (decode);
 
-  /* in reverse playback we cannot destroy the decoder at flush, since
+  /* There could be issues if we avoid the reset() while doing
-   * it will lost the parsing state */
-  reverse = decode->in_segment.rate < 0;
-
-  /* There could be issues if we avoid the reset_full() while doing
    * seeking: we have to reset the internal state */
-  return gst_vaapidecode_reset_full (decode, decode->sinkpad_caps, !reverse);
+  return gst_vaapidecode_reset (decode, decode->sinkpad_caps, TRUE);
 }
 
 static gboolean
@@ -1077,7 +1072,7 @@ gst_vaapidecode_set_format (GstVideoDecoder * vdec, GstVideoCodecState * state)
     return FALSE;
   if (!gst_vaapi_plugin_base_set_caps (plugin, decode->sinkpad_caps, NULL))
     return FALSE;
-  if (!gst_vaapidecode_reset_full (decode, decode->sinkpad_caps, FALSE))
+  if (!gst_vaapidecode_reset (decode, decode->sinkpad_caps, FALSE))
     return FALSE;
 
   return TRUE;